From 17a1210a89fabca5e9c845029466ead2637af482 Mon Sep 17 00:00:00 2001 From: Daniel Petranek Date: Tue, 13 Jun 2023 12:18:38 -0500 Subject: [PATCH 01/11] allow inserting `false` objects during update --- src/fluree/db/query/exec/update.cljc | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/fluree/db/query/exec/update.cljc b/src/fluree/db/query/exec/update.cljc index d568418f1..97b06ed0f 100644 --- a/src/fluree/db/query/exec/update.cljc +++ b/src/fluree/db/query/exec/update.cljc @@ -67,16 +67,13 @@ p (::where/val p-mch) o (::where/val o-mch) dt (::where/datatype o-mch)] - (when (and s p o dt) + (when (and (some? s) (some? p) (some? o) (some? dt)) (let [s* (if-not (number? s) (! error-ch e))))) From 4c15f2719ac84f0b966e3abfd64becf83dad4cb8 Mon Sep 17 00:00:00 2001 From: Daniel Petranek Date: Tue, 13 Jun 2023 12:19:08 -0500 Subject: [PATCH 02/11] allow selecting false objects during subgraph crawl --- src/fluree/db/query/json_ld/response.cljc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fluree/db/query/json_ld/response.cljc b/src/fluree/db/query/json_ld/response.cljc index bcafb5214..0508f541b 100644 --- a/src/fluree/db/query/json_ld/response.cljc +++ b/src/fluree/db/query/json_ld/response.cljc @@ -139,7 +139,7 @@ (not (#{:list :set} (-> context (get p-iri) :container)))) (first acc) acc))))] - (if v + (if (some? v) (recur r (assoc acc p-iri v)) (recur r acc))) (if reverse From 6f7c54c1b532e50081a67ff4f5d6534c84636ac8 Mon Sep 17 00:00:00 2001 From: Daniel Petranek Date: Wed, 14 Jun 2023 10:17:17 -0500 Subject: [PATCH 03/11] add sparql string, numeric, datetime, and hash functions https://www.w3.org/TR/sparql11-query/#SparqlOps 17.4.3 17.4.4 17.4.5 17.4.6 These had existing implementations that needed to be changed to comply: - rand: before it selected a random element from a collection, now returns a random number between 0 and 1. - now: before it returned an integer epoch milliseconds, now returns an ISO8601 datetime string - subStr before it took a start and optionally an end arg, now it takes a start and optionally a length arg These haven't been implemented: - md5, sha1, sha386: We don't have implementations for these and I didn't want to add a dependency for them. Figure we can add later at user request. - encode-for-uri: This seems more complex and I want to see if there's a straightforward way to get it without manually implementing https://www.ietf.org/rfc/rfc3986.txt - langMatches We don't currently support language tags --- src/fluree/db/query/exec/eval.cljc | 199 +++++++++++++++++++----- test/fluree/db/query/fql_test.clj | 4 +- test/fluree/db/transact/update_test.clj | 199 ++++++++++++++++++++++++ 3 files changed, 363 insertions(+), 39 deletions(-) diff --git a/src/fluree/db/query/exec/eval.cljc b/src/fluree/db/query/exec/eval.cljc index fb53c6c2a..67bb7c452 100644 --- a/src/fluree/db/query/exec/eval.cljc +++ b/src/fluree/db/query/exec/eval.cljc @@ -1,11 +1,15 @@ (ns fluree.db.query.exec.eval - (:refer-clojure :exclude [compile rand]) + (:refer-clojure :exclude [compile rand concat replace]) (:require [fluree.db.query.exec.group :as group] [fluree.db.query.exec.where :as where] [fluree.db.util.log :as log] [clojure.set :as set] [clojure.string :as str] - [clojure.walk :refer [postwalk]]) + [clojure.walk :refer [postwalk]] + [clojure.math :as math] + [fluree.db.datatype :as datatype] + [fluree.crypto :as crypto] + [fluree.db.constants :as const]) #?(:clj (:import (java.time Instant)))) (defn sum @@ -54,13 +58,7 @@ (> n 0) (-> n int) (< n 0) (-> n int dec))) -(def groupconcat concat) - -(defn rand - ([coll] - (rand-nth coll)) - ([n coll] - (vec (repeatedly n #(rand-nth coll))))) +(def groupconcat clojure.core/concat) (defn sample [n coll] @@ -106,7 +104,7 @@ (defn now [] - #?(:clj (.toEpochMilli (Instant/now)))) + #?(:clj (str (Instant/now)))) (defn strStarts [s substr] @@ -117,16 +115,166 @@ (str/ends-with? s substr)) (defn subStr - [s start end] - (subs s start end)) + ([s start] + (subStr s start (count s))) + ([s start length] + ;; The index of the first character in a string is 1. + (let [start (dec start)] + (subs s start (min (+ start length) + (count s)))))) + +(defn strLen + [s] + (count s)) + +(defn ucase + [s] + (str/upper-case s)) + +(defn lcase + [s] + (str/lower-case s)) + +(defn contains + [s substr] + (str/includes? s substr)) + +(defn strBefore + [s substr] + (-> (str/split s (re-pattern substr)) + first + str)) + +(defn strAfter + [s substr] + (-> (str/split s (re-pattern substr)) + last + str)) + +(defn concat + [& strs] + (apply str strs)) + +(defn regex + [text pattern] + (boolean (re-find (re-pattern pattern) text))) + +(defn replace + [s pattern replacement] + (str/replace s (re-pattern pattern) replacement)) + +(defn rand + [] + (clojure.core/rand)) + +(defn year + [datetime-string] + (let [datetime (datatype/coerce datetime-string const/$xsd:dateTime)] + (.getYear datetime))) + +(defn month + [datetime-string] + (let [datetime (datatype/coerce datetime-string const/$xsd:dateTime)] + #?(:clj (.getMonthValue datetime) + :cljs (.getMonth datetime)))) + +(defn day + [datetime-string] + (let [datetime (datatype/coerce datetime-string const/$xsd:dateTime)] + #?(:clj (.getDayOfMonth datetime) + :cljs (.getDate datetime)))) + +(defn hours + [datetime-string] + (let [datetime (datatype/coerce datetime-string const/$xsd:dateTime)] + #?(:clj (.getHour datetime) + :cljs (.getHours datetime)))) + +(defn minutes + [datetime-string] + (let [datetime (datatype/coerce datetime-string const/$xsd:dateTime)] + #?(:clj (.getMinute datetime) + :cljs (.getMinutes datetime)))) + +(defn seconds + [datetime-string] + (let [datetime (datatype/coerce datetime-string const/$xsd:dateTime)] + #?(:clj (.getSecond datetime) + :cljs (.getSeconds datetime)))) + +(defn tz + [datetime-string] + (let [datetime (datatype/coerce datetime-string const/$xsd:dateTime)] + #?(:clj (.toString (.getOffset datetime)) + :cljs (.getTimeZoneOffset datetime)))) + +(defn sha256 + [x] + (crypto/sha2-256 x)) + +(defn sha512 + [x] + (crypto/sha2-512 x)) (def allowed-scalar-fns - '#{abs && || ! > < >= <= = + - * / quot and bound coalesce if nil? - not not= now or re-find re-pattern strStarts strEnds subStr}) + '#{ && || ! > < >= <= = + - * / quot and bound coalesce if nil? + not not= or re-find re-pattern + ;; string fns + strStarts strEnds subStr strLen ucase lcase contains strBefore strAfter concat regex replace + ;; numeric fns + abs round ceil floor rand + ;; datetime fns + now year month day hours minutes seconds tz + ;; hash fns + sha256 sha512}) (def allowed-symbols (set/union allowed-aggregate-fns allowed-scalar-fns)) +(def qualified-symbols + '{ + ! fluree.db.query.exec.eval/! + || fluree.db.query.exec.eval/|| + && fluree.db.query.exec.eval/&& + abs clojure.core/abs + avg fluree.db.query.exec.eval/avg + bound fluree.db.query.exec.eval/bound + ceil fluree.db.query.exec.eval/ceil + coalesce fluree.db.query.exec.eval/coalesce + concat fluree.db.query.exec.eval/concat + contains fluree.db.query.exec.eval/contains + count fluree.db.query.exec.eval/count-distinct + floor fluree.db.query.exec.eval/floor + groupconcat fluree.db.query.exec.eval/groupconcat + lcase fluree.db.query.exec.eval/lcase + median fluree.db.query.exec.eval/median + now fluree.db.query.exec.eval/now + rand fluree.db.query.exec.eval/rand + regex fluree.db.query.exec.eval/regex + replace fluree.db.query.exec.eval/replace + round clojure.math/round + sample fluree.db.query.exec.eval/sample + stddev fluree.db.query.exec.eval/stddev + strAfter fluree.db.query.exec.eval/strAfter + strBefore fluree.db.query.exec.eval/strBefore + strEnds fluree.db.query.exec.eval/strEnds + strLen fluree.db.query.exec.eval/strLen + strStarts fluree.db.query.exec.eval/strStarts + subStr fluree.db.query.exec.eval/subStr + sum fluree.db.query.exec.eval/sum + ucase fluree.db.query.exec.eval/ucase + variance fluree.db.query.exec.eval/variance + year fluree.db.query.exec.eval/year + month fluree.db.query.exec.eval/month + day fluree.db.query.exec.eval/day + hours fluree.db.query.exec.eval/hours + minutes fluree.db.query.exec.eval/minutes + seconds fluree.db.query.exec.eval/seconds + tz fluree.db.query.exec.eval/tz + sha256 fluree.db.query.exec.eval/sha256 + sha512 fluree.db.query.exec.eval/sha512 + }) + (defn variable? [sym] (and (symbol? sym) @@ -153,29 +301,6 @@ symbols (filter variable?))) -(def qualified-symbols - '{abs fluree.db.query.exec.eval/abs - avg fluree.db.query.exec.eval/avg - bound fluree.db.query.exec.eval/bound - ceil fluree.db.query.exec.eval/ceil - coalesce fluree.db.query.exec.eval/coalesce - count fluree.db.query.exec.eval/count-distinct - floor fluree.db.query.exec.eval/floor - groupconcat fluree.db.query.exec.eval/groupconcat - median fluree.db.query.exec.eval/median - now fluree.db.query.exec.eval/now - rand fluree.db.query.exec.eval/rand - sample fluree.db.query.exec.eval/sample - stddev fluree.db.query.exec.eval/stddev - strStarts fluree.db.query.exec.eval/strStarts - strEnds fluree.db.query.exec.eval/strEnds - subStr fluree.db.query.exec.eval/subStr - sum fluree.db.query.exec.eval/sum - variance fluree.db.query.exec.eval/variance - ! fluree.db.query.exec.eval/! - && fluree.db.query.exec.eval/&& - || fluree.db.query.exec.eval/||}) - (defn qualify [sym allow-aggregates?] (let [allowed-fns (if allow-aggregates? diff --git a/test/fluree/db/query/fql_test.clj b/test/fluree/db/query/fql_test.clj index 3e0f35e6f..dcd2b5458 100644 --- a/test/fluree/db/query/fql_test.clj +++ b/test/fluree/db/query/fql_test.clj @@ -146,7 +146,7 @@ :where [[?s :schema/age ?age] {:bind {?decadesOld (quot ?age 10)}} [?s :schema/name ?name] - {:bind {?firstLetterOfName (subStr ?name 0 1)}}] + {:bind {?firstLetterOfName (subStr ?name 1 1)}}] :order-by ?firstLetterOfName} res @(fluree/query db q)] (is (= [["A" "Alice" 5] @@ -159,7 +159,7 @@ (let [q '{:select [?firstLetterOfName ?name ?canVote] :where [[?s :schema/age ?age] [?s :schema/name ?name] - {:bind {?firstLetterOfName (subStr ?name 0 1) + {:bind {?firstLetterOfName (subStr ?name 1 1) ?canVote (>= ?age 18)}}] :order-by ?name} res @(fluree/query db q)] diff --git a/test/fluree/db/transact/update_test.clj b/test/fluree/db/transact/update_test.clj index 1a2c740af..146635063 100644 --- a/test/fluree/db/transact/update_test.clj +++ b/test/fluree/db/transact/update_test.clj @@ -120,3 +120,202 @@ '{:select {?s [:*]} :where [[?s :id :ex/jane]]})) "Jane's age should now be updated to 31 (from 30)."))))) + +(deftest transaction-functions + (let [conn @(fluree/connect {:method :memory}) + ledger @(fluree/create conn "functions" {:defaultContext [test-utils/default-str-context + {"ex" "http://example.com/"}]}) + db0 (fluree/db ledger) + db1 @(fluree/stage db0 [{"id" "ex:create-predicates" + ;; string functions + "ex:strLen" 0 + "ex:subStr" 0 + "ex:ucase" 0 + "ex:lcase" 0 + "ex:strStarts" 0 + "ex:strEnds" 0 + "ex:contains" 0 + "ex:strBefore" 0 + "ex:strAfter" 0 + "ex:encodeForUri" 0 + "ex:concat" 0 + "ex:langMatches" 0 + "ex:regex" 0 + "ex:replace" 0 + ;; numeric functions + "ex:abs" 0 + "ex:round" 0 + "ex:ceil" 0 + "ex:floor" 0 + "ex:rand" 0 + ;; date/time functions + "ex:now" 0 + "ex:year" 0 + "ex:month" 0 + "ex:day" 0 + "ex:hours" 0 + "ex:minutes" 0 + "ex:seconds" 0 + "ex:timezone" 0 + "ex:tz" 0 + ;; hash functions + "ex:md5" 0 + "ex:sha1" 0 + "ex:sha256" 0 + "ex:sha384" 0 + "ex:sha512" 0}])] + + (testing "hash functions" + (with-redefs [fluree.db.query.exec.eval/now (fn [] "2023-06-13T19:53:57.234345Z")] + (let [updated (-> @(fluree/stage db1 {"id" "ex:hash-fns" + "ex:message" "abc"}) + (fluree/stage {"delete" [] + "where" [["?s" "id" "ex:hash-fns"] + ["?s" "ex:message" "?message"] + {"bind" {"?sha256" "(sha256 ?message)"}} + {"bind" {"?sha512" "(sha512 ?message)"}}] + "insert" [["?s" "ex:sha256" "?sha256"] + ["?s" "ex:sha512" "?sha512"]]}))] + (is (= {"ex:sha512" "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f" + "ex:sha256" "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"} + @(fluree/query @updated {"where" [["?s" "id" "ex:hash-fns"]] + "selectOne" {"?s" ["ex:sha512" + "ex:sha256"]}})))))) + (testing "datetime functions" + (with-redefs [fluree.db.query.exec.eval/now (fn [] "2023-06-13T19:53:57.234345Z")] + (let [updated (-> @(fluree/stage db1 {"id" "ex:datetime-fns" + "ex:localdatetime" "2023-06-13T14:17:22.435" + "ex:offsetdatetime" "2023-06-13T14:17:22.435-05:00" + "ex:utcdatetime" "2023-06-13T14:17:22.435Z"}) + (fluree/stage {"delete" [] + "where" [["?s" "id" "ex:datetime-fns"] + ["?s" "ex:localdatetime" "?localdatetime"] + ["?s" "ex:offsetdatetime" "?offsetdatetime"] + ["?s" "ex:utcdatetime" "?utcdatetime"] + {"bind" {"?now" "(now)"}} + {"bind" {"?year" "(year ?localdatetime)"}} + {"bind" {"?month" "(month ?localdatetime)"}} + {"bind" {"?day" "(day ?localdatetime)"}} + {"bind" {"?hours" "(hours ?localdatetime)"}} + {"bind" {"?minutes" "(minutes ?localdatetime)"}} + {"bind" {"?seconds" "(seconds ?localdatetime)"}} + {"bind" {"?tz1" "(tz ?utcdatetime)"}} + {"bind" {"?tz2" "(tz ?offsetdatetime)"}}] + "insert" [["?s" "ex:now" "?now"] + ["?s" "ex:year" "?year"] + ["?s" "ex:month" "?month"] + ["?s" "ex:day" "?day"] + ["?s" "ex:hours" "?hours"] + ["?s" "ex:minutes" "?minutes"] + ["?s" "ex:seconds" "?seconds"] + ["?s" "ex:tz" "?tz1"] + ["?s" "ex:tz" "?tz2"]]}))] + (is (= {"ex:now" "2023-06-13T19:53:57.234345Z" + "ex:year" 2023 + "ex:month" 6 + "ex:day" 13 + "ex:hours" 14 + "ex:minutes" 17 + "ex:seconds" 22 + "ex:tz" ["-05:00" "Z"]} + @(fluree/query @updated {"where" [["?s" "id" "ex:datetime-fns"]] + "selectOne" {"?s" ["ex:now" + "ex:year" + "ex:month" + "ex:day" + "ex:hours" + "ex:minutes" + "ex:seconds" + "ex:tz"]}})))))) + + (testing "numeric functions" + (let [updated (-> @(fluree/stage db1 {"id" "ex:numeric-fns" + "ex:pos-int" 2 + "ex:neg-int" -2 + "ex:decimal" 1.4}) + (fluree/stage {"delete" [] + "where" [["?s" "id" "ex:numeric-fns"] + ["?s" "ex:pos-int" "?pos-int"] + ["?s" "ex:neg-int" "?neg-int"] + ["?s" "ex:decimal" "?decimal"] + {"bind" {"?abs" "(abs ?neg-int)"}} + {"bind" {"?round" "(round ?decimal)"}} + {"bind" {"?ceil" "(ceil ?decimal)"}} + {"bind" {"?floor" "(floor ?decimal)"}} + {"bind" {"?rand" "(rand)"}}] + "insert" [["?s" "ex:abs" "?abs"] + ["?s" "ex:round" "?round"] + ["?s" "ex:ceil" "?ceil"] + ["?s" "ex:floor" "?floor"] + ["?s" "ex:rand" "?rand"]]}))] + (is (= {"ex:abs" 2 + "ex:round" 1 + "ex:ceil" 2 + "ex:floor" 1} + @(fluree/query @updated {"where" [["?s" "id" "ex:numeric-fns"]] + "selectOne" {"?s" ["ex:abs" + "ex:round" + "ex:ceil" + "ex:floor"]}}))) + (is (pos? @(fluree/query @updated {"where" [["?s" "id" "ex:numeric-fns"] + ["?s" "ex:rand" "?rand"]] + "selectOne" "?rand"}))))) + + (testing "string functions" + (let [updated (-> @(fluree/stage db1 {"id" "ex:string-fns" "ex:text" "Abcdefg"}) + (fluree/stage {"delete" [] + "where" [["?s" "id" "ex:string-fns"] + ["?s" "ex:text" "?text"] + {"bind" {"?strlen" "(strLen ?text)"}} + {"bind" {"?sub1" "(subStr ?text 5)"}} + {"bind" {"?sub2" "(subStr ?text 1 4)"}} + {"bind" {"?upcased" "(ucase ?text)"}} + {"bind" {"?downcased" "(lcase ?text)"}} + {"bind" {"?a-start" "(strStarts ?text \"x\")"}} + {"bind" {"?a-end" "(strEnds ?text \"x\")"}} + {"bind" {"?contains" "(contains ?text \"x\")"}} + {"bind" {"?strBefore" "(strBefore ?text \"bcd\")"}} + {"bind" {"?strAfter" "(strAfter ?text \"bcd\")"}} + {"bind" {"?concatted" "(concat ?text \" \" \"STR1 \" \"STR2\")"}} + {"bind" {"?matched" "(regex ?text \"^Abc\")"}}] + "insert" [["?s" "ex:strStarts" "?a-start"] + ["?s" "ex:strEnds" "?a-end"] + ["?s" "ex:subStr" "?sub1"] + ["?s" "ex:subStr" "?sub2"] + ["?s" "ex:strLen" "?strlen"] + ["?s" "ex:ucase" "?upcased"] + ["?s" "ex:lcase" "?downcased"] + ["?s" "ex:contains" "?contains"] + ["?s" "ex:strBefore" "?strBefore"] + ["?s" "ex:strAfter" "?strAfter"] + ["?s" "ex:concat" "?concatted"] + ["?s" "ex:regex" "?matched"]]}))] + (is (= {"ex:strEnds" false + "ex:strStarts" false + "ex:contains" false + "ex:regex" true + "ex:subStr" ["Abcd" "efg"] + "ex:strLen" 7 + "ex:ucase" "ABCDEFG" + "ex:lcase" "abcdefg" + "ex:strBefore" "A" + "ex:strAfter" "efg" + "ex:concat" "Abcdefg STR1 STR2"} + @(fluree/query @updated {"where" [["?s" "id" "ex:string-fns"]] + "selectOne" {"?s" ["ex:strLen" + "ex:subStr" + "ex:ucase" + "ex:lcase" + "ex:strStarts" + "ex:strEnds" + "ex:contains" + "ex:strBefore" + "ex:strAfter" + "ex:encodeForUri" + "ex:concat" + "ex:langMatches" + "ex:regex" + "ex:replace"]}}))))) + + #_(testing "functional forms") + #_(testing "scalar functions"))) From 1b9868bba48690e861dbe8cae669b32183f76e43 Mon Sep 17 00:00:00 2001 From: Daniel Petranek Date: Wed, 14 Jun 2023 11:01:13 -0500 Subject: [PATCH 04/11] add reflection warnings and type hints --- src/fluree/db/query/exec/eval.cljc | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/fluree/db/query/exec/eval.cljc b/src/fluree/db/query/exec/eval.cljc index 67bb7c452..74170c944 100644 --- a/src/fluree/db/query/exec/eval.cljc +++ b/src/fluree/db/query/exec/eval.cljc @@ -10,7 +10,10 @@ [fluree.db.datatype :as datatype] [fluree.crypto :as crypto] [fluree.db.constants :as const]) - #?(:clj (:import (java.time Instant)))) + #?(:clj (:import (java.time Instant) + (java.time OffsetDateTime LocalDateTime)))) + +#?(:clj (set! *warn-on-reflection* true)) (defn sum [coll] @@ -169,42 +172,42 @@ (defn year [datetime-string] - (let [datetime (datatype/coerce datetime-string const/$xsd:dateTime)] + (let [datetime ^LocalDateTime (datatype/coerce datetime-string const/$xsd:dateTime)] (.getYear datetime))) (defn month [datetime-string] - (let [datetime (datatype/coerce datetime-string const/$xsd:dateTime)] + (let [datetime ^LocalDateTime (datatype/coerce datetime-string const/$xsd:dateTime)] #?(:clj (.getMonthValue datetime) :cljs (.getMonth datetime)))) (defn day [datetime-string] - (let [datetime (datatype/coerce datetime-string const/$xsd:dateTime)] + (let [datetime ^LocalDateTime (datatype/coerce datetime-string const/$xsd:dateTime)] #?(:clj (.getDayOfMonth datetime) :cljs (.getDate datetime)))) (defn hours [datetime-string] - (let [datetime (datatype/coerce datetime-string const/$xsd:dateTime)] + (let [datetime ^LocalDateTime (datatype/coerce datetime-string const/$xsd:dateTime)] #?(:clj (.getHour datetime) :cljs (.getHours datetime)))) (defn minutes [datetime-string] - (let [datetime (datatype/coerce datetime-string const/$xsd:dateTime)] + (let [datetime ^LocalDateTime (datatype/coerce datetime-string const/$xsd:dateTime)] #?(:clj (.getMinute datetime) :cljs (.getMinutes datetime)))) (defn seconds [datetime-string] - (let [datetime (datatype/coerce datetime-string const/$xsd:dateTime)] + (let [datetime ^LocalDateTime (datatype/coerce datetime-string const/$xsd:dateTime)] #?(:clj (.getSecond datetime) :cljs (.getSeconds datetime)))) (defn tz [datetime-string] - (let [datetime (datatype/coerce datetime-string const/$xsd:dateTime)] + (let [datetime ^OffsetDateTime (datatype/coerce datetime-string const/$xsd:dateTime)] #?(:clj (.toString (.getOffset datetime)) :cljs (.getTimeZoneOffset datetime)))) From 913f051188a0aa55054cf547b793e96d869101e4 Mon Sep 17 00:00:00 2001 From: Daniel Petranek Date: Wed, 14 Jun 2023 13:56:18 -0500 Subject: [PATCH 05/11] move test predicate creation into individual tests --- test/fluree/db/transact/update_test.clj | 99 +++++++------------------ 1 file changed, 28 insertions(+), 71 deletions(-) diff --git a/test/fluree/db/transact/update_test.clj b/test/fluree/db/transact/update_test.clj index 146635063..b4bf64619 100644 --- a/test/fluree/db/transact/update_test.clj +++ b/test/fluree/db/transact/update_test.clj @@ -125,50 +125,14 @@ (let [conn @(fluree/connect {:method :memory}) ledger @(fluree/create conn "functions" {:defaultContext [test-utils/default-str-context {"ex" "http://example.com/"}]}) - db0 (fluree/db ledger) - db1 @(fluree/stage db0 [{"id" "ex:create-predicates" - ;; string functions - "ex:strLen" 0 - "ex:subStr" 0 - "ex:ucase" 0 - "ex:lcase" 0 - "ex:strStarts" 0 - "ex:strEnds" 0 - "ex:contains" 0 - "ex:strBefore" 0 - "ex:strAfter" 0 - "ex:encodeForUri" 0 - "ex:concat" 0 - "ex:langMatches" 0 - "ex:regex" 0 - "ex:replace" 0 - ;; numeric functions - "ex:abs" 0 - "ex:round" 0 - "ex:ceil" 0 - "ex:floor" 0 - "ex:rand" 0 - ;; date/time functions - "ex:now" 0 - "ex:year" 0 - "ex:month" 0 - "ex:day" 0 - "ex:hours" 0 - "ex:minutes" 0 - "ex:seconds" 0 - "ex:timezone" 0 - "ex:tz" 0 - ;; hash functions - "ex:md5" 0 - "ex:sha1" 0 - "ex:sha256" 0 - "ex:sha384" 0 - "ex:sha512" 0}])] + db1 (fluree/db ledger)] (testing "hash functions" (with-redefs [fluree.db.query.exec.eval/now (fn [] "2023-06-13T19:53:57.234345Z")] - (let [updated (-> @(fluree/stage db1 {"id" "ex:hash-fns" - "ex:message" "abc"}) + (let [updated (-> @(fluree/stage db1 [{"id" "ex:create-predicates" + "ex:md5" 0 "ex:sha1" 0 "ex:sha256" 0 "ex:sha384" 0 "ex:sha512" 0} + {"id" "ex:hash-fns" + "ex:message" "abc"}]) (fluree/stage {"delete" [] "where" [["?s" "id" "ex:hash-fns"] ["?s" "ex:message" "?message"] @@ -183,10 +147,13 @@ "ex:sha256"]}})))))) (testing "datetime functions" (with-redefs [fluree.db.query.exec.eval/now (fn [] "2023-06-13T19:53:57.234345Z")] - (let [updated (-> @(fluree/stage db1 {"id" "ex:datetime-fns" - "ex:localdatetime" "2023-06-13T14:17:22.435" - "ex:offsetdatetime" "2023-06-13T14:17:22.435-05:00" - "ex:utcdatetime" "2023-06-13T14:17:22.435Z"}) + (let [updated (-> @(fluree/stage db1 [{"id" "ex:create-predicates" + "ex:now" 0 "ex:year" 0 "ex:month" 0 "ex:day" 0 "ex:hours" 0 + "ex:minutes" 0 "ex:seconds" 0 "ex:timezone" 0 "ex:tz" 0} + {"id" "ex:datetime-fns" + "ex:localdatetime" "2023-06-13T14:17:22.435" + "ex:offsetdatetime" "2023-06-13T14:17:22.435-05:00" + "ex:utcdatetime" "2023-06-13T14:17:22.435Z"}]) (fluree/stage {"delete" [] "where" [["?s" "id" "ex:datetime-fns"] ["?s" "ex:localdatetime" "?localdatetime"] @@ -219,20 +186,16 @@ "ex:seconds" 22 "ex:tz" ["-05:00" "Z"]} @(fluree/query @updated {"where" [["?s" "id" "ex:datetime-fns"]] - "selectOne" {"?s" ["ex:now" - "ex:year" - "ex:month" - "ex:day" - "ex:hours" - "ex:minutes" - "ex:seconds" + "selectOne" {"?s" ["ex:now" "ex:year" "ex:month" "ex:day" "ex:hours" "ex:minutes" "ex:seconds" "ex:tz"]}})))))) (testing "numeric functions" - (let [updated (-> @(fluree/stage db1 {"id" "ex:numeric-fns" - "ex:pos-int" 2 - "ex:neg-int" -2 - "ex:decimal" 1.4}) + (let [updated (-> @(fluree/stage db1 [{"id" "ex:create-predicates" + "ex:abs" 0 "ex:round" 0 "ex:ceil" 0 "ex:floor" 0 "ex:rand" 0} + {"id" "ex:numeric-fns" + "ex:pos-int" 2 + "ex:neg-int" -2 + "ex:decimal" 1.4}]) (fluree/stage {"delete" [] "where" [["?s" "id" "ex:numeric-fns"] ["?s" "ex:pos-int" "?pos-int"] @@ -262,7 +225,12 @@ "selectOne" "?rand"}))))) (testing "string functions" - (let [updated (-> @(fluree/stage db1 {"id" "ex:string-fns" "ex:text" "Abcdefg"}) + (let [updated (-> @(fluree/stage db1 [{"id" "ex:create-predicates" + "ex:strLen" 0 "ex:subStr" 0 "ex:ucase" 0 "ex:lcase" 0 "ex:strStarts" 0 "ex:strEnds" 0 + "ex:contains" 0 "ex:strBefore" 0 "ex:strAfter" 0 "ex:encodeForUri" 0 "ex:concat" 0 + "ex:langMatches" 0 "ex:regex" 0 "ex:replace" 0} + {"id" "ex:string-fns" + "ex:text" "Abcdefg"}]) (fluree/stage {"delete" [] "where" [["?s" "id" "ex:string-fns"] ["?s" "ex:text" "?text"] @@ -302,20 +270,9 @@ "ex:strAfter" "efg" "ex:concat" "Abcdefg STR1 STR2"} @(fluree/query @updated {"where" [["?s" "id" "ex:string-fns"]] - "selectOne" {"?s" ["ex:strLen" - "ex:subStr" - "ex:ucase" - "ex:lcase" - "ex:strStarts" - "ex:strEnds" - "ex:contains" - "ex:strBefore" - "ex:strAfter" - "ex:encodeForUri" - "ex:concat" - "ex:langMatches" - "ex:regex" - "ex:replace"]}}))))) + "selectOne" {"?s" ["ex:strLen" "ex:subStr" "ex:ucase" "ex:lcase" "ex:strStarts" "ex:strEnds" + "ex:contains" "ex:strBefore" "ex:strAfter" "ex:encodeForUri" "ex:concat" + "ex:langMatches" "ex:regex" "ex:replace"]}}))))) #_(testing "functional forms") #_(testing "scalar functions"))) From 24e051677565dfd9382b2176c92579043b7c1448 Mon Sep 17 00:00:00 2001 From: Daniel Petranek Date: Wed, 14 Jun 2023 13:56:42 -0500 Subject: [PATCH 06/11] remove recursive call from 2-arity subStr call --- src/fluree/db/query/exec/eval.cljc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fluree/db/query/exec/eval.cljc b/src/fluree/db/query/exec/eval.cljc index 74170c944..1e69e0582 100644 --- a/src/fluree/db/query/exec/eval.cljc +++ b/src/fluree/db/query/exec/eval.cljc @@ -118,10 +118,10 @@ (str/ends-with? s substr)) (defn subStr + ;; The index of the first character in a string is 1. ([s start] - (subStr s start (count s))) + (subs s (dec start))) ([s start length] - ;; The index of the first character in a string is 1. (let [start (dec start)] (subs s start (min (+ start length) (count s)))))) From cd9ab4e4770e4184fd3acce105c02756816ef4c3 Mon Sep 17 00:00:00 2001 From: Daniel Petranek Date: Wed, 14 Jun 2023 13:57:05 -0500 Subject: [PATCH 07/11] fix strBefore and strAfter on no match case --- src/fluree/db/query/exec/eval.cljc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/fluree/db/query/exec/eval.cljc b/src/fluree/db/query/exec/eval.cljc index 1e69e0582..e876e4250 100644 --- a/src/fluree/db/query/exec/eval.cljc +++ b/src/fluree/db/query/exec/eval.cljc @@ -144,15 +144,17 @@ (defn strBefore [s substr] - (-> (str/split s (re-pattern substr)) - first - str)) + (let [[before :as split] (str/split s (re-pattern substr))] + (if (> (count split) 1) + before + ""))) (defn strAfter [s substr] - (-> (str/split s (re-pattern substr)) - last - str)) + (let [split (str/split s (re-pattern substr))] + (if (> (count split) 1) + (last split) + ""))) (defn concat [& strs] From 53e89a512b2259bd92d4535e4acd2cd547e3028c Mon Sep 17 00:00:00 2001 From: Daniel Petranek Date: Fri, 16 Jun 2023 16:15:30 -0500 Subject: [PATCH 08/11] add some functional forms and rdf term functions These are the functions that can be implemented without altering our existing query fn structure. --- src/fluree/db/query/exec/eval.cljc | 34 +++++++++- test/fluree/db/transact/update_test.clj | 87 ++++++++++++++++++++++++- 2 files changed, 117 insertions(+), 4 deletions(-) diff --git a/src/fluree/db/query/exec/eval.cljc b/src/fluree/db/query/exec/eval.cljc index e876e4250..258c540d4 100644 --- a/src/fluree/db/query/exec/eval.cljc +++ b/src/fluree/db/query/exec/eval.cljc @@ -107,7 +107,8 @@ (defn now [] - #?(:clj (str (Instant/now)))) + #?(:clj (str (Instant/now)) + :cljs (.toISOString (Date.)))) (defn strStarts [s substr] @@ -221,6 +222,27 @@ [x] (crypto/sha2-512 x)) +(defn uuid + [] + (str "urn:uuid:" (random-uuid))) + +(defn struuid + [] + (str (random-uuid))) + +(defn isNumeric + [x] + (number? x)) + +(defn isBlank + [x] + (and (string? x) + (str/starts-with? x "_:"))) + +(defn sparql-str + [x] + (str x)) + (def allowed-scalar-fns '#{ && || ! > < >= <= = + - * / quot and bound coalesce if nil? not not= or re-find re-pattern @@ -231,7 +253,10 @@ ;; datetime fns now year month day hours minutes seconds tz ;; hash fns - sha256 sha512}) + sha256 sha512 + ;; rdf term fns + uuid struuid isNumeric isBlank str + }) (def allowed-symbols (set/union allowed-aggregate-fns allowed-scalar-fns)) @@ -278,6 +303,11 @@ tz fluree.db.query.exec.eval/tz sha256 fluree.db.query.exec.eval/sha256 sha512 fluree.db.query.exec.eval/sha512 + uuid fluree.db.query.exec.eval/uuid + struuid fluree.db.query.exec.eval/struuid + isNumeric fluree.db.query.exec.eval/isNumeric + isBlank fluree.db.query.exec.eval/isBlank + str fluree.db.query.exec.eval/sparql-str }) (defn variable? diff --git a/test/fluree/db/transact/update_test.clj b/test/fluree/db/transact/update_test.clj index b4bf64619..93d76b87e 100644 --- a/test/fluree/db/transact/update_test.clj +++ b/test/fluree/db/transact/update_test.clj @@ -273,6 +273,89 @@ "selectOne" {"?s" ["ex:strLen" "ex:subStr" "ex:ucase" "ex:lcase" "ex:strStarts" "ex:strEnds" "ex:contains" "ex:strBefore" "ex:strAfter" "ex:encodeForUri" "ex:concat" "ex:langMatches" "ex:regex" "ex:replace"]}}))))) + (testing "rdf term functions" + (with-redefs [fluree.db.query.exec.eval/uuid (fn [] "urn:uuid:34bdb25f-9fae-419b-9c50-203b5f306e47") + fluree.db.query.exec.eval/struuid (fn [] "34bdb25f-9fae-419b-9c50-203b5f306e47")] + (let [updated (-> @(fluree/stage db1 [{"id" "ex:create-predicates" + "ex:isBlank" 0 "ex:isNumeric" 0 "ex:str" 0 "ex:uuid" 0 + "ex:struuid" 0 "ex:isNotNumeric" 0 "ex:isNotBlank" 0 + ;; "ex:isIRI" 0 "ex:isURI" 0 "ex:isLiteral" 0 "ex:lang" 0 "ex:IRI" 0 + ;; "ex:datatype" 0 "ex:bnode" 0 "ex:strdt" 0 "ex:strLang" 0 + } + {"id" "ex:rdf-term-fns" + "ex:text" "Abcdefg" + "ex:number" 1 + "ex:ref" {"ex:bool" false}} + {"ex:foo" "bar"}]) + (fluree/stage {"delete" [] + "where" [["?s" "id" "ex:rdf-term-fns"] + ["?s" "ex:text" "?text"] + ["?s" "ex:number" "?num"] + ["?s" "ex:ref" "?r"] + {"bind" {"?str" "(str ?num)" + "?uuid" "(uuid)" + "?struuid" "(struuid)" + "?isBlank" "(isBlank ?s)" + "?isNotBlank" "(isBlank ?num)" + "?isnum" "(isNumeric ?num)" + "?isNotNum" "(isNumeric ?text)"}}] + "insert" [["?s" "ex:uuid" "?uuid"] + ["?s" "ex:struuid" "?struuid"] + ["?s" "ex:str" "?str"] + ["?s" "ex:str" "?str2"] + ["?s" "ex:isNumeric" "?isnum"] + ["?s" "ex:isNotNumeric" "?isNotNum"] + ["?s" "ex:isBlank" "?isBlank"] + ["?s" "ex:isNotBlank" "?isNotBlank"]]}))] + (is (= {"ex:str" "1" + "ex:uuid" "urn:uuid:34bdb25f-9fae-419b-9c50-203b5f306e47" + "ex:struuid" "34bdb25f-9fae-419b-9c50-203b5f306e47", + "ex:isBlank" false + "ex:isNotBlank" false + "ex:isNumeric" true + "ex:isNotNumeric" false} + @(fluree/query @updated {"where" [["?s" "id" "ex:rdf-term-fns"]] + "selectOne" {"?s" ["ex:isIRI" "ex:isURI" "ex:isLiteral" + "ex:lang" "ex:datatype" "ex:IRI" "ex:bnode" "ex:strdt" "ex:strLang" + "ex:isBlank" + "ex:isNotBlank" + "ex:isNumeric" + "ex:isNotNumeric" + "ex:str" + "ex:uuid" + "ex:struuid"]}})))))) - #_(testing "functional forms") - #_(testing "scalar functions"))) + ;; errors: undefined symbol, type error, mismatching parens + (testing "functional forms" + (let [updated (-> @(fluree/stage db1 [{"id" "ex:create-predicates" + "ex:bound" 0 + "ex:if" 0 + "ex:coalesce" 0 + "ex:not-exists" 0 + "ex:exists" 0 + "ex:logical-or" 0 + "ex:logical-and" 0 + "ex:rdfterm-equal" 0 + "ex:sameTerm" 0 + "ex:in" 0 + "ex:not-in" 0} + {"id" "ex:functional-fns" + "ex:text" "Abcdefg"}]) + (fluree/stage {"delete" [] + "where" [["?s" "id" "ex:functional-fns"] + ["?s" "ex:text" "?text"] + {"bind" {"?bound" "(bound ?text)"}}] + "insert" [["?s" "ex:bound" "?bound"]]}))] + (is (= {"ex:bound" true} + @(fluree/query @updated {"where" [["?s" "id" "ex:functional-fns"]] + "selectOne" {"?s" ["ex:bound" + "ex:if" + "ex:coalesce" + "ex:not-exists" + "ex:exists" + "ex:logical-or" + "ex:logical-and" + "ex:rdfterm-equal" + "ex:sameTerm" + "ex:in" + "ex:not-in"]}}))))))) From 3ff9d3525f3a35364bb0d3fa46c7785ded6c1700 Mon Sep 17 00:00:00 2001 From: Daniel Petranek Date: Fri, 16 Jun 2023 16:16:28 -0500 Subject: [PATCH 09/11] consolidate bind statements --- test/fluree/db/transact/update_test.clj | 86 ++++++++++++------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/test/fluree/db/transact/update_test.clj b/test/fluree/db/transact/update_test.clj index 93d76b87e..35cbd3b2e 100644 --- a/test/fluree/db/transact/update_test.clj +++ b/test/fluree/db/transact/update_test.clj @@ -128,23 +128,23 @@ db1 (fluree/db ledger)] (testing "hash functions" - (with-redefs [fluree.db.query.exec.eval/now (fn [] "2023-06-13T19:53:57.234345Z")] - (let [updated (-> @(fluree/stage db1 [{"id" "ex:create-predicates" - "ex:md5" 0 "ex:sha1" 0 "ex:sha256" 0 "ex:sha384" 0 "ex:sha512" 0} - {"id" "ex:hash-fns" - "ex:message" "abc"}]) - (fluree/stage {"delete" [] - "where" [["?s" "id" "ex:hash-fns"] - ["?s" "ex:message" "?message"] - {"bind" {"?sha256" "(sha256 ?message)"}} - {"bind" {"?sha512" "(sha512 ?message)"}}] - "insert" [["?s" "ex:sha256" "?sha256"] - ["?s" "ex:sha512" "?sha512"]]}))] - (is (= {"ex:sha512" "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f" - "ex:sha256" "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"} - @(fluree/query @updated {"where" [["?s" "id" "ex:hash-fns"]] - "selectOne" {"?s" ["ex:sha512" - "ex:sha256"]}})))))) + (with-redefs [fluree.db.query.exec.eval/now (fn [] "2023-06-13T19:53:57.234345Z")] + (let [updated (-> @(fluree/stage db1 [{"id" "ex:create-predicates" + "ex:md5" 0 "ex:sha1" 0 "ex:sha256" 0 "ex:sha384" 0 "ex:sha512" 0} + {"id" "ex:hash-fns" + "ex:message" "abc"}]) + (fluree/stage {"delete" [] + "where" [["?s" "id" "ex:hash-fns"] + ["?s" "ex:message" "?message"] + {"bind" {"?sha256" "(sha256 ?message)" + "?sha512" "(sha512 ?message)"}}] + "insert" [["?s" "ex:sha256" "?sha256"] + ["?s" "ex:sha512" "?sha512"]]}))] + (is (= {"ex:sha512" "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f" + "ex:sha256" "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"} + @(fluree/query @updated {"where" [["?s" "id" "ex:hash-fns"]] + "selectOne" {"?s" ["ex:sha512" + "ex:sha256"]}})))))) (testing "datetime functions" (with-redefs [fluree.db.query.exec.eval/now (fn [] "2023-06-13T19:53:57.234345Z")] (let [updated (-> @(fluree/stage db1 [{"id" "ex:create-predicates" @@ -159,15 +159,15 @@ ["?s" "ex:localdatetime" "?localdatetime"] ["?s" "ex:offsetdatetime" "?offsetdatetime"] ["?s" "ex:utcdatetime" "?utcdatetime"] - {"bind" {"?now" "(now)"}} - {"bind" {"?year" "(year ?localdatetime)"}} - {"bind" {"?month" "(month ?localdatetime)"}} - {"bind" {"?day" "(day ?localdatetime)"}} - {"bind" {"?hours" "(hours ?localdatetime)"}} - {"bind" {"?minutes" "(minutes ?localdatetime)"}} - {"bind" {"?seconds" "(seconds ?localdatetime)"}} - {"bind" {"?tz1" "(tz ?utcdatetime)"}} - {"bind" {"?tz2" "(tz ?offsetdatetime)"}}] + {"bind" {"?now" "(now)" + "?year" "(year ?localdatetime)" + "?month" "(month ?localdatetime)" + "?day" "(day ?localdatetime)" + "?hours" "(hours ?localdatetime)" + "?minutes" "(minutes ?localdatetime)" + "?seconds" "(seconds ?localdatetime)" + "?tz1" "(tz ?utcdatetime)" + "?tz2" "(tz ?offsetdatetime)"}}] "insert" [["?s" "ex:now" "?now"] ["?s" "ex:year" "?year"] ["?s" "ex:month" "?month"] @@ -201,11 +201,11 @@ ["?s" "ex:pos-int" "?pos-int"] ["?s" "ex:neg-int" "?neg-int"] ["?s" "ex:decimal" "?decimal"] - {"bind" {"?abs" "(abs ?neg-int)"}} - {"bind" {"?round" "(round ?decimal)"}} - {"bind" {"?ceil" "(ceil ?decimal)"}} - {"bind" {"?floor" "(floor ?decimal)"}} - {"bind" {"?rand" "(rand)"}}] + {"bind" {"?abs" "(abs ?neg-int)" + "?round" "(round ?decimal)" + "?ceil" "(ceil ?decimal)" + "?floor" "(floor ?decimal)" + "?rand" "(rand)"}}] "insert" [["?s" "ex:abs" "?abs"] ["?s" "ex:round" "?round"] ["?s" "ex:ceil" "?ceil"] @@ -234,18 +234,18 @@ (fluree/stage {"delete" [] "where" [["?s" "id" "ex:string-fns"] ["?s" "ex:text" "?text"] - {"bind" {"?strlen" "(strLen ?text)"}} - {"bind" {"?sub1" "(subStr ?text 5)"}} - {"bind" {"?sub2" "(subStr ?text 1 4)"}} - {"bind" {"?upcased" "(ucase ?text)"}} - {"bind" {"?downcased" "(lcase ?text)"}} - {"bind" {"?a-start" "(strStarts ?text \"x\")"}} - {"bind" {"?a-end" "(strEnds ?text \"x\")"}} - {"bind" {"?contains" "(contains ?text \"x\")"}} - {"bind" {"?strBefore" "(strBefore ?text \"bcd\")"}} - {"bind" {"?strAfter" "(strAfter ?text \"bcd\")"}} - {"bind" {"?concatted" "(concat ?text \" \" \"STR1 \" \"STR2\")"}} - {"bind" {"?matched" "(regex ?text \"^Abc\")"}}] + {"bind" {"?strlen" "(strLen ?text)" + "?sub1" "(subStr ?text 5)" + "?sub2" "(subStr ?text 1 4)" + "?upcased" "(ucase ?text)" + "?downcased" "(lcase ?text)" + "?a-start" "(strStarts ?text \"x\")" + "?a-end" "(strEnds ?text \"x\")" + "?contains" "(contains ?text \"x\")" + "?strBefore" "(strBefore ?text \"bcd\")" + "?strAfter" "(strAfter ?text \"bcd\")" + "?concatted" "(concat ?text \" \" \"STR1 \" \"STR2\")" + "?matched" "(regex ?text \"^Abc\")"}}] "insert" [["?s" "ex:strStarts" "?a-start"] ["?s" "ex:strEnds" "?a-end"] ["?s" "ex:subStr" "?sub1"] From 193c0aa7224bf81a4e4ab25d0840b859a5aef25c Mon Sep 17 00:00:00 2001 From: Daniel Petranek Date: Thu, 22 Jun 2023 09:18:54 -0500 Subject: [PATCH 10/11] handle function parsing errors --- src/fluree/db/json_ld/transact.cljc | 22 ++-- test/fluree/db/transact/update_test.clj | 129 +++++++++++++++--------- 2 files changed, 90 insertions(+), 61 deletions(-) diff --git a/src/fluree/db/json_ld/transact.cljc b/src/fluree/db/json_ld/transact.cljc index f2fe1ff1e..039f8e270 100644 --- a/src/fluree/db/json_ld/transact.cljc +++ b/src/fluree/db/json_ld/transact.cljc @@ -386,17 +386,17 @@ (defn modify [db fuel-tracker json-ld {:keys [t] :as _tx-state}] - (go - (let [mdfn (-> json-ld - syntax/coerce-modification - (q-parse/parse-modification db)) - error-ch (async/chan) - update-ch (->> (where/search db mdfn fuel-tracker error-ch) - (update/modify db mdfn t fuel-tracker error-ch) - (into-flakeset fuel-tracker))] - (async/alt! - error-ch ([e] e) - update-ch ([flakes] flakes))))) + (let [mdfn (-> json-ld + syntax/coerce-modification + (q-parse/parse-modification db))] + (go + (let [error-ch (async/chan) + update-ch (->> (where/search db mdfn fuel-tracker error-ch) + (update/modify db mdfn t fuel-tracker error-ch) + (into-flakeset fuel-tracker))] + (async/alt! + error-ch ([e] e) + update-ch ([flakes] flakes)))))) (defn flakes->final-db "Takes final set of proposed staged flakes and turns them into a new db value diff --git a/test/fluree/db/transact/update_test.clj b/test/fluree/db/transact/update_test.clj index 35cbd3b2e..63340d123 100644 --- a/test/fluree/db/transact/update_test.clj +++ b/test/fluree/db/transact/update_test.clj @@ -128,23 +128,23 @@ db1 (fluree/db ledger)] (testing "hash functions" - (with-redefs [fluree.db.query.exec.eval/now (fn [] "2023-06-13T19:53:57.234345Z")] - (let [updated (-> @(fluree/stage db1 [{"id" "ex:create-predicates" - "ex:md5" 0 "ex:sha1" 0 "ex:sha256" 0 "ex:sha384" 0 "ex:sha512" 0} - {"id" "ex:hash-fns" - "ex:message" "abc"}]) - (fluree/stage {"delete" [] - "where" [["?s" "id" "ex:hash-fns"] - ["?s" "ex:message" "?message"] - {"bind" {"?sha256" "(sha256 ?message)" - "?sha512" "(sha512 ?message)"}}] - "insert" [["?s" "ex:sha256" "?sha256"] - ["?s" "ex:sha512" "?sha512"]]}))] - (is (= {"ex:sha512" "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f" - "ex:sha256" "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"} - @(fluree/query @updated {"where" [["?s" "id" "ex:hash-fns"]] - "selectOne" {"?s" ["ex:sha512" - "ex:sha256"]}})))))) + (with-redefs [fluree.db.query.exec.eval/now (fn [] "2023-06-13T19:53:57.234345Z")] + (let [updated (-> @(fluree/stage db1 [{"id" "ex:create-predicates" + "ex:md5" 0 "ex:sha1" 0 "ex:sha256" 0 "ex:sha384" 0 "ex:sha512" 0} + {"id" "ex:hash-fns" + "ex:message" "abc"}]) + (fluree/stage {"delete" [] + "where" [["?s" "id" "ex:hash-fns"] + ["?s" "ex:message" "?message"] + {"bind" {"?sha256" "(sha256 ?message)" + "?sha512" "(sha512 ?message)"}}] + "insert" [["?s" "ex:sha256" "?sha256"] + ["?s" "ex:sha512" "?sha512"]]}))] + (is (= {"ex:sha512" "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f" + "ex:sha256" "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"} + @(fluree/query @updated {"where" [["?s" "id" "ex:hash-fns"]] + "selectOne" {"?s" ["ex:sha512" + "ex:sha256"]}})))))) (testing "datetime functions" (with-redefs [fluree.db.query.exec.eval/now (fn [] "2023-06-13T19:53:57.234345Z")] (let [updated (-> @(fluree/stage db1 [{"id" "ex:create-predicates" @@ -325,37 +325,66 @@ "ex:uuid" "ex:struuid"]}})))))) - ;; errors: undefined symbol, type error, mismatching parens (testing "functional forms" - (let [updated (-> @(fluree/stage db1 [{"id" "ex:create-predicates" - "ex:bound" 0 - "ex:if" 0 - "ex:coalesce" 0 - "ex:not-exists" 0 - "ex:exists" 0 - "ex:logical-or" 0 - "ex:logical-and" 0 - "ex:rdfterm-equal" 0 - "ex:sameTerm" 0 - "ex:in" 0 - "ex:not-in" 0} - {"id" "ex:functional-fns" - "ex:text" "Abcdefg"}]) - (fluree/stage {"delete" [] - "where" [["?s" "id" "ex:functional-fns"] - ["?s" "ex:text" "?text"] - {"bind" {"?bound" "(bound ?text)"}}] - "insert" [["?s" "ex:bound" "?bound"]]}))] - (is (= {"ex:bound" true} - @(fluree/query @updated {"where" [["?s" "id" "ex:functional-fns"]] - "selectOne" {"?s" ["ex:bound" - "ex:if" - "ex:coalesce" - "ex:not-exists" - "ex:exists" - "ex:logical-or" - "ex:logical-and" - "ex:rdfterm-equal" - "ex:sameTerm" - "ex:in" - "ex:not-in"]}}))))))) + (let [updated (-> @(fluree/stage db1 [{"id" "ex:create-predicates" + "ex:bound" 0 + "ex:if" 0 + "ex:coalesce" 0 + "ex:not-exists" 0 + "ex:exists" 0 + "ex:logical-or" 0 + "ex:logical-and" 0 + "ex:rdfterm-equal" 0 + "ex:sameTerm" 0 + "ex:in" 0 + "ex:not-in" 0} + {"id" "ex:functional-fns" + "ex:text" "Abcdefg"}]) + (fluree/stage {"delete" [] + "where" [["?s" "id" "ex:functional-fns"] + ["?s" "ex:text" "?text"] + {"bind" {"?bound" "(bound ?text)"}}] + "insert" [["?s" "ex:bound" "?bound"]]}))] + (is (= {"ex:bound" true} + @(fluree/query @updated {"where" [["?s" "id" "ex:functional-fns"]] + "selectOne" {"?s" ["ex:bound" + "ex:if" + "ex:coalesce" + "ex:not-exists" + "ex:exists" + "ex:logical-or" + "ex:logical-and" + "ex:rdfterm-equal" + "ex:sameTerm" + "ex:in" + "ex:not-in"]}}))))) + (testing "error handling" + (let [db2 @(fluree/stage db1 [{"id" "ex:create-predicates" + "ex:text" 0 + "ex:error" 0} + {"id" "ex:error" + "ex:text" "Abcdefg"}]) + parse-err @(fluree/stage db2 {"delete" [] + "where" [["?s" "id" "ex:error"] + ["?s" "ex:text" "?text"] + {"bind" {"?err" "(foo ?text)"}}] + "insert" [["?s" "ex:text" "?err"]]}) + + run-err @(fluree/stage db2 {"delete" [] + "where" [["?s" "id" "ex:error"] + ["?s" "ex:text" "?text"] + {"bind" {"?err" "(abs ?text)"}}] + "insert" [["?s" "ex:error" "?err"]]})] + (is (= "Query function references illegal symbol: foo" + (-> parse-err + Throwable->map + :cause)) + "mdfn parse error") + (is (= "Query function references illegal symbol: foo" + (-> @(fluree/query db2 {"where" [["?s" "id" "ex:error"] + ["?s" "ex:text" "?text"] + {"bind" {"?err" "(foo ?text)"}}] + "select" "?err"}) + Throwable->map + :cause)) + "query parse error"))))) From 5912ac8d4772c51cc6e1fe7c3fcb39b745adc4f5 Mon Sep 17 00:00:00 2001 From: Daniel Petranek Date: Fri, 7 Jul 2023 14:43:54 -0500 Subject: [PATCH 11/11] fix now query fn in cljs Also shuffled around some type hints. They didn't cause any problems where they were, but they were misleading. Also, `getYear` does not return what you'd expect in js, we need to rely on `getFullYear` instead. --- src/fluree/db/query/exec/eval.cljc | 34 +++++++++++++++--------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/fluree/db/query/exec/eval.cljc b/src/fluree/db/query/exec/eval.cljc index 258c540d4..abc053bba 100644 --- a/src/fluree/db/query/exec/eval.cljc +++ b/src/fluree/db/query/exec/eval.cljc @@ -10,8 +10,7 @@ [fluree.db.datatype :as datatype] [fluree.crypto :as crypto] [fluree.db.constants :as const]) - #?(:clj (:import (java.time Instant) - (java.time OffsetDateTime LocalDateTime)))) + #?(:clj (:import (java.time Instant OffsetDateTime LocalDateTime)))) #?(:clj (set! *warn-on-reflection* true)) @@ -108,7 +107,7 @@ (defn now [] #?(:clj (str (Instant/now)) - :cljs (.toISOString (Date.)))) + :cljs (.toISOString (js/Date.)))) (defn strStarts [s substr] @@ -175,43 +174,44 @@ (defn year [datetime-string] - (let [datetime ^LocalDateTime (datatype/coerce datetime-string const/$xsd:dateTime)] - (.getYear datetime))) + (let [datetime (datatype/coerce datetime-string const/$xsd:dateTime)] + #?(:clj (.getYear ^LocalDateTime datetime) + :cljs (.getFullYear datetime)))) (defn month [datetime-string] - (let [datetime ^LocalDateTime (datatype/coerce datetime-string const/$xsd:dateTime)] - #?(:clj (.getMonthValue datetime) + (let [datetime (datatype/coerce datetime-string const/$xsd:dateTime)] + #?(:clj (.getMonthValue ^LocalDateTime datetime) :cljs (.getMonth datetime)))) (defn day [datetime-string] - (let [datetime ^LocalDateTime (datatype/coerce datetime-string const/$xsd:dateTime)] - #?(:clj (.getDayOfMonth datetime) + (let [datetime (datatype/coerce datetime-string const/$xsd:dateTime)] + #?(:clj (.getDayOfMonth ^LocalDateTime datetime) :cljs (.getDate datetime)))) (defn hours [datetime-string] - (let [datetime ^LocalDateTime (datatype/coerce datetime-string const/$xsd:dateTime)] - #?(:clj (.getHour datetime) + (let [datetime (datatype/coerce datetime-string const/$xsd:dateTime)] + #?(:clj (.getHour ^LocalDateTime datetime) :cljs (.getHours datetime)))) (defn minutes [datetime-string] - (let [datetime ^LocalDateTime (datatype/coerce datetime-string const/$xsd:dateTime)] - #?(:clj (.getMinute datetime) + (let [datetime (datatype/coerce datetime-string const/$xsd:dateTime)] + #?(:clj (.getMinute ^LocalDateTime datetime) :cljs (.getMinutes datetime)))) (defn seconds [datetime-string] - (let [datetime ^LocalDateTime (datatype/coerce datetime-string const/$xsd:dateTime)] - #?(:clj (.getSecond datetime) + (let [datetime (datatype/coerce datetime-string const/$xsd:dateTime)] + #?(:clj (.getSecond ^LocalDateTime datetime) :cljs (.getSeconds datetime)))) (defn tz [datetime-string] - (let [datetime ^OffsetDateTime (datatype/coerce datetime-string const/$xsd:dateTime)] - #?(:clj (.toString (.getOffset datetime)) + (let [datetime (datatype/coerce datetime-string const/$xsd:dateTime)] + #?(:clj (.toString (.getOffset ^OffsetDateTime datetime)) :cljs (.getTimeZoneOffset datetime)))) (defn sha256