diff --git a/deps.edn b/deps.edn index 908453551..1084ed371 100644 --- a/deps.edn +++ b/deps.edn @@ -9,14 +9,14 @@ byte-streams/byte-streams {:mvn/version "0.2.4"} cheshire/cheshire {:mvn/version "5.11.0"} instaparse/instaparse {:mvn/version "1.4.12"} - metosin/malli {:mvn/version "0.9.2"} + metosin/malli {:mvn/version "0.10.1"} com.fluree/json-ld {:git/url "https://github.com/fluree/json-ld.git" :sha "a909330e33196504ef8a5411aaa0409ab72aaa35"} ;; logging org.clojure/tools.logging {:mvn/version "1.2.4"} ch.qos.logback/logback-classic {:mvn/version "1.4.5"} - org.slf4j/slf4j-api {:mvn/version "2.0.5"} + org.slf4j/slf4j-api {:mvn/version "2.0.6"} ;; Lucene clucie/clucie {:mvn/version "0.4.2"} @@ -44,8 +44,8 @@ :aliases {:build - {:deps {io.github.clojure/tools.build {:git/tag "v0.8.5" - :git/sha "b73ff34"} + {:deps {io.github.clojure/tools.build {:git/tag "v0.9.3" + :git/sha "f37d64e"} slipset/deps-deploy {:mvn/version "0.2.0"}} :ns-default build} @@ -54,12 +54,12 @@ :extra-deps {org.clojure/tools.namespace {:mvn/version "1.3.0"} criterium/criterium {:mvn/version "0.4.6"} figwheel-sidecar/figwheel-sidecar {:mvn/version "0.5.20"} - thheller/shadow-cljs {:mvn/version "2.20.12"} + thheller/shadow-cljs {:mvn/version "2.20.20"} com.magnars/test-with-files {:mvn/version "2021-02-17"}}} :cljtest {:extra-paths ["test" "dev-resources"] - :extra-deps {lambdaisland/kaocha {:mvn/version "1.71.1119"} + :extra-deps {lambdaisland/kaocha {:mvn/version "1.77.1236"} org.clojure/test.check {:mvn/version "1.1.1"} io.github.cap10morgan/test-with-files {:git/tag "v1.0.0" :git/sha "9181a2e"}} @@ -107,5 +107,5 @@ :main-opts ["-m" "antq.core"]} :clj-kondo - {:extra-deps {clj-kondo/clj-kondo {:mvn/version "2022.11.02"}} + {:extra-deps {clj-kondo/clj-kondo {:mvn/version "2023.01.20"}} :main-opts ["-m" "clj-kondo.main" "--lint" "src" "--config" ".clj-kondo/config.edn"]}}} diff --git a/src/fluree/db/json_ld/transact.cljc b/src/fluree/db/json_ld/transact.cljc index 468b1541f..3bad14fe2 100644 --- a/src/fluree/db/json_ld/transact.cljc +++ b/src/fluree/db/json_ld/transact.cljc @@ -20,11 +20,28 @@ [fluree.db.policy.enforce-tx :as policy] [fluree.db.dbproto :as dbproto] [fluree.db.json-ld.credential :as cred] - [fluree.db.util.log :as log]) + [fluree.db.util.log :as log] + [fluree.db.util.validation :as v] + [malli.core :as m]) (:refer-clojure :exclude [vswap!])) #?(:clj (set! *warn-on-reflection* true)) +(def registry + (merge + (m/predicate-schemas) + (m/class-schemas) + (m/comparator-schemas) + (m/type-schemas) + (m/sequence-schemas) + (m/base-schemas) + {::val [:fn v/value?] + ::txn-map [:map-of v/iri-key [:orn [:iri v/iri] + [:val ::val]]] + ::txn [:orn + [:single-map ::txn-map] + [:collection-of-maps [:sequential ::txn-map]]]})) + (declare json-ld-node->flakes) (defn json-ld-type-data diff --git a/src/fluree/db/query/fql/syntax.cljc b/src/fluree/db/query/fql/syntax.cljc index cea411699..9c73c3f63 100644 --- a/src/fluree/db/query/fql/syntax.cljc +++ b/src/fluree/db/query/fql/syntax.cljc @@ -1,6 +1,9 @@ (ns fluree.db.query.fql.syntax - (:require [malli.core :as m] - [fluree.db.util.core :refer [pred-ident?]])) + (:require [clojure.string :as str] + [fluree.db.util.log :as log] + [malli.core :as m] + [fluree.db.util.core :refer [pred-ident?]] + [fluree.db.util.validation :as v])) #?(:clj (set! *warn-on-reflection* true)) @@ -27,8 +30,6 @@ (and (or (string? x) (symbol? x) (keyword? x)) (-> x name first (= \?)))) -(def value? (complement coll?)) - (defn sid? [x] (int? x)) @@ -77,25 +78,32 @@ [:string [:fn fn-string?]] [:list [:fn fn-list?]]] ::wildcard [:fn wildcard?] - ::var [:fn variable?] - ::val [:fn value?] - ::iri [:orn - [:keyword keyword?] - [:string string?]] + ::var [:fn {:decode/json + (fn [v] + (log/debug "decoding var:" v) + (if (string? v) + (symbol v) + v))} + variable?] + ::val [:fn v/value?] + ::iri v/iri + ::iri-pred v/iri-key ::subject [:orn [:sid [:fn sid?]] [:iri ::iri] [:ident [:fn pred-ident?]]] - ::subselect-map [:map-of ::iri [:ref ::subselection]] + ::subselect-map [:map-of + [:orn [:var ::var] [:iri ::iri]] + [:ref ::subselection]] ::subselection [:sequential [:orn [:wildcard ::wildcard] - [:predicate ::iri] + [:predicate ::iri-pred] [:subselect-map [:ref ::subselect-map]]]] ::select-map [:map-of {:max 1} ::var ::subselection] ::selector [:orn [:var ::var] - [:pred ::iri] + [:pred ::iri-pred] [:aggregate ::function] [:select-map ::select-map]] ::select [:orn @@ -125,24 +133,24 @@ ::where-map [:and [:map-of {:max 1} ::where-op :any] [:multi {:dispatch where-op} - [:filter [:map [:filter [:ref ::filter]]]] + [:filter [:map [:filter ::filter]]] [:optional [:map [:optional [:ref ::optional]]]] [:union [:map [:union [:ref ::union]]]] - [:bind [:map [:bind [:ref ::bind]]]]]] + [:bind [:map [:bind ::bind]]]]] ::triple [:catn [:subject [:orn [:var ::var] [:val ::subject]]] [:predicate [:orn [:var ::var] - [:iri ::iri]]] + [:iri ::iri-pred]]] [:object [:orn [:var ::var] + [:iri ::iri] [:ident [:fn pred-ident?]] [:val :any]]]] ::where-tuple [:orn [:triple ::triple] - [:binding [:sequential {:max 2} :any]] [:remote [:sequential {:max 4} :any]]] ::where-pattern [:orn [:where-map ::where-map] @@ -152,7 +160,7 @@ [:collection [:sequential ::where-pattern]]] ::filter [:sequential ::function] ::union [:sequential [:sequential ::where-pattern]] - ::bind [:map-of ::var :any] + ::bind [:map-of ::var ::function] ::where [:sequential [:orn [:where-map ::where-map] [:tuple ::where-tuple]]] @@ -201,7 +209,14 @@ ::multi-query [:map-of [:or :string :keyword] ::analytical-query] ::query [:orn [:single ::analytical-query] - [:multi ::multi-query]]})) + [:multi ::multi-query]] + + ::query-results [:sequential + [:map-of ::iri-pred [:or ::iri ::val + [:sequential + [:or ::iri ::val]]]]] + + ::multi-query-results [:map-of [:or :string :keyword] ::query-results]})) (def query-validator (m/validator ::query {:registry registry})) diff --git a/src/fluree/db/query/history.cljc b/src/fluree/db/query/history.cljc index 1a5a8e0e8..2c56f95ba 100644 --- a/src/fluree/db/query/history.cljc +++ b/src/fluree/db/query/history.cljc @@ -1,23 +1,23 @@ (ns fluree.db.query.history (:require - [clojure.core.async :as async] - [malli.core :as m] - [fluree.json-ld :as json-ld] - [fluree.db.constants :as const] - [fluree.db.datatype :as datatype] - [fluree.db.dbproto :as dbproto] - [fluree.db.flake :as flake] - [fluree.db.query.json-ld.response :as json-ld-resp] - [fluree.db.query.fql.parse :as fql-parse] - [fluree.db.util.async :refer [! error-ch e))))) + (log/error e "Error transforming s-flakes.") + (async/>! error-ch e))))) (defn t-flakes->json-ld "Build a collection of subject maps out of a set of flakes with the same t. @@ -121,7 +142,7 @@ (vals) (async/to-chan!)) - s-out-ch (async/chan)] + s-out-ch (async/chan)] (async/pipeline-async 2 s-out-ch (fn [assert-flakes ch] @@ -137,12 +158,12 @@ [{:id :ex/foo :f/assert [{},,,} :f/retract [{},,,]]}] " [db context error-ch flakes] - (let [fuel (volatile! 0) - cache (volatile! {}) + (let [fuel (volatile! 0) + cache (volatile! {}) - compact (json-ld/compact-fn context) + compact (json-ld/compact-fn context) - out-ch (async/chan) + out-ch (async/chan) t-flakes-ch (->> flakes (sort-by flake/t >) @@ -158,24 +179,24 @@ (fn [t-flakes ch] (-> (async/go (try* - (let [{assert-flakes true + (let [{assert-flakes true retract-flakes false} (group-by flake/op t-flakes) - t (- (flake/t (first t-flakes))) + t (- (flake/t (first t-flakes))) - asserts (->> (t-flakes->json-ld db context compact cache fuel error-ch assert-flakes) - (async/into []) - (async/> (t-flakes->json-ld db context compact cache fuel error-ch assert-flakes) + (async/into []) + (async/> (t-flakes->json-ld db context compact cache fuel error-ch retract-flakes) (async/into []) (async/! error-ch e)))) + (log/error e "Error converting history flakes.") + (async/>! error-ch e)))) (async/pipe ch))) t-flakes-ch) @@ -186,7 +207,7 @@ to subject ids and return the best index to query against." [db context query] (go-try - (let [ ;; parses to [:subject <:id>] or [:flake {:s <> :p <> :o <>}]} + (let [;; parses to [:subject <:id>] or [:flake {:s <> :p <> :o <>}]} [query-type parsed-query] query {:keys [s p o]} (if (= :subject query-type) @@ -245,52 +266,52 @@ retract-flakes :retract-flakes} (group-by (fn [f] (cond - (commit-wrapper-flake? f) :commit-wrapper - (commit-metadata-flake? f) :commit-meta - (and (flake/op f) (not (extra-data-flake? f))) :assert-flakes + (commit-wrapper-flake? f) :commit-wrapper + (commit-metadata-flake? f) :commit-meta + (and (flake/op f) (not (extra-data-flake? f))) :assert-flakes (and (not (flake/op f)) (not (extra-data-flake? f))) :retract-flakes - :else :ignore-flakes)) + :else :ignore-flakes)) t-flakes) commit-wrapper-chan (json-ld-resp/flakes->res db cache context compact fuel 1000000 {:wildcard? true, :depth 0} 0 commit-wrapper-flakes) - commit-meta-chan (json-ld-resp/flakes->res db cache context compact fuel 1000000 - {:wildcard? true, :depth 0} - 0 commit-meta-flakes) + commit-meta-chan (json-ld-resp/flakes->res db cache context compact fuel 1000000 + {:wildcard? true, :depth 0} + 0 commit-meta-flakes) - commit-wrapper (> (t-flakes->json-ld db context compact cache fuel error-ch assert-flakes) - (async/into []) - (async/> (t-flakes->json-ld db context compact cache fuel error-ch retract-flakes) - (async/into []) - (async/> (t-flakes->json-ld db context compact cache fuel error-ch assert-flakes) + (async/into []) + (async/> (t-flakes->json-ld db context compact cache fuel error-ch retract-flakes) + (async/into []) + (async/ {commit-key commit-wrapper} (assoc-in [commit-key data-key] commit-meta) - (assoc-in [commit-key data-key assert-key] asserts) - (assoc-in [commit-key data-key retract-key] retracts))) + (assoc-in [commit-key data-key assert-key] asserts) + (assoc-in [commit-key data-key retract-key] retracts))) (catch* e - (log/error e "Error converting commit flakes.") - (async/>! error-ch e))))) + (log/error e "Error converting commit flakes.") + (async/>! error-ch e))))) (defn commit-flakes->json-ld "Create a collection of commit maps." [db context error-ch flake-slice-ch] - (let [fuel (volatile! 0) - cache (volatile! {}) - compact (json-ld/compact-fn context) + (let [fuel (volatile! 0) + cache (volatile! {}) + compact (json-ld/compact-fn context) t-flakes-ch (async/chan 1 (comp cat (partition-by flake/t))) - out-ch (async/chan)] + out-ch (async/chan)] (async/pipe flake-slice-ch t-flakes-ch) (async/pipeline-async 2 @@ -305,10 +326,10 @@ "Return a transducer that processes a stream of history results and chunk together results with consecutive `t`s. " [t-key] - (let [last-t (volatile! nil) - last-partition-val (volatile! true)] + (let [last-t (volatile! nil) + last-partition-val (volatile! true)] (partition-by (fn [result] - (let [result-t (get result t-key) + (let [result-t (get result t-key) chunk-last-t @last-t] (vreset! last-t result-t) (if (or (nil? chunk-last-t) @@ -324,19 +345,26 @@ Chunks together history results with consecutive `t`s to reduce `time-range` calls. " [db context error-ch history-results-ch] - (let [t-key (json-ld/compact const/iri-t context) - out-ch (async/chan 2 cat) + (let [t-key (json-ld/compact const/iri-t context) + out-ch (async/chan 2 cat) chunked-ch (async/chan 2 (with-consecutive-ts t-key))] (async/pipe history-results-ch chunked-ch) - (async/pipeline-async 2 - out-ch - (fn [chunk ch] - (async/pipe (async/go - (let [to-t (- (-> chunk peek (get t-key))) - from-t (- (-> chunk (nth 0) (get t-key))) - flake-slices-ch (query-range/time-range db :tspo = [] {:from-t from-t :to-t to-t}) - consecutive-commit-details (json-ld db context error-ch flake-slices-ch)))] - (map into chunk consecutive-commit-details))) - ch)) - chunked-ch) + (async/pipeline-async + 2 + out-ch + (fn [chunk ch] + (async/pipe + (async/go + (let [to-t (- (-> chunk peek (get t-key))) + from-t (- (-> chunk (nth 0) (get t-key))) + flake-slices-ch (query-range/time-range + db :tspo = [] {:from-t from-t :to-t to-t}) + consecutive-commit-details (json-ld + db context error-ch + flake-slices-ch)))] + (map into chunk consecutive-commit-details))) + ch)) + chunked-ch) out-ch)) diff --git a/src/fluree/db/util/core.cljc b/src/fluree/db/util/core.cljc index fc378690d..921a9bb9b 100644 --- a/src/fluree/db/util/core.cljc +++ b/src/fluree/db/util/core.cljc @@ -136,8 +136,8 @@ (defn pred-ident? - "Tests if an predicate identity two-tuple - in form of [pred-name-or-id pred-value]" + "Tests if a predicate identity two-tuple in form of + [pred-name-or-id pred-value]" [x] (and (sequential? x) (= 2 (count x)) diff --git a/src/fluree/db/util/validation.cljc b/src/fluree/db/util/validation.cljc new file mode 100644 index 000000000..0f1c02de9 --- /dev/null +++ b/src/fluree/db/util/validation.cljc @@ -0,0 +1,73 @@ +(ns fluree.db.util.validation + (:require [clojure.string :as str] + [malli.core :as m] + [fluree.db.util.log :as log])) + +(def non-empty-string + (m/schema [:string {:min 1}])) + +(def value? (complement coll?)) + +(defn qualified-keyword->json-ld + [kw] + (str (namespace kw) ":" (name kw))) + +(defn compact-iri->keyword + "Converts compact IRI strings to keywords. If there is a colon in the iri, + the part before the colon becomes the keyword's namespace. + E.g. + \"foo\" -> :foo + \"foo:bar\" -> :foo/bar" + [iri] + (-> iri + (str/split #":") + (->> (cons nil) + (take-last 2) + (apply keyword)))) + +(def iri-key + "Decodes all string values to keywords even if they don't look like compact + IRIs. Intended to support e.g. \"id\" -> :id." + (m/schema + [:orn + {:decode/json + (fn [v] + (log/debug "decoding iri key:" v) + (if (string? v) + (if (str/includes? v "://") ; non-compact IRI + v + (compact-iri->keyword v)) + v)) + :encode/json + (fn [v] + (log/debug "encoding iri key:" v) + (if (qualified-keyword? v) + (qualified-keyword->json-ld v) + v))} + [:string non-empty-string] + [:keyword :keyword]])) + +(def iri + "Decodes only compact IRIs to qualified keywords, e.g. \"foo:bar\" -> :foo/bar + but leaves the value as-is otherwise." + (m/schema + [:orn + {:decode/json + (fn [v] + (log/debug "decoding iri:" v) + (if (string? v) + (if (str/includes? v "://") ; non-compact IRI + v + (if (str/includes? v ":") ; compact IRI + (compact-iri->keyword v) + v)) + v)) + :encode/json + (fn [v] + (log/debug "encoding iri:" v) + (if (qualified-keyword? v) + (qualified-keyword->json-ld v) + v))} + [:string non-empty-string] + [:keyword :keyword]])) + diff --git a/test/fluree/db/query/reverse_query_test.clj b/test/fluree/db/query/reverse_query_test.clj index 6f155da41..f48fc8016 100644 --- a/test/fluree/db/query/reverse_query_test.clj +++ b/test/fluree/db/query/reverse_query_test.clj @@ -22,24 +22,26 @@ :schema/name "Cam" :ex/friend [:ex/brian :ex/alice]}])] - (is (= @(fluree/query db '{:context {:friended {:reverse :ex/friend}} + (is (= {:schema/name "Brian" + :friended :ex/cam} + @(fluree/query db '{:context {:friended {:reverse :ex/friend}} :selectOne {?s [:schema/name :friended]} - :where [[?s :id :ex/brian]]}) - {:schema/name "Brian" - :friended :ex/cam})) + :where [[?s :id :ex/brian]]}))) - (is (= @(fluree/query db '{:context {:friended {:reverse :ex/friend}}, + + (is (= {:schema/name "Alice" + :friended [:ex/cam :ex/brian]} + @(fluree/query db '{:context {:friended {:reverse :ex/friend}}, :selectOne {?s [:schema/name :friended]}, - :where [[?s :id :ex/alice]]}) - {:schema/name "Alice" - :friended [:ex/cam :ex/brian]})) + :where [[?s :id :ex/alice]]}))) - (is (= @(fluree/query db '{:context {:friended {:reverse :ex/friend}}, - :selectOne {?s [:schema/name {:friended [:*]}]}, - :where [[?s :id :ex/brian]]}) - {:schema/name "Brian", + (is (= {:schema/name "Brian", :friended {:id :ex/cam, :rdf/type [:ex/User], :schema/name "Cam", - :ex/friend [{:id :ex/brian} {:id :ex/alice}]}}))))) + :ex/friend [{:id :ex/brian} {:id :ex/alice}]}} + @(fluree/query db '{:context {:friended {:reverse :ex/friend}}, + :selectOne {?s [:schema/name {:friended [:*]}]}, + :where [[?s :id :ex/brian]]})))))) +