From 70550ec20f53e87b422173b2876d9e910cdf8cc6 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Fri, 10 Mar 2023 12:23:46 -0700 Subject: [PATCH 01/50] Update deps to latest compatible versions --- deps.edn | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/deps.edn b/deps.edn index 908453551..d8080f5a8 100644 --- a/deps.edn +++ b/deps.edn @@ -9,14 +9,14 @@ byte-streams/byte-streams {:mvn/version "0.2.4"} cheshire/cheshire {:mvn/version "5.11.0"} instaparse/instaparse {:mvn/version "1.4.12"} - metosin/malli {:mvn/version "0.9.2"} + metosin/malli {:mvn/version "0.10.2"} com.fluree/json-ld {:git/url "https://github.com/fluree/json-ld.git" :sha "a909330e33196504ef8a5411aaa0409ab72aaa35"} ;; logging org.clojure/tools.logging {:mvn/version "1.2.4"} ch.qos.logback/logback-classic {:mvn/version "1.4.5"} - org.slf4j/slf4j-api {:mvn/version "2.0.5"} + org.slf4j/slf4j-api {:mvn/version "2.0.6"} ;; Lucene clucie/clucie {:mvn/version "0.4.2"} @@ -44,22 +44,22 @@ :aliases {:build - {:deps {io.github.clojure/tools.build {:git/tag "v0.8.5" - :git/sha "b73ff34"} + {:deps {io.github.clojure/tools.build {:git/tag "v0.9.4" + :git/sha "b0ee58b"} slipset/deps-deploy {:mvn/version "0.2.0"}} :ns-default build} :dev {:extra-paths ["dev" "test" "dev-resources" "src-cljs" "src-nodejs" "src-docs"] - :extra-deps {org.clojure/tools.namespace {:mvn/version "1.3.0"} + :extra-deps {org.clojure/tools.namespace {:mvn/version "1.4.2"} criterium/criterium {:mvn/version "0.4.6"} figwheel-sidecar/figwheel-sidecar {:mvn/version "0.5.20"} - thheller/shadow-cljs {:mvn/version "2.20.12"} + thheller/shadow-cljs {:mvn/version "2.22.0"} com.magnars/test-with-files {:mvn/version "2021-02-17"}}} :cljtest {:extra-paths ["test" "dev-resources"] - :extra-deps {lambdaisland/kaocha {:mvn/version "1.71.1119"} + :extra-deps {lambdaisland/kaocha {:mvn/version "1.80.1274"} org.clojure/test.check {:mvn/version "1.1.1"} io.github.cap10morgan/test-with-files {:git/tag "v1.0.0" :git/sha "9181a2e"}} @@ -107,5 +107,5 @@ :main-opts ["-m" "antq.core"]} :clj-kondo - {:extra-deps {clj-kondo/clj-kondo {:mvn/version "2022.11.02"}} + {:extra-deps {clj-kondo/clj-kondo {:mvn/version "2023.02.17"}} :main-opts ["-m" "clj-kondo.main" "--lint" "src" "--config" ".clj-kondo/config.edn"]}}} From 212c95bea2625996a6c5a09933f1315a58a2a188 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Thu, 23 Mar 2023 11:48:18 -0600 Subject: [PATCH 02/50] Update to malli 0.10.4 Contains some fixes & new features we need (e.g. ::m/default support in :map schemas) --- deps.edn | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/deps.edn b/deps.edn index d8080f5a8..ed55b5a6a 100644 --- a/deps.edn +++ b/deps.edn @@ -9,7 +9,7 @@ byte-streams/byte-streams {:mvn/version "0.2.4"} cheshire/cheshire {:mvn/version "5.11.0"} instaparse/instaparse {:mvn/version "1.4.12"} - metosin/malli {:mvn/version "0.10.2"} + metosin/malli {:mvn/version "0.10.4"} com.fluree/json-ld {:git/url "https://github.com/fluree/json-ld.git" :sha "a909330e33196504ef8a5411aaa0409ab72aaa35"} @@ -55,12 +55,12 @@ criterium/criterium {:mvn/version "0.4.6"} figwheel-sidecar/figwheel-sidecar {:mvn/version "0.5.20"} thheller/shadow-cljs {:mvn/version "2.22.0"} - com.magnars/test-with-files {:mvn/version "2021-02-17"}}} + com.magnars/test-with-files {:mvn/version "2021-02-17"}}} :cljtest {:extra-paths ["test" "dev-resources"] - :extra-deps {lambdaisland/kaocha {:mvn/version "1.80.1274"} - org.clojure/test.check {:mvn/version "1.1.1"} + :extra-deps {lambdaisland/kaocha {:mvn/version "1.80.1274"} + org.clojure/test.check {:mvn/version "1.1.1"} io.github.cap10morgan/test-with-files {:git/tag "v1.0.0" :git/sha "9181a2e"}} :exec-fn kaocha.runner/exec-fn @@ -92,15 +92,15 @@ :main-opts ["-m" "cloverage.coverage" "-p" "src" "-s" "test" "--output" "scanning_results/coverage"]} :eastwood - {:extra-deps {jonase/eastwood {:mvn/version "1.3.0"}} - :main-opts ["-m" "eastwood.lint" - {:source-paths ["src" "src-docs"] - :test-paths ["test"] - ;; TODO: Un-exclude this when it stops triggering false - ;; positives on "UnsupportedOperationException empty is - ;; not supported on Flake" when using the #Flake data - ;; reader - WSM 2023-02-01 - :exclude-linters [:implicit-dependencies]}]} + {:extra-deps {jonase/eastwood {:mvn/version "1.3.0"}} + :main-opts ["-m" "eastwood.lint" + {:source-paths ["src" "src-docs"] + :test-paths ["test"] + ;; TODO: Un-exclude this when it stops triggering false + ;; positives on "UnsupportedOperationException empty is + ;; not supported on Flake" when using the #Flake data + ;; reader - WSM 2023-02-01 + :exclude-linters [:implicit-dependencies]}]} :ancient {:extra-deps {com.github.liquidz/antq {:mvn/version "RELEASE"}} From 549d8e66ab1e02748922bd6f450f4f909247ff9f Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Thu, 23 Mar 2023 12:26:35 -0600 Subject: [PATCH 03/50] Use string keys & @context only in json-ld.api fns ...to better optimize for JSON use cases. Only the f.d.json-ld.api-test and f.d.query.fql-parse-test test namespaces have been updated, so the others fail as of this commit. --- src/fluree/db/api/query.cljc | 190 ++++++++-------- src/fluree/db/conn/file.cljc | 13 +- src/fluree/db/conn/ipfs.cljc | 4 +- src/fluree/db/conn/memory.cljc | 4 +- src/fluree/db/did.cljc | 37 ++-- src/fluree/db/json_ld/api.cljc | 85 +++++--- src/fluree/db/json_ld/commit.cljc | 185 ++++++++-------- src/fluree/db/json_ld/transact.cljc | 10 +- src/fluree/db/json_ld/vocab.cljc | 4 +- src/fluree/db/ledger/json_ld.cljc | 23 +- src/fluree/db/query/exec.cljc | 7 +- src/fluree/db/query/exec/group.cljc | 2 +- src/fluree/db/query/exec/having.cljc | 2 +- src/fluree/db/query/exec/select.cljc | 11 +- src/fluree/db/query/exec/where.cljc | 2 +- src/fluree/db/query/fql/parse.cljc | 82 ++++--- src/fluree/db/query/fql/syntax.cljc | 115 +++++----- .../db/query/subject_crawl/reparse.cljc | 4 +- src/fluree/db/util/core.cljc | 33 ++- src/fluree/db/util/validation.cljc | 18 ++ src/fluree/sdk/browser.cljs | 22 +- src/fluree/sdk/node.cljs | 24 +- test/fluree/db/json_ld/api_test.cljc | 206 +++++++++--------- test/fluree/db/query/fql_parse_test.clj | 118 +++++----- test/fluree/db/test_utils.cljc | 28 +-- 25 files changed, 641 insertions(+), 588 deletions(-) create mode 100644 src/fluree/db/util/validation.cljc diff --git a/src/fluree/db/api/query.cljc b/src/fluree/db/api/query.cljc index 78ca9d734..3c0acfddb 100644 --- a/src/fluree/db/api/query.cljc +++ b/src/fluree/db/api/query.cljc @@ -23,45 +23,44 @@ {:status 400 :error :db/invalid-query})) - (let [{:keys [context history t commit-details] :as _parsed} (history/history-query-parser query-map) + (let [{:strs [context history t commit-details]} + (history/history-query-parser query-map) ;; from and to are positive ints, need to convert to negative or fill in default values - {:keys [from to at]} t - [from-t to-t] (if at - (let [t (cond (= :latest at) (:t db) - (string? at) (t db at)) - (number? at) (- at))] - [t t]) - ;; either (:from or :to) - [(cond (= :latest from) (:t db) - (string? from) (t db from)) - (number? from) (- from) - (nil? from) -1) - (cond (= :latest to) (:t db) - (string? to) (t db to)) - (number? to) (- to) - (nil? to) (:t db))]) - + {:strs [from to at]} t + [from-t to-t] (if at + (let [t (cond (= "latest" at) (:t db) + (string? at) (t db at)) + (number? at) (- at))] + [t t]) + [(cond (= "latest" from) (:t db) + (string? from) (t db from)) + (number? from) (- from) + (nil? from) -1) + (cond (= "latest" to) (:t db) + (string? to) (t db to)) + (number? to) (- to) + (nil? to) (:t db))]) parsed-context (fql-parse/parse-context query-map db) error-ch (async/chan)] (if history ;; filter flakes for history pattern - (let [[pattern idx] (json-ld db parsed-context error-ch flakes)] (if commit-details ;; annotate with commit details (async/alt! - (async/into [] (history/add-commit-details db parsed-context error-ch history-results-chan)) - ([result] result) - error-ch ([e] e)) + (async/into [] (history/add-commit-details db parsed-context error-ch history-results-chan)) + ([result] result) + error-ch ([e] e)) ;; we're already done (async/alt! @@ -79,32 +78,44 @@ "Execute a query against a database source, or optionally additional sources if the query spans multiple data sets. Returns core async channel containing result or exception." - [sources query] + [sources {:strs [opts t] :as query}] (go-try (let [{query :subject, issuer :issuer} (or ( (if t - ( (if t + ( opts (assoc :issuer issuer) (dissoc :meta)) - start #?(:clj (System/nanoTime) - :cljs (util/current-time-millis)) + start #?(:clj (System/nanoTime) + :cljs (util/current-time-millis)) result ( query-opts - :context-type - (or global-context-type)) - meta? (or global-meta query-meta) - remove-meta? (and meta? (not query-meta)) ;; query didn't ask for meta, but multiquery did so must strip it - opts* (-> (:opts query) - (assoc :meta meta? :-remove-meta? remove-meta?) - (cond-> context-type (assoc :context-type context-type))) - query* (assoc query :opts opts*)] - (assoc acc alias query*))) - {} (dissoc flureeQL :opts)) - start-time #?(:clj (System/nanoTime) :cljs (util/current-time-millis)) - ;; kick off all queries in parallel, each alias now mapped to core async channel - pending-resp (map (fn [[alias q]] [alias (query source q)]) queries)] - (loop [[[alias port] & r] pending-resp - status-global nil ;; overall status. - response {}] - (if (nil? port) ;; done? - (if global-meta - {:result response - :status status-global - :time (util/response-time-formatted start-time)} - response) - (let [{:keys [meta -remove-meta?]} (get-in queries [alias :opts]) - res (async/bytes json) hash (crypto/sha2-256 bytes :hex) type-dir (-> data-type name (str "s")) @@ -193,7 +194,7 @@ (defn read-context [conn context-key] - (json/parse (read-address conn context-key) true)) + (json/parse (read-address conn context-key) false)) (defrecord FileConnection [id memory state ledger-defaults parallelism msg-in-ch msg-out-ch lru-cache-atom] @@ -280,8 +281,8 @@ s)) (defn ledger-defaults - [{:keys [context-type context did indexer]}] - {:context (util/normalize-context context-type context) + [{:keys [did indexer context] :as _opts}] + {:context context :did did :indexer (cond (fn? indexer) @@ -305,11 +306,13 @@ state (state-machine/blank-state) cache-size (conn-cache/memory->cache-size memory) - lru-cache-atom (or lru-cache-atom (atom (conn-cache/create-lru-cache cache-size)))] + lru-cache-atom (or lru-cache-atom (atom (conn-cache/create-lru-cache cache-size))) + ld (ledger-defaults defaults)] + (log/debug "connect ledger-defaults:" ld) ;; TODO - need to set up monitor loops for async chans (map->FileConnection {:id conn-id :storage-path storage-path - :ledger-defaults (ledger-defaults defaults) + :ledger-defaults ld :serializer #?(:clj (avro-serde/avro-serde) :cljs (json-serde/json-serde)) :parallelism parallelism diff --git a/src/fluree/db/conn/ipfs.cljc b/src/fluree/db/conn/ipfs.cljc index 3579d4db4..8c5186ba0 100644 --- a/src/fluree/db/conn/ipfs.cljc +++ b/src/fluree/db/conn/ipfs.cljc @@ -158,7 +158,7 @@ (defn ledger-defaults "Normalizes ledger defaults settings" - [ipfs-endpoint {:keys [ipns context-type context did indexer] :as defaults}] + [ipfs-endpoint {:keys [ipns context did indexer] :as _defaults}] (go-try (let [ipns-default-key (or (:key ipns) "self") ipns-default-address (did-map "Returns a complete did map from a private key." - [private-key] - (let [public (crypto/pub-key-from-private private-key) - auth-id (crypto/account-id-from-public public) - did-id (auth-id->did auth-id)] - (did-map did-id public private-key))) + ([private-key] (private->did-map private-key true)) + ([private-key keyword-keys?] + (let [public (crypto/pub-key-from-private private-key) + auth-id (crypto/account-id-from-public public) + did-id (auth-id->did auth-id) + kw-map (did-map did-id public private-key)] + (if keyword-keys? + kw-map + (util/stringify-keys kw-map))))) ;; https://github.com/multiformats/multicodec/blob/master/table.csv (def secp256k1-pub @@ -54,17 +59,17 @@ encoded secp256k1 public key." [did] (let [[_ _ multibase-value] (str/split did #":") - prefix (str (first multibase-value)) - base-key (subs multibase-value 1) - _ (when (not= prefix base58btc) - (throw (ex-info (str "The prefix " (pr-str prefix) " does not map to a supported multibase encoding.") - {:value multibase-value - :prefix prefix}))) - multicodec (alphabase/bytes->hex (base58/decode base-key)) - pubkey-header (subs multicodec 0 2) - pubkey (subs multicodec 2)] + prefix (str (first multibase-value)) + base-key (subs multibase-value 1) + _ (when (not= prefix base58btc) + (throw (ex-info (str "The prefix " (pr-str prefix) " does not map to a supported multibase encoding.") + {:value multibase-value + :prefix prefix}))) + multicodec (alphabase/bytes->hex (base58/decode base-key)) + pubkey-header (subs multicodec 0 2) + pubkey (subs multicodec 2)] (when (not= pubkey-header secp256k1-pub) (throw (ex-info (str "The multicodec header " (pr-str pubkey-header) " does not map to a supported multicodec encoding.") - {:value multicodec + {:value multicodec :header pubkey-header}))) pubkey)) diff --git a/src/fluree/db/json_ld/api.cljc b/src/fluree/db/json_ld/api.cljc index e71c8381e..949b3397f 100644 --- a/src/fluree/db/json_ld/api.cljc +++ b/src/fluree/db/json_ld/api.cljc @@ -46,26 +46,40 @@ Multiple connections to same endpoint will share underlying network connection. - Options include: - - did - (optional) DiD information to use, if storing blocks as verifiable credentials, - or issuing queries against a permissioned database. - - context - (optional) Default @context map to use for ledgers formed with this connection. - - write - (optional) Function to use for all writes, if empty will store in memory until a commit is performed - - read - (optional) Function to use for reads of persisted blocks/data - - commit - (optional) Function to use to write commits. If persistence desired, this must be defined - - push - (optional) Function(s) in a vector that will attempt to push the commit to naming service(s) - " - [{:keys [method parallelism] :as opts}] + Options include (keys are strings): + - defaults: + - did - (optional) DiD information to use, if storing blocks as verifiable credentials, + or issuing queries against a permissioned database. + - context - (optional) Default @context map to use for ledgers formed with this connection." + [{:strs [method parallelism] :as opts}] ;; TODO - do some validation (promise-wrap - (let [opts* (assoc opts :parallelism (or parallelism 4)) - method* (keyword method)] + (let [parallelism* (or parallelism 4) + opts* (assoc opts "parallelism" parallelism*) + shared-opts (-> opts* + (util/update-in-if-contains ["defaults" "did"] + util/assoc-from-str-opts + #{"id" "public" "private"}) + (util/update-in-if-contains ["defaults"] + util/assoc-from-str-opts + #{"did" + {"@context" :context}}) + (util/assoc-from-str-opts + #{"defaults" "lru-cache-atom" "memory" "parallelism"})) + method* (keyword method)] + (log/debug "connect shared opts:" shared-opts) (case method* - :ipfs (ipfs-conn/connect opts*) + :ipfs (ipfs-conn/connect (->> shared-opts + (util/assoc-from-str-opts + opts + #{"server" "memory" "serializer"}))) :file (if platform/BROWSER (throw (ex-info "File connection not supported in the browser" opts)) - (file-conn/connect opts*)) - :memory (memory-conn/connect opts*))))) + (file-conn/connect (->> shared-opts + (util/assoc-from-str-opts + opts + #{"storage-path"})))) + :memory (memory-conn/connect shared-opts))))) (defn connect-ipfs "Forms an ipfs connection using default settings. @@ -74,14 +88,14 @@ - did - (optional) DiD information to use, if storing blocks as verifiable credentials - context - (optional) Default @context map to use for ledgers formed with this connection." [opts] - (connect (assoc opts :method :ipfs))) + (connect (assoc opts "method" :ipfs))) (defn connect-memory "Forms an in-memory connection using default settings. - did - (optional) DiD information to use, if storing blocks as verifiable credentials - context - (optional) Default @context map to use for ledgers formed with this connection." [opts] - (connect (assoc opts :method :memory))) + (connect (assoc opts "method" :memory))) (defn address? "Returns true if the argument is a full ledger address, false if it is just an @@ -90,8 +104,7 @@ (str/starts-with? ledger-alias-or-address "fluree:")) (defn create - "Creates a new json-ld ledger. A connection (conn) - must always be supplied. + "Creates a new json-ld ledger. A connection (conn) must always be supplied. Ledger-name (optional) is a friendly name that is used for: - When publishing to a naming service that allows multiple pointers for the @@ -105,14 +118,19 @@ - When combining multiple ledgers, each ledger becomes an individual named graph which can be referenced by name. - Options map (opts) can include: + Options map (opts) can include (keys are strings): - did - DiD information to use, if storing blocks as verifiable credentials - - context - Default @context map to use for ledgers formed with this connection - " + - @context - Default @context map to use for ledgers formed with this connection" ([conn] (create conn nil nil)) ([conn ledger-alias] (create conn ledger-alias nil)) ([conn ledger-alias opts] - (let [res-ch (jld-ledger/create conn ledger-alias opts)] + (let [opts* (util/assoc-from-str-opts + opts + #{{"@context" :context} "did" "branch" "pub-fn" "ipns" + "indexer" "include" "reindex-min-bytes" "reindex-max-bytes" + "initial-tx"}) + _ (log/debug "create opts*:" opts*) + res-ch (jld-ledger/create conn ledger-alias opts*)] (promise-wrap res-ch)))) (defn load-from-address @@ -188,6 +206,8 @@ "Performs a transaction and queues change if valid (does not commit)" ([db json-ld] (stage db json-ld nil)) ([db json-ld opts] + (log/debug "stage JSON-LD:" json-ld) + (log/debug "staging in db:" db "w/ opts:" opts "\n" json-ld) (let [result-ch (db-proto/-stage db json-ld opts)] (promise-wrap result-ch)))) @@ -203,8 +223,20 @@ (promise-wrap (ledger-proto/-commit! ledger db))) ([ledger db opts] - (promise-wrap - (ledger-proto/-commit! ledger db opts)))) + (log/debug "commit! incoming opts:" opts) + (let [opts* (if (map? opts) + (-> opts + (util/update-in-if-contains ["did"] + util/assoc-from-str-opts + #{"id" "public" "private"}) + (util/assoc-from-str-opts opts + #{"message" "branch" "tag" + {"@context" :context} + "did" "private" "push?"})) + opts)] + (log/debug "commit! decoded opts:" opts*) + (promise-wrap + (ledger-proto/-commit! ledger db opts*))))) (defn status @@ -247,6 +279,7 @@ (if opts (throw (ex-info "DB opts not yet implemented" {:status 500 :error :db/unexpected-error})) + ;; TOOD: When db opts are allowed, convert from string keys to keyword keys (ledger-proto/-db ledger opts)))) @@ -277,7 +310,7 @@ that produced the changes." [ledger query] (let [latest-db (ledger-proto/-db ledger) - res-chan (query-api/history latest-db query)] + res-chan (query-api/history latest-db query)] (promise-wrap res-chan))) (defn range diff --git a/src/fluree/db/json_ld/commit.cljc b/src/fluree/db/json_ld/commit.cljc index 54adf4649..9ca326fec 100644 --- a/src/fluree/db/json_ld/commit.cljc +++ b/src/fluree/db/json_ld/commit.cljc @@ -307,84 +307,84 @@ (flake/create const/$_commitdata:flakes const/$iri const/iri-flakes const/$xsd:string t true nil) (flake/create const/$_commitdata:size const/$iri const/iri-size const/$xsd:string t true nil) (flake/create const/$_commitdata:t const/$iri const/iri-t const/$xsd:string t true nil)] - db* (add-commit-flakes-to-db db schema-flakes)] + db* (add-commit-flakes-to-db db schema-flakes)] (assoc db* :schema (vocab/update-with* schema t schema-flakes)))) (defn add-commit-flakes "Translate commit metadata into flakes and merge them into novelty." [prev-commit {:keys [commit] :as db}] (go-try - (let [last-sid (volatile! (jld-ledger/last-commit-sid db)) - next-sid (fn [] (vswap! last-sid inc)) + (let [last-sid (volatile! (jld-ledger/last-commit-sid db)) + next-sid (fn [] (vswap! last-sid inc)) {:keys [address alias branch data fluree-default-context id issuer message time v]} commit - {db-id :id db-t :t db-address :address :keys [flakes size]} data + {db-id :id db-t :t db-address :address :keys [flakes size]} data {previous-id :id prev-data :data} prev-commit - prev-data-id (:id prev-data) - - t (- db-t) - db* (if (= 1 db-t) - (add-commit-schema-flakes db t) - db) - db-sid (next-sid) - - base-flakes [ ;; link db to associated commit meta: @id - (flake/create t const/$iri id const/$xsd:string t true nil) - - ;; commit flakes - ;; address - (flake/create t const/$_address address const/$xsd:string t true nil) - ;; alias - (flake/create t const/$_ledger:alias alias const/$xsd:string t true nil) - ;; branch - (flake/create t const/$_ledger:branch branch const/$xsd:string t true nil) - ;; fluree-default-context - (flake/create t const/$_ledger:context fluree-default-context const/$xsd:string t true nil) - ;; v - (flake/create t const/$_v v const/$xsd:int t true nil) - ;; time - (flake/create t const/$_commit:time (util/str->epoch-ms time) const/$xsd:dateTime t true nil) ;; data - (flake/create t const/$_commit:data db-sid const/$xsd:anyURI t true nil) - - - - ;; db flakes - ;; @id - (flake/create db-sid const/$iri db-id const/$xsd:string t true nil) - ;; t - (flake/create db-sid const/$_commitdata:t db-t const/$xsd:int t true nil) - ;; address - (flake/create db-sid const/$_address db-address const/$xsd:string t true nil) - ;; size - (flake/create db-sid const/$_commitdata:size size const/$xsd:int t true nil) - ;; flakes - (flake/create db-sid const/$_commitdata:flakes flakes const/$xsd:int t true nil)] + prev-data-id (:id prev-data) + + t (- db-t) + db* (if (= 1 db-t) + (add-commit-schema-flakes db t) + db) + db-sid (next-sid) + + base-flakes [;; link db to associated commit meta: @id + (flake/create t const/$iri id const/$xsd:string t true nil) + + ;; commit flakes + ;; address + (flake/create t const/$_address address const/$xsd:string t true nil) + ;; alias + (flake/create t const/$_ledger:alias alias const/$xsd:string t true nil) + ;; branch + (flake/create t const/$_ledger:branch branch const/$xsd:string t true nil) + ;; fluree-default-context + (flake/create t const/$_ledger:context fluree-default-context const/$xsd:string t true nil) + ;; v + (flake/create t const/$_v v const/$xsd:int t true nil) + ;; time + (flake/create t const/$_commit:time (util/str->epoch-ms time) const/$xsd:dateTime t true nil) ;; data + (flake/create t const/$_commit:data db-sid const/$xsd:anyURI t true nil) + + + + ;; db flakes + ;; @id + (flake/create db-sid const/$iri db-id const/$xsd:string t true nil) + ;; t + (flake/create db-sid const/$_commitdata:t db-t const/$xsd:int t true nil) + ;; address + (flake/create db-sid const/$_address db-address const/$xsd:string t true nil) + ;; size + (flake/create db-sid const/$_commitdata:size size const/$xsd:int t true nil) + ;; flakes + (flake/create db-sid const/$_commitdata:flakes flakes const/$xsd:int t true nil)] prev-commit-flakes (when previous-id (let [prev-sid ( base-flakes - prev-commit-flakes (into prev-commit-flakes) - prev-db-flakes (into prev-db-flakes) - issuer-flakes (into issuer-flakes) - message-flakes (into message-flakes)) - db** (assoc-in db* [:ecount const/$_shard] @last-sid)] + prev-db-flakes (when prev-data-id + (let [prev-sid ( base-flakes + prev-commit-flakes (into prev-commit-flakes) + prev-db-flakes (into prev-db-flakes) + issuer-flakes (into issuer-flakes) + message-flakes (into message-flakes)) + db** (assoc-in db* [:ecount const/$_shard] @last-sid)] (add-commit-flakes-to-db db** flakes)))) (defn link-context-to-commit @@ -393,10 +393,10 @@ back into the commit under the :context key." [{:keys [conn] :as ledger} commit] (go-try - (let [context (get commit (keyword const/iri-default-context)) - context-str (util/stringify-keys context) - {:keys [address]} (tx-state - [db {:keys [bootstrap? issuer context-type] :as _opts}] + [db {:keys [bootstrap? issuer] :as _opts}] (let [{:keys [block ecount schema branch ledger policy], db-t :t} db last-pid (volatile! (jld-ledger/last-pid db)) last-sid (volatile! (jld-ledger/last-sid db)) @@ -218,9 +218,7 @@ :db-before (dbproto/-rootdb db) :policy policy :bootstrap? bootstrap? - :default-ctx (if (= :string context-type) - (:context-str schema) - (:context schema)) + :default-ctx (:context schema) :stage-update? (= t db-t) ;; if a previously staged db is getting updated again before committed :refs (volatile! (or (:refs schema) #{const/$rdf:type})) :t t @@ -363,7 +361,7 @@ (if (empty? (dissoc node :idx :id)) (throw (ex-info (str "Invalid transaction, transaction node contains no properties" (some->> (:id node) - (str " for @id: ") ) + (str " for @id: ")) ".") {:status 400 :error :db/invalid-transaction})) (let [[_node-sid node-flakes] (flakes node tx-state nil))] @@ -415,7 +413,7 @@ (go-try (let [{:keys [delete] :as parsed-query} (-> json-ld - syntax/validate + syntax/validate-query (q-parse/parse-delete db)) [s p o] delete diff --git a/src/fluree/db/json_ld/vocab.cljc b/src/fluree/db/json_ld/vocab.cljc index 27ea33f15..c8ccdc158 100644 --- a/src/fluree/db/json_ld/vocab.cljc +++ b/src/fluree/db/json_ld/vocab.cljc @@ -223,8 +223,8 @@ "Updates the schema map of a db." [db] (go-try - (let [{{:keys [context context-str]} :schema} db + (let [{{:keys [context]} :schema} db _ (log/debug "refresh-schema existing context:" context) schema ( context util/stringify-keys json-ld/parse-context)] - (-> branch - (assoc-in [:latest-db :schema :context] context-kw) - (assoc-in [:latest-db :schema :context-str] context-str)))) + branch (if requested-branch + (get branches requested-branch) + ;; default branch + (get branches branch)) + context (json-ld/parse-context context)] + (log/debug "setting branch context to:" context) + (assoc-in branch [:latest-db :schema :context] context))) ;; TODO - no time travel, only latest db on a branch thus far (defn db @@ -150,7 +148,7 @@ "Creates a new ledger, optionally bootstraps it as permissioned or with default context." [conn ledger-alias opts] (go-try - (let [{:keys [context-type context did branch pub-fn ipns indexer include + (let [{:keys [context did branch pub-fn ipns indexer include reindex-min-bytes reindex-max-bytes initial-tx] :or {branch :main}} opts did* (if did @@ -176,9 +174,7 @@ address (> context - (util/normalize-context context-type) - (merge conn-context)) + context* (merge conn-context context) _ (log/debug "create merged context*:" context*) method-type (conn-proto/-method conn) ;; map of all branches and where they are branched from @@ -239,6 +235,7 @@ :value (->> (jld-reify/load-default-context conn)) > (where/search db q error-ch) - #_(log/debug-async->>vals "where/search results:") (group/combine q) (having/filter q error-ch) (order/arrange q) diff --git a/src/fluree/db/query/exec/group.cljc b/src/fluree/db/query/exec/group.cljc index e2966cf17..e9ef13ae0 100644 --- a/src/fluree/db/query/exec/group.cljc +++ b/src/fluree/db/query/exec/group.cljc @@ -64,7 +64,7 @@ (defn combine "Returns a channel of solutions from `solution-ch` collected into groups defined by the `:group-by` clause specified in the supplied query." - [{:keys [group-by select]} solution-ch] + [{:strs [group-by select]} solution-ch] (if-let [grouping (or group-by (implicit-grouping select))] (-> (async/transduce (map (partial split-solution-by grouping)) diff --git a/src/fluree/db/query/exec/having.cljc b/src/fluree/db/query/exec/having.cljc index 8bb80a373..63217d3db 100644 --- a/src/fluree/db/query/exec/having.cljc +++ b/src/fluree/db/query/exec/having.cljc @@ -6,7 +6,7 @@ (defn filter [q error-ch solution-ch] - (if-let [filter-fn (:having q)] + (if-let [filter-fn (get q "having")] (let [filtered-ch (async/chan)] (async/pipeline-async 2 filtered-ch diff --git a/src/fluree/db/query/exec/select.cljc b/src/fluree/db/query/exec/select.cljc index 3103e8d56..578b700cf 100644 --- a/src/fluree/db/query/exec/select.cljc +++ b/src/fluree/db/query/exec/select.cljc @@ -116,13 +116,14 @@ "Formats each solution within the stream of solutions in `solution-ch` according to the selectors within the select clause of the supplied parsed query `q`." [db q error-ch solution-ch] - (let [context (:context q) + (let [context (get q "@context") + _ (log/debug "format context:" context) compact (json-ld/compact-fn context) - selectors (or (:select q) - (:select-one q) - (:select-distinct q)) + selectors (or (get q "select") + (get q "select-one") + (get q "select-distinct")) iri-cache (volatile! {}) - format-ch (if (contains? q :select-distinct) + format-ch (if (contains? q "select-distinct") (chan 1 (distinct)) (chan))] (async/pipeline-async 1 diff --git a/src/fluree/db/query/exec/where.cljc b/src/fluree/db/query/exec/where.cljc index c52fc84ab..82f522730 100644 --- a/src/fluree/db/query/exec/where.cljc +++ b/src/fluree/db/query/exec/where.cljc @@ -377,7 +377,7 @@ (defn search [db q error-ch] - (let [where-clause (:where q) + (let [where-clause (get q "where") initial-solutions (-> q :values not-empty diff --git a/src/fluree/db/query/fql/parse.cljc b/src/fluree/db/query/fql/parse.cljc index b5a1a5af4..42bdb4f8b 100644 --- a/src/fluree/db/query/fql/parse.cljc +++ b/src/fluree/db/query/fql/parse.cljc @@ -19,13 +19,9 @@ #?(:clj (set! *warn-on-reflection* true)) (defn parse-context - [{:keys [opts] :as q} db] - (let [ctx-key (if (= :string (or (:context-type opts) - (:contextType opts))) - :context-str - :context) - db-ctx (get-in db [:schema ctx-key]) - q-ctx (or (:context q) (get q "@context"))] + [q db] + (let [db-ctx (get-in db [:schema :context]) + q-ctx (get q "@context")] (json-ld/parse-context db-ctx q-ctx))) (defn parse-var-name @@ -49,7 +45,7 @@ (defn parse-values [q] - (when-let [values (:values q)] + (when-let [values (get q "values")] (let [[vars vals] values vars* (util/sequential vars) vals* (mapv util/sequential vals) @@ -278,14 +274,14 @@ (fn [pattern _vars _db _context] (log/debug "parse-pattern pattern:" pattern) (cond - (map? pattern) (->> pattern keys first) + (map? pattern) (->> pattern keys first keyword) (map-entry? pattern) :binding :else :triple))) (defn type-pattern? [typ x] (and (map? x) - (-> x keys first (= typ)))) + (-> x keys first keyword (= typ)))) (def filter-pattern? (partial type-pattern? :filter)) @@ -345,14 +341,14 @@ (parse-triple triple db context)) (defmethod parse-pattern :union - [{:keys [union]} vars db context] + [{:strs [union]} vars db context] (let [parsed (mapv (fn [clause] (parse-where-clause clause vars db context)) union)] (where/->pattern :union parsed))) (defmethod parse-pattern :optional - [{:keys [optional]} vars db context] + [{:strs [optional]} vars db context] (let [clause (if (coll? (first optional)) optional [optional]) @@ -360,7 +356,7 @@ (where/->pattern :optional parsed))) (defmethod parse-pattern :bind - [{:keys [bind]} _vars _db _context] + [{:strs [bind]} _vars _db _context] (let [parsed (parse-bind-map bind) _ (log/debug "parsed bind map:" parsed) pattern (where/->pattern :bind parsed)] @@ -374,7 +370,7 @@ (defn parse-where [q vars db context] - (when-let [where (:where q)] + (when-let [where (get q "where")] (parse-where-clause where vars db context))) (defn parse-selector @@ -395,26 +391,26 @@ (defn parse-select [q db context] - (let [depth (or (:depth q) 0) + (let [depth (or (get q "depth") 0) select-key (some (fn [k] (when (contains? q k) k)) - [:select :selectOne :select-one - :selectDistinct :select-distinct]) + ["select" "selectOne" "select-one" + "selectDistinct" "select-distinct"]) select (-> q (get select-key) (parse-select-clause db context depth))] (case select-key - (:select - :select-one - :select-distinct) (assoc q select-key select) + ("select" + "select-one" + "select-distinct") (assoc q select-key select) - :selectOne (-> q - (dissoc :selectOne) - (assoc :select-one select)) + "selectOne" (-> q + (dissoc "selectOne") + (assoc "select-one" select)) - :selectDistinct (-> q - (dissoc :selectDistinct) - (assoc :select-distinct select))))) + "selectDistinct" (-> q + (dissoc "selectDistinct") + (assoc "select-distinct" select))))) (defn ensure-vector [x] @@ -424,15 +420,15 @@ (defn parse-grouping [q] - (some->> (or (:groupBy q) - (:group-by q)) + (some->> (or (get q "groupBy") + (get q "group-by")) ensure-vector (mapv parse-var-name))) (defn parse-ordering [q] - (some->> (or (:order-by q) - (:orderBy q)) + (some->> (or (get q "order-by") + (get q "orderBy")) ensure-vector (mapv (fn [ord] (if-let [v (parse-var-name ord)] @@ -445,8 +441,8 @@ (defn parse-having [q] - (if-let [code (some-> q :having parse-code)] - (assoc q :having (eval/compile code)) + (if-let [code (some-> q (get "having") parse-code)] + (assoc q "having" (eval/compile code)) q)) (defn parse-analytical-query* @@ -458,11 +454,11 @@ grouping (parse-grouping q) ordering (parse-ordering q)] (-> q - (assoc :context context - :where where) - (cond-> (seq values) (assoc :values values) - grouping (assoc :group-by grouping) - ordering (assoc :order-by ordering)) + (assoc "@context" context + "where" where) + (cond-> (seq values) (assoc "values" values) + grouping (assoc "group-by" grouping) + ordering (assoc "order-by" ordering)) parse-having (parse-select db context)))) @@ -474,17 +470,17 @@ (defn parse [q db] - (syntax/validate q) + (syntax/validate-query q) (parse-analytical-query q db)) (defn parse-delete [q db] - (when (:delete q) + (when (get q "delete") (let [context (parse-context q db) [vars values] (parse-values q) where (parse-where q vars db context)] (-> q - (assoc :context context - :where where) - (cond-> (seq values) (assoc :values values)) - (update :delete parse-triple db context))))) + (assoc "@context" context + "where" where) + (cond-> (seq values) (assoc "values" values)) + (update "delete" parse-triple db context))))) diff --git a/src/fluree/db/query/fql/syntax.cljc b/src/fluree/db/query/fql/syntax.cljc index b7ee13355..fbc021024 100644 --- a/src/fluree/db/query/fql/syntax.cljc +++ b/src/fluree/db/query/fql/syntax.cljc @@ -1,6 +1,9 @@ (ns fluree.db.query.fql.syntax - (:require [malli.core :as m] - [fluree.db.util.core :refer [pred-ident?]])) + (:require [fluree.db.util.core :as util :refer [pred-ident?]] + [fluree.db.util.log :as log] + [malli.core :as m] + [fluree.db.util.validation :as v]) + (:import (clojure.lang ExceptionInfo))) #?(:clj (set! *warn-on-reflection* true)) @@ -27,8 +30,6 @@ (and (or (string? x) (symbol? x) (keyword? x)) (-> x name first (= \?)))) -(def value? (complement coll?)) - (defn sid? [x] (int? x)) @@ -53,6 +54,7 @@ (m/type-schemas) (m/sequence-schemas) (m/base-schemas) + v/registry {::limit pos-int? ::offset nat-int? ::maxFuel number? @@ -62,28 +64,22 @@ ::pretty-print ::prettyPrint ::parseJSON boolean? ::parse-json ::parseJSON - ::contextType [:enum :string :keyword] - ::context-type ::contextType ::issuer [:maybe string?] ::opts [:map - [:maxFuel {:optional true} ::maxFuel] - [:max-fuel {:optional true} ::maxFuel] - [:parseJSON {:optional true} ::parseJSON] - [:parse-json {:optional true} ::parse-json] - [:prettyPrint {:optional true} ::prettyPrint] - [:pretty-print {:optional true} ::pretty-print] - [:contextType {:optional true} ::contextType] - [:context-type {:optional true} ::contextType] - [:issuer {:optional true} ::issuer]] + ["maxFuel" {:optional true} ::maxFuel] + ["max-fuel" {:optional true} ::maxFuel] + ["parseJSON" {:optional true} ::parseJSON] + ["parse-json" {:optional true} ::parse-json] + ["prettyPrint" {:optional true} ::prettyPrint] + ["pretty-print" {:optional true} ::pretty-print] + ["issuer" {:optional true} ::issuer]] ::function [:orn [:string [:fn fn-string?]] [:list [:fn fn-list?]]] ::wildcard [:fn wildcard?] ::var [:fn variable?] - ::val [:fn value?] - ::iri [:orn - [:keyword keyword?] - [:string string?]] + ::val ::v/val + ::iri ::v/iri ::subject [:orn [:sid [:fn sid?]] [:iri ::iri] @@ -123,14 +119,14 @@ [:clause ::var] [:collection [:sequential ::var]]] ::group-by ::groupBy - ::where-op [:enum :filter :optional :union :bind] + ::where-op [:enum "filter" "optional" "union" "bind"] ::where-map [:and [:map-of {:max 1} ::where-op :any] [:multi {:dispatch where-op} - [:filter [:map [:filter [:ref ::filter]]]] - [:optional [:map [:optional [:ref ::optional]]]] - [:union [:map [:union [:ref ::union]]]] - [:bind [:map [:bind [:ref ::bind]]]]]] + ["filter" [:map ["filter" [:ref ::filter]]]] + ["optional" [:map ["optional" [:ref ::optional]]]] + ["union" [:map ["union" [:ref ::union]]]] + ["bind" [:map ["bind" [:ref ::bind]]]]]] ::triple [:catn [:subject [:orn [:var ::var] @@ -140,12 +136,10 @@ [:iri ::iri]]] [:object [:orn [:var ::var] + [:iri ::iri] [:ident [:fn pred-ident?]] [:val :any]]]] - ::where-tuple [:orn - [:triple ::triple] - [:binding [:sequential {:max 2} :any]] - [:remote [:sequential {:max 4} :any]]] + ::where-tuple ::triple ::where-pattern [:orn [:where-map ::where-map] [:tuple ::where-tuple]] @@ -171,51 +165,52 @@ ::t [:or :int :string] ::delete ::triple ::delete-op [:map - [:delete ::delete] - [:where ::where] - [:values {:optional true} ::values]] - ::context [:map-of :any :any] + ["delete" ::delete] + ["where" ::where] + ["values" {:optional true} ::values]] + ::context ::v/context ::analytical-query [:map - [:where ::where] - [:t {:optional true} ::t] - [:context {:optional true} ::context] - [:select {:optional true} ::select] - [:selectOne {:optional true} ::selectOne] - [:select-one {:optional true} ::select-one] - [:selectDistinct {:optional true} ::selectDistinct] - [:select-distinct {:optional true} ::select-distinct] - [:delete {:optional true} ::delete] - [:orderBy {:optional true} ::orderBy] - [:order-by {:optional true} ::order-by] - [:groupBy {:optional true} ::groupBy] - [:group-by {:optional true} ::group-by] - [:filter {:optional true} ::filter] - [:having {:optional true} ::function] - [:values {:optional true} ::values] - [:limit {:optional true} ::limit] - [:offset {:optional true} ::offset] - [:maxFuel {:optional true} ::maxFuel] - [:max-fuel {:optional true} ::max-fuel] - [:depth {:optional true} ::depth] - [:opts {:optional true} ::opts] - [:prettyPrint {:optional true} ::prettyPrint] - [:pretty-print {:optional true} ::pretty-print]] + ["where" ::where] + ["t" {:optional true} ::t] + ["@context" {:optional true} ::context] + ["select" {:optional true} ::select] + ["selectOne" {:optional true} ::selectOne] + ["select-one" {:optional true} ::select-one] + ["selectDistinct" {:optional true} ::selectDistinct] + ["select-distinct" {:optional true} ::select-distinct] + ["delete" {:optional true} ::delete] + ["orderBy" {:optional true} ::orderBy] + ["order-by" {:optional true} ::order-by] + ["groupBy" {:optional true} ::groupBy] + ["group-by" {:optional true} ::group-by] + ["filter" {:optional true} ::filter] + ["having" {:optional true} ::function] + ["values" {:optional true} ::values] + ["limit" {:optional true} ::limit] + ["offset" {:optional true} ::offset] + ["maxFuel" {:optional true} ::maxFuel] + ["max-fuel" {:optional true} ::max-fuel] + ["depth" {:optional true} ::depth] + ["opts" {:optional true} ::opts] + ["prettyPrint" {:optional true} ::prettyPrint] + ["pretty-print" {:optional true} ::pretty-print]] ::multi-query [:map-of [:or :string :keyword] ::analytical-query] ::query [:orn [:single ::analytical-query] [:multi ::multi-query]]})) -(def query-validator +(def valid-query? (m/validator ::query {:registry registry})) (def multi-query? (m/validator ::multi-query {:registry registry})) -(defn validate +(defn validate-query [qry] - (if (query-validator qry) + (if (valid-query? qry) qry (throw (ex-info "Invalid Query" {:status 400 :error :db/invalid-query - :reasons (m/explain ::analytical-query qry {:registry registry})})))) + :reasons (m/explain ::query qry + {:registry registry})})))) diff --git a/src/fluree/db/query/subject_crawl/reparse.cljc b/src/fluree/db/query/subject_crawl/reparse.cljc index a6271e716..ab1e13e2b 100644 --- a/src/fluree/db/query/subject_crawl/reparse.cljc +++ b/src/fluree/db/query/subject_crawl/reparse.cljc @@ -151,8 +151,8 @@ (defn re-parse-as-simple-subj-crawl "Returns true if query contains a single subject crawl. e.g. - {:select {?subjects ['*']} - :where [...]}" + {\"select\" {?subjects ['*']} + \"where\" [...]}" [{:keys [order-by group-by] :as parsed-query}] (when (and (not group-by) (not order-by) diff --git a/src/fluree/db/util/core.cljc b/src/fluree/db/util/core.cljc index fc378690d..1805e1541 100644 --- a/src/fluree/db/util/core.cljc +++ b/src/fluree/db/util/core.cljc @@ -209,13 +209,32 @@ (assoc acc k v))) {} m)) -(defn normalize-context - "Keywordizes string contexts so they merge correctly with other keyword - contexts." - [context-type context] - (if (= :keyword context-type) - context - (keywordize-keys context))) +(defn assoc-from-str-opts + "For each string in str-keys, if the supplied str key exists in the supplied + from opts map, it is assoc'ed into the to map w/ a keyword key. + + Alternatively you can supply single-element maps in str-keys whose key is the + string key to lookup in from and the value is the keyword key to assoc it into + to with. This is useful for when the to key is not just (keyword str-key)." + ([from str-keys] (assoc-from-str-opts from str-keys {})) + ([from str-keys to] + (reduce (fn [t k] + (let [[from-k to-k] (if (map? k) + (first k) + [k (keyword k)])] + (if (contains? from from-k) + (assoc t to-k (get from from-k)) + t))) + to str-keys))) + +(defn update-in-if-contains + "Updates the key-path kp in map m iff m contains the key-path kp. The update + is done with the clojure.core/update-in fn with m kp f args. Returns m + unmodified if it doesn't contain kp." + [m kp f & args] + (if (= ::not-found (get-in m kp ::not-found)) + m + (apply update-in m kp f args))) (defn str->epoch-ms "Takes time as a string and returns epoch millis." diff --git a/src/fluree/db/util/validation.cljc b/src/fluree/db/util/validation.cljc new file mode 100644 index 000000000..05f5c01bc --- /dev/null +++ b/src/fluree/db/util/validation.cljc @@ -0,0 +1,18 @@ +(ns fluree.db.util.validation + (:require [malli.core :as m])) + +(def value? (complement coll?)) + +(def registry + (merge + (m/base-schemas) + (m/type-schemas) + (m/comparator-schemas) + {::iri :string + ::val [:fn value?] + ::at-context [:= "@context"] + ::context-key :string + ::context [:or + :string + [:map-of ::context-key [:or :string :map]]] + ::context-containing-map [:map-of ::at-context ::context]})) diff --git a/src/fluree/sdk/browser.cljs b/src/fluree/sdk/browser.cljs index 9275f1fdb..50dfa4e80 100644 --- a/src/fluree/sdk/browser.cljs +++ b/src/fluree/sdk/browser.cljs @@ -21,12 +21,13 @@ (defn ^:export connect [opts] - (fluree/connect (js->clj opts :keywordize-keys true))) + (fluree/connect (js->clj opts :keywordize-keys false))) (defn ^:export create ([conn] (fluree/create conn)) ([conn ledger-alias] (fluree/create conn ledger-alias)) - ([conn ledger-alias opts] (fluree/create conn ledger-alias (js->clj opts :keywordize-keys true)))) + ([conn ledger-alias opts] (fluree/create conn ledger-alias + (js->clj opts :keywordize-keys false)))) (defn ^:export exists [conn alias-or-address] @@ -41,17 +42,15 @@ (defn ^:export stage ([db-or-ledger json-ld] - (fluree/stage db-or-ledger (js->clj json-ld) {:context-type :string})) + (fluree/stage db-or-ledger (js->clj json-ld) {})) ([db-or-ledger json-ld opts] (fluree/stage db-or-ledger (js->clj json-ld) - (-> (js->clj opts :keywordize-keys true) - (assoc :context-type :string))))) - + (js->clj opts :keywordize-keys false)))) (defn ^:export commit ([ledger db] (.then (fluree/commit! ledger db) (fn [result] (clj->js result)))) ([ledger db opts] (.then (fluree/commit! ledger db - (js->clj opts :keywordize-keys true)) + (js->clj opts :keywordize-keys false)) (fn [result] (clj->js result))))) (defn ^:export status @@ -60,15 +59,12 @@ (defn ^:export db ([ledger] (fluree/db ledger)) - ([ledger opts] (fluree/db ledger (js->clj opts :keywordize-keys true)))) + ([ledger opts] (fluree/db ledger (js->clj opts :keywordize-keys false)))) (defn ^:export query [db query] - (let [query* (->> (js->clj query :keywordize-keys false) - (reduce-kv (fn [acc k v] - (assoc acc (keyword k) v)) - {}))] - (.then (fluree/query db (assoc-in query* [:opts :context-type] :string)) + (let [query* (js->clj query :keywordize-keys false)] + (.then (fluree/query db query*) (fn [result] (clj->js result))))) diff --git a/src/fluree/sdk/node.cljs b/src/fluree/sdk/node.cljs index c2fa9cc40..aa6800c22 100644 --- a/src/fluree/sdk/node.cljs +++ b/src/fluree/sdk/node.cljs @@ -1,6 +1,5 @@ (ns fluree.sdk.node (:require [cljs.nodejs :as node-js] - [clojure.string :as str] [fluree.db.json-ld.api :as fluree] [fluree.db.util.log :as log])) @@ -8,13 +7,13 @@ (defn ^:export connect [opts] - (fluree/connect (js->clj opts :keywordize-keys true))) + (fluree/connect (js->clj opts :keywordize-keys false))) (defn ^:export create ([conn] (fluree/create conn)) ([conn ledger-alias] (fluree/create conn ledger-alias)) ([conn ledger-alias opts] (fluree/create conn ledger-alias - (js->clj opts :keywordize-keys true)))) + (js->clj opts :keywordize-keys false)))) (defn ^:export exists [conn alias-or-address] @@ -30,17 +29,15 @@ (defn ^:export stage ([db-or-ledger json-ld] - (fluree/stage db-or-ledger (js->clj json-ld) {:context-type :string})) + (fluree/stage db-or-ledger (js->clj json-ld))) ([db-or-ledger json-ld opts] (fluree/stage db-or-ledger (js->clj json-ld) - (-> opts - (js->clj :keywordize-keys true) - (assoc :context-type :string))))) + (js->clj opts :keywordize-keys false)))) (defn ^:export commit ([ledger db] (fluree/commit! ledger db)) ([ledger db opts] (fluree/commit! ledger db - (js->clj opts :keywordize-keys true)))) + (js->clj opts :keywordize-keys false)))) (defn ^:export status ([ledger] (clj->js (fluree/status ledger))) @@ -48,17 +45,12 @@ (defn ^:export db ([ledger] (fluree/db ledger)) - ([ledger opts] (fluree/db ledger (js->clj opts :keywordize-keys true)))) + ([ledger opts] (fluree/db ledger (js->clj opts :keywordize-keys false)))) (defn ^:export query [db query] - (let [query* (->> (js->clj query :keywordize-keys false) - (reduce-kv (fn [acc k v] - (assoc acc (if (str/starts-with? k "@") - k - (keyword k)) v)) - {}))] - (.then (fluree/query db (assoc-in query* [:opts :context-type] :string)) + (let [query* (js->clj query :keywordize-keys false)] + (.then (fluree/query db query*) (fn [result] (clj->js result))))) (log/set-level! :warning) diff --git a/test/fluree/db/json_ld/api_test.cljc b/test/fluree/db/json_ld/api_test.cljc index 5278d9cf6..5502b6aa3 100644 --- a/test/fluree/db/json_ld/api_test.cljc +++ b/test/fluree/db/json_ld/api_test.cljc @@ -19,9 +19,9 @@ ledger @(fluree/create conn ledger-alias) check2 @(fluree/exists? conn ledger-alias) _ @(fluree/stage (fluree/db ledger) - [{:id :f/me - :type :schema/Person - :schema/fname "Me"}]) + [{"id" "f:me" + "type" "schema:Person" + "schema:fname" "Me"}]) check3 @(fluree/exists? conn ledger-alias)] (is (every? false? [check1 check2 check3]))))) (testing "returns true after committing data to a ledger" @@ -30,9 +30,9 @@ ledger-alias "testledger" ledger @(fluree/create conn ledger-alias) db @(fluree/stage (fluree/db ledger) - [{:id :f/me - :type :schema/Person - :schema/fname "Me"}])] + [{"id" "f:me" + "type" "schema:Person" + "schema:fname" "Me"}])] @(fluree/commit! ledger db) (is (test-utils/retry-exists? conn ledger-alias 100)) (is (not @(fluree/exists? conn "notaledger")))) @@ -44,9 +44,9 @@ ledger-alias "testledger" ledger ( ?age 45)", "(< ?age 50)"]}]} - {:keys [select where] :as parsed} (parse/parse-analytical-query filter-q db) - {::where/keys [patterns filters]} where] + (let [filter-q {"select" ['?name '?age] + "where" [['?s "rdf:type" "ex:User"] + ['?s "schema:age" '?age] + ['?s "schema:name" '?name] + {"filter" ["(> ?age 45)", "(< ?age 50)"]}]} + {:strs [select where]} (parse/parse-analytical-query filter-q db) + {::where/keys [patterns]} where] (is (= [{:var '?name} {:var '?age}] (de-recordify-select select))) (is (= [[:class @@ -192,12 +192,12 @@ {::where/var '?name}]] patterns)))) (testing "group-by, order-by" - (let [query {:select ['?name '?favNums] - :where [['?s :schema/name '?name] - ['?s :ex/favNums '?favNums]] - :group-by '?name - :order-by '?name} - {:keys [select where group-by order-by] :as parsed} (parse/parse-analytical-query query db)] + (let [query {"select" ['?name '?favNums] + "where" [['?s "schema:name" '?name] + ['?s "ex:favNums" '?favNums]] + "group-by" '?name + "order-by" '?name} + {:strs [group-by order-by]} (parse/parse-analytical-query query db)] (is (= ['?name] group-by)) (is (= [['?name :asc]] diff --git a/test/fluree/db/test_utils.cljc b/test/fluree/db/test_utils.cljc index aaf01a4a2..759675123 100644 --- a/test/fluree/db/test_utils.cljc +++ b/test/fluree/db/test_utils.cljc @@ -6,16 +6,16 @@ [clojure.core.async.interop :refer [did-map default-private-key)}}] - (let [conn-p (fluree/connect-memory {:defaults {:context context - :did did}})] + did (did/private->did-map default-private-key false)}}] + (let [conn-p (fluree/connect-memory {"defaults" {"@context" context + "did" did}})] #?(:clj @conn-p :cljs (go ( Date: Thu, 23 Mar 2023 14:17:53 -0600 Subject: [PATCH 04/50] Add some more kw->string key fixes ...discovered while getting http-api-gateway ready to work with this. --- src/fluree/db/api/query.cljc | 10 +-- src/fluree/db/query/history.cljc | 110 +++++++++++++++-------------- src/fluree/db/util/validation.cljc | 4 +- 3 files changed, 64 insertions(+), 60 deletions(-) diff --git a/src/fluree/db/api/query.cljc b/src/fluree/db/api/query.cljc index 3c0acfddb..92ec04d88 100644 --- a/src/fluree/db/api/query.cljc +++ b/src/fluree/db/api/query.cljc @@ -92,7 +92,7 @@ (assoc-in [:policy :cache] (atom {}))) opts* (-> opts (assoc :issuer issuer) - (dissoc :meta)) + (dissoc "meta")) start #?(:clj (System/nanoTime) :cljs (util/current-time-millis)) result ( Date: Thu, 23 Mar 2023 14:32:36 -0600 Subject: [PATCH 05/50] Fix history query schema Not sure what I was thinking there... --- src/fluree/db/query/history.cljc | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/fluree/db/query/history.cljc b/src/fluree/db/query/history.cljc index 4057bf40b..975269160 100644 --- a/src/fluree/db/query/history.cljc +++ b/src/fluree/db/query/history.cljc @@ -1,6 +1,7 @@ (ns fluree.db.query.history (:require [clojure.core.async :as async] + [clojure.string :as str] [malli.core :as m] [fluree.json-ld :as json-ld] [fluree.db.constants :as const] @@ -15,7 +16,23 @@ [fluree.db.query.range :as query-range] [fluree.db.db.json-ld :as jld-db])) -(def commit-re (re-pattern "^.+[/:#]commit$")) +(def commit-key-re (re-pattern "^.+[/:#]commit$")) + +(defn commit-key + [commit] + (re-matches commit-key-re (-> commit keys first))) + +(defn commit? + [v] + ;; TODO: Move this into malli schema directly when it supports one-to-one + ;; key schema -> value schema mapping in maps. + ;; https://github.com/metosin/malli/issues/881 + (when (map? v) + (when-let [ck (commit-key v)] + (every? (fn [k] (some #(str/ends-with? k %) + #{"id" "address" "alias" "branch" "context" "data" + "time" "v" "issuer"})) + (-> v (get ck) keys))))) (def registry (merge @@ -68,7 +85,7 @@ [:fn {:error/message "Must supply either a \"history\" or \"commit-details\" key."} (fn [{:strs [history commit-details]}] (or history commit-details))]] - ::commit [:re commit-re] + ::commit [:fn commit?] ::history-query-results [:sequential ::commit]})) (def history-query-validator From 5e8d7408310fb901573f9ef120b136ec4dfd6cbd Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Thu, 23 Mar 2023 14:33:26 -0600 Subject: [PATCH 06/50] Add transaction & query result schemas ...for HTTP API validation & documentation --- src/fluree/db/json_ld/transact.cljc | 37 +++- src/fluree/db/query/fql/syntax.cljc | 299 +++++++++++++++------------- 2 files changed, 191 insertions(+), 145 deletions(-) diff --git a/src/fluree/db/json_ld/transact.cljc b/src/fluree/db/json_ld/transact.cljc index 0a021c31c..cf7571abd 100644 --- a/src/fluree/db/json_ld/transact.cljc +++ b/src/fluree/db/json_ld/transact.cljc @@ -1,5 +1,7 @@ (ns fluree.db.json-ld.transact - (:require [fluree.json-ld :as json-ld] + (:require [clojure.string :as str] + [fluree.db.util.validation :as v] + [fluree.json-ld :as json-ld] [fluree.db.constants :as const] [fluree.db.flake :as flake] [fluree.db.json-ld.vocab :as vocab] @@ -20,11 +22,42 @@ [fluree.db.policy.enforce-tx :as policy] [fluree.db.dbproto :as dbproto] [fluree.db.json-ld.credential :as cred] - [fluree.db.util.log :as log]) + [fluree.db.util.log :as log] + [malli.core :as m]) (:refer-clojure :exclude [vswap!])) #?(:clj (set! *warn-on-reflection* true)) +(def retract-key-re (re-pattern "^.+[/:#]retract$")) + +(def registry + (merge + (m/base-schemas) + (m/type-schemas) + v/registry + {::iri ::v/iri + ::val ::v/val + ::context ::v/context + ::txn-val [:orn + [:multiple [:sequential + [:or [:ref ::txn-map] [:ref ::txn-val]]]] + [:node [:ref ::txn-map]] + [:iri ::iri] + [:val ::val]] + ::txn-leaf-map [:map + ["@context" {:optional true} ::context] + [::m/default [:map-of ::iri ::txn-val]]] + ::retract-key [:and ::iri [:re retract-key-re]] + ::txn-map [:orn + [:assert ::txn-leaf-map] + [:retract + [:map + ["@context" {:optional true} ::context] + [::m/default [:map-of ::retract-key ::txn-leaf-map]]]]] + ::txn [:orn + [:single-amp ::txn-map] + [:sequence-of-maps [:sequential ::txn-map]]]})) + (declare json-ld-node->flakes) (defn json-ld-type-data diff --git a/src/fluree/db/query/fql/syntax.cljc b/src/fluree/db/query/fql/syntax.cljc index fbc021024..79f6cc371 100644 --- a/src/fluree/db/query/fql/syntax.cljc +++ b/src/fluree/db/query/fql/syntax.cljc @@ -55,149 +55,162 @@ (m/sequence-schemas) (m/base-schemas) v/registry - {::limit pos-int? - ::offset nat-int? - ::maxFuel number? - ::max-fuel ::maxFuel - ::depth nat-int? - ::prettyPrint boolean? - ::pretty-print ::prettyPrint - ::parseJSON boolean? - ::parse-json ::parseJSON - ::issuer [:maybe string?] - ::opts [:map - ["maxFuel" {:optional true} ::maxFuel] - ["max-fuel" {:optional true} ::maxFuel] - ["parseJSON" {:optional true} ::parseJSON] - ["parse-json" {:optional true} ::parse-json] - ["prettyPrint" {:optional true} ::prettyPrint] - ["pretty-print" {:optional true} ::pretty-print] - ["issuer" {:optional true} ::issuer]] - ::function [:orn - [:string [:fn fn-string?]] - [:list [:fn fn-list?]]] - ::wildcard [:fn wildcard?] - ::var [:fn variable?] - ::val ::v/val - ::iri ::v/iri - ::subject [:orn - [:sid [:fn sid?]] - [:iri ::iri] - [:ident [:fn pred-ident?]]] - ::subselect-map [:map-of ::iri [:ref ::subselection]] - ::subselection [:sequential [:orn - [:wildcard ::wildcard] - [:predicate ::iri] - [:subselect-map [:ref ::subselect-map]]]] - ::select-map [:map-of {:max 1} - ::var ::subselection] - ::selector [:orn - [:var ::var] - [:pred ::iri] - [:aggregate ::function] - [:select-map ::select-map]] - ::select [:orn - [:selector ::selector] - [:collection [:sequential ::selector]]] - ::selectOne ::select - ::select-one ::selectOne - ::select-distinct ::select - ::selectDistinct ::select-distinct - ::direction [:orn - [:asc [:fn asc?]] - [:desc [:fn desc?]]] - ::ordering [:orn - [:scalar ::var] - [:vector [:catn - [:direction ::direction] - [:dimension ::var]]]] - ::orderBy [:orn - [:clause ::ordering] - [:collection [:sequential ::ordering]]] - ::order-by ::orderBy - ::groupBy [:orn - [:clause ::var] - [:collection [:sequential ::var]]] - ::group-by ::groupBy - ::where-op [:enum "filter" "optional" "union" "bind"] - ::where-map [:and - [:map-of {:max 1} ::where-op :any] - [:multi {:dispatch where-op} - ["filter" [:map ["filter" [:ref ::filter]]]] - ["optional" [:map ["optional" [:ref ::optional]]]] - ["union" [:map ["union" [:ref ::union]]]] - ["bind" [:map ["bind" [:ref ::bind]]]]]] - ::triple [:catn - [:subject [:orn - [:var ::var] - [:val ::subject]]] - [:predicate [:orn - [:var ::var] - [:iri ::iri]]] - [:object [:orn - [:var ::var] - [:iri ::iri] - [:ident [:fn pred-ident?]] - [:val :any]]]] - ::where-tuple ::triple - ::where-pattern [:orn - [:where-map ::where-map] - [:tuple ::where-tuple]] - ::optional [:orn - [:single ::where-pattern] - [:collection [:sequential ::where-pattern]]] - ::filter [:sequential ::function] - ::union [:sequential [:sequential ::where-pattern]] - ::bind [:map-of ::var :any] - ::where [:sequential [:orn - [:where-map ::where-map] - [:tuple ::where-tuple]]] - ::var-collection [:sequential ::var] - ::val-collection [:sequential ::val] - ::single-var-binding [:tuple ::var ::val-collection] - ::value-binding [:sequential ::val] - ::multiple-var-binding [:tuple - ::var-collection - [:sequential ::value-binding]] - ::values [:orn - [:single ::single-var-binding] - [:multiple ::multiple-var-binding]] - ::t [:or :int :string] - ::delete ::triple - ::delete-op [:map - ["delete" ::delete] - ["where" ::where] - ["values" {:optional true} ::values]] - ::context ::v/context - ::analytical-query [:map - ["where" ::where] - ["t" {:optional true} ::t] - ["@context" {:optional true} ::context] - ["select" {:optional true} ::select] - ["selectOne" {:optional true} ::selectOne] - ["select-one" {:optional true} ::select-one] - ["selectDistinct" {:optional true} ::selectDistinct] - ["select-distinct" {:optional true} ::select-distinct] - ["delete" {:optional true} ::delete] - ["orderBy" {:optional true} ::orderBy] - ["order-by" {:optional true} ::order-by] - ["groupBy" {:optional true} ::groupBy] - ["group-by" {:optional true} ::group-by] - ["filter" {:optional true} ::filter] - ["having" {:optional true} ::function] - ["values" {:optional true} ::values] - ["limit" {:optional true} ::limit] - ["offset" {:optional true} ::offset] - ["maxFuel" {:optional true} ::maxFuel] - ["max-fuel" {:optional true} ::max-fuel] - ["depth" {:optional true} ::depth] - ["opts" {:optional true} ::opts] - ["prettyPrint" {:optional true} ::prettyPrint] - ["pretty-print" {:optional true} ::pretty-print]] - ::multi-query [:map-of [:or :string :keyword] ::analytical-query] - ::query [:orn - [:single ::analytical-query] - [:multi ::multi-query]]})) + {::limit pos-int? + ::offset nat-int? + ::maxFuel number? + ::max-fuel ::maxFuel + ::depth nat-int? + ::prettyPrint boolean? + ::pretty-print ::prettyPrint + ::parseJSON boolean? + ::parse-json ::parseJSON + ::issuer [:maybe string?] + ::opts [:map + ["maxFuel" {:optional true} ::maxFuel] + ["max-fuel" {:optional true} ::maxFuel] + ["parseJSON" {:optional true} ::parseJSON] + ["parse-json" {:optional true} ::parse-json] + ["prettyPrint" {:optional true} ::prettyPrint] + ["pretty-print" {:optional true} ::pretty-print] + ["issuer" {:optional true} ::issuer]] + ::function [:orn + [:string [:fn fn-string?]] + [:list [:fn fn-list?]]] + ::wildcard [:fn wildcard?] + ::var [:fn variable?] + ::val ::v/val + ::iri ::v/iri + ::subject [:orn + [:sid [:fn sid?]] + [:iri ::iri] + [:ident [:fn pred-ident?]]] + ::subselect-map [:map-of ::iri [:ref ::subselection]] + ::subselection [:sequential [:orn + [:wildcard ::wildcard] + [:predicate ::iri] + [:subselect-map [:ref ::subselect-map]]]] + ::select-map [:map-of {:max 1} + ::var ::subselection] + ::selector [:orn + [:var ::var] + [:pred ::iri] + [:aggregate ::function] + [:select-map ::select-map]] + ::select [:orn + [:selector ::selector] + [:collection [:sequential ::selector]]] + ::selectOne ::select + ::select-one ::selectOne + ::select-distinct ::select + ::selectDistinct ::select-distinct + ::direction [:orn + [:asc [:fn asc?]] + [:desc [:fn desc?]]] + ::ordering [:orn + [:scalar ::var] + [:vector [:catn + [:direction ::direction] + [:dimension ::var]]]] + ::orderBy [:orn + [:clause ::ordering] + [:collection [:sequential ::ordering]]] + ::order-by ::orderBy + ::groupBy [:orn + [:clause ::var] + [:collection [:sequential ::var]]] + ::group-by ::groupBy + ::where-op [:enum "filter" "optional" "union" "bind"] + ::where-map [:and + [:map-of {:max 1} ::where-op :any] + [:multi {:dispatch where-op} + ["filter" [:map ["filter" [:ref ::filter]]]] + ["optional" [:map ["optional" [:ref ::optional]]]] + ["union" [:map ["union" [:ref ::union]]]] + ["bind" [:map ["bind" [:ref ::bind]]]]]] + ::triple [:catn + [:subject [:orn + [:var ::var] + [:val ::subject]]] + [:predicate [:orn + [:var ::var] + [:iri ::iri]]] + [:object [:orn + [:var ::var] + [:iri ::iri] + [:ident [:fn pred-ident?]] + [:val :any]]]] + ::where-tuple ::triple + ::where-pattern [:orn + [:where-map ::where-map] + [:tuple ::where-tuple]] + ::optional [:orn + [:single ::where-pattern] + [:collection [:sequential ::where-pattern]]] + ::filter [:sequential ::function] + ::union [:sequential [:sequential ::where-pattern]] + ::bind [:map-of ::var :any] + ::where [:sequential [:orn + [:where-map ::where-map] + [:tuple ::where-tuple]]] + ::var-collection [:sequential ::var] + ::val-collection [:sequential ::val] + ::single-var-binding [:tuple ::var ::val-collection] + ::value-binding [:sequential ::val] + ::multiple-var-binding [:tuple + ::var-collection + [:sequential ::value-binding]] + ::values [:orn + [:single ::single-var-binding] + [:multiple ::multiple-var-binding]] + ::t [:or :int :string] + ::delete ::triple + ::delete-op [:map + ["delete" ::delete] + ["where" ::where] + ["values" {:optional true} ::values]] + ::context ::v/context + ::analytical-query [:map + ["where" ::where] + ["t" {:optional true} ::t] + ["@context" {:optional true} ::context] + ["select" {:optional true} ::select] + ["selectOne" {:optional true} ::selectOne] + ["select-one" {:optional true} ::select-one] + ["selectDistinct" {:optional true} ::selectDistinct] + ["select-distinct" {:optional true} ::select-distinct] + ["delete" {:optional true} ::delete] + ["orderBy" {:optional true} ::orderBy] + ["order-by" {:optional true} ::order-by] + ["groupBy" {:optional true} ::groupBy] + ["group-by" {:optional true} ::group-by] + ["filter" {:optional true} ::filter] + ["having" {:optional true} ::function] + ["values" {:optional true} ::values] + ["limit" {:optional true} ::limit] + ["offset" {:optional true} ::offset] + ["maxFuel" {:optional true} ::maxFuel] + ["max-fuel" {:optional true} ::max-fuel] + ["depth" {:optional true} ::depth] + ["opts" {:optional true} ::opts] + ["prettyPrint" {:optional true} ::prettyPrint] + ["pretty-print" {:optional true} ::pretty-print]] + ::multi-query [:map-of [:or :string :keyword] ::analytical-query] + ::query [:orn + [:single ::analytical-query] + [:multi ::multi-query]] + ::result-val [:orn + [:string :string] + [:map [:map-of :string [:ref ::result-val]]] + [:multiple [:sequential [:ref ::result-val]]] + [:other :any]] + ::result-map [:map-of :string ::result-val] + ::result-vec [:sequential ::result-val] + ::analytical-query-results [:orn + [:select-one ::result-map] + [:select [:or + [:sequential ::result-map] + [:sequential ::result-vec]]]] + ::multi-query-results [:map-of :string ::analytical-query-results]})) (def valid-query? (m/validator ::query {:registry registry})) From e68ef93ffa30bfbb7408befdbcb2f42aec8745db Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Sun, 26 Mar 2023 09:51:19 -0600 Subject: [PATCH 07/50] Fix history query results schema --- src/fluree/db/query/history.cljc | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/src/fluree/db/query/history.cljc b/src/fluree/db/query/history.cljc index 975269160..540efef8b 100644 --- a/src/fluree/db/query/history.cljc +++ b/src/fluree/db/query/history.cljc @@ -22,17 +22,29 @@ [commit] (re-matches commit-key-re (-> commit keys first))) -(defn commit? +(defn history-query-result? [v] ;; TODO: Move this into malli schema directly when it supports one-to-one ;; key schema -> value schema mapping in maps. ;; https://github.com/metosin/malli/issues/881 (when (map? v) - (when-let [ck (commit-key v)] - (every? (fn [k] (some #(str/ends-with? k %) - #{"id" "address" "alias" "branch" "context" "data" - "time" "v" "issuer"})) - (-> v (get ck) keys))))) + (and + (if-let [ck (commit-key v)] + (every? (fn [k] (some #(re-matches % k) + (map re-pattern + ["^.+[/:#]id$" "^.+[/:#]address$" + "^.+[/:#]alias$" "^.+[/:#]branch$" + "^.+[/:#]context$" + "^.+[/:#]data$" "^.+[/:#]time$" ".+[/:#]v$" + "^.+[/:#]issuer$"]))) + (-> v (get ck) keys)) + true) + (every? (fn [k] (some #(re-matches % k) + (map re-pattern + [".+[/:#]t$" ".+[/:#]assert$" + "^.+[/:#]retract$" + ".+[/:#]commit$"]))) + (keys v))))) (def registry (merge @@ -85,8 +97,8 @@ [:fn {:error/message "Must supply either a \"history\" or \"commit-details\" key."} (fn [{:strs [history commit-details]}] (or history commit-details))]] - ::commit [:fn commit?] - ::history-query-results [:sequential ::commit]})) + ::history-query-result [:fn history-query-result?] + ::history-query-results [:sequential ::history-query-result]})) (def history-query-validator (m/validator ::history-query {:registry registry})) From b414b8b10ebb41ec1a465555265791e09c2c57e8 Mon Sep 17 00:00:00 2001 From: Daniel Petranek Date: Thu, 23 Mar 2023 16:27:28 -0500 Subject: [PATCH 08/50] stringified context - subject crawl reparse --- .../db/query/subject_crawl/reparse.cljc | 15 +- .../db/query/subject_crawl_reparse_test.clj | 133 +++++++++--------- 2 files changed, 75 insertions(+), 73 deletions(-) diff --git a/src/fluree/db/query/subject_crawl/reparse.cljc b/src/fluree/db/query/subject_crawl/reparse.cljc index ab1e13e2b..707a715fd 100644 --- a/src/fluree/db/query/subject_crawl/reparse.cljc +++ b/src/fluree/db/query/subject_crawl/reparse.cljc @@ -66,12 +66,12 @@ (if where-smt (when (and (= :tuple type) (= first-s (clause-subject-var where-smt))) - (let [{::where/keys [val var]} o + (let [{::where/keys [val var]} o f (cond val (fn [flake _] (= val (flake/o flake))) - ;;TODO: filters are not yet supported + ;;TODO: filters are not yet supported #_#_filter (let [{:keys [params variable function]} filter] (if (= 1 (count params)) @@ -101,7 +101,7 @@ [s p o] (if (= :tuple type) pattern (let [[_type-kw tuple] pattern] - tuple)) + tuple)) reparse-component (fn [component] (let [{::where/keys [var val]} component] (cond @@ -116,7 +116,7 @@ "Revises where clause for simple-subject-crawl query to optimize processing. If where does not end up meeting simple-subject-crawl criteria, returns nil so other strategies can be tried." - [{:keys [where vars] :as parsed-query}] + [{:strs [where vars] :as parsed-query}] (let [{::where/keys [patterns]} where [first-pattern & rest-patterns] patterns reparsed-first-clause (re-parse-pattern first-pattern)] @@ -129,11 +129,12 @@ (if-let [subj-filter-map (merge-wheres-to-filter first-s rest-patterns vars)] (assoc parsed-query :where [reparsed-first-clause {:s-filter subj-filter-map}] - :strategy :simple-subject-crawl)))))) + :strategy :simple-subject-crawl)))))) + (defn simple-subject-crawl? "Simple subject crawl is where the same variable is used in the leading position of each where statement." - [{:keys [where select vars] :as _parsed-query}] + [{:strs [where select vars] :as _parsed-query}] (and (instance? SubgraphSelector select) ;;TODO, filtering not supported yet (empty? (::where/filters where)) @@ -153,7 +154,7 @@ e.g. {\"select\" {?subjects ['*']} \"where\" [...]}" - [{:keys [order-by group-by] :as parsed-query}] + [{:strs [order-by group-by] :as parsed-query}] (when (and (not group-by) (not order-by) (simple-subject-crawl? parsed-query)) diff --git a/test/fluree/db/query/subject_crawl_reparse_test.clj b/test/fluree/db/query/subject_crawl_reparse_test.clj index 80ed20aee..80305ad84 100644 --- a/test/fluree/db/query/subject_crawl_reparse_test.clj +++ b/test/fluree/db/query/subject_crawl_reparse_test.clj @@ -10,64 +10,65 @@ (let [conn (test-utils/create-conn) ledger @(fluree/create conn "query/parse" {:context {:ex "http://example.org/ns/"}}) db @(fluree/stage - (fluree/db ledger) - [{:id :ex/brian, - :type :ex/User, - :schema/name "Brian" - :schema/email "brian@example.org" - :schema/age 50 - :ex/favColor "Green" - :ex/favNums 7} - {:id :ex/alice, - :type :ex/User, - :schema/name "Alice" - :schema/email "alice@example.org" - :schema/age 50 - :ex/favColor "Blue" - :ex/favNums [42, 76, 9]} - {:id :ex/cam, - :type :ex/User, - :schema/name "Cam" - :schema/email "cam@example.org" - :schema/age 34 - :ex/favNums [5, 10] - :ex/friend [:ex/brian :ex/alice]}]) - ssc-q1-parsed (parse/parse-analytical-query* {:select {"?s" ["*"]} - :where [["?s" :schema/name "Alice"]]} - db) - ssc-q2-parsed (parse/parse-analytical-query* {:select {"?s" ["*"]} - :where [["?s" :schema/age 50] - ["?s" :ex/favColor "Blue"]]} - db) - not-ssc-parsed (parse/parse-analytical-query* {:context {:ex "http://example.org/ns/"} - :select ['?name '?age '?email] - :where [['?s :schema/name "Cam"] - ['?s :ex/friend '?f] - ['?f :schema/name '?name] - ['?f :schema/age '?age] - ['?f :schema/email '?email]]} - db) - order-group-parsed (parse/parse-analytical-query* {:select ['?name '?favNums] - :where [['?s :schema/name '?name] - ['?s :ex/favNums '?favNums]] - :group-by '?name - :order-by '?name} + (fluree/db ledger) + [{"id" "ex:brian", + "type" "ex:User", + "schema:name" "Brian" + "schema:email" "brian@example.org" + "schema:age" 50 + "ex:favColor" "Green" + "ex:favNums" 7} + {"id" "ex:alice", + "type" "ex:User", + "schema:name" "Alice" + "schema:email" "alice@example.org" + "schema:age" 50 + "ex:favColor" "Blue" + "ex:favNums" [42, 76, 9]} + {"id" "ex:cam", + "type" "ex:User", + "schema:name" "Cam" + "schema:email" "cam@example.org" + "schema:age" 34 + "ex:favNums" [5, 10] + "ex:friend" ["ex:brian" "ex:alice"]}]) + + ssc-q1-parsed (parse/parse-analytical-query* {"select" {"?s" ["*"]} + "where" [["?s" "schema:name" "Alice"]]} + db) + ssc-q2-parsed (parse/parse-analytical-query* {"select" {"?s" ["*"]} + "where" [["?s" "schema:age" 50] + ["?s" "ex:favColor" "Blue"]]} + db) + not-ssc-parsed (parse/parse-analytical-query* {"context" {"ex" "http://example.org/ns/"} + "select" ['?name '?age '?email] + "where" [['?s "schema:name" "Cam"] + ['?s "ex:friend" '?f] + ['?f "schema:name" '?name] + ['?f "schema:age" '?age] + ['?f "schema:email" '?email]]} + db) + order-group-parsed (parse/parse-analytical-query* {"select" ['?name '?favNums] + "where" [['?s "schema:name" '?name] + ['?s "ex:favNums" '?favNums]] + "group-by" '?name + "order-by" '?name} db) - vars-query-parsed (parse/parse-analytical-query* {:select {"?s" ["*"]} - :where [["?s" :schema/name '?name]] - :vars {'?name "Alice"}} + vars-query-parsed (parse/parse-analytical-query* {"select" {"?s" ["*"]} + "where" [["?s" "schema:name" '?name]] + "vars" {'?name "Alice"}} + db) + s+p+o-parsed (parse/parse-analytical-query {"select" {"?s" [:*]} + "where" [["?s" "?p" "?o"]]} + db) + s+p+o2-parsed (parse/parse-analytical-query {"select" {'?s ["*"]} + "where" [['?s "schema:age" 50] + ['?s '?p '?o]]} db) - s+p+o-parsed (parse/parse-analytical-query {:select {"?s" [:*]} - :where [["?s" "?p" "?o"]]} - db) - s+p+o2-parsed (parse/parse-analytical-query {:select {'?s ["*"]} - :where [['?s :schema/age 50] - ['?s '?p '?o]]} - db) - s+p+o3-parsed (parse/parse-analytical-query {:select {'?s ["*"]} - :where [['?s '?p '?o] - ['?s :schema/age 50]]} - db)] + s+p+o3-parsed (parse/parse-analytical-query {"select" {'?s ["*"]} + "where" [['?s '?p '?o] + ['?s "schema:age" 50]]} + db)] (testing "simple-subject-crawl?" (is (= true (reparse/simple-subject-crawl? ssc-q1-parsed))) @@ -80,10 +81,10 @@ (is (not (reparse/simple-subject-crawl? s+p+o2-parsed))) (is (not (reparse/simple-subject-crawl? s+p+o3-parsed)))) (testing "reparse" - (let [ssc-q1-reparsed (reparse/re-parse-as-simple-subj-crawl ssc-q1-parsed) - {:keys [where context]} ssc-q1-reparsed - [pattern] where - {:keys [s p o]} pattern] + (let [ssc-q1-reparsed (reparse/re-parse-as-simple-subj-crawl ssc-q1-parsed) + {:keys [where] context "@context"} ssc-q1-reparsed + [pattern] where + {:keys [s p o]} pattern] (is (not (nil? context))) (is (= {:variable '?s} s)) @@ -92,10 +93,11 @@ (is (= "Alice" value)) (is datatype))) - (let [ssc-q2-reparsed (reparse/re-parse-as-simple-subj-crawl ssc-q2-parsed) - {:keys [where context]} ssc-q2-reparsed - [pattern _s-filter] where - {:keys [s p o]} pattern] + + (let [ssc-q2-reparsed (reparse/re-parse-as-simple-subj-crawl ssc-q2-parsed) + {:keys [where] context "@context"} ssc-q2-reparsed + [pattern _s-filter] where + {:keys [s p o]} pattern] (is (not (nil? context))) (is (= {:variable '?s} s)) @@ -104,5 +106,4 @@ (is (= 50 value)) (is datatype))) - (is (nil? - (reparse/re-parse-as-simple-subj-crawl not-ssc-parsed)))))) + (is (nil? (reparse/re-parse-as-simple-subj-crawl not-ssc-parsed)))))) From f1e42b7683cad69a687e6bf58ec3956c488a2cd3 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Mon, 10 Apr 2023 16:06:11 -0600 Subject: [PATCH 09/50] Update some tests for strings-api changes --- test/fluree/db/json_ld/api_test.cljc | 123 ++++++++-------- test/fluree/db/query/fql_parse_test.clj | 13 +- test/fluree/db/query/fql_test.clj | 182 ++++++++++++------------ test/fluree/db/test_utils.cljc | 62 ++++---- 4 files changed, 196 insertions(+), 184 deletions(-) diff --git a/test/fluree/db/json_ld/api_test.cljc b/test/fluree/db/json_ld/api_test.cljc index b707783fe..663c7c9df 100644 --- a/test/fluree/db/json_ld/api_test.cljc +++ b/test/fluree/db/json_ld/api_test.cljc @@ -39,18 +39,18 @@ :cljs (async done - (go - (let [conn (= (count ?favNums) 2)})) + @(fluree/query db '{"select" [?name ?favNums] + "where" [[?s "schema:name" ?name] + [?s "ex:favNums" ?favNums]] + "group-by" ?name + "having" (>= (count ?favNums) 2)})) "filters results according to the supplied having function code") (is (= [["Liam" [11 42]] ["Alice" [9 42 76]]] - @(fluree/query db '{:select [?name ?favNums] - :where [[?s :schema/name ?name] - [?s :ex/favNums ?favNums]] - :group-by ?name - :having (>= (avg ?favNums) 10)})) + @(fluree/query db '{"select" [?name ?favNums] + "where" [[?s "schema:name" ?name] + [?s "ex:favNums" ?favNums]] + "group-by" ?name + "having" (>= (avg ?favNums) 10)})) "filters results according to the supplied having function code")))))) (deftest ^:integration select-distinct-test @@ -65,11 +65,11 @@ (let [conn (test-utils/create-conn) people (test-utils/load-people conn) db (fluree/db people) - q '{:select-distinct [?name ?email] - :where [[?s :schema/name ?name] - [?s :schema/email ?email] - [?s :ex/favNums ?favNum]] - :order-by ?favNum}] + q '{"select-distinct" [?name ?email] + "where" [[?s "schema:name" ?name] + [?s "schema:email" ?email] + [?s "ex:favNums" ?favNum]] + "order-by" ?favNum}] (is (= [["Cam" "cam@example.org"] ["Brian" "brian@example.org"] ["Alice" "alice@example.org"] @@ -84,43 +84,43 @@ db (fluree/db people)] (testing "binding a single variable" (testing "with a single value" - (let [q '{:select [?name ?age] - :where [[?s :schema/email ?email] - [?s :schema/name ?name] - [?s :schema/age ?age]] - :values [?email ["alice@example.org"]]}] + (let [q '{"select" [?name ?age] + "where" [[?s "schema:email" ?email] + [?s "schema:name" ?name] + [?s "schema:age" ?age]] + "values" [?email ["alice@example.org"]]}] (is (= [["Alice" 50]] @(fluree/query db q)) "returns only the results related to the bound value"))) (testing "with multiple values" - (let [q '{:select [?name ?age] - :where [[?s :schema/email ?email] - [?s :schema/name ?name] - [?s :schema/age ?age]] - :values [?email ["alice@example.org" "cam@example.org"]]}] + (let [q '{"select" [?name ?age] + "where" [[?s "schema:email" ?email] + [?s "schema:name" ?name] + [?s "schema:age" ?age]] + "values" [?email ["alice@example.org" "cam@example.org"]]}] (is (= [["Alice" 50] ["Cam" 34]] @(fluree/query db q)) "returns only the results related to the bound values")))) (testing "binding multiple variables" (testing "with multiple values" - (let [q '{:select [?name ?age] - :where [[?s :schema/email ?email] - [?s :ex/favNums ?favNum] - [?s :schema/name ?name] - [?s :schema/age ?age]] - :values [[?email ?favNum] [["alice@example.org" 42] - ["cam@example.org" 10]]]}] + (let [q '{"select" [?name ?age] + "where" [[?s "schema:email" ?email] + [?s "ex:favNums" ?favNum] + [?s "schema:name" ?name] + [?s "schema:age" ?age]] + "values" [[?email ?favNum] [["alice@example.org" 42] + ["cam@example.org" 10]]]}] (is (= [["Alice" 50] ["Cam" 34]] @(fluree/query db q)) "returns only the results related to the bound values"))) (testing "with some values not present" - (let [q '{:select [?name ?age] - :where [[?s :schema/email ?email] - [?s :ex/favNums ?favNum] - [?s :schema/name ?name] - [?s :schema/age ?age]] - :values [[?email ?favNum] [["alice@example.org" 42] - ["cam@example.org" 37]]]}] + (let [q '{"select" [?name ?age] + "where" [[?s "schema:email" ?email] + [?s "ex:favNums" ?favNum] + [?s "schema:name" ?name] + [?s "schema:age" ?age]] + "values" [[?email ?favNum] [["alice@example.org" 42] + ["cam@example.org" 37]]]}] (is (= [["Alice" 50]] @(fluree/query db q)) "returns only the results related to the existing bound values"))))))) @@ -130,12 +130,12 @@ people (test-utils/load-people conn) db (fluree/db people)] (testing "with 2 separate fn binds" - (let [q '{:select [?firstLetterOfName ?name ?decadesOld] - :where [[?s :schema/age ?age] - {:bind {?decadesOld (quot ?age 10)}} - [?s :schema/name ?name] - {:bind {?firstLetterOfName (subStr ?name 0 1)}}] - :order-by ?firstLetterOfName} + (let [q '{"select" [?firstLetterOfName ?name ?decadesOld] + "where" [[?s "schema:age" ?age] + {"bind" {?decadesOld (quot ?age 10)}} + [?s "schema:name" ?name] + {"bind" {?firstLetterOfName (subStr ?name 0 1)}}] + "order-by" ?firstLetterOfName} res @(fluree/query db q)] (is (= [["A" "Alice" 5] ["B" "Brian" 5] @@ -144,12 +144,12 @@ res)))) (testing "with 2 fn binds in one bind map" - (let [q '{:select [?firstLetterOfName ?name ?canVote] - :where [[?s :schema/age ?age] - [?s :schema/name ?name] - {:bind {?firstLetterOfName (subStr ?name 0 1) - ?canVote (>= ?age 18)}}] - :order-by ?name} + (let [q '{"select" [?firstLetterOfName ?name ?canVote] + "where" [[?s "schema:age" ?age] + [?s "schema:name" ?name] + {"bind" {?firstLetterOfName (subStr ?name 0 1) + ?canVote (>= ?age 18)}}] + "order-by" ?name} res @(fluree/query db q)] (is (= [["A" "Alice" true] ["B" "Brian" true] @@ -158,13 +158,13 @@ res)))) (testing "with invalid aggregate fn" - (let [q '{:select [?sumFavNums ?name ?canVote] - :where [[?s :schema/age ?age] - [?s :ex/favNums ?favNums] - [?s :schema/name ?name] - {:bind {?sumFavNums (sum ?favNums) - ?canVote (>= ?age 18)}}] - :order-by ?name}] + (let [q '{"select" [?sumFavNums ?name ?canVote] + "where" [[?s "schema:age" ?age] + [?s "ex:favNums" ?favNums] + [?s "schema:name" ?name] + {"bind" {?sumFavNums (sum ?favNums) + ?canVote (>= ?age 18)}}] + "order-by" ?name}] (is (re-matches #"Aggregate function sum is only valid for grouped values" (ex-message @(fluree/query db q)))))))) @@ -174,9 +174,9 @@ movies (test-utils/load-movies conn) db (fluree/db movies)] (testing "iri queries" - (let [test-subject @(fluree/query db '{:select [?s] - :where [[?s :id :wiki/Q836821]]})] - (is (= [[:wiki/Q836821]] + (let [test-subject @(fluree/query db '{"select" [?s] + "where" [[?s "id" "wiki:Q836821"]]})] + (is (= [["wiki:Q836821"]] test-subject) "Returns the subject with that iri"))))) @@ -185,22 +185,22 @@ people (test-utils/load-people conn) db (fluree/db people)] (testing "multi queries" - (let [q '{"alice" {:select {?s [:*]} - :where [[?s :schema/email "alice@example.org"]]} - "brian" {:select {?s [:*]} - :where [[?s :schema/email "brian@example.org"]]}} + (let [q '{"alice" {"select" {?s [:*]} + "where" [[?s "schema:email" "alice@example.org"]]} + "brian" {"select" {?s [:*]} + "where" [[?s "schema:email" "brian@example.org"]]}} subject @(fluree/multi-query db q)] - (is (= {"alice" [{:id :ex/alice - :rdf/type [:ex/User] - :ex/favNums [9 42 76] - :schema/age 50 - :schema/email "alice@example.org" - :schema/name "Alice"}] - "brian" [{:id :ex/brian - :rdf/type [:ex/User] - :ex/favNums 7 - :schema/age 50 - :schema/email "brian@example.org" - :schema/name "Brian"}]} + (is (= {"alice" [{"id" "ex:alice" + "rdf:type" ["ex:User"] + "ex:favNums" [9 42 76] + "schema:age" 50 + "schema:email" "alice@example.org" + "schema:name" "Alice"}] + "brian" [{"id" "ex:brian" + "rdf:type" ["ex:User"] + "ex:favNums" 7 + "schema:age" 50 + "schema:email" "brian@example.org" + "schema:name" "Brian"}]} subject) "returns all results in a map keyed by alias."))))) diff --git a/test/fluree/db/test_utils.cljc b/test/fluree/db/test_utils.cljc index a529fa73d..4dc6018dd 100644 --- a/test/fluree/db/test_utils.cljc +++ b/test/fluree/db/test_utils.cljc @@ -53,32 +53,32 @@ "titleEIDR" "10.5240/15F9-F913-FF25-8041-E798-O"}]) (def people - [{:id :ex/brian, - :type :ex/User, - :schema/name "Brian" - :schema/email "brian@example.org" - :schema/age 50 - :ex/favNums 7} - {:id :ex/alice, - :type :ex/User, - :schema/name "Alice" - :schema/email "alice@example.org" - :schema/age 50 - :ex/favNums [42, 76, 9]} - {:id :ex/cam, - :type :ex/User, - :schema/name "Cam" - :schema/email "cam@example.org" - :schema/age 34 - :ex/favNums [5, 10] - :ex/friend [:ex/brian :ex/alice]} - {:id :ex/liam - :type :ex/User - :schema/name "Liam" - :schema/email "liam@example.org" - :schema/age 13 - :ex/favNums [42, 11] - :ex/friend [:ex/brian :ex/alice :ex/cam]}]) + [{"id" "ex:brian", + "type" "ex:User", + "schema:name" "Brian" + "schema:email" "brian@example.org" + "schema:age" 50 + "ex:favNums" 7} + {"id" "ex:alice", + "type" "ex:User", + "schema:name" "Alice" + "schema:email" "alice@example.org" + "schema:age" 50 + "ex:favNums" [42, 76, 9]} + {"id" "ex:cam", + "type" "ex:User", + "schema:name" "Cam" + "schema:email" "cam@example.org" + "schema:age" 34 + "ex:favNums" [5, 10] + "ex:friend" ["ex:brian" "ex:alice"]} + {"id" "ex:liam" + "type" "ex:User" + "schema:name" "Liam" + "schema:email" "liam@example.org" + "schema:age" 13 + "ex:favNums" [42, 11] + "ex:friend" ["ex:brian" "ex:alice" "ex:cam"]}]) (defn create-conn ([] @@ -96,15 +96,17 @@ (doseq [movie movies] (let [staged @(fluree/stage (fluree/db ledger) movie)] @(fluree/commit! ledger staged - {:message (str "Commit " (get movie "name")) - :push? true}))) + {"message" (str "Commit " (get movie "name")) + "push?" true}))) ledger)) (defn load-people [conn] - (let [ledger @(fluree/create conn "test/people" {:defaultContext ["" {:ex "http://example.org/ns/"}]}) + (let [ledger @(fluree/create conn "test/people" + {"default-context" + ["" {"ex" "http://example.org/ns/"}]}) staged @(fluree/stage (fluree/db ledger) people)] - @(fluree/commit! ledger staged {:message "Adding people", :push? true}) + @(fluree/commit! ledger staged {"message" "Adding people", "push?" true}) ledger)) (defn transact From b2958547a05fda74b1843bb418e079b0976e9b00 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Mon, 10 Apr 2023 16:06:59 -0600 Subject: [PATCH 10/50] Fix list containers for strings-api --- src/fluree/db/json_ld/commit.cljc | 62 +++++++++++++++-------------- src/fluree/db/json_ld/reify.cljc | 2 +- src/fluree/db/json_ld/transact.cljc | 4 +- 3 files changed, 35 insertions(+), 33 deletions(-) diff --git a/src/fluree/db/json_ld/commit.cljc b/src/fluree/db/json_ld/commit.cljc index 0e9abfda9..96907a496 100644 --- a/src/fluree/db/json_ld/commit.cljc +++ b/src/fluree/db/json_ld/commit.cljc @@ -294,6 +294,7 @@ (defn add-commit-schema-flakes [{:keys [schema] :as db} t] + (log/debug "add-commit-schema-flakes schema:" schema) (let [schema-flakes [(flake/create const/$_previous const/$iri const/iri-previous const/$xsd:string t true nil) (flake/create const/$_address const/$iri const/iri-address const/$xsd:string t true nil) (flake/create const/$_v const/$iri const/iri-v const/$xsd:string t true nil) @@ -473,33 +474,34 @@ returns a db with an updated :commit." [{:keys [conn indexer context] :as ledger} {:keys [t stats commit] :as db} opts] (go-try - (let [{:keys [id-key did message tag file-data?] :as opts*} (enrich-commit-opts db opts) - ledger-update ( v first key)))) + (contains? v :list))) (defn node? "Returns true if a nested value is itself another node in the graph. diff --git a/src/fluree/db/json_ld/transact.cljc b/src/fluree/db/json_ld/transact.cljc index d71b65dd8..441af4261 100644 --- a/src/fluree/db/json_ld/transact.cljc +++ b/src/fluree/db/json_ld/transact.cljc @@ -125,7 +125,7 @@ "returns true if json-ld value is a list object." [v] (and (map? v) - (= :list (-> v first key)))) + (contains? v "list"))) (defn get-subject-types "Returns a set of all :rdf/type Class subject ids for the provided subject. @@ -210,7 +210,7 @@ (let [list? (list-value? v) retract? (nil? v) v* (if list? - (let [list-vals (:list v)] + (let [list-vals (get v "list")] (when-not (sequential? list-vals) (throw (ex-info (str "List values have to be vectors, provided: " v) {:status 400 :error :db/invalid-transaction}))) From 57a76cfbab900e0ed43c2c6f47aaee13ce78de00 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Mon, 10 Apr 2023 16:07:24 -0600 Subject: [PATCH 11/50] Fix more analytical query handling for strings-api --- src/fluree/db/query/exec/order.cljc | 2 +- src/fluree/db/query/exec/select.cljc | 16 +++++++------- src/fluree/db/query/exec/where.cljc | 2 +- src/fluree/db/query/fql/parse.cljc | 33 +++++++++++++++------------- 4 files changed, 28 insertions(+), 25 deletions(-) diff --git a/src/fluree/db/query/exec/order.cljc b/src/fluree/db/query/exec/order.cljc index ae517d80e..0274d16c8 100644 --- a/src/fluree/db/query/exec/order.cljc +++ b/src/fluree/db/query/exec/order.cljc @@ -39,7 +39,7 @@ ordering specified by the `:order-by` clause of the supplied parsed query. Note that all solutions from `solution-ch` are first loaded into memory before they are sorted in place and placed individually on the output channel." - [{:keys [order-by]} solution-ch] + [{:strs [order-by]} solution-ch] (if order-by (let [comparator (partial compare-solutions order-by) coll-ch (async/into [] solution-ch) diff --git a/src/fluree/db/query/exec/select.cljc b/src/fluree/db/query/exec/select.cljc index 578b700cf..e2a0a2771 100644 --- a/src/fluree/db/query/exec/select.cljc +++ b/src/fluree/db/query/exec/select.cljc @@ -35,8 +35,8 @@ (vswap! iri-cache assoc v iri) iri) (catch* e - (log/error e "Error displaying iri:" v) - (>! error-ch e))))))) + (log/error e "Error displaying iri:" v) + (>! error-ch e))))))) (defprotocol ValueSelector "Format a where search solution (map of pattern matches) by extracting and @@ -63,8 +63,8 @@ [_ db iri-cache context compact error-ch solution] (go (try* (agg-fn solution) (catch* e - (log/error e "Error applying aggregate selector") - (>! error-ch e)))))) + (log/error e "Error applying aggregate selector") + (>! error-ch e)))))) (defn aggregate-selector "Returns a selector that extracts the grouped values bound to the specified @@ -87,8 +87,8 @@ ;; TODO: Replace these nils with fuel values when we turn fuel back on (res db iri-cache context compact nil nil spec 0 flakes))) (catch* e - (log/error e "Error formatting subgraph for subject:" sid) - (>! error-ch e))))))) + (log/error e "Error formatting subgraph for subject:" sid) + (>! error-ch e))))))) (defn subgraph-selector "Returns a selector that extracts the subject id bound to the supplied @@ -103,8 +103,8 @@ according to the selector or collection of selectors specified by `selectors`" [selectors db iri-cache context compact error-ch solution] (if (sequential? selectors) - (go-loop [selectors selectors - values []] + (go-loop [selectors selectors + values []] (if-let [selector (first selectors)] (let [value ( q - :values + (get "values") not-empty (or [blank-solution])) out-ch (async/chan)] diff --git a/src/fluree/db/query/fql/parse.cljc b/src/fluree/db/query/fql/parse.cljc index df0398bf8..082b8d0de 100644 --- a/src/fluree/db/query/fql/parse.cljc +++ b/src/fluree/db/query/fql/parse.cljc @@ -76,15 +76,15 @@ (defn safe-read [code-str] (try* - (let [code (read-string code-str)] - (when-not (list? code) - (throw (ex-info (code-str "Invalid function: " code-str) - {:status 400 :error :db/invalid-query}))) - code) - (catch* e - (log/warn e "Invalid query function attempted: " code-str) - (throw (ex-info (str "Invalid query function: " code-str) - {:status 400 :error :db/invalid-query}))))) + (let [code (read-string code-str)] + (when-not (list? code) + (throw (ex-info (code-str "Invalid function: " code-str) + {:status 400 :error :db/invalid-query}))) + code) + (catch* e + (log/warn e "Invalid query function attempted: " code-str) + (throw (ex-info (str "Invalid query function: " code-str) + {:status 400 :error :db/invalid-query}))))) (defn variables "Returns the set of items within the arbitrary data structure `data` that @@ -320,6 +320,7 @@ (defn parse-triple [[s-pat p-pat o-pat] db context] + (log/debug "parse-triple:" s-pat p-pat o-pat) (let [s (parse-subject-pattern s-pat context) p (parse-predicate-pattern p-pat db context)] (if (and (= const/$rdf:type (::where/val p)) @@ -356,8 +357,8 @@ (defmethod parse-pattern :bind [{:strs [bind]} _vars _db _context] - (let [parsed (parse-bind-map bind) - _ (log/debug "parsed bind map:" parsed) + (let [parsed (parse-bind-map bind) + _ (log/debug "parsed bind map:" parsed) pattern (where/->pattern :bind parsed)] (log/debug "parse-pattern :bind pattern:" pattern) pattern)) @@ -446,9 +447,9 @@ (defn parse-analytical-query* [q db] + (log/debug "parse-analytical-query*:" q) (let [context (parse-context q db) [vars values] (parse-values q) - _ (log/debug "parse-analytical-query*:" q) where (parse-where q vars db context) grouping (parse-grouping q) ordering (parse-ordering q)] @@ -463,9 +464,11 @@ (defn parse-analytical-query [q db] - (let [parsed (parse-analytical-query* q db)] - (or (re-parse-as-simple-subj-crawl parsed) - parsed))) + (let [parsed (parse-analytical-query* q db) + re-parsed (or (re-parse-as-simple-subj-crawl parsed) + parsed)] + (log/debug "parse-analytical-query re-parsed:" re-parsed) + re-parsed)) (defn parse [q db] From c065ba9a135b32fb77bda8092d70e13cdf80b2a1 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Mon, 10 Apr 2023 16:08:08 -0600 Subject: [PATCH 12/50] Update query response formatting for strings-api --- src/fluree/db/query/json_ld/response.cljc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/fluree/db/query/json_ld/response.cljc b/src/fluree/db/query/json_ld/response.cljc index 87a7e5764..fcd08c3d8 100644 --- a/src/fluree/db/query/json_ld/response.cljc +++ b/src/fluree/db/query/json_ld/response.cljc @@ -80,10 +80,11 @@ the requested depth within the select-spec" [db cache context compact-fn fuel-vol max-fuel {:keys [wildcard? _id? depth reverse] :as select-spec} depth-i s-flakes] (go-try + (log/debug "flakes->res select-spec:" select-spec) (when (not-empty s-flakes) (loop [[p-flakes & r] (partition-by flake/p s-flakes) acc (if _id? - {:_id (flake/s (first s-flakes))} + {"_id" (flake/s (first s-flakes))} {})] (if p-flakes (let [ff (first p-flakes) @@ -92,6 +93,7 @@ spec (or (get select-spec p) (when wildcard? (wildcard-spec db cache compact-fn p))) + _ (log/debug "flakes->res spec:" spec) p-iri (:as spec) v (cond (nil? spec) @@ -132,11 +134,12 @@ ;; TODO - we generate id-key here every time, this should be done in the :spec once beforehand and used from there (let [id-key (:as (wildcard-spec db cache compact-fn const/$iri)) c-iri ( context (get p-iri) :container)))) + (not (#{"list" "set"} (-> context (get p-iri) (get "@container"))))) (first acc) acc))))] (if v From 84d0294734c800a8b26c746509a4d770d27ba83e Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Mon, 10 Apr 2023 16:08:43 -0600 Subject: [PATCH 13/50] Fix simple subj crawl queries for strings-api --- .../db/query/subject_crawl/reparse.cljc | 20 +++++++++++++------ .../db/query/subject_crawl/subject.cljc | 2 +- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/fluree/db/query/subject_crawl/reparse.cljc b/src/fluree/db/query/subject_crawl/reparse.cljc index 707a715fd..79d4253c0 100644 --- a/src/fluree/db/query/subject_crawl/reparse.cljc +++ b/src/fluree/db/query/subject_crawl/reparse.cljc @@ -97,6 +97,7 @@ "Re-parses a pattern into the format recognized by downstream simple-subject-crawl code" [pattern] + (log/debug "re-parse-pattern pattern:" pattern) (let [type (where/pattern-type pattern) [s p o] (if (= :tuple type) pattern @@ -117,18 +118,20 @@ If where does not end up meeting simple-subject-crawl criteria, returns nil so other strategies can be tried." [{:strs [where vars] :as parsed-query}] + (log/debug "simple-subject-merge-where parsed-query:" parsed-query) (let [{::where/keys [patterns]} where [first-pattern & rest-patterns] patterns reparsed-first-clause (re-parse-pattern first-pattern)] (when-let [first-s (and (mergeable-where-clause? first-pattern) (clause-subject-var first-pattern))] + (log/debug "simple-subject-merge-where first-s:" first-s) (if (empty? rest-patterns) (assoc parsed-query - :where [reparsed-first-clause] - :strategy :simple-subject-crawl) + "where" [reparsed-first-clause] + :strategy :simple-subject-crawl) (if-let [subj-filter-map (merge-wheres-to-filter first-s rest-patterns vars)] - (assoc parsed-query :where [reparsed-first-clause - {:s-filter subj-filter-map}] + (assoc parsed-query "where" [reparsed-first-clause + {:s-filter subj-filter-map}] :strategy :simple-subject-crawl)))))) (defn simple-subject-crawl? @@ -140,9 +143,11 @@ (empty? (::where/filters where)) ;;TODO: vars support not complete (empty? vars) - (if-let [{select-var :var} select] + (when-let [{select-var :var} select] + (log/debug "select-var:" select-var) (let [{::where/keys [patterns]} where] (every? (fn [pattern] + (log/debug "where pattern:" pattern) (and (mergeable-where-clause? pattern) (let [pred (second pattern)] (and (= select-var (clause-subject-var pattern)) @@ -155,8 +160,11 @@ {\"select\" {?subjects ['*']} \"where\" [...]}" [{:strs [order-by group-by] :as parsed-query}] + (log/debug "re-parse-as-simple-subj-crawl parsed-query:" parsed-query) (when (and (not group-by) (not order-by) (simple-subject-crawl? parsed-query)) ;; following will return nil if parts of where clause exclude it from being a simple-subject-crawl - (simple-subject-merge-where parsed-query))) + (when-let [ssmw (simple-subject-merge-where parsed-query)] + (util/assoc-from-str-opts ssmw #{"select" "where" {"@context" :context}} + (dissoc ssmw "select" "where" "@context"))))) diff --git a/src/fluree/db/query/subject_crawl/subject.cljc b/src/fluree/db/query/subject_crawl/subject.cljc index ba5ef2288..6cf5616af 100644 --- a/src/fluree/db/query/subject_crawl/subject.cljc +++ b/src/fluree/db/query/subject_crawl/subject.cljc @@ -143,7 +143,7 @@ [{:keys [db error-ch f-where limit offset parallelism vars ident-vars finish-fn] :as opts}] (go-try - (log/trace "subj-crawl opts:" opts) + (log/debug "subj-crawl opts:" opts) (let [{:keys [o p-ref?]} f-where vars* (if ident-vars ( Date: Mon, 10 Apr 2023 16:09:09 -0600 Subject: [PATCH 14/50] Reformat dev/user.clj --- dev/user.clj | 56 ++++++++++++++++++++++++++-------------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/dev/user.clj b/dev/user.clj index 5001a85bb..8ab41da07 100644 --- a/dev/user.clj +++ b/dev/user.clj @@ -46,7 +46,7 @@ "8ce4eca704d653dec594703c81a84c403c39f262e54ed014ed857438933a2e1c") - (def did (did/private->did-map default-private-key )) + (def did (did/private->did-map default-private-key)) (def default-context {:id "@id" @@ -71,37 +71,37 @@ (def ledger @(fluree/create file-conn "user/test")) (def db1 @(fluree/stage - (fluree/db ledger) - [{:context {:ex "http://example.org/ns/"} - :id :ex/brian, - :type :ex/User, - :schema/name "Brian" - :schema/email "brian@example.org" - :schema/age 50 - :ex/favNums 7 - } - {:context {:ex "http://example.org/ns/"} - :id :ex/alice, - :type :ex/User, - :schema/name "Alice" - :schema/email "alice@example.org" - :schema/age 50 - :ex/favNums [42, 76, 9] - } - {:context {:ex "http://example.org/ns/"} - :id :ex/cam, - :type :ex/User, - :schema/name "Cam" - :schema/email "cam@example.org" - :schema/age 34 - :ex/favNums [5, 10] - :ex/friend [:ex/brian :ex/alice]}])) + (fluree/db ledger) + [{:context {:ex "http://example.org/ns/"} + :id :ex/brian, + :type :ex/User, + :schema/name "Brian" + :schema/email "brian@example.org" + :schema/age 50 + :ex/favNums 7} + + {:context {:ex "http://example.org/ns/"} + :id :ex/alice, + :type :ex/User, + :schema/name "Alice" + :schema/email "alice@example.org" + :schema/age 50 + :ex/favNums [42, 76, 9]} + + {:context {:ex "http://example.org/ns/"} + :id :ex/cam, + :type :ex/User, + :schema/name "Cam" + :schema/email "cam@example.org" + :schema/age 34 + :ex/favNums [5, 10] + :ex/friend [:ex/brian :ex/alice]}])) (def db2 @(fluree/commit! ledger db1 {:message "hi"})) - @(fluree/load file-conn (-> ledger :alias)) + @(fluree/load file-conn (-> ledger :alias))) + - ) ;; TODO - general ;; if an subject is a component, do we allow it to be explicitly assigned as a ref to another component? From a8e11a8a6be18dd3459282304398cc5364cf7a1c Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Mon, 10 Apr 2023 16:10:05 -0600 Subject: [PATCH 15/50] Fix indentation of f.d.query.jld.resp/flakes->res --- src/fluree/db/query/json_ld/response.cljc | 136 +++++++++++----------- 1 file changed, 68 insertions(+), 68 deletions(-) diff --git a/src/fluree/db/query/json_ld/response.cljc b/src/fluree/db/query/json_ld/response.cljc index fcd08c3d8..103c9a1e6 100644 --- a/src/fluree/db/query/json_ld/response.cljc +++ b/src/fluree/db/query/json_ld/response.cljc @@ -80,71 +80,71 @@ the requested depth within the select-spec" [db cache context compact-fn fuel-vol max-fuel {:keys [wildcard? _id? depth reverse] :as select-spec} depth-i s-flakes] (go-try - (log/debug "flakes->res select-spec:" select-spec) - (when (not-empty s-flakes) - (loop [[p-flakes & r] (partition-by flake/p s-flakes) - acc (if _id? - {"_id" (flake/s (first s-flakes))} - {})] - (if p-flakes - (let [ff (first p-flakes) - p (flake/p ff) - list? (contains? (flake/m ff) :i) - spec (or (get select-spec p) - (when wildcard? - (wildcard-spec db cache compact-fn p))) - _ (log/debug "flakes->res spec:" spec) - p-iri (:as spec) - v (cond - (nil? spec) - nil - - ;; flake's .-o value is an IRI string, JSON-LD compact it before returning - (iri? p) - (-> p-flakes first flake/o compact-fn) - - ;; flake's .-o value is a rdf:type, resolve subject id to IRI then JSON-LD compact it - (rdf-type? p) - (loop [[type-id & rest-types] (map flake/o p-flakes) - acc []] - (if type-id - (recur rest-types - (conj acc (or (get @cache type-id) - (iri db cache compact-fn type-id))))) - acc)) - - :else ;; display all values - (loop [[f & r] (if list? - (sort-by #(:i (flake/m %)) p-flakes) - p-flakes) - acc []] - (if f - (let [res (if (= const/$xsd:anyURI (flake/dt f)) - (cond - ;; have a specified sub-selection (graph crawl) - (:spec spec) - (} for each ref iri - :else - ;; TODO - we generate id-key here every time, this should be done in the :spec once beforehand and used from there - (let [id-key (:as (wildcard-spec db cache compact-fn const/$iri)) - c-iri ( context (get p-iri) (get "@container"))))) - (first acc) - acc))))] - (if v - (recur r (assoc acc p-iri v)) - (recur r acc))) - (if reverse - (merge acc (res select-spec:" select-spec) + (when (not-empty s-flakes) + (loop [[p-flakes & r] (partition-by flake/p s-flakes) + acc (if _id? + {"_id" (flake/s (first s-flakes))} + {})] + (if p-flakes + (let [ff (first p-flakes) + p (flake/p ff) + list? (contains? (flake/m ff) :i) + spec (or (get select-spec p) + (when wildcard? + (wildcard-spec db cache compact-fn p))) + _ (log/debug "flakes->res spec:" spec) + p-iri (:as spec) + v (cond + (nil? spec) + nil + + ;; flake's .-o value is an IRI string, JSON-LD compact it before returning + (iri? p) + (-> p-flakes first flake/o compact-fn) + + ;; flake's .-o value is a rdf:type, resolve subject id to IRI then JSON-LD compact it + (rdf-type? p) + (loop [[type-id & rest-types] (map flake/o p-flakes) + acc []] + (if type-id + (recur rest-types + (conj acc (or (get @cache type-id) + (iri db cache compact-fn type-id))))) + acc)) + + :else ;; display all values + (loop [[f & r] (if list? + (sort-by #(:i (flake/m %)) p-flakes) + p-flakes) + acc []] + (if f + (let [res (if (= const/$xsd:anyURI (flake/dt f)) + (cond + ;; have a specified sub-selection (graph crawl) + (:spec spec) + (} for each ref iri + :else + ;; TODO - we generate id-key here every time, this should be done in the :spec once beforehand and used from there + (let [id-key (:as (wildcard-spec db cache compact-fn const/$iri)) + c-iri ( context (get p-iri) (get "@container"))))) + (first acc) + acc))))] + (if v + (recur r (assoc acc p-iri v)) + (recur r acc))) + (if reverse + (merge acc ( Date: Tue, 11 Apr 2023 12:04:26 -0600 Subject: [PATCH 16/50] Use {"defaults" {"@context" ...}} in ledger create too ...to be consistent with connect opts --- src/fluree/db/json_ld/api.cljc | 16 +++++++---- src/fluree/db/ledger/json_ld.cljc | 12 ++++---- test/fluree/db/json_ld/api_test.cljc | 43 +++++++++++++++++----------- 3 files changed, 43 insertions(+), 28 deletions(-) diff --git a/src/fluree/db/json_ld/api.cljc b/src/fluree/db/json_ld/api.cljc index 031014621..acfc9479b 100644 --- a/src/fluree/db/json_ld/api.cljc +++ b/src/fluree/db/json_ld/api.cljc @@ -120,15 +120,19 @@ Options map (opts) can include (keys are strings): - did - DiD information to use, if storing blocks as verifiable credentials - - default-context - Default default-context map to use for ledgers formed with this connection" + - defaults + - @context - default @context map to use for ledgers formed with this connection" ([conn] (create conn nil nil)) ([conn ledger-alias] (create conn ledger-alias nil)) ([conn ledger-alias opts] - (let [opts* (util/assoc-from-str-opts - opts - #{"default-context" "did" "branch" "pub-fn" "ipns" - "indexer" "include" "reindex-min-bytes" "reindex-max-bytes" - "initial-tx"})] + (let [opts* (-> opts + (util/update-in-if-contains ["defaults"] + util/assoc-from-str-opts + #{{"@context" :context}}) + (util/assoc-from-str-opts + #{"defaults" "did" "branch" "pub-fn" "ipns" + "indexer" "include" "reindex-min-bytes" "reindex-max-bytes" + "initial-tx"}))] (log/debug "create opts*:" opts*) (promise-wrap (jld-ledger/create conn ledger-alias opts*))))) diff --git a/src/fluree/db/ledger/json_ld.cljc b/src/fluree/db/ledger/json_ld.cljc index 1d5810016..404bda46b 100644 --- a/src/fluree/db/ledger/json_ld.cljc +++ b/src/fluree/db/ledger/json_ld.cljc @@ -146,9 +146,10 @@ "Creates a new ledger, optionally bootstraps it as permissioned or with default context." [conn ledger-alias opts] (go-try - (let [{:keys [default-context new-context? did branch pub-fn ipns indexer + (let [{:keys [defaults new-context? did branch pub-fn ipns indexer include reindex-min-bytes reindex-max-bytes initial-tx] :or {branch :main, new-context? true}} opts + default-context (:context defaults) default-context* (if default-context (ctx-util/mapify-context default-context (conn-proto/-default-context conn)) @@ -232,10 +233,11 @@ :value (->> (jld-reify/load-default-context conn)) Date: Tue, 11 Apr 2023 14:13:41 -0600 Subject: [PATCH 17/50] Fix sequential context schema --- src/fluree/db/util/validation.cljc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/fluree/db/util/validation.cljc b/src/fluree/db/util/validation.cljc index 9daf64e15..e99b827c2 100644 --- a/src/fluree/db/util/validation.cljc +++ b/src/fluree/db/util/validation.cljc @@ -8,9 +8,11 @@ (m/base-schemas) (m/type-schemas) (m/comparator-schemas) + (m/predicate-schemas) {::iri :string ::val [:fn value?] ::context [:orn - [:sequence [:orn [:string :string] - [:map map?]]] + [:sequence [:sequential [:orn + [:string :string] + [:map map?]]]] [:map map?]]})) From 6fa009eadc3872b025f9f3d1b3ae4fccb49869c0 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Tue, 11 Apr 2023 14:14:05 -0600 Subject: [PATCH 18/50] Add basic transact opts schema --- src/fluree/db/json_ld/transact.cljc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/fluree/db/json_ld/transact.cljc b/src/fluree/db/json_ld/transact.cljc index 441af4261..26e76ebbd 100644 --- a/src/fluree/db/json_ld/transact.cljc +++ b/src/fluree/db/json_ld/transact.cljc @@ -56,7 +56,8 @@ [::m/default [:map-of ::retract-key ::txn-leaf-map]]]]] ::txn [:orn [:single-amp ::txn-map] - [:sequence-of-maps [:sequential ::txn-map]]]})) + [:sequence-of-maps [:sequential ::txn-map]]] + ::opts [map?]})) (declare json-ld-node->flakes) From a98bcb494b6cceb7c00703b42eea3f96da62702c Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Tue, 11 Apr 2023 14:14:55 -0600 Subject: [PATCH 19/50] Fix commit opts transformation bug --- src/fluree/db/json_ld/api.cljc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/fluree/db/json_ld/api.cljc b/src/fluree/db/json_ld/api.cljc index acfc9479b..f77cf911e 100644 --- a/src/fluree/db/json_ld/api.cljc +++ b/src/fluree/db/json_ld/api.cljc @@ -232,8 +232,7 @@ (util/update-in-if-contains ["did"] util/assoc-from-str-opts #{"id" "public" "private"}) - (util/assoc-from-str-opts opts - #{"message" "branch" "tag" + (util/assoc-from-str-opts #{"message" "branch" "tag" {"@context" :context} "did" "private" "push?"})) opts)] From 845852ace581c79f98998fa2014e568453e4ad22 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Tue, 11 Apr 2023 14:16:19 -0600 Subject: [PATCH 20/50] Update history query tests for strings-api --- test/fluree/db/query/history_test.clj | 839 +++++++++++++------------- 1 file changed, 424 insertions(+), 415 deletions(-) diff --git a/test/fluree/db/query/history_test.clj b/test/fluree/db/query/history_test.clj index 1019e562d..4a20a8bc7 100644 --- a/test/fluree/db/query/history_test.clj +++ b/test/fluree/db/query/history_test.clj @@ -2,485 +2,494 @@ (:require [clojure.test :refer :all] [fluree.db.test-utils :as test-utils] [fluree.db.json-ld.api :as fluree] - [fluree.db.util.core :as util] - [clojure.string :as str])) + [fluree.db.util.core :as util])) (deftest ^:integration history-query (let [ts-primeval (util/current-time-iso) conn (test-utils/create-conn) - ledger @(fluree/create conn "historytest" {:defaultContext ["" {:ex "http://example.org/ns/"}]}) + ledger @(fluree/create conn "historytest" + {"defaults" + {"@context" + ["" {:ex "http://example.org/ns/"}]}}) - db1 @(test-utils/transact ledger [{:id :ex/dan - :ex/x "foo-1" - :ex/y "bar-1"} - {:id :ex/cat - :ex/x "foo-1" - :ex/y "bar-1"} - {:id :ex/dog - :ex/x "foo-1" - :ex/y "bar-1"}]) - db2 @(test-utils/transact ledger {:id :ex/dan - :ex/x "foo-2" - :ex/y "bar-2"}) + db1 @(test-utils/transact ledger [{"id" "ex:dan" + "ex:x" "foo-1" + "ex:y" "bar-1"} + {"id" "ex:cat" + "ex:x" "foo-1" + "ex:y" "bar-1"} + {"id" "ex:dog" + "ex:x" "foo-1" + "ex:y" "bar-1"}]) + db2 @(test-utils/transact ledger {"id" "ex:dan" + "ex:x" "foo-2" + "ex:y" "bar-2"}) ts2 (-> db2 :commit :time) - db3 @(test-utils/transact ledger {:id :ex/dan - :ex/x "foo-3" - :ex/y "bar-3"}) + db3 @(test-utils/transact ledger {"id" "ex:dan" + "ex:x" "foo-3" + "ex:y" "bar-3"}) ts3 (-> db3 :commit :time) - db4 @(test-utils/transact ledger [{:id :ex/cat - :ex/x "foo-cat" - :ex/y "bar-cat"} - {:id :ex/dog - :ex/x "foo-dog" - :ex/y "bar-dog"}]) - db5 @(test-utils/transact ledger {:id :ex/dan - :ex/x "foo-cat" - :ex/y "bar-cat"})] + db4 @(test-utils/transact ledger [{"id" "ex:cat" + "ex:x" "foo-cat" + "ex:y" "bar-cat"} + {"id" "ex:dog" + "ex:x" "foo-dog" + "ex:y" "bar-dog"}]) + db5 @(test-utils/transact ledger {"id" "ex:dan" + "ex:x" "foo-cat" + "ex:y" "bar-cat"})] (testing "subject history" - (is (= [{:f/t 1 - :f/assert [{:id :ex/dan :ex/x "foo-1" :ex/y "bar-1"}] - :f/retract []} - {:f/t 2 - :f/assert [{:id :ex/dan :ex/x "foo-2" :ex/y "bar-2"}] - :f/retract [{:id :ex/dan :ex/x "foo-1" :ex/y "bar-1"}]} - {:f/t 3 - :f/assert [{:id :ex/dan :ex/x "foo-3" :ex/y "bar-3"}] - :f/retract [{:id :ex/dan :ex/x "foo-2" :ex/y "bar-2"}]} - {:f/t 5 - :f/assert [{:id :ex/dan :ex/x "foo-cat" :ex/y "bar-cat"}] - :f/retract [{:id :ex/dan :ex/x "foo-3" :ex/y "bar-3"}]}] - @(fluree/history ledger {:history :ex/dan :t {:from 1}})))) + (is (= [{"f:t" 1 + "f:assert" [{"id" "ex:dan" "ex:x" "foo-1" "ex:y" "bar-1"}] + "f:retract" []} + {"f:t" 2 + "f:assert" [{"id" "ex:dan" "ex:x" "foo-2" "ex:y" "bar-2"}] + "f:retract" [{"id" "ex:dan" "ex:x" "foo-1" "ex:y" "bar-1"}]} + {"f:t" 3 + "f:assert" [{"id" "ex:dan" "ex:x" "foo-3" "ex:y" "bar-3"}] + "f:retract" [{"id" "ex:dan" "ex:x" "foo-2" "ex:y" "bar-2"}]} + {"f:t" 5 + "f:assert" [{"id" "ex:dan" "ex:x" "foo-cat" "ex:y" "bar-cat"}] + "f:retract" [{"id" "ex:dan" "ex:x" "foo-3" "ex:y" "bar-3"}]}] + @(fluree/history ledger {"history" "ex:dan" "t" {"from" 1}})))) (testing "one-tuple flake history" - (is (= [{:f/t 1 - :f/assert [{:id :ex/dan :ex/x "foo-1" :ex/y "bar-1"}] - :f/retract []} - {:f/t 2 - :f/assert [{:ex/x "foo-2" :ex/y "bar-2" :id :ex/dan}] - :f/retract [{:ex/x "foo-1" :ex/y "bar-1" :id :ex/dan}]} - {:f/t 3 - :f/assert [{:ex/x "foo-3" :ex/y "bar-3" :id :ex/dan}] - :f/retract [{:ex/x "foo-2" :ex/y "bar-2" :id :ex/dan}]} - {:f/t 5 - :f/assert [{:ex/x "foo-cat" :ex/y "bar-cat" :id :ex/dan}] - :f/retract [{:ex/x "foo-3" :ex/y "bar-3" :id :ex/dan}]}] - @(fluree/history ledger {:history [:ex/dan] :t {:from 1}})))) + (is (= [{"f:t" 1 + "f:assert" [{"id" "ex:dan" "ex:x" "foo-1" "ex:y" "bar-1"}] + "f:retract" []} + {"f:t" 2 + "f:assert" [{"ex:x" "foo-2" "ex:y" "bar-2" "id" "ex:dan"}] + "f:retract" [{"ex:x" "foo-1" "ex:y" "bar-1" "id" "ex:dan"}]} + {"f:t" 3 + "f:assert" [{"ex:x" "foo-3" "ex:y" "bar-3" "id" "ex:dan"}] + "f:retract" [{"ex:x" "foo-2" "ex:y" "bar-2" "id" "ex:dan"}]} + {"f:t" 5 + "f:assert" [{"ex:x" "foo-cat" "ex:y" "bar-cat" "id" "ex:dan"}] + "f:retract" [{"ex:x" "foo-3" "ex:y" "bar-3" "id" "ex:dan"}]}] + @(fluree/history ledger {"history" ["ex:dan"] "t" {"from" 1}})))) (testing "two-tuple flake history" - (is (= [{:f/t 1 :f/assert [{:ex/x "foo-1" :id :ex/dan}] :f/retract []} - {:f/t 2 - :f/assert [{:ex/x "foo-2" :id :ex/dan}] - :f/retract [{:ex/x "foo-1" :id :ex/dan}]} - {:f/t 3 - :f/assert [{:ex/x "foo-3" :id :ex/dan}] - :f/retract [{:ex/x "foo-2" :id :ex/dan}]} - {:f/t 5 - :f/assert [{:ex/x "foo-cat" :id :ex/dan}] - :f/retract [{:ex/x "foo-3" :id :ex/dan}]}] - @(fluree/history ledger {:history [:ex/dan :ex/x] :t {:from 1}}))) + (is (= [{"f:t" 1 "f:assert" [{"ex:x" "foo-1" "id" "ex:dan"}] "f:retract" []} + {"f:t" 2 + "f:assert" [{"ex:x" "foo-2" "id" "ex:dan"}] + "f:retract" [{"ex:x" "foo-1" "id" "ex:dan"}]} + {"f:t" 3 + "f:assert" [{"ex:x" "foo-3" "id" "ex:dan"}] + "f:retract" [{"ex:x" "foo-2" "id" "ex:dan"}]} + {"f:t" 5 + "f:assert" [{"ex:x" "foo-cat" "id" "ex:dan"}] + "f:retract" [{"ex:x" "foo-3" "id" "ex:dan"}]}] + @(fluree/history ledger {"history" ["ex:dan" "ex:x"] "t" {"from" 1}}))) - (is (= [{:f/t 1 :f/assert [{:ex/x "foo-1" :id :ex/dog} - {:ex/x "foo-1" :id :ex/cat} - {:ex/x "foo-1" :id :ex/dan}] - :f/retract []} - {:f/t 2 - :f/assert [{:ex/x "foo-2" :id :ex/dan}] - :f/retract [{:ex/x "foo-1" :id :ex/dan}]} - {:f/t 3 - :f/assert [{:ex/x "foo-3" :id :ex/dan}] - :f/retract [{:ex/x "foo-2" :id :ex/dan}]} - {:f/t 4 - :f/assert [{:ex/x "foo-dog" :id :ex/dog} - {:ex/x "foo-cat" :id :ex/cat}] - :f/retract [{:ex/x "foo-1" :id :ex/dog} - {:ex/x "foo-1" :id :ex/cat}]} - {:f/t 5 - :f/assert [{:ex/x "foo-cat" :id :ex/dan}] - :f/retract [{:ex/x "foo-3" :id :ex/dan}]}] - @(fluree/history ledger {:history [nil :ex/x] :t {:from 1}})))) + (is (= [{"f:t" 1 "f:assert" [{"ex:x" "foo-1" "id" "ex:dog"} + {"ex:x" "foo-1" "id" "ex:cat"} + {"ex:x" "foo-1" "id" "ex:dan"}] + "f:retract" []} + {"f:t" 2 + "f:assert" [{"ex:x" "foo-2" "id" "ex:dan"}] + "f:retract" [{"ex:x" "foo-1" "id" "ex:dan"}]} + {"f:t" 3 + "f:assert" [{"ex:x" "foo-3" "id" "ex:dan"}] + "f:retract" [{"ex:x" "foo-2" "id" "ex:dan"}]} + {"f:t" 4 + "f:assert" [{"ex:x" "foo-dog" "id" "ex:dog"} + {"ex:x" "foo-cat" "id" "ex:cat"}] + "f:retract" [{"ex:x" "foo-1" "id" "ex:dog"} + {"ex:x" "foo-1" "id" "ex:cat"}]} + {"f:t" 5 + "f:assert" [{"ex:x" "foo-cat" "id" "ex:dan"}] + "f:retract" [{"ex:x" "foo-3" "id" "ex:dan"}]}] + @(fluree/history ledger {"history" [nil "ex:x"] "t" {"from" 1}})))) (testing "three-tuple flake history" - (is (= [{:f/t 4 :f/assert [{:ex/x "foo-cat" :id :ex/cat}] :f/retract []} - {:f/t 5 :f/assert [{:ex/x "foo-cat" :id :ex/dan}] :f/retract []}] - @(fluree/history ledger {:history [nil :ex/x "foo-cat"] :t {:from 1}}))) - (is (= [{:f/t 2 - :f/assert [{:ex/x "foo-2" :id :ex/dan}] - :f/retract []} - {:f/t 3 - :f/assert [] - :f/retract [{:ex/x "foo-2" :id :ex/dan}]}] - @(fluree/history ledger {:history [nil :ex/x "foo-2"] :t {:from 1}}))) - (is (= [{:f/t 5 :f/assert [{:ex/x "foo-cat" :id :ex/dan}] :f/retract []}] - @(fluree/history ledger {:history [:ex/dan :ex/x "foo-cat"] :t {:from 1}})))) + (is (= [{"f:t" 4 "f:assert" [{"ex:x" "foo-cat" "id" "ex:cat"}] "f:retract" []} + {"f:t" 5 "f:assert" [{"ex:x" "foo-cat" "id" "ex:dan"}] "f:retract" []}] + @(fluree/history ledger {"history" [nil "ex:x" "foo-cat"] "t" {"from" 1}}))) + (is (= [{"f:t" 2 + "f:assert" [{"ex:x" "foo-2" "id" "ex:dan"}] + "f:retract" []} + {"f:t" 3 + "f:assert" [] + "f:retract" [{"ex:x" "foo-2" "id" "ex:dan"}]}] + @(fluree/history ledger {"history" [nil "ex:x" "foo-2"] "t" {"from" 1}}))) + (is (= [{"f:t" 5 "f:assert" [{"ex:x" "foo-cat" "id" "ex:dan"}] "f:retract" []}] + @(fluree/history ledger {"history" ["ex:dan" "ex:x" "foo-cat"] "t" {"from" 1}})))) (testing "at-t" - (let [expected [{:f/t 3 - :f/assert [{:ex/x "foo-3" :id :ex/dan}] - :f/retract [{:ex/x "foo-2" :id :ex/dan}]}]] + (let [expected [{"f:t" 3 + "f:assert" [{"ex:x" "foo-3" "id" "ex:dan"}] + "f:retract" [{"ex:x" "foo-2" "id" "ex:dan"}]}]] (is (= expected - @(fluree/history ledger {:history [:ex/dan :ex/x] :t {:from 3 :to 3}}))) + @(fluree/history ledger {"history" ["ex:dan" "ex:x"] "t" {"from" 3 "to" 3}}))) (is (= expected - @(fluree/history ledger {:history [:ex/dan :ex/x] :t {:at 3}}))))) + @(fluree/history ledger {"history" ["ex:dan" "ex:x"] "t" {"at" 3}}))))) (testing "from-t" - (is (= [{:f/t 3 - :f/assert [{:ex/x "foo-3" :id :ex/dan}] - :f/retract [{:ex/x "foo-2" :id :ex/dan}]} - {:f/t 5 - :f/assert [{:ex/x "foo-cat" :id :ex/dan}] - :f/retract [{:ex/x "foo-3" :id :ex/dan}]}] - @(fluree/history ledger {:history [:ex/dan :ex/x] :t {:from 3}})))) + (is (= [{"f:t" 3 + "f:assert" [{"ex:x" "foo-3" "id" "ex:dan"}] + "f:retract" [{"ex:x" "foo-2" "id" "ex:dan"}]} + {"f:t" 5 + "f:assert" [{"ex:x" "foo-cat" "id" "ex:dan"}] + "f:retract" [{"ex:x" "foo-3" "id" "ex:dan"}]}] + @(fluree/history ledger {"history" ["ex:dan" "ex:x"] "t" {"from" 3}})))) (testing "to-t" - (is (= [{:f/t 1 - :f/assert [{:ex/x "foo-1" :id :ex/dan}] - :f/retract []} - {:f/t 2 - :f/assert [{:ex/x "foo-2" :id :ex/dan}] - :f/retract [{:ex/x "foo-1" :id :ex/dan}]} - {:f/t 3 - :f/assert [{:ex/x "foo-3" :id :ex/dan}] - :f/retract [{:ex/x "foo-2" :id :ex/dan}]}] - @(fluree/history ledger {:history [:ex/dan :ex/x] :t {:to 3}})))) + (is (= [{"f:t" 1 + "f:assert" [{"ex:x" "foo-1" "id" "ex:dan"}] + "f:retract" []} + {"f:t" 2 + "f:assert" [{"ex:x" "foo-2" "id" "ex:dan"}] + "f:retract" [{"ex:x" "foo-1" "id" "ex:dan"}]} + {"f:t" 3 + "f:assert" [{"ex:x" "foo-3" "id" "ex:dan"}] + "f:retract" [{"ex:x" "foo-2" "id" "ex:dan"}]}] + @(fluree/history ledger {"history" ["ex:dan" "ex:x"] "t" {"to" 3}})))) (testing "t-range" - (is (= [{:f/t 2 - :f/assert [{:ex/x "foo-2" :id :ex/dan}] - :f/retract [{:ex/x "foo-1" :id :ex/dan}]} - {:f/t 3 - :f/assert [{:ex/x "foo-3" :id :ex/dan}] - :f/retract [{:ex/x "foo-2" :id :ex/dan}]} - {:f/t 4 - :f/assert [{:ex/x "foo-dog" :id :ex/dog} {:ex/x "foo-cat" :id :ex/cat}] - :f/retract [{:ex/x "foo-1" :id :ex/dog} {:ex/x "foo-1" :id :ex/cat}]}] - @(fluree/history ledger {:history [nil :ex/x] :t {:from 2 :to 4}})))) + (is (= [{"f:t" 2 + "f:assert" [{"ex:x" "foo-2" "id" "ex:dan"}] + "f:retract" [{"ex:x" "foo-1" "id" "ex:dan"}]} + {"f:t" 3 + "f:assert" [{"ex:x" "foo-3" "id" "ex:dan"}] + "f:retract" [{"ex:x" "foo-2" "id" "ex:dan"}]} + {"f:t" 4 + "f:assert" [{"ex:x" "foo-dog" "id" "ex:dog"} {"ex:x" "foo-cat" "id" "ex:cat"}] + "f:retract" [{"ex:x" "foo-1" "id" "ex:dog"} {"ex:x" "foo-1" "id" "ex:cat"}]}] + @(fluree/history ledger {"history" [nil "ex:x"] "t" {"from" 2 "to" 4}})))) (testing "datetime-t" - (is (= [{:f/t 2 - :f/assert [{:ex/x "foo-2" :id :ex/dan}] - :f/retract [{:ex/x "foo-1" :id :ex/dan}]} - {:f/t 3 - :f/assert [{:ex/x "foo-3" :id :ex/dan}] - :f/retract [{:ex/x "foo-2" :id :ex/dan}]}] - @(fluree/history ledger {:history [nil :ex/x] :t {:from ts2 :to ts3}})) + (is (= [{"f:t" 2 + "f:assert" [{"ex:x" "foo-2" "id" "ex:dan"}] + "f:retract" [{"ex:x" "foo-1" "id" "ex:dan"}]} + {"f:t" 3 + "f:assert" [{"ex:x" "foo-3" "id" "ex:dan"}] + "f:retract" [{"ex:x" "foo-2" "id" "ex:dan"}]}] + @(fluree/history ledger {"history" [nil "ex:x"] "t" {"from" ts2 "to" ts3}})) "does not include t 1 4 or 5") - (is (= [{:f/t 5 - :f/assert [{:ex/x "foo-cat" :id :ex/dan}] - :f/retract [{:ex/x "foo-3" :id :ex/dan}]}] - @(fluree/history ledger {:history [:ex/dan :ex/x] :t {:from (util/current-time-iso)}})) + (is (= [{"f:t" 5 + "f:assert" [{"ex:x" "foo-cat" "id" "ex:dan"}] + "f:retract" [{"ex:x" "foo-3" "id" "ex:dan"}]}] + @(fluree/history ledger {"history" ["ex:dan" "ex:x"] "t" {"from" (util/current-time-iso)}})) "timestamp translates to first t before ts") (is (= (str "There is no data as of " ts-primeval) - (-> @(fluree/history ledger {:history [:ex/dan :ex/x] :t {:from ts-primeval}}) + (-> @(fluree/history ledger {"history" ["ex:dan" "ex:x"] "t" {"from" ts-primeval}}) (Throwable->map) :cause)))) (testing "invalid query" - (is (= "History query not properly formatted. Provided {:history []}" - (-> @(fluree/history ledger {:history []}) + (is (= "History query not properly formatted. Provided {\"history\" []}" + (-> @(fluree/history ledger {"history" []}) (Throwable->map) :cause)))) (testing "small cache" (let [conn (test-utils/create-conn) - ledger @(fluree/create conn "historycachetest" {:defaultContext ["" {:ex "http://example.org/ns/"}]}) + ledger @(fluree/create conn "historycachetest" + {"defaults" + {"@context" + ["" {"ex" "http://example.org/ns/"}]}}) - db1 @(test-utils/transact ledger [{:id :ex/dan - :ex/x "foo-1" - :ex/y "bar-1"}]) - db2 @(test-utils/transact ledger {:id :ex/dan - :ex/x "foo-2" - :ex/y "bar-2"})] + db1 @(test-utils/transact ledger [{"id" "ex:dan" + "ex:x" "foo-1" + "ex:y" "bar-1"}]) + db2 @(test-utils/transact ledger {"id" "ex:dan" + "ex:x" "foo-2" + "ex:y" "bar-2"})] (testing "no t-range cache collision" - (is (= [{:f/t 2 - :f/assert [{:ex/x "foo-2" :ex/y "bar-2" :id :ex/dan}] - :f/retract [{:ex/x "foo-1" :ex/y "bar-1" :id :ex/dan}]}] - @(fluree/history ledger {:history [:ex/dan] :t {:from 2}})))))))) + (is (= [{"f:t" 2 + "f:assert" [{"ex:x" "foo-2" "ex:y" "bar-2" "id" "ex:dan"}] + "f:retract" [{"ex:x" "foo-1" "ex:y" "bar-1" "id" "ex:dan"}]}] + @(fluree/history ledger {"history" ["ex:dan"] "t" {"from" 2}})))))))) (deftest ^:integration commit-details (with-redefs [fluree.db.util.core/current-time-iso (fn [] "1970-01-01T00:12:00.00000Z")] (let [conn (test-utils/create-conn) - ledger @(fluree/create conn "committest" {:defaultContext ["" {:ex "http://example.org/ns/"}]}) + ledger @(fluree/create conn "committest" + {"defaults" + {"@context" + ["" {"ex" "http://example.org/ns/"}]}}) - db1 @(test-utils/transact ledger {:id :ex/alice - :ex/x "foo-1" - :ex/y "bar-1"}) - db2 @(test-utils/transact ledger {:id :ex/alice - :ex/x "foo-2" - :ex/y "bar-2"}) - db3 @(test-utils/transact ledger {:id :ex/alice - :ex/x "foo-3" - :ex/y "bar-3"}) - db4 @(test-utils/transact ledger {:id :ex/cat - :ex/x "foo-cat" - :ex/y "bar-cat"}) - db5 @(test-utils/transact ledger {:id :ex/alice - :ex/x "foo-cat" - :ex/y "bar-cat"} - {:message "meow"})] + db1 @(test-utils/transact ledger {"id" "ex:alice" + "ex:x" "foo-1" + "ex:y" "bar-1"}) + db2 @(test-utils/transact ledger {"id" "ex:alice" + "ex:x" "foo-2" + "ex:y" "bar-2"}) + db3 @(test-utils/transact ledger {"id" "ex:alice" + "ex:x" "foo-3" + "ex:y" "bar-3"}) + db4 @(test-utils/transact ledger {"id" "ex:cat" + "ex:x" "foo-cat" + "ex:y" "bar-cat"}) + db5 @(test-utils/transact ledger {"id" "ex:alice" + "ex:x" "foo-cat" + "ex:y" "bar-cat"} + {"message" "meow"})] (testing "at time t" - (is (= [{:f/commit {"https://www.w3.org/2018/credentials#issuer" - {:id "did:fluree:TfCzWTrXqF16hvKGjcYiLxRoYJ1B8a6UMH6"} - :f/address "fluree:memory://c2e0047d4d75cad2700d5c8d0db0ad3d7dd2bbbbbdf55ba7c28dd4252a557664" - :f/alias "committest" - :f/branch "main" - :f/context "fluree:memory://b6dcf8968183239ecc7a664025f247de5b7859ac18cdeaace89aafc421eeddee" - :f/data #:f{:address "fluree:memory://bcb581e731a7c0ceadcfbf432b4ee8cf046de377cc33f047bd05b6c47f9da94d" - :assert [{:ex/x "foo-1" - :ex/y "bar-1" - :id :ex/alice}] - :flakes 11 - :retract [] - :size 996 - :t 1} - :f/time 720000 - :f/v 0 - :id "fluree:commit:sha256:bn6sykdmktzuxcavgrsa5ejwdzfae6njj4q3lonb5cexlfhauvpc"}}] - @(fluree/history ledger {:commit-details true :t {:from 1 :to 1}}))) - (let [commit-5 {:f/commit {"https://www.w3.org/2018/credentials#issuer" - {:id "did:fluree:TfCzWTrXqF16hvKGjcYiLxRoYJ1B8a6UMH6"} - :f/address "fluree:memory://2716b3eaef91b763b32daebab8ba3733a8537de37a864f726c5021464b90277f" - :f/alias "committest" - :f/branch "main" - :f/context "fluree:memory://b6dcf8968183239ecc7a664025f247de5b7859ac18cdeaace89aafc421eeddee" - :f/data #:f{:address "fluree:memory://2b8125a4c996f8612ae62f16cc62b167b762c517b0c5aa7b16fa21dfe47e7b2a" - :assert [{:ex/x "foo-cat" - :ex/y "bar-cat" - :id :ex/alice}] - :flakes 102 - :previous {:id "fluree:db:sha256:bbtdwia2mle22abe2z7mmdrs4vufs77yubzxip3chhkmffvfk4npk"} - :retract [{:ex/x "foo-3" - :ex/y "bar-3" - :id :ex/alice}] - :size 9326 - :t 5} - :f/message "meow" - :f/previous {:id "fluree:commit:sha256:bhcunj52uwxshi6jws2ypsjiziyncfa5gal4xptfldj5utb54cpf"} - :f/time 720000 - :f/v 0 - :id "fluree:commit:sha256:bq4r74wu4sru43z5f4byipe5zzkgyi2kksqxlmwuttdcwcgwpsrn"}} - commit-4 {:f/commit {"https://www.w3.org/2018/credentials#issuer" - {:id "did:fluree:TfCzWTrXqF16hvKGjcYiLxRoYJ1B8a6UMH6"} - :f/address "fluree:memory://ffe008a623d3c6920f1d7d7607783893042c8093629f64ffefc8eb8472f542af" - :f/alias "committest" - :f/branch "main" - :f/context "fluree:memory://b6dcf8968183239ecc7a664025f247de5b7859ac18cdeaace89aafc421eeddee" - :f/data #:f{:address "fluree:memory://2cac0ce4036dd82a0c23eb3deca0e7775386ed220411edb271803144b001326c" - :assert [{:ex/x "foo-cat" - :ex/y "bar-cat" - :id :ex/cat}] - :flakes 82 - :previous {:id "fluree:db:sha256:bcl3anjpvmxaciox7inzx4za6teagj7ipacuadzlnwg45y6z77ts"} - :retract [] - :size 7588 - :t 4} - :f/previous {:id "fluree:commit:sha256:b5jlses24wzjhcmqywvcdwgxxzmjlmxruh2duqsgxbxy7522utlg"} - :f/time 720000 - :f/v 0 - :id "fluree:commit:sha256:bhcunj52uwxshi6jws2ypsjiziyncfa5gal4xptfldj5utb54cpf"}}] + (is (= [{"f:commit" {"https://www.w3.org/2018/credentials#issuer" + {"id" "did:fluree:TfCzWTrXqF16hvKGjcYiLxRoYJ1B8a6UMH6"} + "f:address" "fluree:memory://c2e0047d4d75cad2700d5c8d0db0ad3d7dd2bbbbbdf55ba7c28dd4252a557664" + "f:alias" "committest" + "f:branch" "main" + "f:context" "fluree:memory://b6dcf8968183239ecc7a664025f247de5b7859ac18cdeaace89aafc421eeddee" + "f:data" {"f:address" "fluree:memory://bcb581e731a7c0ceadcfbf432b4ee8cf046de377cc33f047bd05b6c47f9da94d" + "f:assert" [{"ex:x" "foo-1" + "ex:y" "bar-1" + "id" "ex:alice"}] + "f:flakes" 11 + "f:retract" [] + "f:size" 996 + "f:t" 1} + "f:time" 720000 + "f:v" 0 + "id" "fluree:commit:sha256:bn6sykdmktzuxcavgrsa5ejwdzfae6njj4q3lonb5cexlfhauvpc"}}] + @(fluree/history ledger {"commit-details" true "t" {"from" 1 "to" 1}}))) + (let [commit-5 {"f:commit" {"https://www.w3.org/2018/credentials#issuer" + {"id" "did:fluree:TfCzWTrXqF16hvKGjcYiLxRoYJ1B8a6UMH6"} + "f:address" "fluree:memory://2716b3eaef91b763b32daebab8ba3733a8537de37a864f726c5021464b90277f" + "f:alias" "committest" + "f:branch" "main" + "f:context" "fluree:memory://b6dcf8968183239ecc7a664025f247de5b7859ac18cdeaace89aafc421eeddee" + "f:data" {"f:address" "fluree:memory://2b8125a4c996f8612ae62f16cc62b167b762c517b0c5aa7b16fa21dfe47e7b2a" + "f:assert" [{"ex:x" "foo-cat" + "ex:y" "bar-cat" + "id" "ex:alice"}] + "f:flakes" 102 + "f:previous" {"id" "fluree:db:sha256:bbtdwia2mle22abe2z7mmdrs4vufs77yubzxip3chhkmffvfk4npk"} + "f:retract" [{"ex:x" "foo-3" + "ex:y" "bar-3" + "id" "ex:alice"}] + "f:size" 9326 + "f:t" 5} + "f:message" "meow" + "f:previous" {"id" "fluree:commit:sha256:bhcunj52uwxshi6jws2ypsjiziyncfa5gal4xptfldj5utb54cpf"} + "f:time" 720000 + "f:v" 0 + "id" "fluree:commit:sha256:bq4r74wu4sru43z5f4byipe5zzkgyi2kksqxlmwuttdcwcgwpsrn"}} + commit-4 {"f:commit" {"https://www.w3.org/2018/credentials#issuer" + {"id" "did:fluree:TfCzWTrXqF16hvKGjcYiLxRoYJ1B8a6UMH6"} + "f:address" "fluree:memory://ffe008a623d3c6920f1d7d7607783893042c8093629f64ffefc8eb8472f542af" + "f:alias" "committest" + "f:branch" "main" + "f:context" "fluree:memory://b6dcf8968183239ecc7a664025f247de5b7859ac18cdeaace89aafc421eeddee" + "f:data" {"f:address" "fluree:memory://2cac0ce4036dd82a0c23eb3deca0e7775386ed220411edb271803144b001326c" + "f:assert" [{"ex:x" "foo-cat" + "ex:y" "bar-cat" + "id" "ex:cat"}] + "f:flakes" 82 + "f:previous" {"id" "fluree:db:sha256:bcl3anjpvmxaciox7inzx4za6teagj7ipacuadzlnwg45y6z77ts"} + "f:retract" [] + "f:size" 7588 + "f:t" 4} + "f:previous" {"id" "fluree:commit:sha256:b5jlses24wzjhcmqywvcdwgxxzmjlmxruh2duqsgxbxy7522utlg"} + "f:time" 720000 + "f:v" 0 + "id" "fluree:commit:sha256:bhcunj52uwxshi6jws2ypsjiziyncfa5gal4xptfldj5utb54cpf"}}] (is (= [commit-4 commit-5] - @(fluree/history ledger {:commit-details true :t {:from 4 :to 5}}))) + @(fluree/history ledger {"commit-details" true "t" {"from" 4 "to" 5}}))) (is (= [commit-5] - @(fluree/history ledger {:commit-details true :t {:at "latest"}}))))) + @(fluree/history ledger {"commit-details" true "t" {"at" "latest"}}))))) (testing "time range" (let [[c2 c3 c4 :as response] @(fluree/history ledger - {:commit-details true - :t {:from 2 :to 4}})] + {"commit-details" true + "t" {"from" 2 "to" 4}})] (testing "all commits in time range are returned" (is (= 3 (count response))) - (is (= {:f/commit {"https://www.w3.org/2018/credentials#issuer" - {:id "did:fluree:TfCzWTrXqF16hvKGjcYiLxRoYJ1B8a6UMH6"} - :f/address "fluree:memory://ffe008a623d3c6920f1d7d7607783893042c8093629f64ffefc8eb8472f542af" - :f/alias "committest" - :f/branch "main" - :f/context "fluree:memory://b6dcf8968183239ecc7a664025f247de5b7859ac18cdeaace89aafc421eeddee" - :f/data #:f{:address "fluree:memory://2cac0ce4036dd82a0c23eb3deca0e7775386ed220411edb271803144b001326c" - :assert [{:ex/x "foo-cat" - :ex/y "bar-cat" - :id :ex/cat}] - :flakes 82 - :previous {:id "fluree:db:sha256:bcl3anjpvmxaciox7inzx4za6teagj7ipacuadzlnwg45y6z77ts"} - :retract [] - :size 7588 - :t 4} - :f/previous {:id "fluree:commit:sha256:b5jlses24wzjhcmqywvcdwgxxzmjlmxruh2duqsgxbxy7522utlg"} - :f/time 720000 - :f/v 0 - :id "fluree:commit:sha256:bhcunj52uwxshi6jws2ypsjiziyncfa5gal4xptfldj5utb54cpf"}} + (is (= {"f:commit" {"https://www.w3.org/2018/credentials#issuer" + {"id" "did:fluree:TfCzWTrXqF16hvKGjcYiLxRoYJ1B8a6UMH6"} + "f:address" "fluree:memory://ffe008a623d3c6920f1d7d7607783893042c8093629f64ffefc8eb8472f542af" + "f:alias" "committest" + "f:branch" "main" + "f:context" "fluree:memory://b6dcf8968183239ecc7a664025f247de5b7859ac18cdeaace89aafc421eeddee" + "f:data" {"f:address" "fluree:memory://2cac0ce4036dd82a0c23eb3deca0e7775386ed220411edb271803144b001326c" + "f:assert" [{"ex:x" "foo-cat" + "ex:y" "bar-cat" + "id" "ex:cat"}] + "f:flakes" 82 + "f:previous" {"id" "fluree:db:sha256:bcl3anjpvmxaciox7inzx4za6teagj7ipacuadzlnwg45y6z77ts"} + "f:retract" [] + "f:size" 7588 + "f:t" 4} + "f:previous" {"id" "fluree:commit:sha256:b5jlses24wzjhcmqywvcdwgxxzmjlmxruh2duqsgxbxy7522utlg"} + "f:time" 720000 + "f:v" 0 + "id" "fluree:commit:sha256:bhcunj52uwxshi6jws2ypsjiziyncfa5gal4xptfldj5utb54cpf"}} c4))) - (is (= {:f/commit {"https://www.w3.org/2018/credentials#issuer" - {:id "did:fluree:TfCzWTrXqF16hvKGjcYiLxRoYJ1B8a6UMH6"} - :f/address "fluree:memory://f29bfb686c834d667dc62f8c46af1802f02c62567b400d50c0202428e489d1fe" - :f/alias "committest" - :f/branch "main" - :f/context "fluree:memory://b6dcf8968183239ecc7a664025f247de5b7859ac18cdeaace89aafc421eeddee" - :f/data #:f{:address "fluree:memory://cb16fc43954b9ed029be2c96c6f73fa5e34e5ca1607111c5e8f8d6e337b648f6" - :assert [{:ex/x "foo-3" - :ex/y "bar-3" - :id :ex/alice}] - :flakes 63 - :previous {:id "fluree:db:sha256:bjufs3dmyea7wzbkrjrh2pzua2mtqgijnl3cqpkktk7wzvy5wlnq"} - :retract [{:ex/x "foo-2" - :ex/y "bar-2" - :id :ex/alice}] - :size 5864 - :t 3} - :f/previous {:id "fluree:commit:sha256:bbvotmnlqkm4xkc27au55rctw5klntegd36j4dbh665qfilem42eq"} - :f/time 720000 - :f/v 0 - :id "fluree:commit:sha256:b5jlses24wzjhcmqywvcdwgxxzmjlmxruh2duqsgxbxy7522utlg"}} + (is (= {"f:commit" {"https://www.w3.org/2018/credentials#issuer" + {"id" "did:fluree:TfCzWTrXqF16hvKGjcYiLxRoYJ1B8a6UMH6"} + "f:address" "fluree:memory://f29bfb686c834d667dc62f8c46af1802f02c62567b400d50c0202428e489d1fe" + "f:alias" "committest" + "f:branch" "main" + "f:context" "fluree:memory://b6dcf8968183239ecc7a664025f247de5b7859ac18cdeaace89aafc421eeddee" + "f:data" {"f:address" "fluree:memory://cb16fc43954b9ed029be2c96c6f73fa5e34e5ca1607111c5e8f8d6e337b648f6" + "f:assert" [{"ex:x" "foo-3" + "ex:y" "bar-3" + "id" "ex:alice"}] + "f:flakes" 63 + "f:previous" {"id" "fluree:db:sha256:bjufs3dmyea7wzbkrjrh2pzua2mtqgijnl3cqpkktk7wzvy5wlnq"} + "f:retract" [{"ex:x" "foo-2" + "ex:y" "bar-2" + "id" "ex:alice"}] + "f:size" 5864 + "f:t" 3} + "f:previous" {"id" "fluree:commit:sha256:bbvotmnlqkm4xkc27au55rctw5klntegd36j4dbh665qfilem42eq"} + "f:time" 720000 + "f:v" 0 + "id" "fluree:commit:sha256:b5jlses24wzjhcmqywvcdwgxxzmjlmxruh2duqsgxbxy7522utlg"}} c3)) - (is (= {:f/commit {"https://www.w3.org/2018/credentials#issuer" - {:id "did:fluree:TfCzWTrXqF16hvKGjcYiLxRoYJ1B8a6UMH6"} - :f/address "fluree:memory://a3f34c963d5724782dc33aae1c9e0d8dc8b1f35092a659cb2431d7204b95288c" - :f/alias "committest" - :f/branch "main" - :f/context "fluree:memory://b6dcf8968183239ecc7a664025f247de5b7859ac18cdeaace89aafc421eeddee" - :f/data #:f{:address "fluree:memory://2fb8ce1aa6771837c7986b1d81dcdb42ca8be8927679fe3814b2fb044b903b9d" - :assert [{:ex/x "foo-2" - :ex/y "bar-2" - :id :ex/alice}] - :flakes 43 - :previous {:id "fluree:db:sha256:bbbi2zkypmbphdnt7ntmtqxtuvayt5izcbfkjaqfrlq2ixxrj5dcu"} - :retract [{:ex/x "foo-1" - :ex/y "bar-1" - :id :ex/alice}] - :size 4134 - :t 2} - :f/previous {:id "fluree:commit:sha256:bn6sykdmktzuxcavgrsa5ejwdzfae6njj4q3lonb5cexlfhauvpc"} - :f/time 720000 - :f/v 0 - :id "fluree:commit:sha256:bbvotmnlqkm4xkc27au55rctw5klntegd36j4dbh665qfilem42eq"}} + (is (= {"f:commit" {"https://www.w3.org/2018/credentials#issuer" + {"id" "did:fluree:TfCzWTrXqF16hvKGjcYiLxRoYJ1B8a6UMH6"} + "f:address" "fluree:memory://a3f34c963d5724782dc33aae1c9e0d8dc8b1f35092a659cb2431d7204b95288c" + "f:alias" "committest" + "f:branch" "main" + "f:context" "fluree:memory://b6dcf8968183239ecc7a664025f247de5b7859ac18cdeaace89aafc421eeddee" + "f:data" {"f:address" "fluree:memory://2fb8ce1aa6771837c7986b1d81dcdb42ca8be8927679fe3814b2fb044b903b9d" + "f:assert" [{"ex:x" "foo-2" + "ex:y" "bar-2" + "id" "ex:alice"}] + "f:flakes" 43 + "f:previous" {"id" "fluree:db:sha256:bbbi2zkypmbphdnt7ntmtqxtuvayt5izcbfkjaqfrlq2ixxrj5dcu"} + "f:retract" [{"ex:x" "foo-1" + "ex:y" "bar-1" + "id" "ex:alice"}] + "f:size" 4134 + "f:t" 2} + "f:previous" {"id" "fluree:commit:sha256:bn6sykdmktzuxcavgrsa5ejwdzfae6njj4q3lonb5cexlfhauvpc"} + "f:time" 720000 + "f:v" 0 + "id" "fluree:commit:sha256:bbvotmnlqkm4xkc27au55rctw5klntegd36j4dbh665qfilem42eq"}} c2)))) (testing "time range from" - (is (= [{:f/commit {"https://www.w3.org/2018/credentials#issuer" - {:id "did:fluree:TfCzWTrXqF16hvKGjcYiLxRoYJ1B8a6UMH6"} - :f/address "fluree:memory://ffe008a623d3c6920f1d7d7607783893042c8093629f64ffefc8eb8472f542af" - :f/alias "committest" - :f/branch "main" - :f/context "fluree:memory://b6dcf8968183239ecc7a664025f247de5b7859ac18cdeaace89aafc421eeddee" - :f/data #:f{:address "fluree:memory://2cac0ce4036dd82a0c23eb3deca0e7775386ed220411edb271803144b001326c" - :assert [{:ex/x "foo-cat" - :ex/y "bar-cat" - :id :ex/cat}] - :flakes 82 - :previous {:id "fluree:db:sha256:bcl3anjpvmxaciox7inzx4za6teagj7ipacuadzlnwg45y6z77ts"} - :retract [] - :size 7588 - :t 4} - :f/previous {:id "fluree:commit:sha256:b5jlses24wzjhcmqywvcdwgxxzmjlmxruh2duqsgxbxy7522utlg"} - :f/time 720000 - :f/v 0 - :id "fluree:commit:sha256:bhcunj52uwxshi6jws2ypsjiziyncfa5gal4xptfldj5utb54cpf"}} - {:f/commit {"https://www.w3.org/2018/credentials#issuer" - {:id "did:fluree:TfCzWTrXqF16hvKGjcYiLxRoYJ1B8a6UMH6"} - :f/address "fluree:memory://2716b3eaef91b763b32daebab8ba3733a8537de37a864f726c5021464b90277f" - :f/alias "committest" - :f/branch "main" - :f/context "fluree:memory://b6dcf8968183239ecc7a664025f247de5b7859ac18cdeaace89aafc421eeddee" - :f/data #:f{:address "fluree:memory://2b8125a4c996f8612ae62f16cc62b167b762c517b0c5aa7b16fa21dfe47e7b2a" - :assert [{:ex/x "foo-cat" - :ex/y "bar-cat" - :id :ex/alice}] - :flakes 102 - :previous {:id "fluree:db:sha256:bbtdwia2mle22abe2z7mmdrs4vufs77yubzxip3chhkmffvfk4npk"} - :retract [{:ex/x "foo-3" - :ex/y "bar-3" - :id :ex/alice}] - :size 9326 - :t 5} - :f/message "meow" - :f/previous {:id "fluree:commit:sha256:bhcunj52uwxshi6jws2ypsjiziyncfa5gal4xptfldj5utb54cpf"} - :f/time 720000 - :f/v 0 - :id "fluree:commit:sha256:bq4r74wu4sru43z5f4byipe5zzkgyi2kksqxlmwuttdcwcgwpsrn"}}] - @(fluree/history ledger {:commit-details true :t {:from 4}})))) + (is (= [{"f:commit" {"https://www.w3.org/2018/credentials#issuer" + {"id" "did:fluree:TfCzWTrXqF16hvKGjcYiLxRoYJ1B8a6UMH6"} + "f:address" "fluree:memory://ffe008a623d3c6920f1d7d7607783893042c8093629f64ffefc8eb8472f542af" + "f:alias" "committest" + "f:branch" "main" + "f:context" "fluree:memory://b6dcf8968183239ecc7a664025f247de5b7859ac18cdeaace89aafc421eeddee" + "f:data" {"f:address" "fluree:memory://2cac0ce4036dd82a0c23eb3deca0e7775386ed220411edb271803144b001326c" + "f:assert" [{"ex:x" "foo-cat" + "ex:y" "bar-cat" + "id" "ex:cat"}] + "f:flakes" 82 + "f:previous" {"id" "fluree:db:sha256:bcl3anjpvmxaciox7inzx4za6teagj7ipacuadzlnwg45y6z77ts"} + "f:retract" [] + "f:size" 7588 + "f:t" 4} + "f:previous" {"id" "fluree:commit:sha256:b5jlses24wzjhcmqywvcdwgxxzmjlmxruh2duqsgxbxy7522utlg"} + "f:time" 720000 + "f:v" 0 + "id" "fluree:commit:sha256:bhcunj52uwxshi6jws2ypsjiziyncfa5gal4xptfldj5utb54cpf"}} + {"f:commit" {"https://www.w3.org/2018/credentials#issuer" + {"id" "did:fluree:TfCzWTrXqF16hvKGjcYiLxRoYJ1B8a6UMH6"} + "f:address" "fluree:memory://2716b3eaef91b763b32daebab8ba3733a8537de37a864f726c5021464b90277f" + "f:alias" "committest" + "f:branch" "main" + "f:context" "fluree:memory://b6dcf8968183239ecc7a664025f247de5b7859ac18cdeaace89aafc421eeddee" + "f:data" {"f:address" "fluree:memory://2b8125a4c996f8612ae62f16cc62b167b762c517b0c5aa7b16fa21dfe47e7b2a" + "f:assert" [{"ex:x" "foo-cat" + "ex:y" "bar-cat" + "id" "ex:alice"}] + "f:flakes" 102 + "f:previous" {"id" "fluree:db:sha256:bbtdwia2mle22abe2z7mmdrs4vufs77yubzxip3chhkmffvfk4npk"} + "f:retract" [{"ex:x" "foo-3" + "ex:y" "bar-3" + "id" "ex:alice"}] + "f:size" 9326 + "f:t" 5} + "f:message" "meow" + "f:previous" {"id" "fluree:commit:sha256:bhcunj52uwxshi6jws2ypsjiziyncfa5gal4xptfldj5utb54cpf"} + "f:time" 720000 + "f:v" 0 + "id" "fluree:commit:sha256:bq4r74wu4sru43z5f4byipe5zzkgyi2kksqxlmwuttdcwcgwpsrn"}}] + @(fluree/history ledger {"commit-details" true "t" {"from" 4}})))) (testing "time range to" - (is (= [{:f/commit {"https://www.w3.org/2018/credentials#issuer" - {:id "did:fluree:TfCzWTrXqF16hvKGjcYiLxRoYJ1B8a6UMH6"} - :f/address "fluree:memory://c2e0047d4d75cad2700d5c8d0db0ad3d7dd2bbbbbdf55ba7c28dd4252a557664" - :f/alias "committest" - :f/branch "main" - :f/context "fluree:memory://b6dcf8968183239ecc7a664025f247de5b7859ac18cdeaace89aafc421eeddee" - :f/data #:f{:address "fluree:memory://bcb581e731a7c0ceadcfbf432b4ee8cf046de377cc33f047bd05b6c47f9da94d" - :assert [{:ex/x "foo-1" - :ex/y "bar-1" - :id :ex/alice}] - :flakes 11 - :retract [] - :size 996 - :t 1} - :f/time 720000 - :f/v 0 - :id "fluree:commit:sha256:bn6sykdmktzuxcavgrsa5ejwdzfae6njj4q3lonb5cexlfhauvpc"}}] - @(fluree/history ledger {:commit-details true :t {:to 1}})))) + (is (= [{"f:commit" {"https://www.w3.org/2018/credentials#issuer" + {"id" "did:fluree:TfCzWTrXqF16hvKGjcYiLxRoYJ1B8a6UMH6"} + "f:address" "fluree:memory://c2e0047d4d75cad2700d5c8d0db0ad3d7dd2bbbbbdf55ba7c28dd4252a557664" + "f:alias" "committest" + "f:branch" "main" + "f:context" "fluree:memory://b6dcf8968183239ecc7a664025f247de5b7859ac18cdeaace89aafc421eeddee" + "f:data" {"f:address" "fluree:memory://bcb581e731a7c0ceadcfbf432b4ee8cf046de377cc33f047bd05b6c47f9da94d" + "f:assert" [{"ex:x" "foo-1" + "ex:y" "bar-1" + "id" "ex:alice"}] + "f:flakes" 11 + "f:retract" [] + "f:size" 996 + "f:t" 1} + "f:time" 720000 + "f:v" 0 + "id" "fluree:commit:sha256:bn6sykdmktzuxcavgrsa5ejwdzfae6njj4q3lonb5cexlfhauvpc"}}] + @(fluree/history ledger {"commit-details" true "t" {"to" 1}})))) (testing "history commit details" - (is (= [#:f{:assert [{:ex/x "foo-3" - :ex/y "bar-3" - :id :ex/alice}] - :commit {"https://www.w3.org/2018/credentials#issuer" - {:id "did:fluree:TfCzWTrXqF16hvKGjcYiLxRoYJ1B8a6UMH6"} - :f/address "fluree:memory://f29bfb686c834d667dc62f8c46af1802f02c62567b400d50c0202428e489d1fe" - :f/alias "committest" - :f/branch "main" - :f/context "fluree:memory://b6dcf8968183239ecc7a664025f247de5b7859ac18cdeaace89aafc421eeddee" - :f/data #:f{:address "fluree:memory://cb16fc43954b9ed029be2c96c6f73fa5e34e5ca1607111c5e8f8d6e337b648f6" - :assert [{:ex/x "foo-3" - :ex/y "bar-3" - :id :ex/alice}] - :flakes 63 - :previous {:id "fluree:db:sha256:bjufs3dmyea7wzbkrjrh2pzua2mtqgijnl3cqpkktk7wzvy5wlnq"} - :retract [{:ex/x "foo-2" - :ex/y "bar-2" - :id :ex/alice}] - :size 5864 - :t 3} - :f/previous {:id "fluree:commit:sha256:bbvotmnlqkm4xkc27au55rctw5klntegd36j4dbh665qfilem42eq"} - :f/time 720000 - :f/v 0 - :id "fluree:commit:sha256:b5jlses24wzjhcmqywvcdwgxxzmjlmxruh2duqsgxbxy7522utlg"} - :retract [{:ex/x "foo-2" - :ex/y "bar-2" - :id :ex/alice}] - :t 3} - #:f{:assert [{:ex/x "foo-cat" - :ex/y "bar-cat" - :id :ex/alice}] - :commit {"https://www.w3.org/2018/credentials#issuer" - {:id "did:fluree:TfCzWTrXqF16hvKGjcYiLxRoYJ1B8a6UMH6"} - :f/address "fluree:memory://2716b3eaef91b763b32daebab8ba3733a8537de37a864f726c5021464b90277f" - :f/alias "committest" - :f/branch "main" - :f/context "fluree:memory://b6dcf8968183239ecc7a664025f247de5b7859ac18cdeaace89aafc421eeddee" - :f/data #:f{:address "fluree:memory://2b8125a4c996f8612ae62f16cc62b167b762c517b0c5aa7b16fa21dfe47e7b2a" - :assert [{:ex/x "foo-cat" - :ex/y "bar-cat" - :id :ex/alice}] - :flakes 102 - :previous {:id "fluree:db:sha256:bbtdwia2mle22abe2z7mmdrs4vufs77yubzxip3chhkmffvfk4npk"} - :retract [{:ex/x "foo-3" - :ex/y "bar-3" - :id :ex/alice}] - :size 9326 - :t 5} - :f/message "meow" - :f/previous {:id "fluree:commit:sha256:bhcunj52uwxshi6jws2ypsjiziyncfa5gal4xptfldj5utb54cpf"} - :f/time 720000 - :f/v 0 - :id "fluree:commit:sha256:bq4r74wu4sru43z5f4byipe5zzkgyi2kksqxlmwuttdcwcgwpsrn"} - :retract [{:ex/x "foo-3" - :ex/y "bar-3" - :id :ex/alice}] - :t 5}] - @(fluree/history ledger {:history :ex/alice :commit-details true :t {:from 3}}))) + (is (= [{"f:assert" [{"ex:x" "foo-3" + "ex:y" "bar-3" + "id" "ex:alice"}] + "f:commit" {"https://www.w3.org/2018/credentials#issuer" + {"id" "did:fluree:TfCzWTrXqF16hvKGjcYiLxRoYJ1B8a6UMH6"} + "f:address" "fluree:memory://f29bfb686c834d667dc62f8c46af1802f02c62567b400d50c0202428e489d1fe" + "f:alias" "committest" + "f:branch" "main" + "f:context" "fluree:memory://b6dcf8968183239ecc7a664025f247de5b7859ac18cdeaace89aafc421eeddee" + "f:data" {"f:address" "fluree:memory://cb16fc43954b9ed029be2c96c6f73fa5e34e5ca1607111c5e8f8d6e337b648f6" + "f:assert" [{"ex:x" "foo-3" + "ex:y" "bar-3" + "id" "ex:alice"}] + "f:flakes" 63 + "f:previous" {"id" "fluree:db:sha256:bjufs3dmyea7wzbkrjrh2pzua2mtqgijnl3cqpkktk7wzvy5wlnq"} + "f:retract" [{"ex:x" "foo-2" + "ex:y" "bar-2" + "id" "ex:alice"}] + "f:size" 5864 + "f:t" 3} + "f:previous" {"id" "fluree:commit:sha256:bbvotmnlqkm4xkc27au55rctw5klntegd36j4dbh665qfilem42eq"} + "f:time" 720000 + "f:v" 0 + "id" "fluree:commit:sha256:b5jlses24wzjhcmqywvcdwgxxzmjlmxruh2duqsgxbxy7522utlg"} + "f:retract" [{"ex:x" "foo-2" + "ex:y" "bar-2" + "id" "ex:alice"}] + "f:t" 3} + {"f:assert" [{"ex:x" "foo-cat" + "ex:y" "bar-cat" + "id" "ex:alice"}] + "f:commit" {"https://www.w3.org/2018/credentials#issuer" + {"id" "did:fluree:TfCzWTrXqF16hvKGjcYiLxRoYJ1B8a6UMH6"} + "f:address" "fluree:memory://2716b3eaef91b763b32daebab8ba3733a8537de37a864f726c5021464b90277f" + "f:alias" "committest" + "f:branch" "main" + "f:context" "fluree:memory://b6dcf8968183239ecc7a664025f247de5b7859ac18cdeaace89aafc421eeddee" + "f:data" {"f:address" "fluree:memory://2b8125a4c996f8612ae62f16cc62b167b762c517b0c5aa7b16fa21dfe47e7b2a" + "f:assert" [{"ex:x" "foo-cat" + "ex:y" "bar-cat" + "id" "ex:alice"}] + "f:flakes" 102 + "f:previous" {"id" "fluree:db:sha256:bbtdwia2mle22abe2z7mmdrs4vufs77yubzxip3chhkmffvfk4npk"} + "f:retract" [{"ex:x" "foo-3" + "ex:y" "bar-3" + "id" "ex:alice"}] + "f:size" 9326 + "f:t" 5} + "f:message" "meow" + "f:previous" {"id" "fluree:commit:sha256:bhcunj52uwxshi6jws2ypsjiziyncfa5gal4xptfldj5utb54cpf"} + "f:time" 720000 + "f:v" 0 + "id" "fluree:commit:sha256:bq4r74wu4sru43z5f4byipe5zzkgyi2kksqxlmwuttdcwcgwpsrn"} + "f:retract" [{"ex:x" "foo-3" + "ex:y" "bar-3" + "id" "ex:alice"}] + "f:t" 5}] + @(fluree/history ledger {"history" "ex:alice" "commit-details" true "t" {"from" 3}}))) + (testing "multiple history results" - (let [history-with-commits @(fluree/history ledger {:history :ex/alice :commit-details true :t {:from 1 :to 5}})] + (let [history-with-commits @(fluree/history ledger {"history" "ex:alice" "commit-details" true "t" {"from" 1 "to" 5}})] (testing "all `t`s with changes to subject are returned" (is (= [1 2 3 5] - (mapv :f/t history-with-commits)))) + (mapv #(get % "f:t") history-with-commits)))) (testing "all expected commits are present and associated with the correct results" (is (= [[1 1] [2 2] [3 3] [5 5]] (map (fn [history-map] - (let [commit-t (get-in history-map [:f/commit :f/data :f/t])] - (vector (:f/t history-map) commit-t))) + (let [commit-t (get-in history-map ["f:commit" "f:data" "f:t"])] + (vector (get history-map "f:t") commit-t))) history-with-commits)))))))))) From dfbd0f82cf0e83596cbc816dd83fc5fd52a75db1 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Tue, 11 Apr 2023 14:16:30 -0600 Subject: [PATCH 21/50] Fix issues in history queries surfaced by tests --- src/fluree/db/api/query.cljc | 6 ++++-- src/fluree/db/query/history.cljc | 9 +++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/fluree/db/api/query.cljc b/src/fluree/db/api/query.cljc index 5e93f14d8..a211666cb 100644 --- a/src/fluree/db/api/query.cljc +++ b/src/fluree/db/api/query.cljc @@ -9,6 +9,7 @@ [fluree.db.query.range :as query-range] [fluree.db.util.core :as util] [fluree.db.util.async :as async-util :refer [json-ld "Build a subject map out a set of flakes with the same subject. - {:id :ex/foo :ex/x 1 :ex/y 2}" + {\"id\" \"ex:foo\" \"ex:x\" 1 \"ex:y\" 2}" [db cache context compact fuel error-ch s-flakes] (async/go (try* @@ -139,7 +139,8 @@ 0 s-flakes)] (-> ( s-flakes first flake/s (->> (dbproto/-iri db)) ! error-ch e))))) @@ -147,7 +148,7 @@ (defn t-flakes->json-ld "Build a collection of subject maps out of a set of flakes with the same t. - [{:id :ex/foo :ex/x 1 :ex/y 2}...] + [{\"id\" \"ex:foo\" \"ex:x\" 1 \"ex:y\" 2}...] " [db context compact cache fuel error-ch t-flakes] (let [s-flakes-ch (->> t-flakes @@ -168,7 +169,7 @@ "Build a collection of maps for each t that contains the t along with the asserted and retracted subject maps. - [{:id :ex/foo :f/assert [{},,,} :f/retract [{},,,]]}] + [{\"id\" \"ex:foo\" \"f:assert\" [{},,,} \"f:retract\" [{},,,]]}] " [db context error-ch flakes] (let [fuel (volatile! 0) From dd8aa8b8593c13e4c8471bbf85027a14ac4f2437 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Tue, 11 Apr 2023 15:44:39 -0600 Subject: [PATCH 22/50] Allow single result vec in query responses This is what I get back from e.g. {"select" ?n "where" [{"union" [[[?s "ex:name" ?n]] [[?s "ex:fname" ?n]]]}]} --- src/fluree/db/query/fql/syntax.cljc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/fluree/db/query/fql/syntax.cljc b/src/fluree/db/query/fql/syntax.cljc index 79f6cc371..c13ac8959 100644 --- a/src/fluree/db/query/fql/syntax.cljc +++ b/src/fluree/db/query/fql/syntax.cljc @@ -208,6 +208,7 @@ ::analytical-query-results [:orn [:select-one ::result-map] [:select [:or + ::result-vec [:sequential ::result-map] [:sequential ::result-vec]]]] ::multi-query-results [:map-of :string ::analytical-query-results]})) From 1a3ba7e8531b8d5dc470e68f720748ea7922e4c9 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Wed, 12 Apr 2023 12:40:44 -0600 Subject: [PATCH 23/50] Put most common query result type first in schema --- src/fluree/db/query/fql/syntax.cljc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fluree/db/query/fql/syntax.cljc b/src/fluree/db/query/fql/syntax.cljc index c13ac8959..284445624 100644 --- a/src/fluree/db/query/fql/syntax.cljc +++ b/src/fluree/db/query/fql/syntax.cljc @@ -206,11 +206,11 @@ ::result-map [:map-of :string ::result-val] ::result-vec [:sequential ::result-val] ::analytical-query-results [:orn - [:select-one ::result-map] [:select [:or ::result-vec [:sequential ::result-map] - [:sequential ::result-vec]]]] + [:sequential ::result-vec]]] + [:select-one ::result-map]] ::multi-query-results [:map-of :string ::analytical-query-results]})) (def valid-query? From 7c13475e120809237d858d3788482fe6496512c4 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Fri, 14 Apr 2023 08:38:57 -0600 Subject: [PATCH 24/50] Convert policy code to use string compact IRIs --- src/fluree/db/api/query.cljc | 8 +- src/fluree/db/db/json_ld.cljc | 17 +- src/fluree/db/json_ld/policy.cljc | 526 ++++++++++-------- src/fluree/db/json_ld/policy_validate.cljc | 80 +-- src/fluree/db/permissions_validate.cljc | 78 +-- src/fluree/db/policy/enforce_tx.cljc | 97 ++-- src/fluree/db/query/range.cljc | 14 +- src/fluree/db/query/subject_crawl/core.cljc | 2 +- .../db/query/subject_crawl/reparse.cljc | 4 +- .../db/query/subject_crawl/subject.cljc | 20 +- test/fluree/db/full_text_test.clj | 4 +- test/fluree/db/policy/basic_test.clj | 440 ++++++++------- test/fluree/db/policy/parsing_test.clj | 152 ++--- test/fluree/db/policy/subj_flakes_test.clj | 109 ++-- test/fluree/db/policy/transact_test.clj | 232 ++++---- 15 files changed, 952 insertions(+), 831 deletions(-) diff --git a/src/fluree/db/api/query.cljc b/src/fluree/db/api/query.cljc index a211666cb..105bd6ed6 100644 --- a/src/fluree/db/api/query.cljc +++ b/src/fluree/db/api/query.cljc @@ -108,7 +108,7 @@ (dissoc "meta")) start #?(:clj (System/nanoTime) :cljs (util/current-time-millis)) - result ( (sid iri:" iri) + (let [iri* (expand-iri db iri) + sid (some-> (sid expanded iri:" iri*) + (log/debug "iri->sid sid:" sid) + sid))) + (defn subid "Returns subject ID of ident as async promise channel. diff --git a/src/fluree/db/json_ld/policy.cljc b/src/fluree/db/json_ld/policy.cljc index dcbc882bc..038b1c57b 100644 --- a/src/fluree/db/json_ld/policy.cljc +++ b/src/fluree/db/json_ld/policy.cljc @@ -13,18 +13,18 @@ ;; that need to get enforced (e.g. as opposed to generically allowing an entire Class, members ;; will need to get evaluated for specific criteria) (def restriction-properties - #{:f/equals :f/contains}) + #{"f:equals" "f:contains"}) ;; These special IRIs (today only one) get replaced with an actual value in the context of the ;; request. ;; - :f/$identity - gets replaced with the subject ID of the identity (DID) that signed ;; the particular request (def special-meaning-properties - #{:f/$identity}) + #{"f:$identity"}) (defn restricted-allow-rule? - "Returns true if an allow rule contains restrictions (e.g. :f/equals or other restriction properties)." + "Returns true if an allow rule contains restrictions (e.g. f:equals or other restriction properties)." [allow-rule] (->> (keys allow-rule) (some restriction-properties))) @@ -33,25 +33,24 @@ [db] (go-try ;; TODO - once supported, use context to always return :f/allow and :f/property as vectors so we don't need to coerce downstream - (> (get policy :f/allow) + (fn [policy] + (log/debug "policies-for-roles* policy:" policy) + (let [class-policies (->> (get policy "f:allow") util/sequential - (filter #(roles (get-in % [:f/targetRole :_id]))) + (filter #(roles (get-in % ["f:targetRole" "_id"]))) not-empty) - prop-policies (when-let [all-prop-policies (get policy :f/property)] + prop-policies (when-let [all-prop-policies (get policy "f:property")] ;; each explicit property can have multiple :f/allow targeting different roles ;; We only want :f/allow that target provided roles, and only want properties ;; returned tht contain at least one relevant :f/allow @@ -59,31 +58,33 @@ util/sequential (filter (fn [prop-policy] - (let [roles-policies (->> (get prop-policy :f/allow) + (let [roles-policies (->> (get prop-policy "f:allow") util/sequential - (filter #(roles (get-in % [:f/targetRole :_id]))) + (filter #(roles (get-in % ["f:targetRole" "_id"]))) not-empty)] (when roles-policies - (assoc prop-policy :f/allow roles-policies))))) + (assoc prop-policy "f:allow" roles-policies))))) not-empty))] (when (or class-policies prop-policies) (cond-> policy - class-policies (assoc :f/allow class-policies) - prop-policies (assoc :f/property prop-policies))))) - all-policies)) + class-policies (assoc "f:allow" class-policies) + prop-policies (assoc "f:property" prop-policies))))) + all-policies)) (defn policies-for-roles "Returns all the rules for the provided roles as compiled functions" [db {:keys [roles]}] ;; TODO - no caching is being done here yet, need to implement for at least all-rules lookup (go-try - (let [all-rules (> (get rule rule-type) condense-property-path) - special-property (first-special-property property-path) - ;; convert path to property-ids so we don't need to lookup keywords with each fn call - property-ids (->> (if special-property - (rest property-path) - property-path) - (subids db) - > (get rule rule-type) condense-property-path) + _ (log/debug "property-path:" property-path) + special-property (first-special-property property-path) + _ (log/debug "property-path special-property:" special-property) + ;; convert path to property-ids so we don't need to lookup keywords with each fn call + property-ids (->> (if special-property + (rest property-path) + property-path) + (subids db) + ] - :f/targetRole ... - :f/equals ... } - :f/modify {:id ... - :f/fn [true ] - :f/targetRole ... - :f/equals ... }" + {f:view {id ... + f:fn [true ] + f:targetRole ... + f:equals ... } + f:modify {id ... + f:fn [true ] + f:targetRole ... + f:equals ... }" [db policy-key-seqs allow-rule] (go-try - (let [fn-tuple (> allow-rule :f/action util/sequential (map :id)) - allow-rule* (-> allow-rule - ;; remove :f/action as we end up keying the rule *per-action* in the final policy map, don't want confusion if this hangs around that it is used - (dissoc :f/action) - ;; associate our compiled function to the existing allow-rule map - (assoc :function fn-tuple))] - (for [policy-key-seq policy-key-seqs - action actions] - ;; for every policy-key-seqs (key sequence that can be used with (assoc-in m ...)) - ;; prepend key-sequence with the policy rule's action(s) - (let [ks* (into [action] policy-key-seq)] - ;; return two-tuple of [full-key-seq updated-allow-rule-map] - [ks* allow-rule*]))))) + (log/debug "compile-allow-rule allow-rule:" allow-rule) + (let [fn-tuple ( allow-rule (get "f:action") util/sequential + (->> (map #(get % "id")))) + allow-rule* (-> allow-rule + ;; remove f:action as we end up keying the rule *per-action* in the final policy map, don't want confusion if this hangs around that it is used + (dissoc "f:action") + ;; associate our compiled function to the existing allow-rule map + (assoc :function fn-tuple))] + (for [policy-key-seq policy-key-seqs + action actions] + ;; for every policy-key-seqs (key sequence that can be used with (assoc-in m ...)) + ;; prepend key-sequence with the policy rule's action(s) + (let [ks* (into [action] policy-key-seq) + ;; return two-tuple of [full-key-seq updated-allow-rule-map] + compiled [ks* allow-rule*]] + (log/debug "compile-allow-rule compiled:" compiled) + compiled))))) (defn compile-prop-policy [db default-key-seqs prop-policy] (go-try - (let [allow-rule (:f/allow prop-policy) - fn-tuple (if (sequential? allow-rule) - (do - (log/warn (str "Multiple role policies for same property is not currently allowed. Using only first " - "allow specification in policy: " prop-policy ".")) - ;; TODO - Multiple conditions existing for a single role/property is probably a valid use case but uncommon. - ;; TODO --- they should be treated as an -OR- condition and could be wrapped into a single fn. - ;; TODO --- the wrapping fn would have to look for async fns and use > allow-rule :f/action util/sequential (map :id)) - allow-rule* (-> allow-rule - ;; remove :f/action as we end up keying the rule *per-action* in the final policy map, don't want confusion if this hangs around that it is used - (dissoc :f/action) - ;; associate our compiled function to the existing allow-rule map - (assoc :function fn-tuple)) - prop-ks (map #(conj % prop-sid) default-key-seqs)] - (for [policy-key-seq prop-ks - action actions] - ;; for every policy-key-seqs (key sequence that can be used with (assoc-in m ...)) - ;; prepend key-sequence with the policy rule's action(s) - (let [ks* (into [action] policy-key-seq)] - ;; return two-tuple of [full-key-seq updated-allow-rule-map] - [ks* allow-rule*]))))) + (log/debug "compile-prop-policy prop-policy:" prop-policy) + (let [allow-rule (get prop-policy "f:allow") + fn-tuple (if (sequential? allow-rule) + (do + (log/warn (str "Multiple role policies for same property is not currently allowed. Using only first " + "allow specification in policy: " prop-policy ".")) + ;; TODO - Multiple conditions existing for a single role/property is probably a valid use case but uncommon. + ;; TODO --- they should be treated as an -OR- condition and could be wrapped into a single fn. + ;; TODO --- the wrapping fn would have to look for async fns and use allow-rule (get "f:action") util/sequential (->> (map #(get % "id")))) + allow-rule* (-> allow-rule + ;; remove f:action as we end up keying the rule *per-action* in the final policy map, don't want confusion if this hangs around that it is used + (dissoc "f:action") + ;; associate our compiled function to the existing allow-rule map + (assoc :function fn-tuple)) + prop-ks (map #(conj % prop-sid) default-key-seqs)] + (for [policy-key-seq prop-ks + action actions] + ;; for every policy-key-seqs (key sequence that can be used with (assoc-in m ...)) + ;; prepend key-sequence with the policy rule's action(s) + (let [ks* (into [action] policy-key-seq)] + ;; return two-tuple of [full-key-seq updated-allow-rule-map] + [ks* allow-rule*]))))) (defn unrestricted-actions "A policy will be a 'root' (all access) policy if these 3 conditions apply: - 1) :f/targetNode value of :f/allNodes - 2) top-level (default) :f/allow rule that has no restrictions - 3) no additional :f/property restrictions for the respective :f/action granted + 1) f/targetNode value of f/allNodes + 2) top-level (default) f/allow rule that has no restrictions + 3) no additional f/property restrictions for the respective f/action granted Where a root policy exists for a specific action, returns the key sequences for each of the - actions., e.g. [ [[:f/view :root?], true], [[:f/modify :root?], true] ] which will end up in + actions., e.g. [ [[f/view :root?], true], [[f/modify :root?], true] ] which will end up in the policy map like: - {:f/view {:root? true} - :f/modify {:root? true}} + {f/view {:root? true} + f/modify {:root? true}} - At this stage, these root policies exist at the top-level :f/allow rule, but if we - discover there is a more restrictive policy for a :f/property, they will be removed (due to + At this stage, these root policies exist at the top-level f/allow rule, but if we + discover there is a more restrictive policy for a f/property, they will be removed (due to condition #3 above not being met)." [node-policy] (let [root-actions (reduce @@ -289,39 +301,39 @@ (if (restricted-allow-rule? next-policy) ;; allow policy is restricted, therefore not 'root' actions - (into actions (map :id (:f/action next-policy))))) - #{} (:f/allow node-policy))] + (into actions (map #(get % "id") (get next-policy "f:action"))))) + #{} (get node-policy "f:allow"))] (not-empty root-actions))) (defn non-allNodes - "Removes :f/allNodes from list of nodes. + "Removes f/allNodes from list of nodes. Returns nil if no other nodes exist." [nodes] (not-empty - (remove #(= :f/allNodes %) nodes))) + (remove #(= "f:allNodes" %) nodes))) (defn remove-root-default-allow - "Removes :f/allow top-level policies that have already been granted - root access as they no longer need to be considered in evaulation." + "Removes f/allow top-level policies that have already been granted + root access as they no longer need to be considered in evaluation." [node-policy root-actions] - (let [default-allow (get node-policy :f/allow) + (let [default-allow (get node-policy "f:allow") non-root-default-allow (reduce (fn [acc allow-policy] - (let [actions (:f/action allow-policy) + (let [actions (get allow-policy "f:action") ;; root-actions is a set, remove any actions that are already root, which is likely is all of them - non-root-actions (remove #(root-actions (:id %)) actions)] + non-root-actions (remove #(root-actions (get % "id")) actions)] ;; if any actions are left, leave the policy with the remaining actions - else remove policy entirely (if (empty? non-root-actions) acc - (conj acc (assoc allow-policy :f/action non-root-actions))))) + (conj acc (assoc allow-policy "f:action" non-root-actions))))) [] default-allow)] ;; if there are no non-root default allow policies, remove :f/allow ;; else, replace with only non-root default allow policies (if (empty non-root-default-allow) - (dissoc node-policy :f/allow) - (assoc node-policy :f/allow non-root-default-allow)))) + (dissoc node-policy "f:allow") + (assoc node-policy "f:allow" non-root-default-allow)))) ;; TODO - exceptions in here won't be caught! (defn default-allow-restrictions @@ -330,16 +342,18 @@ land in the final policy map." [db base-key-seq default-allow] (go-try - (let [default-allow-keys (map #(conj % :default) base-key-seq)] - (->> default-allow ;; calling function assumed to force into sequential if wasn't already - (map #(compile-allow-rule db default-allow-keys %)) - async/merge - (async/reduce - (fn [acc result] - (if (instance? Throwable result) - (reduced result) - (into acc result))) []) - > default-allow ; calling function assumed to force into sequential if wasn't already + (map #(compile-allow-rule db default-allow-keys %)) + async/merge + (async/reduce + (fn [acc result] + (if (instance? Throwable result) + (reduced result) + (into acc result))) []) + > prop-policies - (map #(compile-prop-policy db base-key-seq %)) - async/merge - (async/reduce - (fn [acc result] - (if (instance? Throwable result) - (reduced result) - (into acc result))) []) - > prop-policies + (map #(compile-prop-policy db base-key-seq %)) + async/merge + (async/reduce + (fn [acc result] + (if (instance? Throwable result) + (reduced result) + (into acc result))) []) + (:f/allow non-root-policy) util/sequential)] - ( (:f/property policy) util/sequential)] - ( [] - root-policies (into root-policies) - default-restrictions (into default-restrictions) - property-restrictions (into property-restrictions))))) + (log/debug "compile-node-policy policy:" policy) + (log/debug "compile-node-policy nodes:" nodes) + (let [all-nodes? (some #(= "f:allNodes" %) nodes) + nodes* (when all-nodes? + (non-allNodes nodes) + nodes) + _ (log/debug "compile-node-policy nodes*:" nodes*) + root-actions (when all-nodes? + (unrestricted-actions policy)) + _ (log/debug "compile-node-policy root-actions:" root-actions) + non-root-policy (if root-actions ;; if there was a root policy defined, remove default allows that contain it + (remove-root-default-allow policy root-actions) + policy) + _ (log/debug "compile-node-policy non-root-policy:" non-root-policy) + node-sids (when nodes* ;; if this policy was only to define root, this will be empty - so no need to create async chans + ( non-root-policy + (get "f:allow") + util/sequential)] + ( policy + (get "f:property") + util/sequential)] + ( [] + root-policies (into root-policies) + default-restrictions (into default-restrictions) + property-restrictions (into property-restrictions))))) (defn compile-class-policy - "Compiles a class rule (where :f/targetClass is used)" + "Compiles a class rule (where f:targetClass is used)" [db policy classes] (go-try - (let [class-sids ( (:f/allow policy) util/sequential)] - ( (:f/property policy) util/sequential)] - ( policy + (get "f:allow") + util/sequential)] + ( policy + (get "f:property") + util/sequential)] + (> policy :f/targetClass util/sequential (mapv :id)) - nodes (some->> policy :f/targetNode util/sequential (mapv :id))] - (cond-> [] - classes (into ( policy (get "f:targetClass") util/sequential + (->> (mapv #(get % "id")))) + nodes (some-> policy (get "f:targetNode") util/sequential + (->> (mapv #(get % "id"))))] + (log/debug "compile-policy classes:" classes) + (log/debug "compile-policy nodes:" nodes) + (cond-> [] + classes (into (> identity - (dbproto/-subid db) - (> (> ( compiled-policies - root-access? (assoc-in [:f/view :root?] true))) - (catch* e - (if (= :db/invalid-query (:error (ex-data e))) - (throw (ex-info (str "There are no Fluree rules in the db, a policy-driven database cannot be retrieved. " - "If you have created rules, make sure they are of @type f:Rule.") - {:status 400 - :error :db/invalid-policy})) - (throw e)))))) + (try* + (let [_ (log/debug "policy-map role:" role) + ident-sid (some->> identity + (dbproto/-subid db) + (> (> role-policies + (compile-policies db) + compiled-policies + root-access? (assoc-in ["f:view" :root?] true))) + (catch* e + (if (= :db/invalid-query (:error (ex-data e))) + (throw (ex-info (str "There are no Fluree rules in the db, a policy-driven database cannot be retrieved. " + "If you have created rules, make sure they are of @type f:Rule.") + {:status 400 + :error :db/invalid-policy})) + (throw e)))))) (defn policy-opts [opts] - (-> (select-keys opts [:did :role :credential]) - not-empty)) + (-> opts (select-keys ["did" "role" "credential"]) not-empty)) (defn wrap-policy "Given a db object and a map containing the identity, wraps specified policy permissions" - [{:keys [policy] :as db} {:keys [did role credential]}] + [{:keys [policy] :as db} {:strs [did role credential]}] ;; TODO - not yet paying attention to verifiable credentials that are present (go-try - (cond - (or (:ident policy) (:roles policy)) (throw (ex-info (str "Policy already in place for this db. " - "Applying policy more than once, via multiple uses of `wrap-policy` and/or supplying an identity via `:opts`, is not supported.") - {:status 400 - :error :db/policy-exception})) - (not role) (throw (ex-info "Applying policy without a role is not yet supported." - {:status 400 - :error :db/policy-exception})) - :else (assoc db :policy - ( (count next-res) 1) - (log/warn (str "f:equals used for identity " ident " and path: " equals-rule - " however the query produces more than one result, the first one " - " is being used which can product unpredictable results. " - "Prefer f:contains when comparing with multiple results."))) - (recur r next-val)) - (do - (swap! cache assoc equals-rule last-result) - last-result)))))) + (log/debug "resolve-equals-rule policy:" policy) + (log/debug "resolve-equals-rule path-pids:" path-pids) + (let [{:keys [cache ident]} policy + db-root (dbproto/-rootdb db)] + (loop [[next-pid & r] path-pids + last-result ident] + (if next-pid + (let [next-res ( (count next-res) 1) + (log/warn (str "f:equals used for identity " ident " and path: " equals-rule + " however the query produces more than one result, the first one " + " is being used which can product unpredictable results. " + "Prefer f:contains when comparing with multiple results."))) + (recur r next-val)) + (do + (swap! cache assoc equals-rule last-result) + last-result)))))) (defn cache-store-value "Caches path lookup result into the policy map cache. Returns original value." @@ -68,7 +73,7 @@ (get @(get-in db [:policy :cache]) cache-key)) (defn generate-equals-fn - "Returns validating function for :f/equals rule. + "Returns validating function for f:equals rule. Validating functions take two arguments, the db and the flake to be validated. @@ -78,21 +83,30 @@ All policy functions are evaluated for a truthy or falsey result which determines if the provided flake can be operated on/viewed." [rule property-path] - (if (= :f/$identity (first property-path)) - ;; make certain first element of path is :f/$identity which following fn only considers. Will support other path constructs in the future - (let [path-no-identity (rest property-path) ;; remove :f/$identity - following logic will "substitute" the user's actual identity in its place + (log/debug "generate-equals-fn property-path:" property-path) + (if (= "f:$identity" (first property-path)) + ;; make certain first element of path is f:$identity which following fn only considers. Will support other path constructs in the future + (let [path-no-identity (rest property-path) ; remove f:$identity - following logic will "substitute" the user's actual identity in its place f (fn [db flake] (go-try + (log/debug "equals-fn flake:" flake) + (log/debug "equals-fn path-no-identity:" path-no-identity) ;; because same 'path' is likely used in many flake evaluations, keep a local cache of results so expensive lookup only happens once per query/transaction. - (let [path-val (or (cache-get-value db property-path) - (->> (async/> (async/> class-ids (keep #(get-in policy [action :class %])) (mapcat identity) @@ -100,19 +103,20 @@ be a two-tuple where the second position is the default policy map." [db flake default-allow-policies] (go-try - (loop [[[async? f] & r] (eduction - (map second) (map :function) - default-allow-policies)] - ;; return first truthy response, else false - (if f - (let [f-res (if async? - (> flake flake/p (get policies-by-pid))] - (let [allow? (loop [[[async? f] & r] p-policies] - ;; return first truthy response, else false - (if f - (let [res (if async? - (> flake flake/p (get policies-by-pid))] + (let [allow? (loop [[[async? f] & r] p-policies] + ;; return first truthy response, else false + (if f + (let [res (if async? + (! error-ch e))))) + (log/error e + "Error authorizing flake in ledger" + (select-keys db [:network :ledger-id :t])) + (>! error-ch e))))) (defn authorize-flakes "Authorize each flake in the supplied `flakes` collection asynchronously, @@ -190,7 +190,7 @@ flake-slices ; Note this bypasses all permissions in CLJS for now! :clj - (if (true? (get-in policy [:f/view :root?])) + (if (true? (get-in policy ["f:view" :root?])) flake-slices (let [auth-fn (fn [flakes ch] (-> (authorize-flakes db error-ch flakes) @@ -340,7 +340,9 @@ (let [s1* (if (or (number? s1) (nil? s1)) s1 (> (> (>val "flakes-xf flakes:") + filter-map (filter-subject vars filter-map) + permissioned? (filter-subject-flakes db) + permissioned? did-map "8ce4eca704d653dec594703c81a84c403c39f262e54ed014ed857438933a2e1c")) alice-did (:id (did/private->did-map "c0459840c334ca9f20c257bed971da88bd9b1b5d4fca69d4e3f4b8504f981c07")) db @(fluree/stage - (fluree/db ledger) - [{:id :ex/alice, - :type :ex/User, - :schema/name "Alice" - :schema/email "alice@flur.ee" - :schema/birthDate "2022-08-17" - :schema/ssn "111-11-1111" - :ex/location {:ex/state "NC" - :ex/country "USA"}} - {:id :ex/john, - :type :ex/User, - :schema/name "John" - :schema/email "john@flur.ee" - :schema/birthDate "2021-08-17" - :schema/ssn "888-88-8888"} - {:id :ex/widget, - :type :ex/Product, - :schema/name "Widget" - :schema/price 99.99 - :schema/priceCurrency "USD"} - ;; assign root-did to :ex/rootRole - {:id root-did - :f/role :ex/rootRole} - ;; assign alice-did to :ex/userRole and also link the did to :ex/alice via :ex/user - {:id alice-did - :ex/user :ex/alice - :f/role :ex/userRole}]) + (fluree/db ledger) + [{"id" "ex:alice", + "type" "ex:User", + "schema:name" "Alice" + "schema:email" "alice@flur.ee" + "schema:birthDate" "2022-08-17" + "schema:ssn" "111-11-1111" + "ex:location" {"ex:state" "NC" + "ex:country" "USA"}} + {"id" "ex:john", + "type" "ex:User", + "schema:name" "John" + "schema:email" "john@flur.ee" + "schema:birthDate" "2021-08-17" + "schema:ssn" "888-88-8888"} + {"id" "ex:widget", + "type" "ex:Product", + "schema:name" "Widget" + "schema:price" 99.99 + "schema:priceCurrency" "USD"} + ;; assign root-did to :ex/rootRole + {"id" root-did + "f:role" {"id" "ex:rootRole"}} + ;; assign alice-did to :ex/userRole and also link the did to :ex/alice via :ex/user + {"id" alice-did + "ex:user" {"id" "ex:alice"} + "f:role" {"id" "ex:userRole"}}]) db+policy @(fluree/stage - db - ;; add policy targeting :ex/rootRole that can view and modify everything - [{:id :ex/rootPolicy, - :type [:f/Policy], ;; must be of type :f/Policy, else it won't be treated as a policy - :f/targetNode :f/allNodes ;; :f/allNodes special keyword meaning every node (everything) - :f/allow [{:id :ex/rootAccessAllow - :f/targetRole :ex/rootRole ;; our name for global / root role - :f/action [:f/view :f/modify]}]} - ;; add a policy targeting :ex/userRole that can see all users, but only SSN if belonging to themselves - {:id :ex/UserPolicy, - :type [:f/Policy], - :f/targetClass :ex/User - :f/allow [{:id :ex/globalViewAllow - :f/targetRole :ex/userRole ;; our assigned name for standard user's role (given to Alice above) - :f/action [:f/view]}] - :f/property [{:f/path :schema/ssn - :f/allow [{:id :ex/ssnViewRule - :f/targetRole :ex/userRole - :f/action [:f/view] - :f/equals {:list [:f/$identity :ex/user]}}]}]}])] - (let [root-wrapped-db @(fluree/wrap-policy db+policy {:did root-did - :role :ex/rootRole}) - double-policy-query-result @(fluree/query root-wrapped-db {:select {'?s [:* {:ex/location [:*]}]} - :where [['?s :rdf/type :ex/User]] - :opts {:did root-did - :role :ex/rootRole}})] - (is (util/exception? double-policy-query-result) - "Should be an error to try to apply policy twice on one db.") + db + ;; add policy targeting "ex:rootRole" that can view and modify everything + [{"id" "ex:rootPolicy" + "type" ["f:Policy"] ;; must be of type "f:Policy", else it won't be treated as a policy + "f:targetNode" {"id" "f:allNodes"} ;; "f:allNodes" special keyword meaning every node (everything) + "f:allow" [{"id" "ex:rootAccessAllow" + "f:targetRole" {"id" "ex:rootRole"} ;; our name for global / root role + "f:action" [{"id" "f:view"} {"id" "f:modify"}]}]} + ;; add a policy targeting "ex:userRole" that can see all users, but only SSN if belonging to themselves + {"id" "ex:UserPolicy" + "type" ["f:Policy"] + "f:targetClass" {"id" "ex:User"} + "f:allow" [{"id" "ex:globalViewAllow" + "f:targetRole" {"id" "ex:userRole"} ;; our assigned name for standard user's role (given to Alice above) + "f:action" [{"id" "f:view"}]}] + "f:property" [{"f:path" {"id" "schema:ssn"} + "f:allow" [{"id" "ex:ssnViewRule" + "f:targetRole" {"id" "ex:userRole"} + "f:action" [{"id" "f:view"}] + "f:equals" {"list" [{"id" "f:$identity"} + {"id" "ex:user"}]}}]}]}])] + (let [root-wrapped-db @(fluree/wrap-policy + db+policy + {"did" root-did + "role" "ex:rootRole"}) + double-policy-query-result @(fluree/query + root-wrapped-db + {"select" {'?s ["*" {"ex:location" ["*"]}]} + "where" [['?s "rdf:type" "ex:User"]] + "opts" {"did" root-did + "role" "ex:rootRole"}})] + (is (util/exception? double-policy-query-result) + "Should be an error to try to apply policy twice on one db.") (is (str/includes? (ex-message double-policy-query-result) "Policy already in place"))) ;; root can see all user data - (is (= [{:id :ex/john, - :rdf/type [:ex/User], - :schema/name "John", - :schema/email "john@flur.ee", - :schema/birthDate "2021-08-17", - :schema/ssn "888-88-8888"} - {:id :ex/alice, - :rdf/type [:ex/User], - :schema/name "Alice", - :schema/email "alice@flur.ee", - :schema/birthDate "2022-08-17", - :schema/ssn "111-11-1111", - :ex/location {:id "_:f211106232532993", - :ex/state "NC", - :ex/country "USA"}}] - @(fluree/query db+policy {:select {'?s [:* {:ex/location [:*]}]} - :where [['?s :rdf/type :ex/User]] - :opts {:did root-did - :role :ex/rootRole}})) + (is (= [{"id" "ex:john", + "rdf:type" ["ex:User"], + "schema:name" "John", + "schema:email" "john@flur.ee", + "schema:birthDate" "2021-08-17", + "schema:ssn" "888-88-8888"} + {"id" "ex:alice", + "rdf:type" ["ex:User"], + "schema:name" "Alice", + "schema:email" "alice@flur.ee", + "schema:birthDate" "2022-08-17", + "schema:ssn" "111-11-1111", + "ex:location" {"id" "_:f211106232532993", + "ex:state" "NC", + "ex:country" "USA"}}] + @(fluree/query db+policy {"select" {'?s ["*" {"ex:location" ["*"]}]} + "where" [['?s "rdf:type" "ex:User"]] + "opts" {"did" root-did + "role" "ex:rootRole"}})) "Both user records + all attributes should show") - (is (= [{:id :ex/john, - :rdf/type [:ex/User], - :schema/name "John", - :schema/email "john@flur.ee", - :schema/birthDate "2021-08-17", - :schema/ssn "888-88-8888"} - {:id :ex/alice, - :rdf/type [:ex/User], - :schema/name "Alice", - :schema/email "alice@flur.ee", - :schema/birthDate "2022-08-17", - :schema/ssn "111-11-1111", - :ex/location {:id "_:f211106232532993", - :ex/state "NC", - :ex/country "USA"}}] - @(fluree/query db+policy {:select {'?s [:* {:ex/location [:*]}]} - :where [['?s :rdf/type :ex/User]] - :opts {:did root-did - :role :ex/rootRole}})) + (is (= [{"id" "ex:john", + "rdf:type" ["ex:User"], + "schema:name" "John", + "schema:email" "john@flur.ee", + "schema:birthDate" "2021-08-17", + "schema:ssn" "888-88-8888"} + {"id" "ex:alice", + "rdf:type" ["ex:User"], + "schema:name" "Alice", + "schema:email" "alice@flur.ee", + "schema:birthDate" "2022-08-17", + "schema:ssn" "111-11-1111", + "ex:location" {"id" "_:f211106232532993", + "ex:state" "NC", + "ex:country" "USA"}}] + @(fluree/query db+policy {"select" {'?s ["*" {"ex:location" ["*"]}]} + "where" [['?s "rdf:type" "ex:User"]] + "opts" {"did" root-did + "role" "ex:rootRole"}})) "Both user records + all attributes should show") ;; root role can see all product data, without identity - (is (= [{:id :ex/widget, - :rdf/type [:ex/Product], - :schema/name "Widget", - :schema/price 99.99, - :schema/priceCurrency "USD"}] - @(fluree/query db+policy {:select {'?s [:* {:ex/location [:*]}]} - :where [['?s :rdf/type :ex/Product]] - :opts {:role :ex/rootRole}})) + (is (= [{"id" "ex:widget", + "rdf:type" ["ex:Product"], + "schema:name" "Widget", + "schema:price" 99.99, + "schema:priceCurrency" "USD"}] + @(fluree/query db+policy {"select" {'?s ["*" {"ex:location" ["*"]}]} + "where" [['?s "rdf:type" "ex:Product"]] + "opts" {"role" "ex:rootRole"}})) "The product record should show with all attributes") - (is (= [{:id :ex/john, - :rdf/type [:ex/User], - :schema/name "John", - :schema/email "john@flur.ee", - :schema/birthDate "2021-08-17"} - {:id :ex/alice, - :rdf/type [:ex/User], - :schema/name "Alice", - :schema/email "alice@flur.ee", - :schema/birthDate "2022-08-17"}] - @(fluree/query db+policy {:select {'?s [:* {:ex/location [:*]}]} - :where [['?s :rdf/type :ex/User]] - :opts {:role :ex/userRole}})) + (is (= [{"id" "ex:john", + "rdf:type" ["ex:User"], + "schema:name" "John", + "schema:email" "john@flur.ee", + "schema:birthDate" "2021-08-17"} + {"id" "ex:alice", + "rdf:type" ["ex:User"], + "schema:name" "Alice", + "schema:email" "alice@flur.ee", + "schema:birthDate" "2022-08-17"}] + @(fluree/query db+policy {"select" {'?s ["*" {"ex:location" ["*"]}]} + "where" [['?s "rdf:type" "ex:User"]] + "opts" {"role" "ex:userRole"}})) "Both users should show, but no SSNs because no identity was provided") ;; Alice cannot see product data as it was not explicitly allowed (is (= [] - @(fluree/query db+policy {:select {'?s [:*]} - :where [['?s :rdf/type :ex/Product]] - :opts {:did alice-did - :role :ex/userRole}}))) + @(fluree/query db+policy {"select" {'?s ["*"]} + "where" [['?s "rdf:type" "ex:Product"]] + "opts" {"did" alice-did + "role" "ex:userRole"}}))) ;; Alice can see all users, but can only see SSN for herself, and can't see the nested location - (is (= [{:id :ex/john, - :rdf/type [:ex/User], - :schema/name "John", - :schema/email "john@flur.ee", - :schema/birthDate "2021-08-17"} - {:id :ex/alice, - :rdf/type [:ex/User], - :schema/name "Alice", - :schema/email "alice@flur.ee", - :schema/birthDate "2022-08-17", - :schema/ssn "111-11-1111"}] - @(fluree/query db+policy {:select {'?s [:* {:ex/location [:*]}]} - :where [['?s :rdf/type :ex/User]] - :opts {:did alice-did - :role :ex/userRole}})) + (is (= [{"id" "ex:john", + "rdf:type" ["ex:User"], + "schema:name" "John", + "schema:email" "john@flur.ee", + "schema:birthDate" "2021-08-17"} + {"id" "ex:alice", + "rdf:type" ["ex:User"], + "schema:name" "Alice", + "schema:email" "alice@flur.ee", + "schema:birthDate" "2022-08-17", + "schema:ssn" "111-11-1111"}] + @(fluree/query db+policy {"select" {'?s ["*" {"ex:location" ["*"]}]} + "where" [['?s "rdf:type" "ex:User"]] + "opts" {"did" alice-did + "role" "ex:userRole"}})) "Both users should show, but only SSN for Alice") ;; Alice can only see her allowed data in a non-graph-crawl query too (is (= [["John" nil] ["Alice" "111-11-1111"]] - @(fluree/query db+policy {:select '[?name ?ssn] - :where '[[?p :schema/name ?name] - {:optional [?p :schema/ssn ?ssn]}] - :opts {:did alice-did - :role :ex/userRole}})) + @(fluree/query db+policy {"select" '[?name ?ssn] + "where" '[[?p "schema:name" ?name] + {"optional" [?p "schema:ssn" ?ssn]}] + "opts" {"did" alice-did + "role" "ex:userRole"}})) "Both user names should show, but only SSN for Alice") (testing "multi-query" - (is (= {"john" [["john@flur.ee" nil]] + (is (= {"john" [["john@flur.ee" nil]] "alice" [["alice@flur.ee" "111-11-1111"]]} - @(fluree/multi-query db+policy {"john" '{:select [?email ?ssn] - :where [[?p :schema/name "John"] - [?p :schema/email ?email] - {:optional [?p :schema/ssn ?ssn]}]} - "alice" '{:select [?email ?ssn] - :where [[?p :schema/name "Alice"] - [?p :schema/email ?email] - {:optional [?p :schema/ssn ?ssn]}]} - :opts {:did alice-did - :role :ex/userRole}})) + @(fluree/multi-query + db+policy + {"john" '{"select" [?email ?ssn] + "where" [[?p "schema:name" "John"] + [?p "schema:email" ?email] + {"optional" [?p "schema:ssn" ?ssn]}]} + "alice" '{"select" [?email ?ssn] + "where" [[?p "schema:name" "Alice"] + [?p "schema:email" ?email] + {"optional" [?p "schema:ssn" ?ssn]}]} + "opts" {"did" alice-did + "role" "ex:userRole"}})) "Both emails should show, but only SSN for Alice") - (let [illegal-opts-multi-result @(fluree/multi-query db+policy {"john" {:select '[?ssn] - :where '[[?p :schema/name "John"] - {:optional [?p :schema/ssn ?ssn]}] - ;;supplying an identity here is not supported - :opts {:did root-did - :role :ex/rootRole}} - "alice" '{:select [?ssn] - :where [[?p :schema/name "Alice"] - {:optional [?p :schema/ssn ?ssn]}]}})] + (let [illegal-opts-multi-result @(fluree/multi-query + db+policy + {"john" {"select" '[?ssn] + "where" '[[?p "schema:name" "John"] + {"optional" [?p "schema:ssn" ?ssn]}] + ;; supplying an identity here is not supported + "opts" {"did" root-did + "role" "ex:rootRole"}} + "alice" '{"select" [?ssn] + "where" [[?p "schema:name" "Alice"] + {"optional" [?p "schema:ssn" ?ssn]}]}})] (is (util/exception? illegal-opts-multi-result)) (is (str/includes? (ex-message illegal-opts-multi-result) "Applying policy via `:opts` on individual queries")))) @@ -208,59 +219,60 @@ (testing "history query" (let [_ @(fluree/commit! ledger db+policy)] (is (= [] - @(fluree/history ledger {:history [:ex/john :schema/ssn] :t {:from 1} - :commit-details true - :opts {:did alice-did - :role :ex/userRole}})) + @(fluree/history ledger {"history" ["ex:john" "schema:ssn"] + "t" {"from" 1} + "commit-details" true + "opts" {"did" alice-did + "role" "ex:userRole"}})) "Alice should not be able to see any history for John's ssn") - (is (= [{:f/t 1, - :f/assert [{:schema/ssn "111-11-1111", :id :ex/alice}], - :f/retract []}] - @(fluree/history ledger {:history [:ex/alice :schema/ssn] :t {:from 1} - :opts {:did alice-did - :role :ex/userRole}})) + (is (= [{"f:t" 1, + "f:assert" [{"schema:ssn" "111-11-1111", "id" "ex:alice"}], + "f:retract" []}] + @(fluree/history ledger {"history" ["ex:alice" "schema:ssn"] "t" {"from" 1} + "opts" {"did" alice-did + "role" "ex:userRole"}})) "Alice should be able to see history for her own ssn.") - (let [[history-result] @(fluree/history ledger {:history [:ex/alice :schema/ssn] :t {:from 1} - :commit-details true - :opts {:did alice-did - :role :ex/userRole}}) - commit-details-asserts (get-in history-result [:f/commit :f/data :f/assert])] - (is (= [{:rdf/type [:ex/User], - :schema/name "John", - :schema/email "john@flur.ee", - :schema/birthDate "2021-08-17", - :id :ex/john} - {:rdf/type [:ex/User], - :schema/name "Alice", - :schema/email "alice@flur.ee", - :schema/birthDate "2022-08-17", - :schema/ssn "111-11-1111", - :ex/location {:id nil}, - :id :ex/alice}] + (let [[history-result] @(fluree/history ledger {"history" ["ex:alice" "schema:ssn"] "t" {"from" 1} + "commit-details" true + "opts" {"did" alice-did + "role" "ex:userRole"}}) + commit-details-asserts (get-in history-result ["f:commit" "f:data" "f:assert"])] + (is (= [{"rdf:type" ["ex:User"], + "schema:name" "John", + "schema:email" "john@flur.ee", + "schema:birthDate" "2021-08-17", + "id" "ex:john"} + {"rdf:type" ["ex:User"], + "schema:name" "Alice", + "schema:email" "alice@flur.ee", + "schema:birthDate" "2022-08-17", + "schema:ssn" "111-11-1111", + "ex:location" {"id" nil}, + "id" "ex:alice"}] commit-details-asserts) "Alice should be able to see her own ssn in commit details, but not John's.")) - (let [[history-result] @(fluree/history ledger {:history [:ex/alice :schema/ssn] :t {:from 1} - :commit-details true - :opts {:did root-did - :role :ex/rootRole}}) - commit-details-asserts (get-in history-result [:f/commit :f/data :f/assert])] + (let [[history-result] @(fluree/history ledger {"history" ["ex:alice" "schema:ssn"] "t" {"from" 1} + "commit-details" true + "opts" {"did" root-did + "role" "ex:rootRole"}}) + commit-details-asserts (get-in history-result ["f:commit" "f:data" "f:assert"])] (is (contains? (into #{} commit-details-asserts) - {:rdf/type [:ex/User], - :schema/name "John", - :schema/email "john@flur.ee", - :schema/birthDate "2021-08-17", - :schema/ssn "888-88-8888", - :id :ex/john}) + {"rdf:type" ["ex:User"], + "schema:name" "John", + "schema:email" "john@flur.ee", + "schema:birthDate" "2021-08-17", + "schema:ssn" "888-88-8888", + "id" "ex:john"}) "Root can see John's ssn in commit details.")) - (let [_ @(test-utils/transact ledger {:id :ex/john - :schema/name "Jack"})] - (is (= [{:f/t 1, - :f/assert [{:schema/name "John", :id :ex/john}], - :f/retract []} - {:f/t 2, - :f/assert [{:schema/name "Jack", :id :ex/john}], - :f/retract [{:schema/name "John", :id :ex/john}]}] - @(fluree/history ledger {:history [:ex/john :schema/name] :t {:from 1} - :opts {:did alice-did - :role :ex/userRole}})) + (let [_ @(test-utils/transact ledger {"id" "ex:john" + "schema:name" "Jack"})] + (is (= [{"f:t" 1, + "f:assert" [{"schema:name" "John", "id" "ex:john"}], + "f:retract" []} + {"f:t" 2, + "f:assert" [{"schema:name" "Jack", "id" "ex:john"}], + "f:retract" [{"schema:name" "John", "id" "ex:john"}]}] + @(fluree/history ledger {"history" ["ex:john" "schema:name"] "t" {"from" 1} + "opts" {"did" alice-did + "role" "ex:userRole"}})) "Alice should be able to see all history for John's name"))))))) diff --git a/test/fluree/db/policy/parsing_test.clj b/test/fluree/db/policy/parsing_test.clj index 21a306334..6c1f80e7b 100644 --- a/test/fluree/db/policy/parsing_test.clj +++ b/test/fluree/db/policy/parsing_test.clj @@ -1,11 +1,11 @@ (ns fluree.db.policy.parsing-test (:require - [clojure.test :refer :all] - [fluree.db.test-utils :as test-utils] - [fluree.db.json-ld.api :as fluree] - [fluree.db.did :as did] - [fluree.db.json-ld.policy :as policy] - [fluree.db.util.async :refer [did-map "8ce4eca704d653dec594703c81a84c403c39f262e54ed014ed857438933a2e1c")) alice-did (:id (did/private->did-map "c0459840c334ca9f20c257bed971da88bd9b1b5d4fca69d4e3f4b8504f981c07")) customer-did (:id (did/private->did-map "854358f6cb3a78ff81febe0786010d6e22839ea6bd52e03365a728d7b693b5a0")) db @(fluree/stage (fluree/db ledger) - [;; assign root-did to :ex/rootRole - {:id root-did - :f/role :ex/rootRole} - ;; assign alice-did to :ex/userRole and also link the did to :ex/alice via :ex/user - {:id alice-did - :ex/user :ex/alice - :f/role :ex/userRole} - {:id customer-did - :ex/user :ex/bob - :f/role :ex/customerRole} - {:id :ex/rootPolicy, - :type [:f/Policy], - :f/targetNode :f/allNodes ;; :f/allNodes special keyword meaning every node (everything) - :f/allow [{:id :ex/rootAccessAllow - :f/targetRole :ex/rootRole - :f/action [:f/view :f/modify]}]} - {:id :ex/UserPolicy, - :type [:f/Policy], - :f/targetClass :ex/User - :f/allow [{:id :ex/globalViewAllow - :f/targetRole :ex/userRole - :f/action [:f/view]} - {:f/targetRole :ex/userRole - :f/action [:f/modify] - ;; by default, user can modify their own user profile (following relationship from identity/DID -> :ex/user to User object - :f/equals {:list [:f/$identity :ex/user]}}] - :f/property [{:f/path :schema/ssn - :f/allow [{:id :ex/ssnViewRule - :f/targetRole :ex/userRole - :f/action [:f/view] - :f/equals {:list [:f/$identity :ex/user]}}]}]}])] + [;; assign root-did to "ex:rootRole" + {"id" root-did + "f:role" {"id" "ex:rootRole"}} + ;; assign alice-did to "ex:userRole" and also link the did to "ex:alice" via "ex:user" + {"id" alice-did + "ex:user" {"id" "ex:alice"} + "f:role" {"id" "ex:userRole"}} + {"id" customer-did + "ex:user" {"id" "ex:bob"} + "f:role" {"id" "ex:customerRole"}} + {"id" "ex:rootPolicy", + "type" ["f:Policy"], + "f:targetNode" {"id" "f:allNodes"} ; "f:allNodes" special keyword meaning every node (everything) + "f:allow" [{"id" "ex:rootAccessAllow" + "f:targetRole" {"id" "ex:rootRole"} + "f:action" [{"id" "f:view"} + {"id" "f:modify"}]}]} + {"id" "ex:UserPolicy", + "type" ["f:Policy"], + "f:targetClass" {"id" "ex:User"} + "f:allow" [{"id" "ex:globalViewAllow" + "f:targetRole" {"id" "ex:userRole"} + "f:action" [{"id" "f:view"}]} + {"f:targetRole" {"id" "ex:userRole"} + "f:action" [{"id" "f:modify"}] + ;; by default, user can modify their own user profile (following relationship from identity/DID -> "ex:user" to User object + "f:equals" {"list" [{"id" "f:$identity"} + {"id" "ex:user"}]}}] + "f:property" [{"f:path" {"id" "schema:ssn"} + "f:allow" [{"id" "ex:ssnViewRule" + "f:targetRole" {"id" "ex:userRole"} + "f:action" [{"id" "f:view"}] + "f:equals" {"list" [{"id" "f:$identity"} + {"id" "ex:user"}]}}]}]}])] (testing "Policy map for classes and props within classes is properly formed" - (let [policy-alice (-> @(fluree/promise-wrap (policy/policy-map db alice-did :ex/userRole nil)) + (let [policy-alice (-> @(fluree/promise-wrap (policy/policy-map db alice-did "ex:userRole" nil)) replace-policy-fns) - sid-User @(fluree/internal-id db :ex/User) - sid-ssn @(fluree/internal-id db :schema/ssn) + sid-User @(fluree/internal-id db "ex:User") + _ (log/debug "sid-User:" sid-User) + sid-ssn @(fluree/internal-id db "schema:ssn") sid-alice-did @(fluree/internal-id db alice-did) - sid-userRole @(fluree/internal-id db :ex/userRole)] - (is (= {:f/modify {:class {sid-User {:default {:f/equals [{:id :f/$identity} - {:id :ex/user}] - :f/targetRole {:_id sid-userRole} - :function [true - ::replaced-policy-function] - :id "_:f211106232533008"}}}} - :f/view {:class {sid-User {sid-ssn {:f/equals [{:id :f/$identity} - {:id :ex/user}] - :f/targetRole {:_id sid-userRole} - :function [true - ::replaced-policy-function] - :id :ex/ssnViewRule} - :default {:f/targetRole {:_id sid-userRole} - :function [false - ::replaced-policy-function] - :id :ex/globalViewAllow}}}} + sid-userRole @(fluree/internal-id db "ex:userRole")] + (is (= {"f:modify" {:class {sid-User {:default {"f:equals" [{"id" "f:$identity"} + {"id" "ex:user"}] + "f:targetRole" {"_id" sid-userRole} + :function [true + ::replaced-policy-function] + "id" "_:f211106232533008"}}}} + "f:view" {:class {sid-User {sid-ssn {"f:equals" [{"id" "f:$identity"} + {"id" "ex:user"}] + "f:targetRole" {"_id" sid-userRole} + :function [true + ::replaced-policy-function] + "id" "ex:ssnViewRule"} + :default {"f:targetRole" {"_id" sid-userRole} + :function [false + ::replaced-policy-function] + "id" "ex:globalViewAllow"}}}} :ident sid-alice-did :roles #{sid-userRole}} policy-alice) - "Policies for only :ex/userRole should return"))) + "Policies for only ex:userRole should return"))) (testing "Root policy contains {:root? true} for each applicable :f/action" - (let [policy-root (-> @(fluree/promise-wrap (policy/policy-map db root-did :ex/rootRole nil)) + (let [policy-root (-> @(fluree/promise-wrap (policy/policy-map db root-did "ex:rootRole" nil)) replace-policy-fns) sid-root-did @(fluree/internal-id db root-did) - sid-rootRole @(fluree/internal-id db :ex/rootRole)] - (is (= {:f/modify {:root? true} - :f/view {:root? true} - :ident sid-root-did - :roles #{sid-rootRole}} + sid-rootRole @(fluree/internal-id db "ex:rootRole")] + (is (= {"f:modify" {:root? true} + "f:view" {:root? true} + :ident sid-root-did + :roles #{sid-rootRole}} policy-root)))))) (testing "Testing query policy with strings" (let [conn (test-utils/create-conn) - ledger @(fluree/create conn "policy-parse/a" {:defaultContext ["" {"ex" "http://example.org/ns/"}] - :context-type :string}) + ledger @(fluree/create conn "policy-parse/a" {"defaults" + {"@context" ["" {"ex" "http://example.org/ns/"}]}}) root-did (:id (did/private->did-map "8ce4eca704d653dec594703c81a84c403c39f262e54ed014ed857438933a2e1c")) db @(fluree/stage (fluree/db ledger) @@ -137,12 +143,12 @@ "f:targetRole" {"id" "ex:rootRole"} "f:action" [{"id" "f:view"} {"id" "f:modify"}]}]}])] (testing "Root policy contains {:root? true} for each applicable :f/action" - (let [policy-root (-> @(fluree/promise-wrap (policy/policy-map db root-did :ex/rootRole nil)) + (let [policy-root (-> @(fluree/promise-wrap (policy/policy-map db root-did "ex:rootRole" nil)) replace-policy-fns) sid-root-did @(fluree/internal-id db root-did) - sid-rootRole @(fluree/internal-id db :ex/rootRole)] - (is (= {:f/modify {:root? true} - :f/view {:root? true} - :ident sid-root-did - :roles #{sid-rootRole}} + sid-rootRole @(fluree/internal-id db "ex:rootRole")] + (is (= {"f:modify" {:root? true} + "f:view" {:root? true} + :ident sid-root-did + :roles #{sid-rootRole}} policy-root))))))) diff --git a/test/fluree/db/policy/subj_flakes_test.clj b/test/fluree/db/policy/subj_flakes_test.clj index df69b6c43..3cd1a44ae 100644 --- a/test/fluree/db/policy/subj_flakes_test.clj +++ b/test/fluree/db/policy/subj_flakes_test.clj @@ -13,66 +13,69 @@ (deftest ^:integration subject-flakes-policy (testing "Policy enforcement for groups of flakes by subject." (let [conn (test-utils/create-conn) - ledger @(fluree/create conn "policy/a" {:defaultContext ["" {:ex "http://example.org/ns/"}]}) + ledger @(fluree/create conn "policy/a" {"defaults" + {"@context" ["" {"ex" "http://example.org/ns/"}]}}) root-did (:id (did/private->did-map "8ce4eca704d653dec594703c81a84c403c39f262e54ed014ed857438933a2e1c")) alice-did (:id (did/private->did-map "c0459840c334ca9f20c257bed971da88bd9b1b5d4fca69d4e3f4b8504f981c07")) db @(fluree/stage (fluree/db ledger) - [{:id :ex/alice, - :type :ex/User, - :schema/name "Alice" - :schema/email "alice@flur.ee" - :schema/birthDate "2022-08-17" - :schema/ssn "111-11-1111" - :ex/location {:ex/state "NC" - :ex/country "USA"}} - {:id :ex/john, - :type :ex/User, - :schema/name "John" - :schema/email "john@flur.ee" - :schema/birthDate "2021-08-17" - :schema/ssn "888-88-8888"} - {:id :ex/widget, - :type :ex/Product, - :schema/name "Widget" - :schema/price 99.99 - :schema/priceCurrency "USD"} - ;; assign root-did to :ex/rootRole - {:id root-did - :f/role :ex/rootRole} - ;; assign alice-did to :ex/userRole and also link the did to :ex/alice via :ex/user - {:id alice-did - :ex/user :ex/alice - :f/role :ex/userRole}]) + [{"id" "ex:alice", + "type" "ex:User", + "schema:name" "Alice" + "schema:email" "alice@flur.ee" + "schema:birthDate" "2022-08-17" + "schema:ssn" "111-11-1111" + "ex:location" {"ex:state" "NC" + "ex:country" "USA"}} + {"id" "ex:john", + "type" "ex:User", + "schema:name" "John" + "schema:email" "john@flur.ee" + "schema:birthDate" "2021-08-17" + "schema:ssn" "888-88-8888"} + {"id" "ex:widget", + "type" "ex:Product", + "schema:name" "Widget" + "schema:price" 99.99 + "schema:priceCurrency" "USD"} + ;; assign root-did to "ex:rootRole" + {"id" root-did + "f:role" {"id" "ex:rootRole"}} + ;; assign alice-did to "ex:userRole" and also link the did to "ex:alice" via "ex:user" + {"id" alice-did + "ex:user" {"id" "ex:alice"} + "f:role" {"id" "ex:userRole"}}]) db+policy @(fluree/stage - db - ;; add policy targeting :ex/rootRole that can view and modify everything - [{:id :ex/rootPolicy, - :type [:f/Policy], ;; must be of type :f/Policy, else it won't be treated as a policy - :f/targetNode :f/allNodes ;; :f/allNodes special keyword meaning every node (everything) - :f/allow [{:id :ex/rootAccessAllow - :f/targetRole :ex/rootRole ;; our name for global / root role - :f/action [:f/view :f/modify]}]} - ;; add a policy targeting :ex/userRole that can see all users, but only SSN if belonging to themselves - {:id :ex/UserPolicy, - :type [:f/Policy], - :f/targetClass :ex/User - :f/allow [{:id :ex/globalViewAllow - :f/targetRole :ex/userRole ;; our assigned name for standard user's role (given to Alice above) - :f/action [:f/view]}] - :f/property [{:f/path :schema/ssn - :f/allow [{:id :ex/ssnViewRule - :f/targetRole :ex/userRole - :f/action [:f/view] - :f/equals {:list [:f/$identity :ex/user]}}]}]}]) + db + ;; add policy targeting "ex:rootRole" that can view and modify everything + [{"id" "ex:rootPolicy", + "type" ["f:Policy"], ;; must be of type "f:Policy", else it won't be treated as a policy + "f:targetNode" {"id" "f:allNodes"} ;; "f:allNodes" special keyword meaning every node (everything) + "f:allow" [{"id" "ex:rootAccessAllow" + "f:targetRole" {"id" "ex:rootRole"} ;; our name for global / root role + "f:action" [{"id" "f:view"} + {"id" "f:modify"}]}]} + ;; add a policy targeting "ex:userRole" that can see all users, but only SSN if belonging to themselves + {"id" "ex:UserPolicy", + "type" ["f:Policy"], + "f:targetClass" {"id" "ex:User"} + "f:allow" [{"id" "ex:globalViewAllow" + "f:targetRole" {"id" "ex:userRole"} ;; our assigned name for standard user's role (given to Alice above) + "f:action" [{"id" "f:view"}]}] + "f:property" [{"f:path" {"id" "schema:ssn"} + "f:allow" [{"id" "ex:ssnViewRule" + "f:targetRole" {"id" "ex:userRole"} + "f:action" [{"id" "f:view"}] + "f:equals" {"list" [{"id" "f:$identity"} + {"id" "ex:user"}]}}]}]}]) ;; get a group of flakes that we know will have different permissions for different users. - john-flakes @(fluree/range db+policy :spot = [:ex/john]) - alice-flakes @(fluree/range db+policy :spot = [(fluree/expand-iri db+policy :ex/alice)]) - widget-flakes @(fluree/range db+policy :spot = [(fluree/expand-iri db+policy :ex/widget)]) + john-flakes @(fluree/range db+policy :spot = ["ex:john"]) + alice-flakes @(fluree/range db+policy :spot = [(fluree/expand-iri db+policy "ex:alice")]) + widget-flakes @(fluree/range db+policy :spot = [(fluree/expand-iri db+policy "ex:widget")]) - alice-db @(fluree/wrap-policy db+policy {:did alice-did - :role :ex/userRole}) + alice-db @(fluree/wrap-policy db+policy {"did" alice-did + "role" "ex:userRole"}) ;; john's flakes filtered using alice's policy-enforced db alice-db-john (->> john-flakes @@ -107,4 +110,4 @@ (is (= [] alice-db-widget) - "Alice wasn't given any permissions for class :ex/Product, so should see nothing for widget.")))) + "Alice wasn't given any permissions for class ex:Product, so should see nothing for widget.")))) diff --git a/test/fluree/db/policy/transact_test.clj b/test/fluree/db/policy/transact_test.clj index a8afa0ec4..afb292f7f 100644 --- a/test/fluree/db/policy/transact_test.clj +++ b/test/fluree/db/policy/transact_test.clj @@ -1,140 +1,146 @@ (ns fluree.db.policy.transact-test (:require - [clojure.test :refer :all] - [fluree.db.test-utils :as test-utils] - [fluree.db.json-ld.api :as fluree] - [fluree.db.did :as did] - [fluree.db.util.core :as util])) + [clojure.test :refer :all] + [fluree.db.test-utils :as test-utils] + [fluree.db.json-ld.api :as fluree] + [fluree.db.did :as did] + [fluree.db.util.core :as util])) (deftest ^:integration policy-enforcement (testing "Testing basic policy enforcement." (let [conn (test-utils/create-conn) - ledger @(fluree/create conn "policy/tx-a" {:defaultContext ["" {:ex "http://example.org/ns/"}]}) + ledger @(fluree/create conn "policy/tx-a" {"defaults" + {"@context" + ["" {"ex" "http://example.org/ns/"}]}}) root-did (:id (did/private->did-map "8ce4eca704d653dec594703c81a84c403c39f262e54ed014ed857438933a2e1c")) alice-did (:id (did/private->did-map "c0459840c334ca9f20c257bed971da88bd9b1b5d4fca69d4e3f4b8504f981c07")) db @(fluree/stage - (fluree/db ledger) - [{:id :ex/alice, - :type :ex/User, - :schema/name "Alice" - :schema/email "alice@flur.ee" - :schema/birthDate "2022-08-17" - :schema/ssn "111-11-1111" - :ex/location {:ex/state "NC" - :ex/country "USA"}} - {:id :ex/john, - :type :ex/User, - :schema/name "John" - :schema/email "john@flur.ee" - :schema/birthDate "2021-08-17" - :schema/ssn "888-88-8888"} - {:id :ex/widget, - :type :ex/Product, - :schema/name "Widget" - :schema/price 99.99 - :schema/priceCurrency "USD"} - ;; assign root-did to :ex/rootRole - {:id root-did - :f/role :ex/rootRole} - ;; assign alice-did to :ex/userRole and also link the did to :ex/alice via :ex/user - {:id alice-did - :ex/user :ex/alice - :f/role [:ex/userRole :ex/otherRole]}]) + (fluree/db ledger) + [{"id" "ex:alice", + "type" "ex:User", + "schema:name" "Alice" + "schema:email" "alice@flur.ee" + "schema:birthDate" "2022-08-17" + "schema:ssn" "111-11-1111" + "ex:location" {"ex:state" "NC" + "ex:country" "USA"}} + {"id" "ex:john", + "type" "ex:User", + "schema:name" "John" + "schema:email" "john@flur.ee" + "schema:birthDate" "2021-08-17" + "schema:ssn" "888-88-8888"} + {"id" "ex:widget", + "type" "ex:Product", + "schema:name" "Widget" + "schema:price" 99.99 + "schema:priceCurrency" "USD"} + ;; assign root-did to "ex:rootRole" + {"id" root-did + "f:role" {"id" "ex:rootRole"}} + ;; assign alice-did to "ex:userRole" and also link the did to "ex:alice" via "ex:user" + {"id" alice-did + "ex:user" {"id" "ex:alice"} + "f:role" [{"id" "ex:userRole"} {"id" "ex:otherRole"}]}]) db+policy @(fluree/stage - db - ;; add policy targeting :ex/rootRole that can view and modify everything - [{:id :ex/rootPolicy, - :type [:f/Policy], ;; must be of type :f/Policy, else it won't be treated as a policy - :f/targetNode :f/allNodes ;; :f/allNodes special keyword meaning every node (everything) - :f/allow [{:id :ex/rootAccessAllow - :f/targetRole :ex/rootRole ;; our name for global / root role - :f/action [:f/view :f/modify]}]} - ;; add a policy targeting :ex/userRole that can see all users, but only SSN if belonging to themselves - {:id :ex/UserPolicy, - :type [:f/Policy], - :f/targetClass :ex/User - :f/allow [{:id :ex/globalViewAllow - :f/targetRole :ex/userRole ;; our assigned name for standard user's role (given to Alice above) - :f/action [:f/view]}] - :f/property [{:f/path :schema/ssn - :f/allow [{:id :ex/ssnViewRule - :f/targetRole :ex/userRole - :f/action [:f/view] - :f/equals {:list [:f/$identity :ex/user]}}]} - {:f/path :schema/email - :f/allow [{:id :ex/emailChangeRule - :f/targetRole :ex/userRole - :f/action [:f/view :f/modify] - :f/equals {:list [:f/$identity :ex/user]}}]}]} - ;; add a :ex/Product policy allows view & modify for only :schema/name - {:id :ex/ProductPolicy, - :type [:f/Policy], - :f/targetClass :ex/Product - :f/property [{:f/path :rdf/type - :f/allow [{:f/targetRole :ex/userRole - :f/action [:f/view]}]} - {:f/path :schema/name - :f/allow [{:f/targetRole :ex/userRole - :f/action [:f/view :f/modify]}]}]}])] + db + ;; add policy targeting "ex:rootRole" that can view and modify everything + [{"id" "ex:rootPolicy", + "type" ["f:Policy"], ;; must be of type "f:Policy", else it won't be treated as a policy + "f:targetNode" {"id" "f:allNodes"} ;; "f:allNodes" special keyword meaning every node (everything) + "f:allow" [{"id" "ex:rootAccessAllow" + "f:targetRole" {"id" "ex:rootRole"} ;; our name for global / root role + "f:action" [{"id" "f:view"} {"id" "f:modify"}]}]} + ;; add a policy targeting "ex:userRole" that can see all users, but only SSN if belonging to themselves + {"id" "ex:UserPolicy", + "type" ["f:Policy"], + "f:targetClass" {"id" "ex:User"} + "f:allow" [{"id" "ex:globalViewAllow" + "f:targetRole" {"id" "ex:userRole"} ;; our assigned name for standard user's role (given to Alice above) + "f:action" [{"id" "f:view"}]}] + "f:property" [{"f:path" {"id" "schema:ssn"} + "f:allow" [{"id" "ex:ssnViewRule" + "f:targetRole" {"id" "ex:userRole"} + "f:action" [{"id" "f:view"}] + "f:equals" {"list" [{"id" "f:$identity"} + {"id" "ex:user"}]}}]} + {"f:path" {"id" "schema:email"} + "f:allow" [{"id" "ex:emailChangeRule" + "f:targetRole" {"id" "ex:userRole"} + "f:action" [{"id" "f:view"} {"id" "f:modify"}] + "f:equals" {"list" [{"id" "f:$identity"} + {"id" "ex:user"}]}}]}]} + ;; add a "ex:Product" policy allows view & modify for only "schema:name" + {"id" "ex:ProductPolicy", + "type" ["f:Policy"], + "f:targetClass" {"id" "ex:Product"} + "f:property" [{"f:path" {"id" "rdf:type"} + "f:allow" [{"f:targetRole" {"id" "ex:userRole"} + "f:action" [{"id" "f:view"}]}]} + {"f:path" {"id" "schema:name"} + "f:allow" [{"f:targetRole" {"id" "ex:userRole"} + "f:action" [{"id" "f:view"} {"id" "f:modify"}]}]}]}])] (testing "Policy allowed modification" (testing "using role + id" - (let [update-name @(fluree/stage db+policy {:id :ex/alice - :schema/email "alice@foo.bar"} - {:did alice-did - :role :ex/userRole})] + (let [update-email @(fluree/stage db+policy + {"id" "ex:alice" + "schema:email" "alice@foo.bar"} + {"did" alice-did + "role" "ex:userRole"})] - (is (= [{:id :ex/alice, - :rdf/type [:ex/User], - :schema/name "Alice", - :schema/email "alice@foo.bar", - :schema/birthDate "2022-08-17", - :schema/ssn "111-11-1111", - :ex/location {:id nil}}] - @(fluree/query update-name - {:select {'?s [:*]} - :where [['?s :schema/name "Alice"]]})) + (is (= [{"id" "ex:alice", + "rdf:type" ["ex:User"], + "schema:name" "Alice", + "schema:email" "alice@foo.bar", + "schema:birthDate" "2022-08-17", + "schema:ssn" "111-11-1111", + "ex:location" {"id" nil}}] + @(fluree/query update-email + {"select" {'?s ["*"]} + "where" [['?s "schema:name" "Alice"]]})) "Alice should be allowed to update her own name."))) + (testing "using role only" - (let [update-price @(fluree/stage db+policy {:id :ex/widget - :schema/price 105.99} - {:role :ex/rootRole})] + (let [update-price @(fluree/stage db+policy {"id" "ex:widget" + "schema:price" 105.99} + {"role" "ex:rootRole"})] - (is (= [{:id :ex/widget, - :rdf/type [:ex/Product], - :schema/name "Widget", - :schema/price 105.99, - :schema/priceCurrency "USD"}] + (is (= [{"id" "ex:widget", + "rdf:type" ["ex:Product"], + "schema:name" "Widget", + "schema:price" 105.99, + "schema:priceCurrency" "USD"}] @(fluree/query update-price - {:select {'?s [:*]} - :where [['?s :rdf/type :ex/Product]]})) - "Updated :schema/price should have been allowed, and entire product is visible in query.")) - (let [update-name @(fluree/stage db+policy {:id :ex/widget - :schema/name "Widget2"} - {:role :ex/userRole})] + {"select" {'?s ["*"]} + "where" [['?s "rdf:type" "ex:Product"]]})) + "Updated schema:price should have been allowed, and entire product is visible in query.")) + (let [update-name @(fluree/stage db+policy {"id" "ex:widget" + "schema:name" "Widget2"} + {"role" "ex:userRole"})] - (is (= [{:rdf/type [:ex/Product] - :schema/name "Widget2"}] + (is (= [{"rdf:type" ["ex:Product"] + "schema:name" "Widget2"}] @(fluree/query update-name - {:select {'?s [:*]} - :where [['?s :rdf/type :ex/Product]]})) - "Updated :schema/name should have been allowed, and only name is visible in query.")))) + {"select" {'?s ["*"]} + "where" [['?s "rdf:type" "ex:Product"]]})) + "Updated schema:name should have been allowed, and only name is visible in query.")))) (testing "Policy doesn't allow a modification" - (let [update-price @(fluree/stage db+policy {:id :ex/widget - :schema/price 42.99} - {:did root-did - :role :ex/userRole})] + (let [update-price @(fluree/stage db+policy {"id" "ex:widget" + "schema:price" 42.99} + {"did" root-did + "role" "ex:userRole"})] (is (util/exception? update-price) - "Attempted update should have thrown an exception, `:ex/userRole` cannot modify product prices regardless of identity") + "Attempted update should have thrown an exception, ex:userRole cannot modify product prices regardless of identity") (is (= :db/policy-exception (:error (ex-data update-price))) "Exception should be of type :db/policy-exception")) - (let [update-email @(fluree/stage db+policy {:id :ex/john - :schema/email "john@foo.bar"} - {:role :ex/user})] + (let [update-email @(fluree/stage db+policy {"id" "ex:john" + "schema:email" "john@foo.bar"} + {"role" "ex:user"})] (is (util/exception? update-email) "attempted update should have thrown an exception, no identity was provided") @@ -142,10 +148,10 @@ (is (= :db/policy-exception (:error (ex-data update-email))) "exception should be of type :db/policy-exception")) - (let [update-name-other-role @(fluree/stage db+policy {:id :ex/widget - :schema/name "Widget2"} - {:did alice-did - :role :ex/otherRole})] + (let [update-name-other-role @(fluree/stage db+policy {"id" "ex:widget" + "schema:name" "Widget2"} + {"did" alice-did + "role" "ex:otherRole"})] (is (util/exception? update-name-other-role) "Attempted update should have thrown an exception, this role cannot modify product names") From 697e8c588ddc23983f527fda0dc87a1fbe5b3d26 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Mon, 17 Apr 2023 10:54:39 -0600 Subject: [PATCH 25/50] Stringify delete-where transactions --- src/fluree/db/json_ld/reify.cljc | 4 ++-- src/fluree/db/json_ld/transact.cljc | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/fluree/db/json_ld/reify.cljc b/src/fluree/db/json_ld/reify.cljc index 328f43f82..97d4bf3e3 100644 --- a/src/fluree/db/json_ld/reify.cljc +++ b/src/fluree/db/json_ld/reify.cljc @@ -289,13 +289,13 @@ (defn db-assert [db-data] - (let [commit-assert (get-in db-data [const/iri-assert])] + (let [commit-assert (get db-data const/iri-assert)] ;; TODO - any basic validation required commit-assert)) (defn db-retract [db-data] - (let [commit-retract (get-in db-data [const/iri-retract])] + (let [commit-retract (get db-data const/iri-retract)] ;; TODO - any basic validation required commit-retract)) diff --git a/src/fluree/db/json_ld/transact.cljc b/src/fluree/db/json_ld/transact.cljc index 059317009..88552e1f0 100644 --- a/src/fluree/db/json_ld/transact.cljc +++ b/src/fluree/db/json_ld/transact.cljc @@ -383,13 +383,13 @@ "Executes a delete statement" [db max-fuel json-ld {:keys [t] :as _tx-state}] (go-try - (let [{:keys [delete] :as parsed-query} + (let [{:strs [delete] :as parsed-query} (-> json-ld syntax/validate-query (q-parse/parse-delete db)) [s p o] delete - parsed-query (assoc parsed-query :delete [s p o]) + parsed-query (assoc parsed-query "delete" [s p o]) error-ch (async/chan) flake-ch (async/chan) where-ch (where/search db parsed-query error-ch)] @@ -446,8 +446,8 @@ (tx-state db* (assoc opts :issuer issuer)) - flakes (if (and (contains? tx :delete) - (contains? tx :where)) + flakes (if (and (contains? tx "delete") + (contains? tx "where")) (final-db tx-state flakes))))) From 3c8a14254253253b73a5e5bdb599f8df4c72a73e Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Mon, 17 Apr 2023 10:55:06 -0600 Subject: [PATCH 26/50] Use :list for jld.transact/list-value? --- src/fluree/db/json_ld/transact.cljc | 149 ++++++++++++++-------------- 1 file changed, 76 insertions(+), 73 deletions(-) diff --git a/src/fluree/db/json_ld/transact.cljc b/src/fluree/db/json_ld/transact.cljc index 88552e1f0..c02337728 100644 --- a/src/fluree/db/json_ld/transact.cljc +++ b/src/fluree/db/json_ld/transact.cljc @@ -126,7 +126,7 @@ "returns true if json-ld value is a list object." [v] (and (map? v) - (contains? v "list"))) + (contains? v :list))) (defn get-subject-types "Returns a set of all :rdf/type Class subject ids for the provided subject. @@ -180,66 +180,67 @@ {:keys [t next-pid next-sid iris db-before subj-mods] :as tx-state} referring-pid] (go-try - (let [existing-sid (when id - ( [] - new-subj? (conj (flake/create sid const/$xsd:anyURI id* const/$xsd:string t true nil)) - new-type-sids (into (map #(flake/create sid const/$rdf:type % const/$xsd:anyURI t true nil) new-type-sids)))] - ;; save SHACL, class data into atom for later validation - checks that same @id not being updated in multiple spots - (register-node subj-mods node sid {:iri-only? (iri-only? node) - :shacl shacl-map - :new? new-subj? - :classes classes}) - (loop [[[k v] & r] (dissoc node :id :idx :type) - property-flakes type-flakes ;; only used if generating new Class and Property flakes - subj-flakes base-flakes] - (if k - (let [list? (list-value? v) - retract? (nil? v) - v* (if list? - (let [list-vals (get v "list")] - (when-not (sequential? list-vals) - (throw (ex-info (str "List values have to be vectors, provided: " v) - {:status 400 :error :db/invalid-transaction}))) - list-vals) - (util/sequential v)) - ref? (not (:value (first v*))) ;; either a ref or a value - existing-pid (> ( flakes* - property-flakes (into property-flakes)))))] - (recur r property-flakes* flakes*)) - ;; return two-tuple of node's final sid (needed to link nodes together) and the resulting flakes - [sid (into subj-flakes property-flakes)]))))) + (log/debug "json-ld-node->flakes node:" node) + (let [existing-sid (when id + ( [] + new-subj? (conj (flake/create sid const/$xsd:anyURI id* const/$xsd:string t true nil)) + new-type-sids (into (map #(flake/create sid const/$rdf:type % const/$xsd:anyURI t true nil) new-type-sids)))] + ;; save SHACL, class data into atom for later validation - checks that same @id not being updated in multiple spots + (register-node subj-mods node sid {:iri-only? (iri-only? node) + :shacl shacl-map + :new? new-subj? + :classes classes}) + (loop [[[k v] & r] (dissoc node :id :idx :type) + property-flakes type-flakes ;; only used if generating new Class and Property flakes + subj-flakes base-flakes] + (if k + (let [list? (list-value? v) + retract? (nil? v) + v* (if list? + (let [list-vals (get v :list)] + (when-not (sequential? list-vals) + (throw (ex-info (str "List values have to be vectors, provided: " v) + {:status 400 :error :db/invalid-transaction}))) + list-vals) + (util/sequential v)) + ref? (not (:value (first v*))) ;; either a ref or a value + existing-pid (> ( flakes* + property-flakes (into property-flakes)))))] + (recur r property-flakes* flakes*)) + ;; return two-tuple of node's final sid (needed to link nodes together) and the resulting flakes + [sid (into subj-flakes property-flakes)]))))) (defn ->tx-state [db {:keys [bootstrap? issuer] :as _opts}] @@ -327,18 +328,20 @@ (defn stage-flakes [flakeset tx-state nodes] (go-try - (loop [[node & r] nodes - flakes* flakeset] - (if node - (if (empty? (dissoc node :idx :id)) - (throw (ex-info (str "Invalid transaction, transaction node contains no properties" - (some->> (:id node) - (str " for @id: ")) - ".") - {:status 400 :error :db/invalid-transaction})) - (let [[_node-sid node-flakes] (flakes node tx-state nil))] - (recur r (into flakes* node-flakes)))) - flakes*)))) + (log/debug "stage-flakes flakeset:" flakeset) + (log/debug "stage-flakes nodes:" nodes) + (loop [[node & r] nodes + flakes* flakeset] + (if node + (if (empty? (dissoc node :idx :id)) + (throw (ex-info (str "Invalid transaction, transaction node contains no properties" + (some->> (:id node) + (str " for @id: ")) + ".") + {:status 400 :error :db/invalid-transaction})) + (let [[_node-sid node-flakes] (flakes node tx-state nil))] + (recur r (into flakes* node-flakes)))) + flakes*)))) (defn validate-rules [{:keys [db-after add] :as staged-map} {:keys [subj-mods] :as _tx-state}] From 01966d56dbc49a31f7b45fca25a784cf30973ac6 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Mon, 17 Apr 2023 10:56:42 -0600 Subject: [PATCH 27/50] Use "@list" instead of "list" in tests --- test/fluree/db/json_ld/api_test.cljc | 33 +++++++++++----------- test/fluree/db/policy/basic_test.clj | 4 +-- test/fluree/db/policy/parsing_test.clj | 8 +++--- test/fluree/db/policy/subj_flakes_test.clj | 4 +-- test/fluree/db/policy/transact_test.clj | 8 +++--- 5 files changed, 29 insertions(+), 28 deletions(-) diff --git a/test/fluree/db/json_ld/api_test.cljc b/test/fluree/db/json_ld/api_test.cljc index ca85fc110..844dd60b9 100644 --- a/test/fluree/db/json_ld/api_test.cljc +++ b/test/fluree/db/json_ld/api_test.cljc @@ -247,11 +247,11 @@ (fluree/db ledger) [{"id" "ex:alice", "type" "ex:User", - "ex:friends" {"list" [{"id" "ex:john"} - {"id" "ex:cam"}]}} + "ex:friends" {"@list" [{"id" "ex:john"} + {"id" "ex:cam"}]}} {"id" "ex:cam", "type" "ex:User" - "ex:numList" {"list" [7 8 9 10]}} + "ex:numList" {"@list" [7 8 9 10]}} {"id" "ex:john", "type" "ex:User"}]) db @(fluree/commit! ledger db) @@ -312,8 +312,8 @@ [{"id" "ex:ssnViewRule" "f:targetRole" {"id" "ex:userRole"} "f:action" [{"id" "f:view"}] - "f:equals" {"list" [{"id" "f:$identity"} - {"id" "ex:user"}]}}]}]}]) + "f:equals" {"@list" [{"id" "f:$identity"} + {"id" "ex:user"}]}}]}]}]) db+policy @(fluree/commit! ledger db+policy) loaded (test-utils/retry-load conn ledger-alias 100) loaded-db (fluree/db loaded)] @@ -334,12 +334,13 @@ "f:equals" [{"id" "f:$identity"} {"id" "ex:user"}]}, "f:path" {"id" "schema:ssn"}}, "f:targetClass" {"id" "ex:User"}}] - @(fluree/query loaded-db - '{"select" {?s ["*" - {"rdf:type" ["_id"]} - {"f:allow" ["*" {"f:targetRole" ["_id"]}]} - {"f:property" ["*" {"f:allow" ["*" {"f:targetRole" ["_id"]}]}]}]} - "where" [[?s "rdf:type" "f:Policy"]]})))))))))) + @(fluree/query + loaded-db + '{"select" {?s ["*" + {"rdf:type" ["_id"]} + {"f:allow" ["*" {"f:targetRole" ["_id"]}]} + {"f:property" ["*" {"f:allow" ["*" {"f:targetRole" ["_id"]}]}]}]} + "where" [[?s "rdf:type" "f:Policy"]]})))))))))) #?(:clj (deftest load-from-memory-test @@ -512,11 +513,11 @@ (fluree/db ledger) [{"id" "ex:alice", "type" "ex:User", - "ex:friends" {"list" [{"id" "ex:john"} - {"id" "ex:cam"}]}} + "ex:friends" {"@list" [{"id" "ex:john"} + {"id" "ex:cam"}]}} {"id" "ex:cam", "type" "ex:User" - "ex:numList" {"list" [7 8 9 10]}} + "ex:numList" {"@list" [7 8 9 10]}} {"id" "ex:john", "type" "ex:User"}]) db @(fluree/commit! ledger db) @@ -569,8 +570,8 @@ "f:allow" [{"id" "ex:ssnViewRule" "f:targetRole" {"id" "ex:userRole"} "f:action" [{"id" "f:view"}] - "f:equals" {"list" [{"id" "f:$identity"} - {"id" "ex:user"}]}}]}]}]) + "f:equals" {"@list" [{"id" "f:$identity"} + {"id" "ex:user"}]}}]}]}]) db+policy @(fluree/commit! ledger db+policy) loaded (test-utils/retry-load conn ledger-alias 100) loaded-db (fluree/db loaded)] diff --git a/test/fluree/db/policy/basic_test.clj b/test/fluree/db/policy/basic_test.clj index 3f8d12a23..3e0be898c 100644 --- a/test/fluree/db/policy/basic_test.clj +++ b/test/fluree/db/policy/basic_test.clj @@ -65,8 +65,8 @@ "f:allow" [{"id" "ex:ssnViewRule" "f:targetRole" {"id" "ex:userRole"} "f:action" [{"id" "f:view"}] - "f:equals" {"list" [{"id" "f:$identity"} - {"id" "ex:user"}]}}]}]}])] + "f:equals" {"@list" [{"id" "f:$identity"} + {"id" "ex:user"}]}}]}]}])] (let [root-wrapped-db @(fluree/wrap-policy db+policy {"did" root-did diff --git a/test/fluree/db/policy/parsing_test.clj b/test/fluree/db/policy/parsing_test.clj index 6c1f80e7b..b4c21da6a 100644 --- a/test/fluree/db/policy/parsing_test.clj +++ b/test/fluree/db/policy/parsing_test.clj @@ -80,14 +80,14 @@ {"f:targetRole" {"id" "ex:userRole"} "f:action" [{"id" "f:modify"}] ;; by default, user can modify their own user profile (following relationship from identity/DID -> "ex:user" to User object - "f:equals" {"list" [{"id" "f:$identity"} - {"id" "ex:user"}]}}] + "f:equals" {"@list" [{"id" "f:$identity"} + {"id" "ex:user"}]}}] "f:property" [{"f:path" {"id" "schema:ssn"} "f:allow" [{"id" "ex:ssnViewRule" "f:targetRole" {"id" "ex:userRole"} "f:action" [{"id" "f:view"}] - "f:equals" {"list" [{"id" "f:$identity"} - {"id" "ex:user"}]}}]}]}])] + "f:equals" {"@list" [{"id" "f:$identity"} + {"id" "ex:user"}]}}]}]}])] (testing "Policy map for classes and props within classes is properly formed" (let [policy-alice (-> @(fluree/promise-wrap (policy/policy-map db alice-did "ex:userRole" nil)) diff --git a/test/fluree/db/policy/subj_flakes_test.clj b/test/fluree/db/policy/subj_flakes_test.clj index 3cd1a44ae..606ba81b1 100644 --- a/test/fluree/db/policy/subj_flakes_test.clj +++ b/test/fluree/db/policy/subj_flakes_test.clj @@ -67,8 +67,8 @@ "f:allow" [{"id" "ex:ssnViewRule" "f:targetRole" {"id" "ex:userRole"} "f:action" [{"id" "f:view"}] - "f:equals" {"list" [{"id" "f:$identity"} - {"id" "ex:user"}]}}]}]}]) + "f:equals" {"@list" [{"id" "f:$identity"} + {"id" "ex:user"}]}}]}]}]) ;; get a group of flakes that we know will have different permissions for different users. john-flakes @(fluree/range db+policy :spot = ["ex:john"]) alice-flakes @(fluree/range db+policy :spot = [(fluree/expand-iri db+policy "ex:alice")]) diff --git a/test/fluree/db/policy/transact_test.clj b/test/fluree/db/policy/transact_test.clj index afb292f7f..2dd203e9b 100644 --- a/test/fluree/db/policy/transact_test.clj +++ b/test/fluree/db/policy/transact_test.clj @@ -63,14 +63,14 @@ "f:allow" [{"id" "ex:ssnViewRule" "f:targetRole" {"id" "ex:userRole"} "f:action" [{"id" "f:view"}] - "f:equals" {"list" [{"id" "f:$identity"} - {"id" "ex:user"}]}}]} + "f:equals" {"@list" [{"id" "f:$identity"} + {"id" "ex:user"}]}}]} {"f:path" {"id" "schema:email"} "f:allow" [{"id" "ex:emailChangeRule" "f:targetRole" {"id" "ex:userRole"} "f:action" [{"id" "f:view"} {"id" "f:modify"}] - "f:equals" {"list" [{"id" "f:$identity"} - {"id" "ex:user"}]}}]}]} + "f:equals" {"@list" [{"id" "f:$identity"} + {"id" "ex:user"}]}}]}]} ;; add a "ex:Product" policy allows view & modify for only "schema:name" {"id" "ex:ProductPolicy", "type" ["f:Policy"], From 7384f3325fa934b4545fd5805f17a3e7030d986c Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Mon, 17 Apr 2023 10:57:12 -0600 Subject: [PATCH 28/50] Stringify most query tests --- test/fluree/db/query/aggregate_test.clj | 12 +- test/fluree/db/query/datatype_test.clj | 80 ++- test/fluree/db/query/filter_query_test.clj | 200 +++---- test/fluree/db/query/history_test.clj | 88 ++- test/fluree/db/query/index_range_test.clj | 72 +-- test/fluree/db/query/json_ld_basic_test.clj | 485 ++++++++--------- .../fluree/db/query/json_ld_compound_test.clj | 268 ++++----- test/fluree/db/query/misc_queries_test.clj | 509 +++++++++--------- test/fluree/db/query/optional_query_test.clj | 93 ++-- test/fluree/db/query/reverse_query_test.clj | 62 +-- test/fluree/db/query/subclass_test.clj | 26 +- .../db/query/subject_crawl_reparse_test.clj | 21 +- test/fluree/db/query/time_travel_test.clj | 39 +- test/fluree/db/query/union_query_test.clj | 95 ++-- 14 files changed, 1069 insertions(+), 981 deletions(-) diff --git a/test/fluree/db/query/aggregate_test.clj b/test/fluree/db/query/aggregate_test.clj index b2c79d63f..d9a1e3b76 100644 --- a/test/fluree/db/query/aggregate_test.clj +++ b/test/fluree/db/query/aggregate_test.clj @@ -10,17 +10,17 @@ people (test-utils/load-people conn) db (fluree/db people)] (testing "with explicit grouping" - (let [qry '{:select [?name (count ?favNums)] - :where [[?s :schema/name ?name] - [?s :ex/favNums ?favNums]] - :group-by ?name} + (let [qry '{"select" [?name (count ?favNums)] + "where" [[?s "schema:name" ?name] + [?s "ex:favNums" ?favNums]] + "group-by" ?name} subject @(fluree/query db qry)] (is (= [["Liam" 2] ["Cam" 2] ["Alice" 3] ["Brian" 1]] subject) "aggregates bindings within each group"))) (testing "with implicit grouping" - (let [qry '{:select [(count ?name)] - :where [[?s :schema/name ?name]]} + (let [qry '{"select" [(count ?name)] + "where" [[?s "schema:name" ?name]]} subject @(fluree/query db qry)] (is (= [[4]] subject) "aggregates bindings for all results")))))) diff --git a/test/fluree/db/query/datatype_test.clj b/test/fluree/db/query/datatype_test.clj index d0289741e..dd12014e4 100644 --- a/test/fluree/db/query/datatype_test.clj +++ b/test/fluree/db/query/datatype_test.clj @@ -1,54 +1,52 @@ (ns fluree.db.query.datatype-test - (:require [clojure.string :as str] - [clojure.test :refer :all] + (:require [clojure.test :refer :all] [fluree.db.test-utils :as test-utils] - [fluree.db.json-ld.api :as fluree] - [fluree.db.util.log :as log])) + [fluree.db.json-ld.api :as fluree])) -(def default-context - {:id "@id" - :type "@type" - :schema "http://schema.org/" - :ex "http://example.org/ns/" - :rdfs "http://www.w3.org/2000/01/rdf-schema#" - :rdf "http://www.w3.org/1999/02/22-rdf-syntax-ns#"}) +(def context + {"id" "@id" + "type" "@type" + "schema" "http://schema.org/" + "ex" "http://example.org/ns/" + "rdfs" "http://www.w3.org/2000/01/rdf-schema#" + "rdf" "http://www.w3.org/1999/02/22-rdf-syntax-ns#"}) (deftest ^:integration datatype-test - (let [conn (test-utils/create-conn) - ledger @(fluree/create conn "ledger/datatype")] + (let [conn (test-utils/create-conn) + ledger @(fluree/create conn "ledger/datatype")] (testing "Querying predicates with mixed datatypes" - (let [mixed-db @(fluree/stage (fluree/db ledger) - [{:context default-context - :id :ex/coco - :type :schema/Person - :schema/name "Coco"} - {:context default-context - :id :ex/halie - :type :schema/Person - :schema/name "Halie"} - {:context default-context - :id :ex/john - :type :schema/Person - :schema/name 3}])] - (is (= [{:id :ex/halie - :rdf/type [:schema/Person] - :schema/name "Halie"}] + (let [mixed-db @(fluree/stage (fluree/db ledger) + [{"@context" context + "id" "ex:coco" + "type" "schema:Person" + "schema:name" "Coco"} + {"@context" context + "id" "ex:halie" + "type" "schema:Person" + "schema:name" "Halie"} + {"@context" context + "id" "ex:john" + "type" "schema:Person" + "schema:name" 3}])] + (is (= [{"id" "ex:halie" + "rdf:type" ["schema:Person"] + "schema:name" "Halie"}] @(fluree/query mixed-db - {:context default-context - :select {'?u [:*]} - :where [['?u :schema/name "Halie"]]})) + {"@context" context + "select" {'?u ["*"]} + "where" [['?u "schema:name" "Halie"]]})) "only returns the data type queried") (is (= [] @(fluree/query mixed-db - {:context default-context - :select {'?u [:*]} - :where [['?u :schema/name "a"]]})) + {"context" context + "select" {'?u ["*"]} + "where" [['?u "schema:name" "a"]]})) "does not return results without matching subjects") - (is (= [{:id :ex/john - :rdf/type [:schema/Person] - :schema/name 3}] + (is (= [{"id" "ex:john" + "rdf:type" ["schema:Person"] + "schema:name" 3}] @(fluree/query mixed-db - {:context default-context - :select {'?u [:*]} - :where [['?u :schema/name 3]]})) + {"@context" context + "select" {'?u ["*"]} + "where" [['?u "schema:name" 3]]})) "only returns the data type queried"))))) diff --git a/test/fluree/db/query/filter_query_test.clj b/test/fluree/db/query/filter_query_test.clj index 439a84858..04177b9f6 100644 --- a/test/fluree/db/query/filter_query_test.clj +++ b/test/fluree/db/query/filter_query_test.clj @@ -1,120 +1,122 @@ (ns fluree.db.query.filter-query-test (:require - [clojure.test :refer :all] - [fluree.db.test-utils :as test-utils] - [fluree.db.json-ld.api :as fluree])) + [clojure.test :refer :all] + [fluree.db.test-utils :as test-utils] + [fluree.db.json-ld.api :as fluree])) (deftest ^:integration filter-test (let [conn (test-utils/create-conn) - ledger @(fluree/create conn "query/filter" {:defaultContext ["" {:ex "http://example.org/ns/"}]}) + ledger @(fluree/create conn "query/filter" {"defaults" + {"@context" + ["" {"ex" "http://example.org/ns/"}]}}) db @(fluree/stage - (fluree/db ledger) - [{:id :ex/brian, - :type :ex/User, - :schema/name "Brian" - :ex/last "Smith" - :schema/email "brian@example.org" - :schema/age 50 - :ex/favNums 7} - {:id :ex/alice, - :type :ex/User, - :schema/name "Alice" - :ex/last "Smith" - :schema/email "alice@example.org" - :ex/favColor "Green" - :schema/age 42 - :ex/favNums [42, 76, 9]} - {:id :ex/cam, - :type :ex/User, - :schema/name "Cam" - :ex/last "Jones" - :schema/email "cam@example.org" - :schema/age 34 - :ex/favColor "Blue" - :ex/favNums [5, 10] - :ex/friend [:ex/brian :ex/alice]} - {:id :ex/david, - :type :ex/User, - :schema/name "David" - :ex/last "Jones" - :schema/email "david@example.org" - :schema/age 46 - :ex/favNums [15 70] - :ex/friend [:ex/cam]}])] + (fluree/db ledger) + [{"id" "ex:brian", + "type" "ex:User", + "schema:name" "Brian" + "ex:last" "Smith" + "schema:email" "brian@example.org" + "schema:age" 50 + "ex:favNums" 7} + {"id" "ex:alice", + "type" "ex:User", + "schema:name" "Alice" + "ex:last" "Smith" + "schema:email" "alice@example.org" + "ex:favColor" "Green" + "schema:age" 42 + "ex:favNums" [42, 76, 9]} + {"id" "ex:cam", + "type" "ex:User", + "schema:name" "Cam" + "ex:last" "Jones" + "schema:email" "cam@example.org" + "schema:age" 34 + "ex:favColor" "Blue" + "ex:favNums" [5, 10] + "ex:friend" [{"id" "ex:brian"} {"id" "ex:alice"}]} + {"id" "ex:david", + "type" "ex:User", + "schema:name" "David" + "ex:last" "Jones" + "schema:email" "david@example.org" + "schema:age" 46 + "ex:favNums" [15 70] + "ex:friend" [{"id" "ex:cam"}]}])] (testing "single filter" (is (= [["David" 46] ["Brian" 50]] - @(fluree/query db {:select ['?name '?age] - :where [['?s :rdf/type :ex/User] - ['?s :schema/age '?age] - ['?s :schema/name '?name] - {:filter ["(> ?age 45)"]}]})))) + @(fluree/query db {"select" ['?name '?age] + "where" [['?s "rdf:type" "ex:User"] + ['?s "schema:age" '?age] + ['?s "schema:name" '?name] + {"filter" ["(> ?age 45)"]}]})))) (testing "multiple filters on same var" (is (= [["David" 46]] - @(fluree/query db {:select ['?name '?age] - :where [['?s :rdf/type :ex/User] - ['?s :schema/age '?age] - ['?s :schema/name '?name] - {:filter ["(> ?age 45)", "(< ?age 50)"]}]})))) + @(fluree/query db {"select" ['?name '?age] + "where" [['?s "rdf:type" "ex:User"] + ['?s "schema:age" '?age] + ['?s "schema:name" '?name] + {"filter" ["(> ?age 45)", "(< ?age 50)"]}]})))) (testing "multiple filters, different vars" (is (= [["Brian" "Smith"]] - @(fluree/query db {:select ['?name '?last] - :where [['?s :rdf/type :ex/User] - ['?s :schema/age '?age] - ['?s :schema/name '?name] - ['?s :ex/last '?last] - {:filter ["(> ?age 45)", "(strEnds ?last \"ith\")"]}]})))) + @(fluree/query db {"select" ['?name '?last] + "where" [['?s "rdf:type" "ex:User"] + ['?s "schema:age" '?age] + ['?s "schema:name" '?name] + ['?s "ex:last" '?last] + {"filter" ["(> ?age 45)", "(strEnds ?last \"ith\")"]}]})))) (testing "nested filters" (is (= [["Brian" 50]] - @(fluree/query db '{:select [?name ?age] - :where [[?s :rdf/type :ex/User] - [?s :schema/age ?age] - [?s :schema/name ?name] - {:filter ["(> ?age (/ (+ ?age 47) 2))"]}]})))) + @(fluree/query db '{"select" [?name ?age] + "where" [[?s "rdf:type" "ex:User"] + [?s "schema:age" ?age] + [?s "schema:name" ?name] + {"filter" ["(> ?age (/ (+ ?age 47) 2))"]}]})))) ;;TODO: simple-subject-crawl does not yet support filters. ;;these are being run as regular analytial queries (testing "simple-subject-crawl" - (is (= [{:id :ex/david, - :rdf/type [:ex/User], - :schema/name "David", - :ex/last "Jones", - :schema/email "david@example.org", - :schema/age 46, - :ex/favNums [15 70], - :ex/friend {:id :ex/cam}} - {:id :ex/brian, - :rdf/type [:ex/User], - :schema/name "Brian", - :ex/last "Smith", - :schema/email "brian@example.org", - :schema/age 50, - :ex/favNums 7}] - @(fluree/query db {:select {"?s" ["*"]} - :where [["?s" :schema/age "?age"] - {:filter ["(> ?age 45)"]}]}))) - (is (= [{:id :ex/david, - :rdf/type [:ex/User], - :schema/name "David", - :ex/last "Jones", - :schema/email "david@example.org", - :schema/age 46, - :ex/favNums [15 70], - :ex/friend {:id :ex/cam}}] - @(fluree/query db {:select {"?s" ["*"]} - :where [["?s" :schema/age "?age"] - {:filter ["(> ?age 45)", "(< ?age 50)"]}]}))) - (is (= [{:rdf/type [:ex/User] - :schema/email "cam@example.org" - :ex/favNums [5 10] - :schema/age 34 - :ex/last "Jones" - :schema/name "Cam" - :id :ex/cam - :ex/friend [{:id :ex/brian} {:id :ex/alice}] - :ex/favColor "Blue"}] - @(fluree/query db {:select {"?s" ["*"]} - :where [["?s" :ex/favColor "?color"] - {:filter ["(strStarts ?color \"B\")"]}]})))))) + (is (= [{"id" "ex:david", + "rdf:type" ["ex:User"], + "schema:name" "David", + "ex:last" "Jones", + "schema:email" "david@example.org", + "schema:age" 46, + "ex:favNums" [15 70], + "ex:friend" {"id" "ex:cam"}} + {"id" "ex:brian", + "rdf:type" ["ex:User"], + "schema:name" "Brian", + "ex:last" "Smith", + "schema:email" "brian@example.org", + "schema:age" 50, + "ex:favNums" 7}] + @(fluree/query db {"select" {"?s" ["*"]} + "where" [["?s" "schema:age" "?age"] + {"filter" ["(> ?age 45)"]}]}))) + (is (= [{"id" "ex:david", + "rdf:type" ["ex:User"], + "schema:name" "David", + "ex:last" "Jones", + "schema:email" "david@example.org", + "schema:age" 46, + "ex:favNums" [15 70], + "ex:friend" {"id" "ex:cam"}}] + @(fluree/query db {"select" {"?s" ["*"]} + "where" [["?s" "schema:age" "?age"] + {"filter" ["(> ?age 45)", "(< ?age 50)"]}]}))) + (is (= [{"rdf:type" ["ex:User"] + "schema:email" "cam@example.org" + "ex:favNums" [5 10] + "schema:age" 34 + "ex:last" "Jones" + "schema:name" "Cam" + "id" "ex:cam" + "ex:friend" [{"id" "ex:brian"} {"id" "ex:alice"}] + "ex:favColor" "Blue"}] + @(fluree/query db {"select" {"?s" ["*"]} + "where" [["?s" "ex:favColor" "?color"] + {"filter" ["(strStarts ?color \"B\")"]}]})))))) diff --git a/test/fluree/db/query/history_test.clj b/test/fluree/db/query/history_test.clj index 4a20a8bc7..02973734c 100644 --- a/test/fluree/db/query/history_test.clj +++ b/test/fluree/db/query/history_test.clj @@ -480,16 +480,80 @@ "ex:y" "bar-3" "id" "ex:alice"}] "f:t" 5}] - @(fluree/history ledger {"history" "ex:alice" "commit-details" true "t" {"from" 3}}))) + @(fluree/history ledger {"history" "ex:alice" "commit-details" true "t" {"from" 3}})))) - (testing "multiple history results" - (let [history-with-commits @(fluree/history ledger {"history" "ex:alice" "commit-details" true "t" {"from" 1 "to" 5}})] - (testing "all `t`s with changes to subject are returned" - (is (= [1 2 3 5] - (mapv #(get % "f:t") history-with-commits)))) - (testing "all expected commits are present and associated with the correct results" - (is (= [[1 1] [2 2] [3 3] [5 5]] - (map (fn [history-map] - (let [commit-t (get-in history-map ["f:commit" "f:data" "f:t"])] - (vector (get history-map "f:t") commit-t))) - history-with-commits)))))))))) + ;; TODO: Fix this https://github.com/fluree/db/issues/451 + #_(testing "history commit details on a loaded ledger" + (is (= [{"f:assert" [{"ex:x" "foo-3" + "ex:y" "bar-3" + "id" "ex:alice"}] + "f:commit" {"https://www.w3.org/2018/credentials#issuer" + {"id" "did:fluree:TfCzWTrXqF16hvKGjcYiLxRoYJ1B8a6UMH6"} + "f:address" "fluree:memory://f29bfb686c834d667dc62f8c46af1802f02c62567b400d50c0202428e489d1fe" + "f:alias" "committest" + "f:branch" "main" + "f:context" "fluree:memory://b6dcf8968183239ecc7a664025f247de5b7859ac18cdeaace89aafc421eeddee" + "f:data" {"f:address" "fluree:memory://cb16fc43954b9ed029be2c96c6f73fa5e34e5ca1607111c5e8f8d6e337b648f6" + "f:assert" [{"ex:x" "foo-3" + "ex:y" "bar-3" + "id" "ex:alice"}] + "f:flakes" 63 + "f:previous" {"id" "fluree:db:sha256:bjufs3dmyea7wzbkrjrh2pzua2mtqgijnl3cqpkktk7wzvy5wlnq"} + "f:retract" [{"ex:x" "foo-2" + "ex:y" "bar-2" + "id" "ex:alice"}] + "f:size" 5864 + "f:t" 3} + "f:previous" {"id" "fluree:commit:sha256:bbvotmnlqkm4xkc27au55rctw5klntegd36j4dbh665qfilem42eq"} + "f:time" 720000 + "f:v" 0 + "id" "fluree:commit:sha256:b5jlses24wzjhcmqywvcdwgxxzmjlmxruh2duqsgxbxy7522utlg"} + "f:retract" [{"ex:x" "foo-2" + "ex:y" "bar-2" + "id" "ex:alice"}] + "f:t" 3} + {"f:assert" [{"ex:x" "foo-cat" + "ex:y" "bar-cat" + "id" "ex:alice"}] + "f:commit" {"https://www.w3.org/2018/credentials#issuer" + {"id" "did:fluree:TfCzWTrXqF16hvKGjcYiLxRoYJ1B8a6UMH6"} + "f:address" "fluree:memory://2716b3eaef91b763b32daebab8ba3733a8537de37a864f726c5021464b90277f" + "f:alias" "committest" + "f:branch" "main" + "f:context" "fluree:memory://b6dcf8968183239ecc7a664025f247de5b7859ac18cdeaace89aafc421eeddee" + "f:data" {"f:address" "fluree:memory://2b8125a4c996f8612ae62f16cc62b167b762c517b0c5aa7b16fa21dfe47e7b2a" + "f:assert" [{"ex:x" "foo-cat" + "ex:y" "bar-cat" + "id" "ex:alice"}] + "f:flakes" 102 + "f:previous" {"id" "fluree:db:sha256:bbtdwia2mle22abe2z7mmdrs4vufs77yubzxip3chhkmffvfk4npk"} + "f:retract" [{"ex:x" "foo-3" + "ex:y" "bar-3" + "id" "ex:alice"}] + "f:size" 9326 + "f:t" 5} + "f:message" "meow" + "f:previous" {"id" "fluree:commit:sha256:bhcunj52uwxshi6jws2ypsjiziyncfa5gal4xptfldj5utb54cpf"} + "f:time" 720000 + "f:v" 0 + "id" "fluree:commit:sha256:bq4r74wu4sru43z5f4byipe5zzkgyi2kksqxlmwuttdcwcgwpsrn"} + "f:retract" [{"ex:x" "foo-3" + "ex:y" "bar-3" + "id" "ex:alice"}] + "f:t" 5}] + (let [loaded-ledger @(fluree/load conn "committest")] + @(fluree/history loaded-ledger {"history" "ex:alice" + "commit-details" true + "t" {"from" 3}}))))) + + (testing "multiple history results" + (let [history-with-commits @(fluree/history ledger {"history" "ex:alice" "commit-details" true "t" {"from" 1 "to" 5}})] + (testing "all `t`s with changes to subject are returned" + (is (= [1 2 3 5] + (mapv #(get % "f:t") history-with-commits)))) + (testing "all expected commits are present and associated with the correct results" + (is (= [[1 1] [2 2] [3 3] [5 5]] + (map (fn [history-map] + (let [commit-t (get-in history-map ["f:commit" "f:data" "f:t"])] + (vector (get history-map "f:t") commit-t))) + history-with-commits))))))))) diff --git a/test/fluree/db/query/index_range_test.clj b/test/fluree/db/query/index_range_test.clj index 16815aeb9..a3213e4df 100644 --- a/test/fluree/db/query/index_range_test.clj +++ b/test/fluree/db/query/index_range_test.clj @@ -1,42 +1,42 @@ (ns fluree.db.query.index-range-test (:require - [clojure.string :as str] [clojure.test :refer :all] [fluree.db.test-utils :as test-utils] [fluree.db.json-ld.api :as fluree] - [fluree.db.util.log :as log] [fluree.db.flake :as flake])) (deftest ^:integration index-range-scans (testing "Various index range scans using the API." (let [conn (test-utils/create-conn) ledger @(fluree/create conn "query/index-range" - {:defaultContext ["" {:ex "http://example.org/ns/"}]}) + {"defaults" + {"@context" + ["" {"ex" "http://example.org/ns/"}]}}) db @(fluree/stage (fluree/db ledger) - [{:id :ex/brian, - :type :ex/User, - :schema/name "Brian" - :schema/email "brian@example.org" - :schema/age 50 - :ex/favNums 7} - {:id :ex/alice, - :type :ex/User, - :schema/name "Alice" - :schema/email "alice@example.org" - :schema/age 50 - :ex/favNums [42, 76, 9]} - {:id :ex/cam, - :type :ex/User, - :schema/name "Cam" - :schema/email "cam@example.org" - :schema/age 34 - :ex/favNums [5, 10] - :ex/friend [:ex/brian :ex/alice]}]) - cam-sid @(fluree/internal-id db :ex/cam)] + [{"id" "ex:brian", + "type" "ex:User", + "schema:name" "Brian" + "schema:email" "brian@example.org" + "schema:age" 50 + "ex:favNums" 7} + {"id" "ex:alice", + "type" "ex:User", + "schema:name" "Alice" + "schema:email" "alice@example.org" + "schema:age" 50 + "ex:favNums" [42, 76, 9]} + {"id" "ex:cam", + "type" "ex:User", + "schema:name" "Cam" + "schema:email" "cam@example.org" + "schema:age" 34 + "ex:favNums" [5, 10] + "ex:friend" ["ex:brian" "ex:alice"]}]) + cam-sid @(fluree/internal-id db "ex:cam")] (is (= "http://example.org/ns/cam" - (fluree/expand-iri db :ex/cam)) + (fluree/expand-iri db "ex:cam")) "Expanding compact IRI is broken, likely other tests will fail.") (is (int? cam-sid) @@ -44,7 +44,7 @@ (testing "Slice operations" (testing "Slice for subject id only" - (let [alice-sid @(fluree/internal-id db :ex/alice)] + (let [alice-sid @(fluree/internal-id db "ex:alice")] (is (= [[alice-sid 0 "http://example.org/ns/alice" 1 -1 true nil] [alice-sid 200 1001 0 -1 true nil] [alice-sid 1002 "Alice" 1 -1 true nil] @@ -58,8 +58,8 @@ "Slice should return a vector of flakes for only Alice"))) (testing "Slice for subject + predicate" - (let [alice-sid @(fluree/internal-id db :ex/alice) - favNums-pid @(fluree/internal-id db :ex/favNums)] + (let [alice-sid @(fluree/internal-id db "ex:alice") + favNums-pid @(fluree/internal-id db "ex:favNums")] (is (= [[alice-sid favNums-pid 9 8 -1 true nil] [alice-sid favNums-pid 42 8 -1 true nil] [alice-sid favNums-pid 76 8 -1 true nil]] @@ -68,24 +68,24 @@ "Slice should only return Alice's favNums (multi-cardinality)"))) (testing "Slice for subject + predicate + value" - (let [alice-sid @(fluree/internal-id db :ex/alice) - favNums-pid @(fluree/internal-id db :ex/favNums)] + (let [alice-sid @(fluree/internal-id db "ex:alice") + favNums-pid @(fluree/internal-id db "ex:favNums")] (is (= [[alice-sid favNums-pid 42 8 -1 true nil]] (->> @(fluree/slice db :spot [alice-sid favNums-pid 42]) (mapv flake/Flake->parts))) "Slice should only return the specified favNum value"))) (testing "Slice for subject + predicate + value + datatype" - (let [alice-sid @(fluree/internal-id db :ex/alice) - favNums-pid @(fluree/internal-id db :ex/favNums)] + (let [alice-sid @(fluree/internal-id db "ex:alice") + favNums-pid @(fluree/internal-id db "ex:favNums")] (is (= [[alice-sid favNums-pid 42 8 -1 true nil]] (->> @(fluree/slice db :spot [alice-sid favNums-pid [42 8]]) (mapv flake/Flake->parts))) "Slice should only return the specified favNum value with matching datatype"))) (testing "Slice for subject + predicate + value + mismatch datatype" - (let [alice-sid @(fluree/internal-id db :ex/alice) - favNums-pid @(fluree/internal-id db :ex/favNums)] + (let [alice-sid @(fluree/internal-id db "ex:alice") + favNums-pid @(fluree/internal-id db "ex:favNums")] (is (= [] (->> @(fluree/slice db :spot [alice-sid favNums-pid [42 7]]) (mapv flake/Flake->parts))) @@ -93,9 +93,9 @@ (testing "Subject IRI resolution for index-range automatically happens" - (let [with-compact-iri @(fluree/range db :spot = [:ex/alice]) - with-full-iri @(fluree/range db :spot = [(fluree/expand-iri db :ex/alice)]) - with-sid @(fluree/range db :spot = [@(fluree/internal-id db :ex/alice)])] + (let [with-compact-iri @(fluree/range db :spot = ["ex:alice"]) + with-full-iri @(fluree/range db :spot = [(fluree/expand-iri db "ex:alice")]) + with-sid @(fluree/range db :spot = [@(fluree/internal-id db "ex:alice")])] (is (= with-compact-iri with-full-iri with-sid) diff --git a/test/fluree/db/query/json_ld_basic_test.clj b/test/fluree/db/query/json_ld_basic_test.clj index 023fcb31a..a8d508655 100644 --- a/test/fluree/db/query/json_ld_basic_test.clj +++ b/test/fluree/db/query/json_ld_basic_test.clj @@ -9,69 +9,69 @@ movies (test-utils/load-movies conn) db (fluree/db movies)] (testing "basic wildcard single subject query" - (let [query-res @(fluree/query db '{:select {?s [:*]} - :where [[?s :id :wiki/Q836821]]})] - (is (= query-res [{:id :wiki/Q836821, - :rdf/type [:schema/Movie], - :schema/name "The Hitchhiker's Guide to the Galaxy", - :schema/disambiguatingDescription "2005 British-American comic science fiction film directed by Garth Jennings", - :schema/titleEIDR "10.5240/B752-5B47-DBBE-E5D4-5A3F-N", - :schema/isBasedOn {:id :wiki/Q3107329}}]) + (let [query-res @(fluree/query db '{"select" {?s ["*"]} + "where" [[?s "id" "wiki:Q836821"]]})] + (is (= query-res [{"id" "wiki:Q836821", + "rdf:type" ["schema:Movie"], + "schema:name" "The Hitchhiker's Guide to the Galaxy", + "schema:disambiguatingDescription" "2005 British-American comic science fiction film directed by Garth Jennings", + "schema:titleEIDR" "10.5240/B752-5B47-DBBE-E5D4-5A3F-N", + "schema:isBasedOn" {"id" "wiki:Q3107329"}}]) "Basic select * is working will context normalization"))) (testing "basic single subject query with explicit field selection" - (let [query-res @(fluree/query db '{:select {?s [:id :schema/name]} - :where [[?s :id :wiki/Q836821]]})] - (is (= query-res [{:id :wiki/Q836821, :schema/name "The Hitchhiker's Guide to the Galaxy"}])))) + (let [query-res @(fluree/query db '{"select" {?s ["id" "schema:name"]} + "where" [[?s "id" "wiki:Q836821"]]})] + (is (= query-res [{"id" "wiki:Q836821", "schema:name" "The Hitchhiker's Guide to the Galaxy"}])))) (testing "basic single subject query with selectOne" - (let [query-res @(fluree/query db '{:selectOne {?s [:id :schema/name]} - :where [[?s :id :wiki/Q836821]]})] - (is (= query-res {:id :wiki/Q836821, :schema/name "The Hitchhiker's Guide to the Galaxy"})))) + (let [query-res @(fluree/query db '{"selectOne" {?s ["id" "schema:name"]} + "where" [[?s "id" "wiki:Q836821"]]})] + (is (= query-res {"id" "wiki:Q836821", "schema:name" "The Hitchhiker's Guide to the Galaxy"})))) (testing "basic single subject query with graph crawl" - (let [query-res @(fluree/query db '{:selectOne {?s [:* {:schema/isBasedOn [:*]}]} - :where [[?s :id :wiki/Q836821]]})] - (is (= query-res {:id :wiki/Q836821, - :rdf/type [:schema/Movie], - :schema/name "The Hitchhiker's Guide to the Galaxy", - :schema/disambiguatingDescription "2005 British-American comic science fiction film directed by Garth Jennings", - :schema/titleEIDR "10.5240/B752-5B47-DBBE-E5D4-5A3F-N", - :schema/isBasedOn {:id :wiki/Q3107329, - :rdf/type [:schema/Book], - :schema/name "The Hitchhiker's Guide to the Galaxy", - :schema/isbn "0-330-25864-8", - :schema/author {:id :wiki/Q42}}})))) + (let [query-res @(fluree/query db '{"selectOne" {?s ["*" {"schema:isBasedOn" ["*"]}]} + "where" [[?s "id" "wiki:Q836821"]]})] + (is (= query-res {"id" "wiki:Q836821", + "rdf:type" ["schema:Movie"], + "schema:name" "The Hitchhiker's Guide to the Galaxy", + "schema:disambiguatingDescription" "2005 British-American comic science fiction film directed by Garth Jennings", + "schema:titleEIDR" "10.5240/B752-5B47-DBBE-E5D4-5A3F-N", + "schema:isBasedOn" {"id" "wiki:Q3107329", + "rdf:type" ["schema:Book"], + "schema:name" "The Hitchhiker's Guide to the Galaxy", + "schema:isbn" "0-330-25864-8", + "schema:author" {"id" "wiki:Q42"}}})))) (testing "basic single subject query using depth graph crawl" (testing "using only wildcard" - (let [query-res @(fluree/query db '{:selectOne {?s [:*]} - :where [[?s :id :wiki/Q836821]] - :depth 3})] - (is (= query-res {:id :wiki/Q836821, - :rdf/type [:schema/Movie], - :schema/name "The Hitchhiker's Guide to the Galaxy", - :schema/disambiguatingDescription "2005 British-American comic science fiction film directed by Garth Jennings", - :schema/titleEIDR "10.5240/B752-5B47-DBBE-E5D4-5A3F-N", - :schema/isBasedOn {:id :wiki/Q3107329, - :rdf/type [:schema/Book], - :schema/name "The Hitchhiker's Guide to the Galaxy", - :schema/isbn "0-330-25864-8", - :schema/author {:id :wiki/Q42, - :rdf/type [:schema/Person], - :schema/name "Douglas Adams"}}})))) + (let [query-res @(fluree/query db '{"selectOne" {?s ["*"]} + "where" [[?s "id" "wiki:Q836821"]] + "depth" 3})] + (is (= query-res {"id" "wiki:Q836821", + "rdf:type" ["schema:Movie"], + "schema:name" "The Hitchhiker's Guide to the Galaxy", + "schema:disambiguatingDescription" "2005 British-American comic science fiction film directed by Garth Jennings", + "schema:titleEIDR" "10.5240/B752-5B47-DBBE-E5D4-5A3F-N", + "schema:isBasedOn" {"id" "wiki:Q3107329", + "rdf:type" ["schema:Book"], + "schema:name" "The Hitchhiker's Guide to the Galaxy", + "schema:isbn" "0-330-25864-8", + "schema:author" {"id" "wiki:Q42", + "rdf:type" ["schema:Person"], + "schema:name" "Douglas Adams"}}})))) (testing "using graph sub-selection" - (let [query-res @(fluree/query db '{:selectOne {?s [:* {:schema/isBasedOn [:*]}]} - :where [[?s :id :wiki/Q836821]] - :depth 3})] - (is (= query-res {:id :wiki/Q836821, - :rdf/type [:schema/Movie], - :schema/name "The Hitchhiker's Guide to the Galaxy", - :schema/disambiguatingDescription "2005 British-American comic science fiction film directed by Garth Jennings", - :schema/titleEIDR "10.5240/B752-5B47-DBBE-E5D4-5A3F-N", - :schema/isBasedOn {:id :wiki/Q3107329, - :rdf/type [:schema/Book], - :schema/name "The Hitchhiker's Guide to the Galaxy", - :schema/isbn "0-330-25864-8", - :schema/author {:id :wiki/Q42, - :rdf/type [:schema/Person], - :schema/name "Douglas Adams"}}})))))))) + (let [query-res @(fluree/query db '{"selectOne" {?s ["*" {"schema:isBasedOn" ["*"]}]} + "where" [[?s "id" "wiki:Q836821"]] + "depth" 3})] + (is (= query-res {"id" "wiki:Q836821", + "rdf:type" ["schema:Movie"], + "schema:name" "The Hitchhiker's Guide to the Galaxy", + "schema:disambiguatingDescription" "2005 British-American comic science fiction film directed by Garth Jennings", + "schema:titleEIDR" "10.5240/B752-5B47-DBBE-E5D4-5A3F-N", + "schema:isBasedOn" {"id" "wiki:Q3107329", + "rdf:type" ["schema:Book"], + "schema:name" "The Hitchhiker's Guide to the Galaxy", + "schema:isbn" "0-330-25864-8", + "schema:author" {"id" "wiki:Q42", + "rdf:type" ["schema:Person"], + "schema:name" "Douglas Adams"}}})))))))) (deftest ^:integration json-ld-rdf-type-query (testing "json-ld rdf type queries" @@ -79,33 +79,33 @@ movies (test-utils/load-movies conn) db (fluree/db movies)] (testing "basic analytical RFD type query" - (let [query-res @(fluree/query db '{:select {?s [:* {:schema/isBasedOn [:*]}]} - :where [[?s :rdf/type :schema/Movie]]})] - (is (= query-res ;; :id is a DID and will be unique per DB so exclude from comparison - [{:id :wiki/Q230552, - :rdf/type [:schema/Movie], - :schema/name "Back to the Future Part III", - :schema/disambiguatingDescription "1990 film by Robert Zemeckis", - :schema/titleEIDR "10.5240/15F9-F913-FF25-8041-E798-O"} - {:id :wiki/Q109331, :rdf/type [:schema/Movie], - :schema/name "Back to the Future Part II", - :schema/titleEIDR "10.5240/5DA5-C386-2911-7E2B-1782-L", - :schema/followedBy {:id :wiki/Q230552}} - {:id :wiki/Q91540, - :rdf/type [:schema/Movie], - :schema/name "Back to the Future", - :schema/disambiguatingDescription "1985 film by Robert Zemeckis", - :schema/titleEIDR "10.5240/09A3-1F6E-3538-DF46-5C6F-I", - :schema/followedBy {:id :wiki/Q109331}} - {:id :wiki/Q836821, :rdf/type [:schema/Movie], - :schema/name "The Hitchhiker's Guide to the Galaxy", - :schema/disambiguatingDescription "2005 British-American comic science fiction film directed by Garth Jennings", - :schema/titleEIDR "10.5240/B752-5B47-DBBE-E5D4-5A3F-N", - :schema/isBasedOn {:id :wiki/Q3107329, - :rdf/type [:schema/Book], - :schema/name "The Hitchhiker's Guide to the Galaxy", - :schema/isbn "0-330-25864-8", - :schema/author {:id :wiki/Q42}}}]) + (let [query-res @(fluree/query db '{"select" {?s ["*" {"schema:isBasedOn" ["*"]}]} + "where" [[?s "rdf:type" "schema:Movie"]]})] + (is (= query-res ;; "id" is a DID and will be unique per DB so exclude from comparison + [{"id" "wiki:Q230552", + "rdf:type" ["schema:Movie"], + "schema:name" "Back to the Future Part III", + "schema:disambiguatingDescription" "1990 film by Robert Zemeckis", + "schema:titleEIDR" "10.5240/15F9-F913-FF25-8041-E798-O"} + {"id" "wiki:Q109331", "rdf:type" ["schema:Movie"], + "schema:name" "Back to the Future Part II", + "schema:titleEIDR" "10.5240/5DA5-C386-2911-7E2B-1782-L", + "schema:followedBy" {"id" "wiki:Q230552"}} + {"id" "wiki:Q91540", + "rdf:type" ["schema:Movie"], + "schema:name" "Back to the Future", + "schema:disambiguatingDescription" "1985 film by Robert Zemeckis", + "schema:titleEIDR" "10.5240/09A3-1F6E-3538-DF46-5C6F-I", + "schema:followedBy" {"id" "wiki:Q109331"}} + {"id" "wiki:Q836821", "rdf:type" ["schema:Movie"], + "schema:name" "The Hitchhiker's Guide to the Galaxy", + "schema:disambiguatingDescription" "2005 British-American comic science fiction film directed by Garth Jennings", + "schema:titleEIDR" "10.5240/B752-5B47-DBBE-E5D4-5A3F-N", + "schema:isBasedOn" {"id" "wiki:Q3107329", + "rdf:type" ["schema:Book"], + "schema:name" "The Hitchhiker's Guide to the Galaxy", + "schema:isbn" "0-330-25864-8", + "schema:author" {"id" "wiki:Q42"}}}]) "Standard bootstrap data isn't matching.")))))) @@ -115,176 +115,179 @@ movies (test-utils/load-movies conn)] (testing "define @list container in context" (let [db @(fluree/stage (fluree/db movies) - {:context {:id "@id" - :ex "http://example.org/ns#" - :ex/list {"@container" "@list"}} - :id "list-test" - :ex/list [42 2 88 1]}) - query-res @(fluree/query db '{:context ["" {:ex "http://example.org/ns#"}] - :selectOne {?s [:*]}, - :where [[?s :id "list-test"]]})] - (is (= query-res - {:id "list-test" - :ex/list [42 2 88 1]}) + {"@context" {"id" "@id" + "ex" "http://example.org/ns#" + "ex:list" {"@container" "@list"}} + "id" "list-test" + "ex:list" [42 2 88 1]}) + query-res @(fluree/query db '{"@context" ["" {"ex" "http://example.org/ns#"}] + "selectOne" {?s ["*"]}, + "where" [[?s "id" "list-test"]]})] + (is (= {"id" "list-test" + "ex:list" [42 2 88 1]} + query-res) "Order of query result is different from transaction."))) (testing "define @list directly on subject" (let [db @(fluree/stage (fluree/db movies) - {:context {:id "@id" - :ex "http://example.org/ns#"} - :id "list-test2" - :ex/list {"@list" [42 2 88 1]}}) - query-res @(fluree/query db '{:context ["" {:ex "http://example.org/ns#"}], - :selectOne {?s [:*]}, - :where [[?s :id "list-test2"]]})] + {"@context" {"id" "@id" + "ex" "http://example.org/ns#"} + "id" "list-test2" + "ex:list" {"@list" [42 2 88 1]}}) + query-res @(fluree/query db '{"@context" ["" {"ex" "http://example.org/ns#"}], + "selectOne" {?s ["*"]}, + "where" [[?s "id" "list-test2"]]})] (is (= query-res - {:id "list-test2" - :ex/list [42 2 88 1]}) + {"id" "list-test2" + "ex:list" [42 2 88 1]}) "Order of query result is different from transaction.")))))) (deftest ^:integration simple-subject-crawl-test (let [conn (test-utils/create-conn) - ledger @(fluree/create conn "query/simple-subject-crawl" {:defaultContext ["" {:ex "http://example.org/ns/"}]}) + ledger @(fluree/create conn "query/simple-subject-crawl" + {"defaults" + {"@context" + ["" {"ex" "http://example.org/ns/"}]}}) db @(fluree/stage - (fluree/db ledger) - [{:id :ex/brian, - :type :ex/User, - :schema/name "Brian" - :ex/last "Smith" - :schema/email "brian@example.org" - :schema/age 50 - :ex/favColor "Green" - :ex/favNums 7} - {:id :ex/alice, - :type :ex/User, - :schema/name "Alice" - :ex/last "Smith" - :schema/email "alice@example.org" - :ex/favColor "Green" - :schema/age 42 - :ex/favNums [42, 76, 9]} - {:id :ex/cam, - :type :ex/User, - :schema/name "Cam" - :ex/last "Jones" - :schema/email "cam@example.org" - :schema/age 34 - :ex/favColor "Blue" - :ex/favNums [5, 10] - :ex/friend [:ex/brian :ex/alice]} - {:id :ex/david, - :type :ex/User, - :schema/name "David" - :ex/last "Jones" - :schema/email "david@example.org" - :schema/age 46 - :ex/favNums [15 70] - :ex/friend [:ex/cam]}])] + (fluree/db ledger) + [{"id" "ex:brian", + "type" "ex:User", + "schema:name" "Brian" + "ex:last" "Smith" + "schema:email" "brian@example.org" + "schema:age" 50 + "ex:favColor" "Green" + "ex:favNums" 7} + {"id" "ex:alice", + "type" "ex:User", + "schema:name" "Alice" + "ex:last" "Smith" + "schema:email" "alice@example.org" + "ex:favColor" "Green" + "schema:age" 42 + "ex:favNums" [42, 76, 9]} + {"id" "ex:cam", + "type" "ex:User", + "schema:name" "Cam" + "ex:last" "Jones" + "schema:email" "cam@example.org" + "schema:age" 34 + "ex:favColor" "Blue" + "ex:favNums" [5, 10] + "ex:friend" [{"id" "ex:brian"} {"id" "ex:alice"}]} + {"id" "ex:david", + "type" "ex:User", + "schema:name" "David" + "ex:last" "Jones" + "schema:email" "david@example.org" + "schema:age" 46 + "ex:favNums" [15 70] + "ex:friend" [{"id" "ex:cam"}]}])] (testing "using `where`" (testing "id" ;;TODO not getting reparsed as ssc - (is (= [{:id :ex/brian, - :rdf/type [:ex/User] - :schema/name "Brian" - :ex/last "Smith" - :schema/email "brian@example.org" - :schema/age 50 - :ex/favColor "Green" - :ex/favNums 7}] - @(fluree/query db {:select {"?s" ["*"]} - :where [["?s" :id :ex/brian]]})))) + (is (= [{"id" "ex:brian", + "rdf:type" ["ex:User"] + "schema:name" "Brian" + "ex:last" "Smith" + "schema:email" "brian@example.org" + "schema:age" 50 + "ex:favColor" "Green" + "ex:favNums" 7}] + @(fluree/query db {"select" {"?s" ["*"]} + "where" [["?s" "id" "ex:brian"]]})))) ;;TODO not getting reparsed as ssc (testing "iri" - (is (= [{:id :ex/david - :rdf/type [:ex/User] - :schema/name "David" - :ex/last "Jones" - :schema/email "david@example.org" - :schema/age 46 - :ex/favNums [15 70] - :ex/friend {:id :ex/cam}} - {:rdf/type [:ex/User] - :schema/email "cam@example.org" - :ex/favNums [5 10] - :schema/age 34 - :ex/last "Jones" - :schema/name "Cam" - :id :ex/cam - :ex/friend [{:id :ex/brian} {:id :ex/alice}] - :ex/favColor "Blue"} - {:id :ex/alice - :rdf/type [:ex/User] - :schema/name "Alice" - :ex/last "Smith" - :schema/email "alice@example.org" - :schema/age 42 - :ex/favNums [9 42 76] - :ex/favColor "Green"} - {:id :ex/brian - :rdf/type [:ex/User] - :schema/name "Brian" - :ex/last "Smith" - :schema/email "brian@example.org" - :schema/age 50 - :ex/favColor "Green" - :ex/favNums 7}] - @(fluree/query db {:select {"?s" ["*"]} - :where [["?s" :type :ex/User]]})))) + (is (= [{"id" "ex:david" + "rdf:type" ["ex:User"] + "schema:name" "David" + "ex:last" "Jones" + "schema:email" "david@example.org" + "schema:age" 46 + "ex:favNums" [15 70] + "ex:friend" {"id" "ex:cam"}} + {"rdf:type" ["ex:User"] + "schema:email" "cam@example.org" + "ex:favNums" [5 10] + "schema:age" 34 + "ex:last" "Jones" + "schema:name" "Cam" + "id" "ex:cam" + "ex:friend" [{"id" "ex:brian"} {"id" "ex:alice"}] + "ex:favColor" "Blue"} + {"id" "ex:alice" + "rdf:type" ["ex:User"] + "schema:name" "Alice" + "ex:last" "Smith" + "schema:email" "alice@example.org" + "schema:age" 42 + "ex:favNums" [9 42 76] + "ex:favColor" "Green"} + {"id" "ex:brian" + "rdf:type" ["ex:User"] + "schema:name" "Brian" + "ex:last" "Smith" + "schema:email" "brian@example.org" + "schema:age" 50 + "ex:favColor" "Green" + "ex:favNums" 7}] + @(fluree/query db {"select" {"?s" ["*"]} + "where" [["?s" "type" "ex:User"]]})))) (testing "tuple" - (is (= [{:id :ex/alice - :rdf/type [:ex/User] - :schema/name "Alice" - :ex/last "Smith" - :schema/email "alice@example.org" - :schema/age 42 - :ex/favNums [9 42 76] - :ex/favColor "Green"}] - @(fluree/query db {:select {"?s" ["*"]} - :where [["?s" :schema/name "Alice"]]}))) - (is (= [{:rdf/type [:ex/User] - :schema/email "cam@example.org" - :ex/favNums [5 10] - :schema/age 34 - :ex/last "Jones" - :schema/name "Cam" - :id :ex/cam - :ex/friend [{:id :ex/brian} {:id :ex/alice}] - :ex/favColor "Blue"} - {:id :ex/alice - :rdf/type [:ex/User] - :schema/name "Alice" - :ex/last "Smith" - :schema/email "alice@example.org" - :schema/age 42 - :ex/favNums [9 42 76] - :ex/favColor "Green"} - {:id :ex/brian, - :rdf/type [:ex/User], - :ex/favNums 7, - :ex/favColor "Green", - :schema/age 50, - :ex/last "Smith", - :schema/email "brian@example.org", - :schema/name "Brian"}] - @(fluree/query db {:select {"?s" ["*"]} - :where [["?s" :ex/favColor "?color"]]}))) - (is (= [{:id :ex/alice - :rdf/type [:ex/User] - :schema/name "Alice" - :ex/last "Smith" - :schema/email "alice@example.org" - :schema/age 42 - :ex/favNums [9 42 76] - :ex/favColor "Green"}] - @(fluree/query db {:select {"?s" ["*"]} - :where [["?s" :schema/age 42]]}))) - (is (= [{:id :ex/alice, - :rdf/type [:ex/User], - :ex/favNums [9 42 76], - :ex/favColor "Green", - :schema/age 42, - :ex/last "Smith", - :schema/email "alice@example.org", - :schema/name "Alice"}] - @(fluree/query db {:select {"?s" ["*"]} - :where [["?s" :schema/age 42] - ["?s" :ex/favColor "Green"]]}))))))) + (is (= [{"id" "ex:alice" + "rdf:type" ["ex:User"] + "schema:name" "Alice" + "ex:last" "Smith" + "schema:email" "alice@example.org" + "schema:age" 42 + "ex:favNums" [9 42 76] + "ex:favColor" "Green"}] + @(fluree/query db {"select" {"?s" ["*"]} + "where" [["?s" "schema:name" "Alice"]]}))) + (is (= [{"rdf:type" ["ex:User"] + "schema:email" "cam@example.org" + "ex:favNums" [5 10] + "schema:age" 34 + "ex:last" "Jones" + "schema:name" "Cam" + "id" "ex:cam" + "ex:friend" [{"id" "ex:brian"} {"id" "ex:alice"}] + "ex:favColor" "Blue"} + {"id" "ex:alice" + "rdf:type" ["ex:User"] + "schema:name" "Alice" + "ex:last" "Smith" + "schema:email" "alice@example.org" + "schema:age" 42 + "ex:favNums" [9 42 76] + "ex:favColor" "Green"} + {"id" "ex:brian", + "rdf:type" ["ex:User"], + "ex:favNums" 7, + "ex:favColor" "Green", + "schema:age" 50, + "ex:last" "Smith", + "schema:email" "brian@example.org", + "schema:name" "Brian"}] + @(fluree/query db {"select" {"?s" ["*"]} + "where" [["?s" "ex:favColor" "?color"]]}))) + (is (= [{"id" "ex:alice" + "rdf:type" ["ex:User"] + "schema:name" "Alice" + "ex:last" "Smith" + "schema:email" "alice@example.org" + "schema:age" 42 + "ex:favNums" [9 42 76] + "ex:favColor" "Green"}] + @(fluree/query db {"select" {"?s" ["*"]} + "where" [["?s" "schema:age" 42]]}))) + (is (= [{"id" "ex:alice", + "rdf:type" ["ex:User"], + "ex:favNums" [9 42 76], + "ex:favColor" "Green", + "schema:age" 42, + "ex:last" "Smith", + "schema:email" "alice@example.org", + "schema:name" "Alice"}] + @(fluree/query db {"select" {"?s" ["*"]} + "where" [["?s" "schema:age" 42] + ["?s" "ex:favColor" "Green"]]}))))))) diff --git a/test/fluree/db/query/json_ld_compound_test.clj b/test/fluree/db/query/json_ld_compound_test.clj index ca7f11d02..97ed254ce 100644 --- a/test/fluree/db/query/json_ld_compound_test.clj +++ b/test/fluree/db/query/json_ld_compound_test.clj @@ -7,145 +7,151 @@ (deftest ^:integration simple-compound-queries (testing "Simple compound queries." (let [conn (test-utils/create-conn) - ledger @(fluree/create conn "query/compounda" {:defaultContext ["" {:ex "http://example.org/ns/"}]}) + ledger @(fluree/create conn "query/compounda" + {"defaults" + {"@context" + ["" {"ex" "http://example.org/ns/"}]}}) db @(fluree/stage (fluree/db ledger) - [{:id :ex/brian, - :type :ex/User, - :schema/name "Brian" - :schema/email "brian@example.org" - :schema/age 50 - :ex/favNums 7} - {:id :ex/alice, - :type :ex/User, - :schema/name "Alice" - :schema/email "alice@example.org" - :schema/age 50 - :ex/favNums [42, 76, 9]} - {:id :ex/cam, - :type :ex/User, - :schema/name "Cam" - :schema/email "cam@example.org" - :schema/age 34 - :ex/favNums [5, 10] - :ex/friend [:ex/brian :ex/alice]}]) + [{"id" "ex:brian", + "type" "ex:User", + "schema:name" "Brian" + "schema:email" "brian@example.org" + "schema:age" 50 + "ex:favNums" 7} + {"id" "ex:alice", + "type" "ex:User", + "schema:name" "Alice" + "schema:email" "alice@example.org" + "schema:age" 50 + "ex:favNums" [42, 76, 9]} + {"id" "ex:cam", + "type" "ex:User", + "schema:name" "Cam" + "schema:email" "cam@example.org" + "schema:age" 34 + "ex:favNums" [5, 10] + "ex:friend" [{"id" "ex:brian"} {"id" "ex:alice"}]}]) two-tuple-select-with-crawl - @(fluree/query db '{:select [?age {?f [:*]}] - :where [[?s :schema/name "Cam"] - [?s :ex/friend ?f] - [?f :schema/age ?age]]}) + @(fluree/query db '{"select" [?age {?f ["*"]}] + "where" [[?s "schema:name" "Cam"] + [?s "ex:friend" ?f] + [?f "schema:age" ?age]]}) two-tuple-select-with-crawl+var - @(fluree/query db '{:select [?age {?f [:*]}] - :where [[?s :schema/name ?name] - [?s :ex/friend ?f] - [?f :schema/age ?age]] - :values [?name ["Cam"]]})] - - (is (= two-tuple-select-with-crawl - two-tuple-select-with-crawl+var - [[50 {:id :ex/brian, - :rdf/type [:ex/User], - :schema/name "Brian", - :schema/email "brian@example.org", - :schema/age 50, - :ex/favNums 7}] - [50 {:id :ex/alice, - :rdf/type [:ex/User], - :schema/name "Alice", - :schema/email "alice@example.org", - :schema/age 50, - :ex/favNums [9, 42, 76]}]])) + @(fluree/query db '{"select" [?age {?f ["*"]}] + "where" [[?s "schema:name" ?name] + [?s "ex:friend" ?f] + [?f "schema:age" ?age]] + "values" [?name ["Cam"]]})] + + (let [expected [[50 {"id" "ex:brian", + "rdf:type" ["ex:User"], + "schema:name" "Brian", + "schema:email" "brian@example.org", + "schema:age" 50, + "ex:favNums" 7}] + [50 {"id" "ex:alice", + "rdf:type" ["ex:User"], + "schema:name" "Alice", + "schema:email" "alice@example.org", + "schema:age" 50, + "ex:favNums" [9, 42, 76]}]]] + + (is (= expected two-tuple-select-with-crawl)) + + (is (= expected two-tuple-select-with-crawl+var))) + ;; here we have pass-through variables (?name and ?age) which must get "passed through" ;; the last where statements into the select statement - (is (= @(fluree/query db {:context {:schema "http://schema.org/" - :ex "http://example.org/ns/"} - :select ['?name '?age '?email] - :where [['?s :schema/name "Cam"] - ['?s :ex/friend '?f] - ['?f :schema/name '?name] - ['?f :schema/age '?age] - ['?f :schema/email '?email]]}) + (is (= @(fluree/query db {"@context" {"schema" "http://schema.org/" + "ex" "http://example.org/ns/"} + "select" ['?name '?age '?email] + "where" [['?s "schema:name" "Cam"] + ['?s "ex:friend" '?f] + ['?f "schema:name" '?name] + ['?f "schema:age" '?age] + ['?f "schema:email" '?email]]}) [["Brian" 50 "brian@example.org"] ["Alice" 50 "alice@example.org"]]) "Prior where statement variables may not be passing through to select results") ;; same as prior query, but using selectOne - (is (= @(fluree/query db {:context {:schema "http://schema.org/" - :ex "http://example.org/ns/"} - :selectOne ['?name '?age '?email] - :where [['?s :schema/name "Cam"] - ['?s :ex/friend '?f] - ['?f :schema/name '?name] - ['?f :schema/age '?age] - ['?f :schema/email '?email]]}) + (is (= @(fluree/query db {"@context" {"schema" "http://schema.org/" + "ex" "http://example.org/ns/"} + "selectOne" ['?name '?age '?email] + "where" [['?s "schema:name" "Cam"] + ['?s "ex:friend" '?f] + ['?f "schema:name" '?name] + ['?f "schema:age" '?age] + ['?f "schema:email" '?email]]}) ["Brian" 50 "brian@example.org"]) "selectOne should only return a single result, like (first ...)") ;; if mixing multi-cardinality results along with single cardinality, there ;; should be a result output for every multi-cardinality value and the single ;; cardinality values should duplicate - (is (= @(fluree/query db {:context {:schema "http://schema.org/" - :ex "http://example.org/ns/"} - :select ['?name '?favNums] - :where [['?s :schema/name '?name] - ['?s :ex/favNums '?favNums]]}) + (is (= @(fluree/query db {"context" {"schema" "http://schema.org/" + "ex" "http://example.org/ns/"} + "select" ['?name '?favNums] + "where" [['?s "schema:name" '?name] + ['?s "ex:favNums" '?favNums]]}) [["Cam" 5] ["Cam" 10] ["Alice" 9] ["Alice" 42] ["Alice" 76] ["Brian" 7]]) "Multi-cardinality values should duplicate non-multicardinality values ") ;; ordering by a single variable - (is (= @(fluree/query db {:context {:schema "http://schema.org/" - :ex "http://example.org/ns/"} - :select ['?name '?favNums] - :where [['?s :schema/name '?name] - ['?s :ex/favNums '?favNums]] - :orderBy '?favNums}) + (is (= @(fluree/query db {"@context" {"schema" "http://schema.org/" + "ex" "http://example.org/ns/"} + "select" ['?name '?favNums] + "where" [['?s "schema:name" '?name] + ['?s "ex:favNums" '?favNums]] + "orderBy" '?favNums}) [["Cam" 5] ["Brian" 7] ["Alice" 9] ["Cam" 10] ["Alice" 42] ["Alice" 76]]) "Ordering of favNums not in ascending order.") ;; ordering by a single variable descending - (is (= @(fluree/query db {:context {:schema "http://schema.org/" - :ex "http://example.org/ns/"} - :select ['?name '?favNums] - :where [['?s :schema/name '?name] - ['?s :ex/favNums '?favNums]] - :orderBy '(desc ?favNums)}) + (is (= @(fluree/query db {"@context" {"schema" "http://schema.org/" + "ex" "http://example.org/ns/"} + "select" ['?name '?favNums] + "where" [['?s "schema:name" '?name] + ['?s "ex:favNums" '?favNums]] + "orderBy" '(desc ?favNums)}) [["Alice" 76] ["Alice" 42] ["Cam" 10] ["Alice" 9] ["Brian" 7] ["Cam" 5]]) "Ordering of favNums not in descending order.") ;; ordering by multiple variables - (is (= @(fluree/query db {:context {:schema "http://schema.org/" - :ex "http://example.org/ns/"} - :select ['?name '?favNums] - :where [['?s :schema/name '?name] - ['?s :ex/favNums '?favNums]] - :orderBy ['?name '(desc ?favNums)]}) + (is (= @(fluree/query db {"@context" {"schema" "http://schema.org/" + "ex" "http://example.org/ns/"} + "select" ['?name '?favNums] + "where" [['?s "schema:name" '?name] + ['?s "ex:favNums" '?favNums]] + "orderBy" ['?name '(desc ?favNums)]}) [["Alice" 76] ["Alice" 42] ["Alice" 9] ["Brian" 7] ["Cam" 10] ["Cam" 5]]) "Ordering of multiple variables not working.") ;; ordering by multiple variables where some are equal, and not all carried to 'select' - (is (= @(fluree/query db {:context {:schema "http://schema.org/" - :ex "http://example.org/ns/"} - :select ['?name '?favNums] - :where [['?s :schema/name '?name] - ['?s :schema/age '?age] - ['?s :ex/favNums '?favNums]] - :orderBy ['?age '?name '(desc ?favNums)]}) + (is (= @(fluree/query db {"@context" {"schema" "http://schema.org/" + "ex" "http://example.org/ns/"} + "select" ['?name '?favNums] + "where" [['?s "schema:name" '?name] + ['?s "schema:age" '?age] + ['?s "ex:favNums" '?favNums]] + "orderBy" ['?age '?name '(desc ?favNums)]}) [["Cam" 10] ["Cam" 5] ["Alice" 76] ["Alice" 42] ["Alice" 9] ["Brian" 7]]) "Ordering of multiple variables where some are equal working.") ;; group-by with a multicardinality value, but not using any aggregate function - (is (= @(fluree/query db {:context {:schema "http://schema.org/" - :ex "http://example.org/ns/"} - :select ['?name '?favNums] - :where [['?s :schema/name '?name] - ['?s :ex/favNums '?favNums]] - :group-by '?name - :order-by '?name}) + (is (= @(fluree/query db {"@context" {"schema" "http://schema.org/" + "ex" "http://example.org/ns/"} + "select" ['?name '?favNums] + "where" [['?s "schema:name" '?name] + ['?s "ex:favNums" '?favNums]] + "group-by" '?name + "order-by" '?name}) [["Alice" [9 42 76]] ["Brian" [7]] ["Cam" [5 10]]]) "Sums of favNums by person are not accurate.") @@ -153,42 +159,42 @@ ;; checking s, p, o values all pulled correctly and all IRIs are resolved from sid integer & compacted (is (= @(fluree/query db - {:select ['?s '?p '?o] - :where [['?s :schema/age 34] - ['?s '?p '?o]]}) - [[:ex/cam :id "http://example.org/ns/cam"] - [:ex/cam :rdf/type :ex/User] - [:ex/cam :schema/name "Cam"] - [:ex/cam :schema/email "cam@example.org"] - [:ex/cam :schema/age 34] - [:ex/cam :ex/favNums 5] - [:ex/cam :ex/favNums 10] - [:ex/cam :ex/friend :ex/brian] - [:ex/cam :ex/friend :ex/alice]]) + {"select" ['?s '?p '?o] + "where" [['?s "schema:age" 34] + ['?s '?p '?o]]}) + [["ex:cam" "id" "http://example.org/ns/cam"] + ["ex:cam" "rdf:type" "ex:User"] + ["ex:cam" "schema:name" "Cam"] + ["ex:cam" "schema:email" "cam@example.org"] + ["ex:cam" "schema:age" 34] + ["ex:cam" "ex:favNums" 5] + ["ex:cam" "ex:favNums" 10] + ["ex:cam" "ex:friend" "ex:brian"] + ["ex:cam" "ex:friend" "ex:alice"]]) "IRIs are resolved from subj ids, whether s, p, or o vals.") ;; checking object-subject joins (is (= @(fluree/query db - '{:select {?s ["*" {:ex/friend ["*"]}]} - :where [[?s :ex/friend ?o] - [?o :schema/name "Alice"]]}) - [{:id :ex/cam, - :rdf/type [:ex/User], - :schema/name "Cam", - :schema/email "cam@example.org", - :schema/age 34, - :ex/favNums [5 10], - :ex/friend - [{:id :ex/brian, - :rdf/type [:ex/User], - :schema/name "Brian", - :schema/email "brian@example.org", - :schema/age 50, - :ex/favNums 7} - {:id :ex/alice, - :rdf/type [:ex/User], - :schema/name "Alice", - :schema/email "alice@example.org", - :schema/age 50, - :ex/favNums [9 42 76]}]}]) + '{"select" {?s ["*" {"ex:friend" ["*"]}]} + "where" [[?s "ex:friend" ?o] + [?o "schema:name" "Alice"]]}) + [{"id" "ex:cam", + "rdf:type" ["ex:User"], + "schema:name" "Cam", + "schema:email" "cam@example.org", + "schema:age" 34, + "ex:favNums" [5 10], + "ex:friend" + [{"id" "ex:brian", + "rdf:type" ["ex:User"], + "schema:name" "Brian", + "schema:email" "brian@example.org", + "schema:age" 50, + "ex:favNums" 7} + {"id" "ex:alice", + "rdf:type" ["ex:User"], + "schema:name" "Alice", + "schema:email" "alice@example.org", + "schema:age" 50, + "ex:favNums" [9 42 76]}]}]) "Subjects appearing as objects should be referenceable.")))) diff --git a/test/fluree/db/query/misc_queries_test.clj b/test/fluree/db/query/misc_queries_test.clj index cc1c9acb7..a20a07b90 100644 --- a/test/fluree/db/query/misc_queries_test.clj +++ b/test/fluree/db/query/misc_queries_test.clj @@ -7,232 +7,240 @@ (deftest ^:integration select-sid (testing "Select index's subject id in query using special keyword" (let [conn (test-utils/create-conn) - ledger @(fluree/create conn "query/subid" {:defaultContext ["" {:ex "http://example.org/ns/"}]}) + ledger @(fluree/create conn "query/subid" + {"defaults" + {"@context" + ["" {"ex" "http://example.org/ns/"}]}}) db @(fluree/stage - (fluree/db ledger) - {:graph [{:id :ex/alice, - :type :ex/User, - :schema/name "Alice"} - {:id :ex/bob, - :type :ex/User, - :schema/name "Bob" - :ex/favArtist {:id :ex/picasso - :schema/name "Picasso"}}]})] - (is (= [{:_id 211106232532993, - :id :ex/bob, - :rdf/type [:ex/User], - :schema/name "Bob", - :ex/favArtist {:_id 211106232532994 - :schema/name "Picasso"}} - {:_id 211106232532992, - :id :ex/alice, - :rdf/type [:ex/User], - :schema/name "Alice"}] - @(fluree/query db {:select {'?s [:_id :* {:ex/favArtist [:_id :schema/name]}]} - :where [['?s :type :ex/User]]})))))) + (fluree/db ledger) + {"@graph" [{"id" "ex:alice", + "type" "ex:User", + "schema:name" "Alice"} + {"id" "ex:bob", + "type" "ex:User", + "schema:name" "Bob" + "ex:favArtist" {"id" "ex:picasso" + "schema:name" "Picasso"}}]})] + (is (= [{"_id" 211106232532993, + "id" "ex:bob", + "rdf:type" ["ex:User"], + "schema:name" "Bob", + "ex:favArtist" {"_id" 211106232532994 + "schema:name" "Picasso"}} + {"_id" 211106232532992, + "id" "ex:alice", + "rdf:type" ["ex:User"], + "schema:name" "Alice"}] + @(fluree/query db {"select" {'?s ["_id" "*" {"ex:favArtist" ["_id" "schema:name"]}]} + "where" [['?s "type" "ex:User"]]})))))) (deftest ^:integration result-formatting (let [conn (test-utils/create-conn) - ledger @(fluree/create conn "query-context" {:defaultContext ["" {:ex "http://example.org/ns/"}]}) - db @(test-utils/transact ledger [{:id :ex/dan - :ex/x 1}])] - (is (= [{:id :foo/dan - :foo/x 1}] - @(fluree/query db {"@context" ["" {:foo "http://example.org/ns/"}] - :where [['?s :id :foo/dan]] - :select {'?s [:*]}})) + ledger @(fluree/create conn "query-context" + {"defaults" + {"@context" ["" {"ex" "http://example.org/ns/"}]}}) + db @(test-utils/transact ledger [{"id" "ex:dan" + "ex:x" 1}])] + (is (= [{"id" "foo:dan" + "foo:x" 1}] + @(fluree/query db {"@context" ["" {"foo" "http://example.org/ns/"}] + "where" [['?s "id" "foo:dan"]] + "select" {'?s ["*"]}})) "default unwrapped objects") - (is (= [{:id :foo/dan - :foo/x [1]}] - @(fluree/query db {"@context" ["" {:foo "http://example.org/ns/" - :foo/x {:container :set}}] - :where [['?s :id :foo/dan]] - :select {'?s [:*]}})) + (is (= [{"id" "foo:dan" + "foo:x" [1]}] + @(fluree/query db {"@context" ["" {"foo" "http://example.org/ns/" + "foo:x" {"@container" "set"}}] + "where" [['?s "id" "foo:dan"]] + "select" {'?s ["*"]}})) "override unwrapping with :set") - (is (= [{:id :ex/dan + (is (= [{"id" "ex:dan" "foo:x" [1]}] @(fluree/query db {"@context" ["" {"foo" "http://example.org/ns/" "foo:x" {"@container" "@list"}}] - :where [['?s "@id" "foo:dan"]] - :select {'?s ["*"]}})) + "where" [['?s "@id" "foo:dan"]] + "select" {'?s ["*"]}})) "override unwrapping with @list"))) (deftest ^:integration s+p+o-full-db-queries (with-redefs [fluree.db.util.core/current-time-iso (fn [] "1970-01-01T00:12:00.00000Z")] (let [conn (test-utils/create-conn) - ledger @(fluree/create conn "query/everything" {:defaultContext ["" {:ex "http://example.org/ns/"}]}) + ledger @(fluree/create conn "query/everything" + {"defaults" + {"@context" + ["" {"ex" "http://example.org/ns/"}]}}) db @(fluree/stage (fluree/db ledger) - {:graph [{:id :ex/alice, - :type :ex/User, - :schema/name "Alice" - :schema/email "alice@flur.ee" - :schema/age 42} - {:id :ex/bob, - :type :ex/User, - :schema/name "Bob" - :schema/age 22} - {:id :ex/jane, - :type :ex/User, - :schema/name "Jane" - :schema/email "jane@flur.ee" - :schema/age 30}]})] + {"@graph" [{"id" "ex:alice", + "type" "ex:User", + "schema:name" "Alice" + "schema:email" "alice@flur.ee" + "schema:age" 42} + {"id" "ex:bob", + "type" "ex:User", + "schema:name" "Bob" + "schema:age" 22} + {"id" "ex:jane", + "type" "ex:User", + "schema:name" "Jane" + "schema:email" "jane@flur.ee" + "schema:age" 30}]})] (testing "Query that pulls entire database." - (is (= [[:ex/jane :id "http://example.org/ns/jane"] - [:ex/jane :rdf/type :ex/User] - [:ex/jane :schema/name "Jane"] - [:ex/jane :schema/email "jane@flur.ee"] - [:ex/jane :schema/age 30] - [:ex/bob :id "http://example.org/ns/bob"] - [:ex/bob :rdf/type :ex/User] - [:ex/bob :schema/name "Bob"] - [:ex/bob :schema/age 22] - [:ex/alice :id "http://example.org/ns/alice"] - [:ex/alice :rdf/type :ex/User] - [:ex/alice :schema/name "Alice"] - [:ex/alice :schema/email "alice@flur.ee"] - [:ex/alice :schema/age 42] - [:schema/age :id "http://schema.org/age"] - [:schema/email :id "http://schema.org/email"] - [:schema/name :id "http://schema.org/name"] - [:ex/User :id "http://example.org/ns/User"] - [:ex/User :rdf/type :rdfs/Class] - [:rdfs/Class :id "http://www.w3.org/2000/01/rdf-schema#Class"] - [:rdf/type :id "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"] - [:id :id "@id"]] - @(fluree/query db {:select ['?s '?p '?o] - :where [['?s '?p '?o]]})) + (is (= [["ex:jane" "id" "http://example.org/ns/jane"] + ["ex:jane" "rdf:type" "ex:User"] + ["ex:jane" "schema:name" "Jane"] + ["ex:jane" "schema:email" "jane@flur.ee"] + ["ex:jane" "schema:age" 30] + ["ex:bob" "id" "http://example.org/ns/bob"] + ["ex:bob" "rdf:type" "ex:User"] + ["ex:bob" "schema:name" "Bob"] + ["ex:bob" "schema:age" 22] + ["ex:alice" "id" "http://example.org/ns/alice"] + ["ex:alice" "rdf:type" "ex:User"] + ["ex:alice" "schema:name" "Alice"] + ["ex:alice" "schema:email" "alice@flur.ee"] + ["ex:alice" "schema:age" 42] + ["schema:age" "id" "http://schema.org/age"] + ["schema:email" "id" "http://schema.org/email"] + ["schema:name" "id" "http://schema.org/name"] + ["ex:User" "id" "http://example.org/ns/User"] + ["ex:User" "rdf:type" "rdfs:Class"] + ["rdfs:Class" "id" "http://www.w3.org/2000/01/rdf-schema#Class"] + ["rdf:type" "id" "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"] + ["id" "id" "@id"]] + @(fluree/query db {"select" ['?s '?p '?o] + "where" [['?s '?p '?o]]})) "Entire database should be pulled.") - (is (= [{:id :ex/jane, - :rdf/type [:ex/User], - :schema/name "Jane", - :schema/email "jane@flur.ee", - :schema/age 30} - {:id :ex/jane, - :rdf/type [:ex/User], - :schema/name "Jane", - :schema/email "jane@flur.ee", - :schema/age 30} - {:id :ex/jane, - :rdf/type [:ex/User], - :schema/name "Jane", - :schema/email "jane@flur.ee", - :schema/age 30} - {:id :ex/jane, - :rdf/type [:ex/User], - :schema/name "Jane", - :schema/email "jane@flur.ee", - :schema/age 30} - {:id :ex/jane, - :rdf/type [:ex/User], - :schema/name "Jane", - :schema/email "jane@flur.ee", - :schema/age 30} - {:id :ex/bob, - :rdf/type [:ex/User], - :schema/name "Bob", - :schema/age 22} - {:id :ex/bob, - :rdf/type [:ex/User], - :schema/name "Bob", - :schema/age 22} - {:id :ex/bob, - :rdf/type [:ex/User], - :schema/name "Bob", - :schema/age 22} - {:id :ex/bob, - :rdf/type [:ex/User], - :schema/name "Bob", - :schema/age 22} - {:id :ex/alice, - :rdf/type [:ex/User], - :schema/name "Alice", - :schema/email "alice@flur.ee", - :schema/age 42} - {:id :ex/alice, - :rdf/type [:ex/User], - :schema/name "Alice", - :schema/email "alice@flur.ee", - :schema/age 42} - {:id :ex/alice, - :rdf/type [:ex/User], - :schema/name "Alice", - :schema/email "alice@flur.ee", - :schema/age 42} - {:id :ex/alice, - :rdf/type [:ex/User], - :schema/name "Alice", - :schema/email "alice@flur.ee", - :schema/age 42} - {:id :ex/alice, - :rdf/type [:ex/User], - :schema/name "Alice", - :schema/email "alice@flur.ee", - :schema/age 42} - {:id :schema/age} - {:id :schema/email} - {:id :schema/name} - {:id :ex/User, :rdf/type [:rdfs/Class]} - {:id :ex/User, :rdf/type [:rdfs/Class]} - {:id :rdfs/Class} - {:id :rdf/type} - {:id :id}] - @(fluree/query db {:select {'?s ["*"]} - :where [['?s '?p '?o]]})) + (is (= [{"id" "ex:jane", + "rdf:type" ["ex:User"], + "schema:name" "Jane", + "schema:email" "jane@flur.ee", + "schema:age" 30} + {"id" "ex:jane", + "rdf:type" ["ex:User"], + "schema:name" "Jane", + "schema:email" "jane@flur.ee", + "schema:age" 30} + {"id" "ex:jane", + "rdf:type" ["ex:User"], + "schema:name" "Jane", + "schema:email" "jane@flur.ee", + "schema:age" 30} + {"id" "ex:jane", + "rdf:type" ["ex:User"], + "schema:name" "Jane", + "schema:email" "jane@flur.ee", + "schema:age" 30} + {"id" "ex:jane", + "rdf:type" ["ex:User"], + "schema:name" "Jane", + "schema:email" "jane@flur.ee", + "schema:age" 30} + {"id" "ex:bob", + "rdf:type" ["ex:User"], + "schema:name" "Bob", + "schema:age" 22} + {"id" "ex:bob", + "rdf:type" ["ex:User"], + "schema:name" "Bob", + "schema:age" 22} + {"id" "ex:bob", + "rdf:type" ["ex:User"], + "schema:name" "Bob", + "schema:age" 22} + {"id" "ex:bob", + "rdf:type" ["ex:User"], + "schema:name" "Bob", + "schema:age" 22} + {"id" "ex:alice", + "rdf:type" ["ex:User"], + "schema:name" "Alice", + "schema:email" "alice@flur.ee", + "schema:age" 42} + {"id" "ex:alice", + "rdf:type" ["ex:User"], + "schema:name" "Alice", + "schema:email" "alice@flur.ee", + "schema:age" 42} + {"id" "ex:alice", + "rdf:type" ["ex:User"], + "schema:name" "Alice", + "schema:email" "alice@flur.ee", + "schema:age" 42} + {"id" "ex:alice", + "rdf:type" ["ex:User"], + "schema:name" "Alice", + "schema:email" "alice@flur.ee", + "schema:age" 42} + {"id" "ex:alice", + "rdf:type" ["ex:User"], + "schema:name" "Alice", + "schema:email" "alice@flur.ee", + "schema:age" 42} + {"id" "schema:age"} + {"id" "schema:email"} + {"id" "schema:name"} + {"id" "ex:User", "rdf:type" ["rdfs:Class"]} + {"id" "ex:User", "rdf:type" ["rdfs:Class"]} + {"id" "rdfs:Class"} + {"id" "rdf:type"} + {"id" "id"}] + @(fluree/query db {"select" {'?s ["*"]} + "where" [['?s '?p '?o]]})) "Every triple should be returned.") (let [db* @(fluree/commit! ledger db)] - (is (= [[:ex/jane :id "http://example.org/ns/jane"] - [:ex/jane :rdf/type :ex/User] - [:ex/jane :schema/name "Jane"] - [:ex/jane :schema/email "jane@flur.ee"] - [:ex/jane :schema/age 30] - [:ex/bob :id "http://example.org/ns/bob"] - [:ex/bob :rdf/type :ex/User] - [:ex/bob :schema/name "Bob"] - [:ex/bob :schema/age 22] - [:ex/alice :id "http://example.org/ns/alice"] - [:ex/alice :rdf/type :ex/User] - [:ex/alice :schema/name "Alice"] - [:ex/alice :schema/email "alice@flur.ee"] - [:ex/alice :schema/age 42] - ["did:fluree:TfCzWTrXqF16hvKGjcYiLxRoYJ1B8a6UMH6" :id "did:fluree:TfCzWTrXqF16hvKGjcYiLxRoYJ1B8a6UMH6"] - ["fluree:db:sha256:beh3as6tvb6shyvxs5w4zdt4gbpaa7xterlvqe42sc7r6u625ank" :id "fluree:db:sha256:beh3as6tvb6shyvxs5w4zdt4gbpaa7xterlvqe42sc7r6u625ank"] - ["fluree:db:sha256:beh3as6tvb6shyvxs5w4zdt4gbpaa7xterlvqe42sc7r6u625ank" :f/address "fluree:memory://2921503e8e62f47b598d4504f990a25bc7f7b841c11367599d87884d4bf5f0f8"] - ["fluree:db:sha256:beh3as6tvb6shyvxs5w4zdt4gbpaa7xterlvqe42sc7r6u625ank" :f/flakes 25] - ["fluree:db:sha256:beh3as6tvb6shyvxs5w4zdt4gbpaa7xterlvqe42sc7r6u625ank" :f/size 1888] - ["fluree:db:sha256:beh3as6tvb6shyvxs5w4zdt4gbpaa7xterlvqe42sc7r6u625ank" :f/t 1] - [:schema/age :id "http://schema.org/age"] - [:schema/email :id "http://schema.org/email"] - [:schema/name :id "http://schema.org/name"] - [:ex/User :id "http://example.org/ns/User"] - [:ex/User :rdf/type :rdfs/Class] - [:rdfs/Class :id "http://www.w3.org/2000/01/rdf-schema#Class"] - [:rdf/type :id "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"] - [:f/t :id "https://ns.flur.ee/ledger#t"] - [:f/size :id "https://ns.flur.ee/ledger#size"] - [:f/flakes :id "https://ns.flur.ee/ledger#flakes"] - [:f/context :id "https://ns.flur.ee/ledger#context"] - [:f/branch :id "https://ns.flur.ee/ledger#branch"] - [:f/alias :id "https://ns.flur.ee/ledger#alias"] - [:f/data :id "https://ns.flur.ee/ledger#data"] - [:f/address :id "https://ns.flur.ee/ledger#address"] - [:f/v :id "https://ns.flur.ee/ledger#v"] - ["https://www.w3.org/2018/credentials#issuer" :id "https://www.w3.org/2018/credentials#issuer"] - [:f/time :id "https://ns.flur.ee/ledger#time"] - [:f/message :id "https://ns.flur.ee/ledger#message"] - [:f/previous :id "https://ns.flur.ee/ledger#previous"] - [:id :id "@id"] - ["fluree:commit:sha256:bbjjp52yfifntgso6dtpbzynb3qdyjxb3kwiykksiwei45izv2up3" :id "fluree:commit:sha256:bbjjp52yfifntgso6dtpbzynb3qdyjxb3kwiykksiwei45izv2up3"] - ["fluree:commit:sha256:bbjjp52yfifntgso6dtpbzynb3qdyjxb3kwiykksiwei45izv2up3" :f/time 720000] + (is (= [["ex:jane" "id" "http://example.org/ns/jane"] + ["ex:jane" "rdf:type" "ex:User"] + ["ex:jane" "schema:name" "Jane"] + ["ex:jane" "schema:email" "jane@flur.ee"] + ["ex:jane" "schema:age" 30] + ["ex:bob" "id" "http://example.org/ns/bob"] + ["ex:bob" "rdf:type" "ex:User"] + ["ex:bob" "schema:name" "Bob"] + ["ex:bob" "schema:age" 22] + ["ex:alice" "id" "http://example.org/ns/alice"] + ["ex:alice" "rdf:type" "ex:User"] + ["ex:alice" "schema:name" "Alice"] + ["ex:alice" "schema:email" "alice@flur.ee"] + ["ex:alice" "schema:age" 42] + ["did:fluree:TfCzWTrXqF16hvKGjcYiLxRoYJ1B8a6UMH6" "id" "did:fluree:TfCzWTrXqF16hvKGjcYiLxRoYJ1B8a6UMH6"] + ["fluree:db:sha256:beh3as6tvb6shyvxs5w4zdt4gbpaa7xterlvqe42sc7r6u625ank" "id" "fluree:db:sha256:beh3as6tvb6shyvxs5w4zdt4gbpaa7xterlvqe42sc7r6u625ank"] + ["fluree:db:sha256:beh3as6tvb6shyvxs5w4zdt4gbpaa7xterlvqe42sc7r6u625ank" "f:address" "fluree:memory://2921503e8e62f47b598d4504f990a25bc7f7b841c11367599d87884d4bf5f0f8"] + ["fluree:db:sha256:beh3as6tvb6shyvxs5w4zdt4gbpaa7xterlvqe42sc7r6u625ank" "f:flakes" 25] + ["fluree:db:sha256:beh3as6tvb6shyvxs5w4zdt4gbpaa7xterlvqe42sc7r6u625ank" "f:size" 1888] + ["fluree:db:sha256:beh3as6tvb6shyvxs5w4zdt4gbpaa7xterlvqe42sc7r6u625ank" "f:t" 1] + ["schema:age" "id" "http://schema.org/age"] + ["schema:email" "id" "http://schema.org/email"] + ["schema:name" "id" "http://schema.org/name"] + ["ex:User" "id" "http://example.org/ns/User"] + ["ex:User" "rdf:type" "rdfs:Class"] + ["rdfs:Class" "id" "http://www.w3.org/2000/01/rdf-schema#Class"] + ["rdf:type" "id" "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"] + ["f:t" "id" "https://ns.flur.ee/ledger#t"] + ["f:size" "id" "https://ns.flur.ee/ledger#size"] + ["f:flakes" "id" "https://ns.flur.ee/ledger#flakes"] + ["f:context" "id" "https://ns.flur.ee/ledger#context"] + ["f:branch" "id" "https://ns.flur.ee/ledger#branch"] + ["f:alias" "id" "https://ns.flur.ee/ledger#alias"] + ["f:data" "id" "https://ns.flur.ee/ledger#data"] + ["f:address" "id" "https://ns.flur.ee/ledger#address"] + ["f:v" "id" "https://ns.flur.ee/ledger#v"] + ["https://www.w3.org/2018/credentials#issuer" "id" "https://www.w3.org/2018/credentials#issuer"] + ["f:time" "id" "https://ns.flur.ee/ledger#time"] + ["f:message" "id" "https://ns.flur.ee/ledger#message"] + ["f:previous" "id" "https://ns.flur.ee/ledger#previous"] + ["id" "id" "@id"] + ["fluree:commit:sha256:bbjjp52yfifntgso6dtpbzynb3qdyjxb3kwiykksiwei45izv2up3" "id" "fluree:commit:sha256:bbjjp52yfifntgso6dtpbzynb3qdyjxb3kwiykksiwei45izv2up3"] + ["fluree:commit:sha256:bbjjp52yfifntgso6dtpbzynb3qdyjxb3kwiykksiwei45izv2up3" "f:time" 720000] ["fluree:commit:sha256:bbjjp52yfifntgso6dtpbzynb3qdyjxb3kwiykksiwei45izv2up3" "https://www.w3.org/2018/credentials#issuer" "did:fluree:TfCzWTrXqF16hvKGjcYiLxRoYJ1B8a6UMH6"] - ["fluree:commit:sha256:bbjjp52yfifntgso6dtpbzynb3qdyjxb3kwiykksiwei45izv2up3" :f/v 0] - ["fluree:commit:sha256:bbjjp52yfifntgso6dtpbzynb3qdyjxb3kwiykksiwei45izv2up3" :f/address "fluree:memory://1b766e9d594d0afb754aeb7b9bc62158ff319273552a0b7fda9bb77b001e8acb"] - ["fluree:commit:sha256:bbjjp52yfifntgso6dtpbzynb3qdyjxb3kwiykksiwei45izv2up3" :f/data "fluree:db:sha256:beh3as6tvb6shyvxs5w4zdt4gbpaa7xterlvqe42sc7r6u625ank"] - ["fluree:commit:sha256:bbjjp52yfifntgso6dtpbzynb3qdyjxb3kwiykksiwei45izv2up3" :f/alias "query/everything"] - ["fluree:commit:sha256:bbjjp52yfifntgso6dtpbzynb3qdyjxb3kwiykksiwei45izv2up3" :f/branch "main"] - ["fluree:commit:sha256:bbjjp52yfifntgso6dtpbzynb3qdyjxb3kwiykksiwei45izv2up3" :f/context "fluree:memory://b6dcf8968183239ecc7a664025f247de5b7859ac18cdeaace89aafc421eeddee"]] - @(fluree/query db* {:select ['?s '?p '?o] - :where [['?s '?p '?o]]})))))))) + ["fluree:commit:sha256:bbjjp52yfifntgso6dtpbzynb3qdyjxb3kwiykksiwei45izv2up3" "f:v" 0] + ["fluree:commit:sha256:bbjjp52yfifntgso6dtpbzynb3qdyjxb3kwiykksiwei45izv2up3" "f:address" "fluree:memory://1b766e9d594d0afb754aeb7b9bc62158ff319273552a0b7fda9bb77b001e8acb"] + ["fluree:commit:sha256:bbjjp52yfifntgso6dtpbzynb3qdyjxb3kwiykksiwei45izv2up3" "f:data" "fluree:db:sha256:beh3as6tvb6shyvxs5w4zdt4gbpaa7xterlvqe42sc7r6u625ank"] + ["fluree:commit:sha256:bbjjp52yfifntgso6dtpbzynb3qdyjxb3kwiykksiwei45izv2up3" "f:alias" "query/everything"] + ["fluree:commit:sha256:bbjjp52yfifntgso6dtpbzynb3qdyjxb3kwiykksiwei45izv2up3" "f:branch" "main"] + ["fluree:commit:sha256:bbjjp52yfifntgso6dtpbzynb3qdyjxb3kwiykksiwei45izv2up3" "f:context" "fluree:memory://b6dcf8968183239ecc7a664025f247de5b7859ac18cdeaace89aafc421eeddee"]] + @(fluree/query db* {"select" ['?s '?p '?o] + "where" [['?s '?p '?o]]})))))))) (deftest ^:integration illegal-reference-test (testing "Illegal reference queries" @@ -240,16 +248,16 @@ people (test-utils/load-people conn) db (fluree/db people)] (testing "with non-string objects" - (let [test-subject @(fluree/query db {:select ['?s '?p] - :where [['?s '?p 22]]})] + (let [test-subject @(fluree/query db {"select" ['?s '?p] + "where" [['?s '?p 22]]})] (is (util/exception? test-subject) "return errors") (is (= :db/invalid-query (-> test-subject ex-data :error)) "have 'invalid query' error codes"))) (testing "with string objects" - (let [test-subject @(fluree/query db {:select ['?s '?p] - :where [['?s '?p "Bob"]]})] + (let [test-subject @(fluree/query db {"select" ['?s '?p] + "where" [['?s '?p "Bob"]]})] (is (util/exception? test-subject) "return errors") (is (= :db/invalid-query @@ -258,47 +266,50 @@ (deftest ^:integration class-queries (let [conn (test-utils/create-conn) - ledger @(fluree/create conn "query/class" {:defaultContext ["" {:ex "http://example.org/ns/"}]}) + ledger @(fluree/create conn "query/class" + {"defaults" + {"@context" + ["" {"ex" "http://example.org/ns/"}]}}) db @(fluree/stage - (fluree/db ledger) - [{:id :ex/alice, - :type :ex/User, - :schema/name "Alice" - :schema/email "alice@flur.ee" - :schema/age 42} - {:id :ex/bob, - :type :ex/User, - :schema/name "Bob" - :schema/age 22} - {:id :ex/jane, - :type :ex/User, - :schema/name "Jane" - :schema/email "jane@flur.ee" - :schema/age 30} - {:id :ex/dave - :type :ex/nonUser - :schema/name "Dave"}])] + (fluree/db ledger) + [{"id" "ex:alice", + "type" "ex:User", + "schema:name" "Alice" + "schema:email" "alice@flur.ee" + "schema:age" 42} + {"id" "ex:bob", + "type" "ex:User", + "schema:name" "Bob" + "schema:age" 22} + {"id" "ex:jane", + "type" "ex:User", + "schema:name" "Jane" + "schema:email" "jane@flur.ee" + "schema:age" 30} + {"id" "ex:dave" + "type" "ex:nonUser" + "schema:name" "Dave"}])] (testing "rdf/type" - (is (= [[:ex/User]] - @(fluree/query db '{:select [?class] - :where [[:ex/jane :rdf/type ?class]]}))) - (is (= [[:ex/dave :ex/nonUser] - [:ex/jane :ex/User] - [:ex/bob :ex/User] - [:ex/alice :ex/User] - [:ex/nonUser :rdfs/Class] - [:ex/User :rdfs/Class]] - @(fluree/query db '{:select [?s ?class] - :where [[?s :rdf/type ?class]]})))) + (is (= [["ex:User"]] + @(fluree/query db '{"select" [?class] + "where" [["ex:jane" "rdf:type" ?class]]}))) + (is (= [["ex:dave" "ex:nonUser"] + ["ex:jane" "ex:User"] + ["ex:bob" "ex:User"] + ["ex:alice" "ex:User"] + ["ex:nonUser" "rdfs:Class"] + ["ex:User" "rdfs:Class"]] + @(fluree/query db '{"select" [?s ?class] + "where" [[?s "rdf:type" ?class]]})))) (testing "shacl targetClass" (let [shacl-db @(fluree/stage - (fluree/db ledger) - {:context {:ex "http://example.org/ns/"} - :id :ex/UserShape, - :type [:sh/NodeShape], - :sh/targetClass :ex/User - :sh/property [{:sh/path :schema/name - :sh/datatype :xsd/string}]})] - (is (= [[:ex/User]] - @(fluree/query shacl-db '{:select [?class] - :where [[:ex/UserShape :sh/targetClass ?class]]}))))))) + (fluree/db ledger) + {"@context" {"ex" "http://example.org/ns/"} + "id" "ex:UserShape", + "type" ["sh:NodeShape"], + "sh:targetClass" "ex:User" + "sh:property" [{"sh:path" "schema:name" + "sh:datatype" "xsd:string"}]})] + (is (= [["ex:User"]] + @(fluree/query shacl-db '{"select" [?class] + "where" [["ex:UserShape" "sh:targetClass" ?class]]}))))))) diff --git a/test/fluree/db/query/optional_query_test.clj b/test/fluree/db/query/optional_query_test.clj index 1b0f2fcf2..435e2aa84 100644 --- a/test/fluree/db/query/optional_query_test.clj +++ b/test/fluree/db/query/optional_query_test.clj @@ -1,75 +1,76 @@ (ns fluree.db.query.optional-query-test (:require - [clojure.string :as str] [clojure.test :refer :all] - [fluree.db.test-utils :as test-utils] [fluree.db.json-ld.api :as fluree] - [fluree.db.util.log :as log])) + [fluree.db.test-utils :as test-utils])) (deftest ^:integration optional-queries (testing "Testing various 'optional' query clauses." (let [conn (test-utils/create-conn) - ledger @(fluree/create conn "query/optional" {:defaultContext ["" {:ex "http://example.org/ns/"}]}) + ledger @(fluree/create conn "query/optional" + {"defaults" + {"@context" + ["" {"ex" "http://example.org/ns/"}]}}) db @(fluree/stage (fluree/db ledger) - [{:id :ex/brian, - :type :ex/User, - :schema/name "Brian" - :ex/friend [:ex/alice]} - {:id :ex/alice, - :type :ex/User, - :ex/favColor "Green" - :schema/email "alice@flur.ee" - :schema/name "Alice"} - {:id :ex/cam, - :type :ex/User, - :schema/name "Cam" - :schema/email "cam@flur.ee" - :ex/friend [:ex/brian :ex/alice]}])] + [{"id" "ex:brian", + "type" "ex:User", + "schema:name" "Brian" + "ex:friend" ["ex:alice"]} + {"id" "ex:alice", + "type" "ex:User", + "ex:favColor" "Green" + "schema:email" "alice@flur.ee" + "schema:name" "Alice"} + {"id" "ex:cam", + "type" "ex:User", + "schema:name" "Cam" + "schema:email" "cam@flur.ee" + "ex:friend" ["ex:brian" "ex:alice"]}])] ;; basic single optional statement - (is (= @(fluree/query db '{:select [?name ?favColor] - :where [[?s :rdf/type :ex/User] - [?s :schema/name ?name] - {:optional [?s :ex/favColor ?favColor]}]}) + (is (= @(fluree/query db '{"select" [?name ?favColor] + "where" [[?s "rdf:type" "ex:User"] + [?s "schema:name" ?name] + {"optional" [?s "ex:favColor" ?favColor]}]}) [["Cam" nil] ["Alice" "Green"] ["Brian" nil]]) "Cam, Alice and Brian should all return, but only Alica has a favColor") ;; including another pass-through variable - note Brian doesn't have an email - (is (= @(fluree/query db '{:select [?name ?favColor ?email] - :where [[?s :rdf/type :ex/User] - [?s :schema/name ?name] - [?s :schema/email ?email] - {:optional [?s :ex/favColor ?favColor]}]}) + (is (= @(fluree/query db '{"select" [?name ?favColor ?email] + "where" [[?s "rdf:type" "ex:User"] + [?s "schema:name" ?name] + [?s "schema:email" ?email] + {"optional" [?s "ex:favColor" ?favColor]}]}) [["Cam" nil "cam@flur.ee"] ["Alice" "Green" "alice@flur.ee"]])) ;; including another pass-through variable, but with 'optional' sandwiched - (is (= @(fluree/query db '{:select [?name ?favColor ?email] - :where [[?s :rdf/type :ex/User] - [?s :schema/name ?name] - {:optional [?s :ex/favColor ?favColor]} - [?s :schema/email ?email]]}) + (is (= @(fluree/query db '{"select" [?name ?favColor ?email] + "where" [[?s "rdf:type" "ex:User"] + [?s "schema:name" ?name] + {"optional" [?s "ex:favColor" ?favColor]} + [?s "schema:email" ?email]]}) [["Cam" nil "cam@flur.ee"] ["Alice" "Green" "alice@flur.ee"]])) ;; query with two optionals! - (is (= @(fluree/query db '{:select [?name ?favColor ?email] - :where [[?s :rdf/type :ex/User] - [?s :schema/name ?name] - {:optional [?s :ex/favColor ?favColor]} - {:optional [?s :schema/email ?email]}]}) + (is (= @(fluree/query db '{"select" [?name ?favColor ?email] + "where" [[?s "rdf:type" "ex:User"] + [?s "schema:name" ?name] + {"optional" [?s "ex:favColor" ?favColor]} + {"optional" [?s "schema:email" ?email]}]}) [["Cam" nil "cam@flur.ee"] ["Alice" "Green" "alice@flur.ee"] ["Brian" nil nil]])) ;; optional with unnecessary embedded vector statement - (is (= @(fluree/query db '{:select [?name ?favColor] - :where [[?s :rdf/type :ex/User] - [?s :schema/name ?name] - {:optional [[?s :ex/favColor ?favColor]]}]}) + (is (= @(fluree/query db '{"select" [?name ?favColor] + "where" [[?s "rdf:type" "ex:User"] + [?s "schema:name" ?name] + {"optional" [[?s "ex:favColor" ?favColor]]}]}) [["Cam" nil] ["Alice" "Green"] ["Brian" nil]]) @@ -79,9 +80,9 @@ (is (= [["Cam" nil nil] ["Alice" "Green" "alice@flur.ee"] ["Brian" nil nil]] - @(fluree/query db '{:select [?name ?favColor ?email] - :where [[?s :rdf/type :ex/User] - [?s :schema/name ?name] - {:optional [[?s :ex/favColor ?favColor] - [?s :schema/email ?email]]}]})) + @(fluree/query db '{"select" [?name ?favColor ?email] + "where" [[?s "rdf:type" "ex:User"] + [?s "schema:name" ?name] + {"optional" [[?s "ex:favColor" ?favColor] + [?s "schema:email" ?email]]}]})) "Multiple optional clauses should work as a left outer join between them")))) diff --git a/test/fluree/db/query/reverse_query_test.clj b/test/fluree/db/query/reverse_query_test.clj index ca120bc03..369406ff0 100644 --- a/test/fluree/db/query/reverse_query_test.clj +++ b/test/fluree/db/query/reverse_query_test.clj @@ -7,39 +7,41 @@ (deftest ^:integration context-reverse-test (testing "Test that the @reverse context values pulls select values back correctly." (let [conn (test-utils/create-conn) - ledger @(fluree/create conn "query/reverse" {:defaultContext ["" {:ex "http://example.org/ns/"}]}) + ledger @(fluree/create conn "query/reverse" {"defaults" + {"@context" + ["" {"ex" "http://example.org/ns/"}]}}) db @(fluree/stage (fluree/db ledger) - [{:id :ex/brian, - :type :ex/User, - :schema/name "Brian" - :ex/friend [:ex/alice]} - {:id :ex/alice, - :type :ex/User, - :schema/name "Alice"} - {:id :ex/cam, - :type :ex/User, - :schema/name "Cam" - :ex/friend [:ex/brian :ex/alice]}])] + [{"id" "ex:brian", + "type" "ex:User", + "schema:name" "Brian" + "ex:friend" [{"id" "ex:alice"}]} + {"id" "ex:alice", + "type" "ex:User", + "schema:name" "Alice"} + {"id" "ex:cam", + "type" "ex:User", + "schema:name" "Cam" + "ex:friend" [{"id" "ex:brian"} {"id" "ex:alice"}]}])] - (is (= @(fluree/query db '{:context ["" {:friended {:reverse :ex/friend}}] - :selectOne {?s [:schema/name :friended]} - :where [[?s :id :ex/brian]]}) - {:schema/name "Brian" - :friended :ex/cam})) + (is (= @(fluree/query db '{"@context" ["" {"friended" {"@reverse" "ex:friend"}}] + "selectOne" {?s ["schema:name" "friended"]} + "where" [[?s "id" "ex:brian"]]}) + {"schema:name" "Brian" + "friended" "ex:cam"})) - (is (= @(fluree/query db '{:context ["" {:friended {:reverse :ex/friend}}], - :selectOne {?s [:schema/name :friended]}, - :where [[?s :id :ex/alice]]}) - {:schema/name "Alice" - :friended [:ex/cam :ex/brian]})) + (is (= @(fluree/query db '{"@context" ["" {"friended" {"@reverse" "ex:friend"}}], + "selectOne" {?s ["schema:name" "friended"]}, + "where" [[?s "id" "ex:alice"]]}) + {"schema:name" "Alice" + "friended" ["ex:cam" "ex:brian"]})) - (is (= @(fluree/query db '{:context ["" {:friended {:reverse :ex/friend}}], - :selectOne {?s [:schema/name {:friended [:*]}]}, - :where [[?s :id :ex/brian]]}) - {:schema/name "Brian", - :friended {:id :ex/cam, - :rdf/type [:ex/User], - :schema/name "Cam", - :ex/friend [{:id :ex/brian} {:id :ex/alice}]}}))))) + (is (= @(fluree/query db '{"@context" ["" {"friended" {"@reverse" "ex:friend"}}], + "selectOne" {?s ["schema:name" {"friended" ["*"]}]}, + "where" [[?s "id" "ex:brian"]]}) + {"schema:name" "Brian", + "friended" {"id" "ex:cam", + "rdf:type" ["ex:User"], + "schema:name" "Cam", + "ex:friend" [{"id" "ex:brian"} {"id" "ex:alice"}]}}))))) diff --git a/test/fluree/db/query/subclass_test.clj b/test/fluree/db/query/subclass_test.clj index c83a26df6..917f8d595 100644 --- a/test/fluree/db/query/subclass_test.clj +++ b/test/fluree/db/query/subclass_test.clj @@ -45,17 +45,17 @@ "rdfs:subClassOf" {"@id" "schema:CreativeWork"}}]})] (is (= @(fluree/query db3 - {:select {'?s [:*]} - :where [['?s :rdf/type :schema/CreativeWork]]}) - [{:id :wiki/Q836821, - :rdf/type [:schema/Movie], - :schema/name "The Hitchhiker's Guide to the Galaxy", - :schema/disambiguatingDescription "2005 British-American comic science fiction film directed by Garth Jennings", - :schema/titleEIDR "10.5240/B752-5B47-DBBE-E5D4-5A3F-N", - :schema/isBasedOn {:id :wiki/Q3107329}} - {:id :wiki/Q3107329, - :rdf/type [:schema/Book], - :schema/name "The Hitchhiker's Guide to the Galaxy", - :schema/isbn "0-330-25864-8", - :schema/author {:id :wiki/Q42}}]) + {"select" {'?s ["*"]} + "where" [['?s "rdf:type" "schema:CreativeWork"]]}) + [{"id" "wiki:Q836821", + "rdf:type" ["schema:Movie"], + "schema:name" "The Hitchhiker's Guide to the Galaxy", + "schema:disambiguatingDescription" "2005 British-American comic science fiction film directed by Garth Jennings", + "schema:titleEIDR" "10.5240/B752-5B47-DBBE-E5D4-5A3F-N", + "schema:isBasedOn" {"id" "wiki:Q3107329"}} + {"id" "wiki:Q3107329", + "rdf:type" ["schema:Book"], + "schema:name" "The Hitchhiker's Guide to the Galaxy", + "schema:isbn" "0-330-25864-8", + "schema:author" {"id" "wiki:Q42"}}]) "CreativeWork query should return both Book and Movie")))) diff --git a/test/fluree/db/query/subject_crawl_reparse_test.clj b/test/fluree/db/query/subject_crawl_reparse_test.clj index 024bda29d..80b540a0c 100644 --- a/test/fluree/db/query/subject_crawl_reparse_test.clj +++ b/test/fluree/db/query/subject_crawl_reparse_test.clj @@ -9,8 +9,9 @@ (deftest test-reparse-as-ssc (let [conn (test-utils/create-conn) ledger @(fluree/create conn "query/parse" - {"default-context" - ["" {"ex" "http://example.org/ns/"}]}) + {"defaults" + {"@context" + ["" {"ex" "http://example.org/ns/"}]}}) db @(fluree/stage (fluree/db ledger) [{"id" "ex:brian", @@ -33,13 +34,13 @@ "schema:email" "cam@example.org" "schema:age" 34 "ex:favNums" [5, 10] - "ex:friend" ["ex:brian" "ex:alice"]}]) + "ex:friend" [{"id" "ex:brian"} {"id" "ex:alice"}]}]) - ssc-q1-parsed (parse/parse-analytical-query* - {"select" {"?s" ["*"]} - "where" [["?s" "schema:name" "Alice"]]} - db) - ssc-q2-parsed (parse/parse-analytical-query* + ssc-q1-parsed (parse/parse-analytical-query* + {"select" {"?s" ["*"]} + "where" [["?s" "schema:name" "Alice"]]} + db) + ssc-q2-parsed (parse/parse-analytical-query* {"select" {"?s" ["*"]} "where" [["?s" "schema:age" 50] ["?s" "ex:favColor" "Blue"]]} @@ -91,7 +92,7 @@ (is (not (reparse/simple-subject-crawl? s+p+o3-parsed)))) (testing "reparse" (let [ssc-q1-reparsed (reparse/re-parse-as-simple-subj-crawl ssc-q1-parsed) - {:keys [where] context "@context"} ssc-q1-reparsed + {:keys [where context]} ssc-q1-reparsed [pattern] where {:keys [s p o]} pattern] (is (not (nil? context))) @@ -104,7 +105,7 @@ (is datatype))) (let [ssc-q2-reparsed (reparse/re-parse-as-simple-subj-crawl ssc-q2-parsed) - {:keys [where] context "@context"} ssc-q2-reparsed + {:keys [where context]} ssc-q2-reparsed [pattern _s-filter] where {:keys [s p o]} pattern] (is (not (nil? context))) diff --git a/test/fluree/db/query/time_travel_test.clj b/test/fluree/db/query/time_travel_test.clj index f5bb1201e..01c321023 100644 --- a/test/fluree/db/query/time_travel_test.clj +++ b/test/fluree/db/query/time_travel_test.clj @@ -9,21 +9,20 @@ (let [conn (test-utils/create-conn) ledger (test-utils/load-movies conn) db (fluree/db ledger) - movies @(fluree/query db {:select '{?s [:*]} - :where '[[?s :rdf/type :schema/Movie]] - :t 2})] + movies @(fluree/query db {"select" '{?s ["*"]} + "where" '[[?s "rdf:type" "schema:Movie"]] + "t" 2})] (is (= 3 (count movies))) (is (every? #{"The Hitchhiker's Guide to the Galaxy" "Back to the Future" "Back to the Future Part II"} - (map :schema/name movies)))))) + (map #(get % "schema:name") movies)))))) (deftest query-with-iso8601-string-t-value-test (testing "only gets results from before that time" (let [;conn (test-utils/create-conn) ; doesn't work see comment below - conn @(fluree/connect {:method :memory - :defaults - {:context test-utils/default-context - :context-type :keyword}}) + conn @(fluree/connect {"method" "memory" + "defaults" + {"@context" test-utils/default-context}}) ;; if the :did default below is present on the conn ;; (as it is w/ test-utils/create-conn) ;; then the tests below fail at the last check @@ -51,8 +50,8 @@ (fn [] start-iso)} (fn [] (let [db1 @(fluree/stage - (fluree/db ledger) - (first test-utils/movies))] + (fluree/db ledger) + (first test-utils/movies))] @(fluree/commit! ledger db1)))) _ (with-redefs-fn {#'util/current-time-millis (fn [] three-loaded-millis) @@ -60,8 +59,8 @@ (fn [] three-loaded-iso)} (fn [] (let [db2 @(fluree/stage - (fluree/db ledger) - (second test-utils/movies))] + (fluree/db ledger) + (second test-utils/movies))] @(fluree/commit! ledger db2)))) _ (with-redefs-fn {#'util/current-time-millis (fn [] all-loaded-millis) @@ -69,20 +68,20 @@ (fn [] all-loaded-iso)} (fn [] (let [db3 @(fluree/stage - (fluree/db ledger) - (nth test-utils/movies 2))] + (fluree/db ledger) + (nth test-utils/movies 2))] @(fluree/commit! ledger db3)))) db (fluree/db ledger) - base-query {:select '{?s [:*]} - :where '[[?s :rdf/type :schema/Movie]]} + base-query {"select" '{?s ["*"]} + "where" '[[?s "rdf:type" "schema:Movie"]]} one-movie @(fluree/query db (assoc base-query - :t after-one-loaded-iso)) + "t" after-one-loaded-iso)) three-movies @(fluree/query db (assoc base-query - :t after-three-loaded-iso)) + "t" after-three-loaded-iso)) all-movies @(fluree/query db (assoc base-query - :t after-all-loaded-iso)) + "t" after-all-loaded-iso)) too-early @(fluree/query db (assoc base-query - :t too-early-iso))] + "t" too-early-iso))] (is (= 1 (count one-movie))) (is (= 3 (count three-movies))) (is (= 4 (count all-movies))) diff --git a/test/fluree/db/query/union_query_test.clj b/test/fluree/db/query/union_query_test.clj index 616cbec21..57edbb3b8 100644 --- a/test/fluree/db/query/union_query_test.clj +++ b/test/fluree/db/query/union_query_test.clj @@ -7,65 +7,66 @@ (deftest ^:integration union-queries (testing "Testing various 'union' query clauses." (let [conn (test-utils/create-conn) - ledger @(fluree/create conn "query/union" {:defaultContext ["" {:ex "http://example.org/ns/"}]}) + ledger @(fluree/create conn "query/union" {"defaults" + {"@context" + ["" {"ex" "http://example.org/ns/"}]}}) db @(fluree/stage (fluree/db ledger) - [{:id :ex/brian, - :type :ex/User, - :schema/name "Brian" - :ex/last "Smith" - :schema/email "brian@example.org" - :schema/age 50 - :ex/favNums 7 - :ex/scores [76 80 15]} - {:id :ex/alice, - :type :ex/User, - :schema/name "Alice" - :ex/last "Smith" - :schema/email "alice@example.org" - :ex/favColor "Green" - :schema/age 42 - :ex/favNums [42, 76, 9] - :ex/scores [102 92.5 90]} - {:id :ex/cam, - :type :ex/User, - :schema/name "Cam" - :ex/last "Jones" - :ex/email "cam@example.org" - :schema/age 34 - :ex/favNums [5, 10] - :ex/scores [97.2 100 80] - :ex/friend [:ex/brian :ex/alice]}])] + [{"id" "ex:brian", + "type" "ex:User", + "schema:name" "Brian" + "ex:last" "Smith" + "schema:email" "brian@example.org" + "schema:age" 50 + "ex:favNums" 7 + "ex:scores" [76 80 15]} + {"id" "ex:alice", + "type" "ex:User", + "schema:name" "Alice" + "ex:last" "Smith" + "schema:email" "alice@example.org" + "ex:favColor" "Green" + "schema:age" 42 + "ex:favNums" [42, 76, 9] + "ex:scores" [102 92.5 90]} + {"id" "ex:cam", + "type" "ex:User", + "schema:name" "Cam" + "ex:last" "Jones" + "ex:email" "cam@example.org" + "schema:age" 34 + "ex:favNums" [5, 10] + "ex:scores" [97.2 100 80] + "ex:friend" [{"id" "ex:brian"} {"id" "ex:alice"}]}])] - ;; basic combine :schema/email and :ex/email into same result variable - (is (= @(fluree/query db {:select ['?name '?email] - :where [['?s :rdf/type :ex/User] - ['?s :schema/name '?name] - {:union [[['?s :ex/email '?email]] - [['?s :schema/email '?email]]]}]}) + ;; basic combine schema:email and ex:email into same result variable + (is (= @(fluree/query db {"select" ['?name '?email] + "where" [['?s "rdf:type" "ex:User"] + ['?s "schema:name" '?name] + {"union" [[['?s "ex:email" '?email]] + [['?s "schema:email" '?email]]]}]}) [["Cam" "cam@example.org"] ["Alice" "alice@example.org"] ["Brian" "brian@example.org"]]) - "Emails for all 3 users should return, even though some are :schema/email and others :ex/email") + "Emails for all 3 users should return, even though some are schema:email and others ex:email") ;; basic union that uses different variables for output - (is (= @(fluree/query db {:select ['?s '?email1 '?email2] - :where [['?s :rdf/type :ex/User] - {:union [[['?s :ex/email '?email1]] - [['?s :schema/email '?email2]]]}]}) - [[:ex/cam "cam@example.org" nil] - [:ex/alice nil "alice@example.org"] - [:ex/brian nil "brian@example.org"]]) + (is (= @(fluree/query db {"select" ['?s '?email1 '?email2] + "where" [['?s "rdf:type" "ex:User"] + {"union" [[['?s "ex:email" '?email1]] + [['?s "schema:email" '?email2]]]}]}) + [["ex:cam" "cam@example.org" nil] + ["ex:alice" nil "alice@example.org"] + ["ex:brian" nil "brian@example.org"]]) "Emails for all 3 users should return, but wil each using their own var and nils for others") ;; basic union that uses different variables for output and has a passthrough variable - (is (= @(fluree/query db {:select ['?name '?email1 '?email2] - :where [['?s :rdf/type :ex/User] - ['?s :schema/name '?name] - {:union [[['?s :ex/email '?email1]] - [['?s :schema/email '?email2]]]}]}) + (is (= @(fluree/query db {"select" ['?name '?email1 '?email2] + "where" [['?s "rdf:type" "ex:User"] + ['?s "schema:name" '?name] + {"union" [[['?s "ex:email" '?email1]] + [['?s "schema:email" '?email2]]]}]}) [["Cam" "cam@example.org" nil] ["Alice" nil "alice@example.org"] ["Brian" nil "brian@example.org"]]) "Emails for all 3 users should return using different vars, but also passing through a variable")))) - From 4a5347ce4933e4b849ca80ced3a7072e1807eb81 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Mon, 17 Apr 2023 10:57:26 -0600 Subject: [PATCH 29/50] Stringify transact tests --- test/fluree/db/transact/delete_test.clj | 101 ++++++------ test/fluree/db/transact/retraction_test.clj | 50 +++--- .../db/transact/stable_context_test.clj | 145 ++++++++++-------- 3 files changed, 159 insertions(+), 137 deletions(-) diff --git a/test/fluree/db/transact/delete_test.clj b/test/fluree/db/transact/delete_test.clj index 133fd5792..50fb6c07e 100644 --- a/test/fluree/db/transact/delete_test.clj +++ b/test/fluree/db/transact/delete_test.clj @@ -7,68 +7,75 @@ (deftest ^:integration deleting-data (testing "Deletions of entire subjects." (let [conn (test-utils/create-conn) - ledger @(fluree/create conn "tx/delete" {:defaultContext ["" {:ex "http://example.org/ns/"}]}) + ledger @(fluree/create conn "tx/delete" + {"defaults" + {"@context" + ["" {"ex" "http://example.org/ns/"}]}}) db @(fluree/stage (fluree/db ledger) - {:graph [{:id :ex/alice, - :type :ex/User, - :schema/name "Alice" - :schema/email "alice@flur.ee" - :schema/age 42} - {:id :ex/bob, - :type :ex/User, - :schema/name "Bob" - :schema/age 22} - {:id :ex/jane, - :type :ex/User, - :schema/name "Jane" - :schema/email "jane@flur.ee" - :schema/age 30}]}) + {"@graph" [{"id" "ex:alice", + "type" "ex:User", + "schema:name" "Alice" + "schema:email" "alice@flur.ee" + "schema:age" 42} + {"id" "ex:bob", + "type" "ex:User", + "schema:name" "Bob" + "schema:age" 22} + {"id" "ex:jane", + "type" "ex:User", + "schema:name" "Jane" + "schema:email" "jane@flur.ee" + "schema:age" 30}]}) - ;; delete everything for :ex/alice + ;; delete everything for "ex:alice" db-subj-delete @(fluree/stage db - '{:delete [:ex/alice ?p ?o] - :where [[:ex/alice ?p ?o]]}) + '{"delete" ["ex:alice" ?p ?o] + "where" [["ex:alice" ?p ?o]]}) - ;; delete any :schema/age values for :ex/bob + ;; delete any "schema:age" values for "ex:bob" db-subj-pred-del @(fluree/stage db - '{:delete [:ex/bob :schema/age ?o] - :where [[:ex/bob :schema/age ?o]]}) + '{"delete" ["ex:bob" "schema:age" ?o] + "where" [["ex:bob" "schema:age" ?o]]}) - ;; delete all subjects with a :schema/email predicate + ;; delete all subjects with a "schema:email" predicate db-all-preds @(fluree/stage db - '{:delete [?s ?p ?o] - :where [[?s :schema/email ?x] - [?s ?p ?o]]}) + '{"delete" [?s ?p ?o] + "where" [[?s "schema:email" ?x] + [?s ?p ?o]]}) - ;; delete all subjects where :schema/age = 30 + ;; delete all subjects where "schema:age" = 30 db-age-delete @(fluree/stage db - '{:delete [?s ?p ?o] - :where [[?s :schema/age 30] - [?s ?p ?o]]})] + '{"delete" [?s ?p ?o] + "where" [[?s "schema:age" 30] + [?s ?p ?o]]})] + + (is (= ["Jane" "Bob"] + @(fluree/query db-subj-delete + '{"select" ?name + "where" [[?s "schema:name" ?name]]})) - (is (= @(fluree/query db-subj-delete - '{:select ?name - :where [[?s :schema/name ?name]]}) - ["Jane" "Bob"]) "Only Jane and Bob should be left in the db.") - (is (= @(fluree/query db-subj-pred-del - '{:selectOne {?s [:*]} - :where [[?s :id :ex/bob]]}) - {:id :ex/bob, - :rdf/type [:ex/User], - :schema/name "Bob"}) + (is (= {"id" "ex:bob", + "rdf:type" ["ex:User"], + "schema:name" "Bob"} + @(fluree/query db-subj-pred-del + '{"selectOne" {?s ["*"]} + "where" [[?s "id" "ex:bob"]]})) + "Bob should no longer have an age property.") - (is (= @(fluree/query db-all-preds - '{:select ?name - :where [[?s :schema/name ?name]]}) - ["Bob"]) + (is (= ["Bob"] + @(fluree/query db-all-preds + '{"select" ?name + "where" [[?s "schema:name" ?name]]})) + "Only Bob should be left, as he is the only one without an email.") - (is (= @(fluree/query db-age-delete - '{:select ?name - :where [[?s :schema/name ?name]]}) - ["Bob" "Alice"]) + (is (= ["Bob" "Alice"] + @(fluree/query db-age-delete + '{"select" ?name + "where" [[?s "schema:name" ?name]]})) + "Only Bob and Alice should be left in the db.")))) diff --git a/test/fluree/db/transact/retraction_test.clj b/test/fluree/db/transact/retraction_test.clj index d54efc0ba..e1312b91f 100644 --- a/test/fluree/db/transact/retraction_test.clj +++ b/test/fluree/db/transact/retraction_test.clj @@ -8,31 +8,31 @@ (let [conn (test-utils/create-conn) ledger @(fluree/create conn "tx/retract") db @(fluree/stage - (fluree/db ledger) - {:context ["" {:ex "http://example.org/ns/"}] - :graph [{:id :ex/alice, - :type :ex/User, - :schema/name "Alice" - :schema/age 42} - {:id :ex/bob, - :type :ex/User, - :schema/name "Bob" - :schema/age 22} - {:id :ex/jane, - :type :ex/User, - :schema/name "Jane" - :schema/age 30}]}) + (fluree/db ledger) + {"@context" ["" {"ex" "http://example.org/ns/"}] + "@graph" [{"id" "ex:alice", + "type" "ex:User", + "schema:name" "Alice" + "schema:age" 42} + {"id" "ex:bob", + "type" "ex:User", + "schema:name" "Bob" + "schema:age" 22} + {"id" "ex:jane", + "type" "ex:User", + "schema:name" "Jane" + "schema:age" 30}]}) ;; retract Alice's age attribute by using nil db-age-retract @(fluree/stage - db - {:context ["" {:ex "http://example.org/ns/"}] - :id :ex/alice, - :schema/age nil})] - (is (= @(fluree/query db-age-retract - '{:context ["" {:ex "http://example.org/ns/"}], - :select {?s [:*]}, - :where [[?s :id :ex/alice]]}) - [{:id :ex/alice, - :rdf/type [:ex/User], - :schema/name "Alice"}]) + db + {"@context" ["" {"ex" "http://example.org/ns/"}] + "id" "ex:alice", + "schema:age" nil})] + (is (= [{"id" "ex:alice", + "rdf:type" ["ex:User"], + "schema:name" "Alice"}] + @(fluree/query db-age-retract + '{"@context" ["" {"ex" "http://example.org/ns/"}], + "select" {?s ["*"]}, + "where" [[?s "id" "ex:alice"]]})) "Alice should no longer have an age property")))) diff --git a/test/fluree/db/transact/stable_context_test.clj b/test/fluree/db/transact/stable_context_test.clj index 04feba8c2..0ec030483 100644 --- a/test/fluree/db/transact/stable_context_test.clj +++ b/test/fluree/db/transact/stable_context_test.clj @@ -10,25 +10,31 @@ (deftest ^:integration default-context-stability-memory-from-load (let [conn (test-utils/create-conn) - ledger @(fluree/create conn "ctx/stability-mem-ld" {:defaultContext {:id "@id" - :type "@type" - :ex "http://example.org/ns/" - :blah "http://blah.me/wow/ns/"}}) - db1 @(test-utils/transact ledger [{:id :blah/one - :ex/name "One"}]) - db1-load (fluree/db @(fluree/load conn "ctx/stability-mem-ld")) - - db2 (->> @(fluree/stage db1-load [{:id :blah/two - :ex/name "Two"}]) + ledger @(fluree/create conn "ctx/stability-mem-ld" + {"defaults" + {"@context" + {"id" "@id" + "type" "@type" + "ex" "http://example.org/ns/" + "blah" "http://blah.me/wow/ns/"}}}) + db1 @(test-utils/transact ledger [{"id" "blah:one" + "ex:name" "One"}]) + db1-load (fluree/db (test-utils/retry-load conn "ctx/stability-mem-ld" + 100)) + + db2 (->> @(fluree/stage db1-load [{"id" "blah:two" + "ex:name" "Two"}]) (fluree/commit! ledger) deref) - db2-load (fluree/db @(fluree/load conn "ctx/stability-mem-ld")) + db2-load (fluree/db (test-utils/retry-load conn "ctx/stability-mem-ld" + 100)) - db3 (->> @(fluree/stage db2-load [{:id :blah/three - :ex/name "Three"}]) + db3 (->> @(fluree/stage db2-load [{"id" "blah:three" + "ex:name" "Three"}]) (fluree/commit! ledger) deref) - db3-load (fluree/db @(fluree/load conn "ctx/stability-mem-ld"))] + db3-load (fluree/db (test-utils/retry-load conn "ctx/stability-mem-ld" + 100))] (testing "Loaded default context is same as initial db's" (is (= (dbproto/-default-context db1-load) @@ -43,31 +49,37 @@ (dbproto/-default-context db1)))) (testing "Query after the 3rd load is using original default context" - (is (= [{:id :blah/three - :ex/name "Three"} - {:id :blah/two - :ex/name "Two"} - {:id :blah/one - :ex/name "One"}] - @(fluree/query db3-load '{:select {?s [:*]} - :where [[?s :ex/name nil]]})))))) + (is (= [{"id" "blah:three" + "ex:name" "Three"} + {"id" "blah:two" + "ex:name" "Two"} + {"id" "blah:one" + "ex:name" "One"}] + @(fluree/query db3-load '{"select" {?s ["*"]} + "where" [[?s "ex:name" nil]]})))))) (deftest ^:integration default-context-stability-memory (let [conn (test-utils/create-conn) - ledger @(fluree/create conn "ctx/stability-mem" {:defaultContext {:id "@id" - :type "@type" - :ex "http://example.org/ns/" - :blah "http://blah.me/wow/ns/"}}) - db1 @(test-utils/transact ledger [{:id :blah/one - :ex/name "One"}]) - db1-load (fluree/db @(fluree/load conn "ctx/stability-mem")) - db2 @(test-utils/transact ledger [{:id :blah/two - :ex/name "Two"}]) - db2-load (fluree/db @(fluree/load conn "ctx/stability-mem")) - db3 @(test-utils/transact ledger [{:id :blah/three - :ex/name "Three"}]) - db3-load (fluree/db @(fluree/load conn "ctx/stability-mem"))] + ledger @(fluree/create conn "ctx/stability-mem" + {"defaults" + {"@context" + {"id" "@id" + "type" "@type" + "ex" "http://example.org/ns/" + "blah" "http://blah.me/wow/ns/"}}}) + db1 @(test-utils/transact ledger [{"id" "blah:one" + "ex:name" "One"}]) + db1-load (fluree/db (test-utils/retry-load conn "ctx/stability-mem" + 100)) + db2 @(test-utils/transact ledger [{"id" "blah:two" + "ex:name" "Two"}]) + db2-load (fluree/db (test-utils/retry-load conn "ctx/stability-mem" + 100)) + db3 @(test-utils/transact ledger [{"id" "blah:three" + "ex:name" "Three"}]) + db3-load (fluree/db (test-utils/retry-load conn "ctx/stability-mem" + 100))] (testing "Loaded default context is same as initial db's" (is (= (dbproto/-default-context db1-load) @@ -82,35 +94,38 @@ (dbproto/-default-context db1)))) (testing "Query after the 3rd load is using original default context" - (is (= [{:id :blah/three - :ex/name "Three"} - {:id :blah/two - :ex/name "Two"} - {:id :blah/one - :ex/name "One"}] - @(fluree/query db3-load '{:select {?s [:*]} - :where [[?s :ex/name nil]]})))))) + (is (= [{"id" "blah:three" + "ex:name" "Three"} + {"id" "blah:two" + "ex:name" "Two"} + {"id" "blah:one" + "ex:name" "One"}] + @(fluree/query db3-load '{"select" {?s ["*"]} + "where" [[?s "ex:name" nil]]})))))) (deftest ^:integration default-context-stability-file (with-tmp-dir storage-path (let [conn @(fluree/connect - {:method :file :storage-path storage-path - :defaults - {:context test-utils/default-context - :context-type :keyword}}) - ledger @(fluree/create conn "ctx/stability" {:defaultContext {:id "@id" - :type "@type" - :ex "http://example.org/ns/" - :blah "http://blah.me/wow/ns/"}}) - db1 @(test-utils/transact ledger [{:id :blah/one - :ex/name "One"}]) + {"method" "file" + "storage-path" storage-path + "defaults" + {"@context" test-utils/default-context}}) + ledger @(fluree/create conn "ctx/stability" + {"defaults" + {"@context" + {"id" "@id" + "type" "@type" + "ex" "http://example.org/ns/" + "blah" "http://blah.me/wow/ns/"}}}) + db1 @(test-utils/transact ledger [{"id" "blah:one" + "ex:name" "One"}]) db1-load (fluree/db (test-utils/retry-load conn "ctx/stability" 100)) - db2 @(test-utils/transact ledger [{:id :blah/two - :ex/name "Two"}]) + db2 @(test-utils/transact ledger [{"id" "blah:two" + "ex:name" "Two"}]) db2-load (fluree/db (test-utils/retry-load conn "ctx/stability" 100)) - db3 @(test-utils/transact ledger [{:id :blah/three - :ex/name "Three"}]) + db3 @(test-utils/transact ledger [{"id" "blah:three" + "ex:name" "Three"}]) db3-load (fluree/db (test-utils/retry-load conn "ctx/stability" 100))] (testing "Loaded default context is same as initial db's" @@ -126,11 +141,11 @@ (dbproto/-default-context db1)))) (testing "Query after the 3rd load is using original default context" - (is (= [{:id :blah/three - :ex/name "Three"} - {:id :blah/two - :ex/name "Two"} - {:id :blah/one - :ex/name "One"}] - @(fluree/query db3-load '{:select {?s [:*]} - :where [[?s :ex/name nil]]}))))))) + (is (= [{"id" "blah:three" + "ex:name" "Three"} + {"id" "blah:two" + "ex:name" "Two"} + {"id" "blah:one" + "ex:name" "One"}] + @(fluree/query db3-load '{"select" {?s ["*"]} + "where" [[?s "ex:name" nil]]}))))))) From f0d8dc1363fe45385043a7cc3dab93d52d643b85 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Mon, 17 Apr 2023 10:57:35 -0600 Subject: [PATCH 30/50] Stringify SHACL tests --- test/fluree/db/shacl/shacl_basic_test.clj | 1025 +++++++++++---------- 1 file changed, 523 insertions(+), 502 deletions(-) diff --git a/test/fluree/db/shacl/shacl_basic_test.clj b/test/fluree/db/shacl/shacl_basic_test.clj index 83fda1bc6..da4c0d657 100644 --- a/test/fluree/db/shacl/shacl_basic_test.clj +++ b/test/fluree/db/shacl/shacl_basic_test.clj @@ -9,684 +9,705 @@ (deftest ^:integration using-pre-defined-types-as-classes (testing "Class not used as class initially can still be used as one." (let [conn (test-utils/create-conn) - ledger @(fluree/create conn "class/testing" {:defaultContext ["" {:ex "http://example.org/ns/"}]}) + ledger @(fluree/create conn "class/testing" + {"defaults" + {"@context" + ["" {"ex" "http://example.org/ns/"}]}}) db1 @(fluree/stage - (fluree/db ledger) - {:id :ex/MyClass, - :schema/description "Just a basic object not used as a class"}) + (fluree/db ledger) + {"id" "ex:MyClass", + "schema:description" "Just a basic object not used as a class"}) db2 @(fluree/stage - db1 - {:id :ex/myClassInstance, - :type [:ex/MyClass] - :schema/description "Now a new subject uses MyClass as a Class"}) - query-res @(fluree/query db2 '{:select {?s [:*]}, - :where [[?s :id :ex/myClassInstance]]})] - (is (= query-res - [{:id :ex/myClassInstance, - :rdf/type [:ex/MyClass], - :schema/description "Now a new subject uses MyClass as a Class"}]))))) + db1 + {"id" "ex:myClassInstance", + "type" ["ex:MyClass"] + "schema:description" "Now a new subject uses MyClass as a Class"}) + query-res @(fluree/query db2 '{"select" {?s ["*"]}, + "where" [[?s "id" "ex:myClassInstance"]]})] + (is (= [{"id" "ex:myClassInstance", + "rdf:type" ["ex:MyClass"], + "schema:description" "Now a new subject uses MyClass as a Class"}] + query-res))))) + (deftest ^:integration shacl-cardinality-constraints (testing "shacl minimum and maximum cardinality" (let [conn (test-utils/create-conn) - ledger @(fluree/create conn "shacl/a" {:defaultContext ["" {:ex "http://example.org/ns/"}]}) - user-query {:select {'?s [:*]} - :where [['?s :rdf/type :ex/User]]} + ledger @(fluree/create conn "shacl/a" + {"defaults" + {"@context" + ["" {"ex" "http://example.org/ns/"}]}}) + user-query {"select" {'?s ["*"]} + "where" [['?s "rdf:type" "ex:User"]]} db @(fluree/stage - (fluree/db ledger) - {:id :ex/UserShape, - :type [:sh/NodeShape], - :sh/targetClass :ex/User - :sh/property [{:sh/path :schema/name - :sh/minCount 1 - :sh/maxCount 1 - :sh/datatype :xsd/string}]}) + (fluree/db ledger) + {"id" "ex:UserShape", + "type" ["sh:NodeShape"], + "sh:targetClass" {"id" "ex:User"} + "sh:property" [{"sh:path" {"id" "schema:name"} + "sh:minCount" 1 + "sh:maxCount" 1 + "sh:datatype" {"id" "xsd:string"}}]}) db-ok @(fluree/stage - db - {:id :ex/john, - :type [:ex/User], - :schema/name "John" - :schema/callSign "j-rock"}) - ; no :schema/name + db + {"id" "ex:john", + "type" ["ex:User"], + "schema:name" "John" + "schema:callSign" "j-rock"}) + ; no "schema:name" db-no-names (try @(fluree/stage - db - {:id :ex/john, - :type [:ex/User], - :schema/callSign "j-rock"}) + db + {"id" "ex:john", + "type" ["ex:User"], + "schema:callSign" "j-rock"}) (catch Exception e e)) db-two-names (try @(fluree/stage - db - {:id :ex/john, - :type [:ex/User], - :schema/name ["John", "Johnny"] - :schema/callSign "j-rock"}) + db + {"id" "ex:john", + "type" ["ex:User"], + "schema:name" ["John", "Johnny"] + "schema:callSign" "j-rock"}) (catch Exception e e))] (is (util/exception? db-no-names) - "Exception, because :schema/name requires at least 1 value.") + "Exception, because schema:name requires at least 1 value.") (is (str/starts-with? (ex-message db-no-names) "Required properties not present:")) (is (util/exception? db-two-names) "Exception, because :schema/name can have at most 1 value.") - (is (= (ex-message db-two-names) - "SHACL PropertyShape exception - sh:maxCount of 1 lower than actual count of 2.")) - (is (= @(fluree/query db-ok user-query) - [{:id :ex/john, - :rdf/type [:ex/User], - :schema/name "John", - :schema/callSign "j-rock"}]) + (is (= "SHACL PropertyShape exception - sh:maxCount of 1 lower than actual count of 2." + (ex-message db-two-names))) + (is (= [{"id" "ex:john", + "rdf:type" ["ex:User"], + "schema:name" "John", + "schema:callSign" "j-rock"}] + @(fluree/query db-ok user-query)) "basic rdf:type query response not correct")))) (deftest ^:integration shacl-datatype-constraints (testing "shacl datatype errors" (let [conn (test-utils/create-conn) - ledger @(fluree/create conn "shacl/b" {:defaultContext ["" {:ex "http://example.org/ns/"}]}) - user-query {:select {'?s [:*]} - :where [['?s :rdf/type :ex/User]]} + ledger @(fluree/create conn "shacl/b" + {"defaults" + {"@context" + ["" {"ex" "http://example.org/ns/"}]}}) + user-query {"select" {'?s ["*"]} + "where" [['?s "rdf:type" "ex:User"]]} db @(fluree/stage - (fluree/db ledger) - {:id :ex/UserShape, - :type [:sh/NodeShape], - :sh/targetClass :ex/User - :sh/property [{:sh/path :schema/name - :sh/datatype :xsd/string}]}) + (fluree/db ledger) + {"id" "ex:UserShape", + "type" ["sh:NodeShape"], + "sh:targetClass" {"id" "ex:User"} + "sh:property" [{"sh:path" {"id" "schema:name"} + "sh:datatype" {"id" "xsd:string"}}]}) db-ok @(fluree/stage - db - {:id :ex/john, - :type [:ex/User], - :schema/name "John"}) - ; no :schema/name + db + {"id" "ex:john", + "type" ["ex:User"], + "schema:name" "John"}) + ; no "schema:name" db-int-name (try @(fluree/stage - db - {:id :ex/john, - :type [:ex/User], - :schema/name 42}) + db + {"id" "ex:john", + "type" ["ex:User"], + "schema:name" 42}) (catch Exception e e)) db-bool-name (try @(fluree/stage - db - {:id :ex/john, - :type [:ex/User], - :schema/name true}) + db + {"id" "ex:john", + "type" ["ex:User"], + "schema:name" true}) (catch Exception e e))] (is (util/exception? db-int-name) - "Exception, because :schema/name is an integer and not a string.") + "Exception, because schema:name is an integer and not a string.") (is (str/starts-with? (ex-message db-int-name) "Required data type")) (is (util/exception? db-bool-name) - "Exception, because :schema/name is a boolean and not a string.") + "Exception, because schema:name is a boolean and not a string.") (is (str/starts-with? (ex-message db-bool-name) "Required data type")) - (is (= @(fluree/query db-ok user-query) - [{:id :ex/john, - :rdf/type [:ex/User], - :schema/name "John"}]) + (is (= [{"id" "ex:john", + "rdf:type" ["ex:User"], + "schema:name" "John"}] + @(fluree/query db-ok user-query)) "basic rdf:type query response not correct")))) (deftest ^:integration shacl-closed-shape (testing "shacl closed shape" (let [conn (test-utils/create-conn) - ledger @(fluree/create conn "shacl/c" {:defaultContext ["" {:ex "http://example.org/ns/"}]}) - user-query {:select {'?s [:*]} - :where [['?s :rdf/type :ex/User]]} + ledger @(fluree/create conn "shacl/c" + {"defaults" + {"@context" + ["" {"ex" "http://example.org/ns/"}]}}) + user-query {"select" {'?s ["*"]} + "where" [['?s "rdf:type" "ex:User"]]} db @(fluree/stage - (fluree/db ledger) - {:id :ex/UserShape, - :type [:sh/NodeShape], - :sh/targetClass :ex/User - :sh/property [{:sh/path :schema/name - :sh/datatype :xsd/string}] - :sh/ignoredProperties [:rdf/type] - :sh/closed true}) + (fluree/db ledger) + {"id" "ex:UserShape", + "type" ["sh:NodeShape"], + "sh:targetClass" {"id" "ex:User"} + "sh:property" [{"sh:path" {"id" "schema:name"} + "sh:datatype" {"id" "xsd:string"}}] + "sh:ignoredProperties" [{"id" "rdf:type"}] + "sh:closed" true}) db-ok @(fluree/stage - db - {:id :ex/john, - :type [:ex/User], - :schema/name "John"}) - ; no :schema/name + db + {"id" "ex:john", + "type" ["ex:User"], + "schema:name" "John"}) + ; no "schema:name" db-extra-prop (try @(fluree/stage - db - {:id :ex/john, - :type [:ex/User], - :schema/name "John" - :schema/email "john@flur.ee"}) + db + {"id" "ex:john", + "type" ["ex:User"], + "schema:name" "John" + "schema:email" "john@flur.ee"}) (catch Exception e e))] (is (util/exception? db-extra-prop) - "Exception, because :schema/name is an integer and not a string.") + "Exception, because schema:name is an integer and not a string.") (is (str/starts-with? (ex-message db-extra-prop) "SHACL shape is closed")) - (is (= @(fluree/query db-ok user-query) - [{:id :ex/john, - :rdf/type [:ex/User], - :schema/name "John"}]) + (is (= [{"id" "ex:john", + "rdf:type" ["ex:User"], + "schema:name" "John"}] + @(fluree/query db-ok user-query)) "basic rdf:type query response not correct")))) (deftest ^:integration shacl-property-pairs (testing "shacl property pairs" - (let [conn (test-utils/create-conn) - ledger @(fluree/create conn "shacl/pairs" {:defaultContext ["" {:ex "http://example.org/ns/"}]}) - user-query {:select {'?s [:*]} - :where [['?s :rdf/type :ex/User]]} ] + (let [conn (test-utils/create-conn) + ledger @(fluree/create conn "shacl/pairs" + {"defaults" + {"@context" + ["" {"ex" "http://example.org/ns/"}]}}) + user-query {"select" {'?s ["*"]} + "where" [['?s "rdf:type" "ex:User"]]}] (testing "single-cardinality equals" - (let [db @(fluree/stage - (fluree/db ledger) - {:id :ex/EqualNamesShape - :type [:sh/NodeShape], - :sh/targetClass :ex/User - :sh/property [{:sh/path :schema/name - :sh/equals :ex/firstName}]}) - db-ok @(fluree/stage - db - {:id :ex/alice, - :type [:ex/User], - :schema/name "Alice" - :ex/firstName "Alice"}) + (let [db @(fluree/stage + (fluree/db ledger) + {"id" "ex:EqualNamesShape" + "type" ["sh:NodeShape"], + "sh:targetClass" {"id" "ex:User"} + "sh:property" [{"sh:path" {"id" "schema:name"} + "sh:equals" {"id" "ex:firstName"}}]}) + db-ok @(fluree/stage + db + {"id" "ex:alice", + "type" ["ex:User"], + "schema:name" "Alice" + "ex:firstName" "Alice"}) db-not-equal (try @(fluree/stage db - {:id :ex/john, - :type [:ex/User], - :schema/name "John" - :ex/firstName "Jack"}) + {"id" "ex:john", + "type" ["ex:User"], + "schema:name" "John" + "ex:firstName" "Jack"}) (catch Exception e e))] (is (util/exception? db-not-equal) - "Exception, because :schema/name does not equal :ex/firstName") + "Exception, because schema:name does not equal ex:firstName") (is (str/starts-with? (ex-message db-not-equal) "SHACL PropertyShape exception - sh:equals")) - (is (= [{:id :ex/alice, - :rdf/type [:ex/User], - :schema/name "Alice" - :ex/firstName "Alice"}] + (is (= [{"id" "ex:alice", + "rdf:type" ["ex:User"], + "schema:name" "Alice" + "ex:firstName" "Alice"}] @(fluree/query db-ok user-query))))) (testing "multi-cardinality equals" - (let [db @(fluree/stage - (fluree/db ledger) - {:id :ex/EqualNamesShape - :type [:sh/NodeShape], - :sh/targetClass :ex/User - :sh/property [{:sh/path :ex/favNums - :sh/equals :ex/luckyNums}]}) - db-ok @(fluree/stage - db - {:id :ex/alice, - :type [:ex/User], - :schema/name "Alice" - :ex/favNums [11 17] - :ex/luckyNums [11 17]}) - - db-ok2 @(fluree/stage - db - {:id :ex/alice, - :type [:ex/User], - :schema/name "Alice" - :ex/favNums [11 17] - :ex/luckyNums [17 11]}) - - db-not-equal1 (try - @(fluree/stage - db - {:id :ex/brian - :type [:ex/User], - :schema/name "Brian" - :ex/favNums [11 17] - :ex/luckyNums [13 18]}) - (catch Exception e e)) - db-not-equal2 (try - @(fluree/stage - db - {:id :ex/brian - :type [:ex/User], - :schema/name "Brian" - :ex/favNums [11 17] - :ex/luckyNums [11]}) - (catch Exception e e)) - db-not-equal3 (try - @(fluree/stage - db - {:id :ex/brian - :type [:ex/User], - :schema/name "Brian" - :ex/favNums [11 17] - :ex/luckyNums [11 17 18]}) - (catch Exception e e)) - db-not-equal4 (try - @(fluree/stage - db - {:id :ex/brian - :type [:ex/User], - :schema/name "Brian" - :ex/favNums [11 17] - :ex/luckyNums ["11" "17"]}) - (catch Exception e e))] - (is (util/exception? db-not-equal1) - "Exception, because :ex/favNums does not equal :ex/luckyNums") - (is (str/starts-with? (ex-message db-not-equal1) - "SHACL PropertyShape exception - sh:equals")) - (is (util/exception? db-not-equal2) - "Exception, because :ex/favNums does not equal :ex/luckyNums") - (is (str/starts-with? (ex-message db-not-equal2) - "SHACL PropertyShape exception - sh:equals")) - (is (util/exception? db-not-equal3) - "Exception, because :ex/favNums does not equal :ex/luckyNums") - (is (str/starts-with? (ex-message db-not-equal3) - "SHACL PropertyShape exception - sh:equals")) - (is (util/exception? db-not-equal4) - "Exception, because :ex/favNums does not equal :ex/luckyNums") - (is (str/starts-with? (ex-message db-not-equal4) - "SHACL PropertyShape exception - sh:equals")) - (is (= [{:id :ex/alice, - :rdf/type [:ex/User], - :schema/name "Alice" - :ex/favNums [11 17] - :ex/luckyNums [11 17]}] - @(fluree/query db-ok user-query))) - (is (= [{:id :ex/alice, - :rdf/type [:ex/User], - :schema/name "Alice" - :ex/favNums [11 17] - :ex/luckyNums [11 17]}] - @(fluree/query db-ok2 user-query))))) - (testing "disjoint" (let [db @(fluree/stage (fluree/db ledger) - {:id :ex/DisjointShape - :type [:sh/NodeShape], - :sh/targetClass :ex/User - :sh/property [{:sh/path :ex/favNums - :sh/disjoint :ex/luckyNums}]}) + {"id" "ex:EqualNamesShape" + "type" ["sh:NodeShape"], + "sh:targetClass" {"id" "ex:User"} + "sh:property" [{"sh:path" {"id" "ex:favNums"} + "sh:equals" {"id" "ex:luckyNums"}}]}) db-ok @(fluree/stage db - {:id :ex/alice, - :type [:ex/User], - :schema/name "Alice" - :ex/favNums [11 17] - :ex/luckyNums 1}) + {"id" "ex:alice", + "type" ["ex:User"], + "schema:name" "Alice" + "ex:favNums" [11 17] + "ex:luckyNums" [11 17]}) + + db-ok2 @(fluree/stage + db + {"id" "ex:alice", + "type" ["ex:User"], + "schema:name" "Alice" + "ex:favNums" [11 17] + "ex:luckyNums" [17 11]}) + + db-not-equal1 (try + @(fluree/stage + db + {"id" "ex:brian" + "type" ["ex:User"], + "schema:name" "Brian" + "ex:favNums" [11 17] + "ex:luckyNums" [13 18]}) + (catch Exception e e)) + db-not-equal2 (try + @(fluree/stage + db + {"id" "ex:brian" + "type" ["ex:User"], + "schema:name" "Brian" + "ex:favNums" [11 17] + "ex:luckyNums" [11]}) + (catch Exception e e)) + db-not-equal3 (try + @(fluree/stage + db + {"id" "ex:brian" + "type" ["ex:User"], + "schema:name" "Brian" + "ex:favNums" [11 17] + "ex:luckyNums" [11 17 18]}) + (catch Exception e e)) + db-not-equal4 (try + @(fluree/stage + db + {"id" "ex:brian" + "type" ["ex:User"], + "schema:name" "Brian" + "ex:favNums" [11 17] + "ex:luckyNums" ["11" "17"]}) + (catch Exception e e))] + (is (util/exception? db-not-equal1) + "Exception, because ex:favNums does not equal ex:luckyNums") + (is (str/starts-with? (ex-message db-not-equal1) + "SHACL PropertyShape exception - sh:equals")) + (is (util/exception? db-not-equal2) + "Exception, because ex:favNums does not equal ex:luckyNums") + (is (str/starts-with? (ex-message db-not-equal2) + "SHACL PropertyShape exception - sh:equals")) + (is (util/exception? db-not-equal3) + "Exception, because ex:favNums does not equal ex:luckyNums") + (is (str/starts-with? (ex-message db-not-equal3) + "SHACL PropertyShape exception - sh:equals")) + (is (util/exception? db-not-equal4) + "Exception, because ex:favNums does not equal ex:luckyNums") + (is (str/starts-with? (ex-message db-not-equal4) + "SHACL PropertyShape exception - sh:equals")) + (is (= [{"id" "ex:alice", + "rdf:type" ["ex:User"], + "schema:name" "Alice" + "ex:favNums" [11 17] + "ex:luckyNums" [11 17]}] + @(fluree/query db-ok user-query))) + (is (= [{"id" "ex:alice", + "rdf:type" ["ex:User"], + "schema:name" "Alice" + "ex:favNums" [11 17] + "ex:luckyNums" [11 17]}] + @(fluree/query db-ok2 user-query))))) + (testing "disjoint" + (let [db @(fluree/stage + (fluree/db ledger) + {"id" "ex:DisjointShape" + "type" ["sh:NodeShape"], + "sh:targetClass" {"id" "ex:User"} + "sh:property" [{"sh:path" {"id" "ex:favNums"} + "sh:disjoint" {"id" "ex:luckyNums"}}]}) + db-ok @(fluree/stage + db + {"id" "ex:alice", + "type" ["ex:User"], + "schema:name" "Alice" + "ex:favNums" [11 17] + "ex:luckyNums" 1}) db-not-disjoint1 (try @(fluree/stage db - {:id :ex/brian - :type [:ex/User], - :schema/name "Brian" - :ex/favNums 11 - :ex/luckyNums 11}) + {"id" "ex:brian" + "type" ["ex:User"], + "schema:name" "Brian" + "ex:favNums" 11 + "ex:luckyNums" 11}) (catch Exception e e)) db-not-disjoint2 (try @(fluree/stage db - {:id :ex/brian - :type [:ex/User], - :schema/name "Brian" - :ex/favNums [11 17 31] - :ex/luckyNums 11}) + {"id" "ex:brian" + "type" ["ex:User"], + "schema:name" "Brian" + "ex:favNums" [11 17 31] + "ex:luckyNums" 11}) (catch Exception e e)) db-not-disjoint3 (try @(fluree/stage db - {:id :ex/brian - :type [:ex/User], - :schema/name "Brian" - :ex/favNums [11 17 31] - :ex/luckyNums [13 18 11]}) + {"id" "ex:brian" + "type" ["ex:User"], + "schema:name" "Brian" + "ex:favNums" [11 17 31] + "ex:luckyNums" [13 18 11]}) (catch Exception e e))] (is (util/exception? db-not-disjoint1) - "Exception, because :ex/favNums is not disjoint from :ex/luckyNums") + "Exception, because ex:favNums is not disjoint from ex:luckyNums") (is (str/starts-with? (ex-message db-not-disjoint1) "SHACL PropertyShape exception - sh:disjoint")) (is (util/exception? db-not-disjoint2) - "Exception, because :ex/favNums is not disjoint from :ex/luckyNums") + "Exception, because ex:favNums is not disjoint from ex:luckyNums") (is (str/starts-with? (ex-message db-not-disjoint2) "SHACL PropertyShape exception - sh:disjoint")) (is (util/exception? db-not-disjoint3) - "Exception, because :ex/favNums is not disjoint from :ex/luckyNums") + "Exception, because ex:favNums is not disjoint from ex:luckyNums") (is (str/starts-with? (ex-message db-not-disjoint3) "SHACL PropertyShape exception - sh:disjoint")) - (is (= [{:id :ex/alice, - :rdf/type [:ex/User], - :schema/name "Alice" - :ex/favNums [11 17] - :ex/luckyNums 1}] + (is (= [{"id" "ex:alice", + "rdf:type" ["ex:User"], + "schema:name" "Alice" + "ex:favNums" [11 17] + "ex:luckyNums" 1}] @(fluree/query db-ok user-query))))) (testing "lessThan" - (let [db @(fluree/stage - (fluree/db ledger) - {:id :ex/LessThanShape - :type [:sh/NodeShape], - :sh/targetClass :ex/User - :sh/property [{:sh/path :ex/p1 - :sh/lessThan :ex/p2}]}) - db-ok1 @(fluree/stage - db - {:id :ex/alice, - :type [:ex/User], - :schema/name "Alice" - :ex/p1 [11 17] - :ex/p2 [18 19]}) + (let [db @(fluree/stage + (fluree/db ledger) + {"id" "ex:LessThanShape" + "type" ["sh:NodeShape"], + "sh:targetClass" {"id" "ex:User"} + "sh:property" [{"sh:path" {"id" "ex:p1"} + "sh:lessThan" {"id" "ex:p2"}}]}) + db-ok1 @(fluree/stage + db + {"id" "ex:alice", + "type" ["ex:User"], + "schema:name" "Alice" + "ex:p1" [11 17] + "ex:p2" [18 19]}) + + + db-ok2 @(fluree/stage + db + {"id" "ex:alice", + "type" ["ex:User"], + "schema:name" "Alice" + "ex:p1" [11 17] + "ex:p2" [18]}) + + db-fail1 (try + @(fluree/stage + db + {"id" "ex:alice", + "type" ["ex:User"], + "schema:name" "Alice" + "ex:p1" [11 17] + "ex:p2" 17}) + (catch Exception e e)) + db-fail2 (try + @(fluree/stage + db + {"id" "ex:alice", + "type" ["ex:User"], + "schema:name" "Alice" + "ex:p1" [11 17] + "ex:p2" ["18" "19"]}) + (catch Exception e e)) - db-ok2 @(fluree/stage - db - {:id :ex/alice, - :type [:ex/User], - :schema/name "Alice" - :ex/p1 [11 17] - :ex/p2 [18]}) - - db-fail1 (try - @(fluree/stage - db - {:id :ex/alice, - :type [:ex/User], - :schema/name "Alice" - :ex/p1 [11 17] - :ex/p2 17}) - (catch Exception e e)) - - db-fail2 (try - @(fluree/stage - db - {:id :ex/alice, - :type [:ex/User], - :schema/name "Alice" - :ex/p1 [11 17] - :ex/p2 ["18" "19"]}) - (catch Exception e e)) - - - db-fail3 (try - @(fluree/stage - db - {:id :ex/alice, - :type [:ex/User], - :schema/name "Alice" - :ex/p1 [12 17] - :ex/p2 [10 18]}) - (catch Exception e e)) - - db-fail4 (try - @(fluree/stage - db - {:id :ex/alice, - :type [:ex/User], - :schema/name "Alice" - :ex/p1 [11 17] - :ex/p2 [12 16]}) - (catch Exception e e)) - db-iris (try @(fluree/stage - db - {:id :ex/alice, - :type [:ex/User], - :schema/name "Alice" - :ex/p1 :ex/brian - :ex/p2 :ex/john}) - (catch Exception e e))] + + db-fail3 (try + @(fluree/stage + db + {"id" "ex:alice", + "type" ["ex:User"], + "schema:name" "Alice" + "ex:p1" [12 17] + "ex:p2" [10 18]}) + (catch Exception e e)) + + db-fail4 (try + @(fluree/stage + db + {"id" "ex:alice", + "type" ["ex:User"], + "schema:name" "Alice" + "ex:p1" [11 17] + "ex:p2" [12 16]}) + (catch Exception e e)) + db-iris (try @(fluree/stage + db + {"id" "ex:alice", + "type" ["ex:User"], + "schema:name" "Alice" + "ex:p1" {"id" "ex:brian"} + "ex:p2" {"id" "ex:john"}}) + (catch Exception e e))] (is (util/exception? db-fail1) - "Exception, because :ex/p1 is not less than :ex/p2") + "Exception, because ex:p1 is not less than ex:p2") (is (str/starts-with? (ex-message db-fail1) "SHACL PropertyShape exception - sh:lessThan")) (is (util/exception? db-fail2) - "Exception, because :ex/p1 is not less than :ex/p2") + "Exception, because ex:p1 is not less than ex:p2") (is (str/starts-with? (ex-message db-fail2) "SHACL PropertyShape exception - sh:lessThan")) (is (util/exception? db-fail3) - "Exception, because :ex/p1 is not less than :ex/p2") + "Exception, because ex:p1 is not less than ex:p2") (is (str/starts-with? (ex-message db-fail3) "SHACL PropertyShape exception - sh:lessThan")) (is (util/exception? db-fail4) - "Exception, because :ex/p1 is not less than :ex/p2") + "Exception, because ex:p1 is not less than ex:p2") (is (str/starts-with? (ex-message db-fail4) "SHACL PropertyShape exception - sh:lessThan")) (is (util/exception? db-iris) - "Exception, because :ex/p1 and :ex/p2 are iris, and not valid for comparison") + "Exception, because ex:p1 and ex:p2 are iris, and not valid for comparison") (is (str/starts-with? (ex-message db-iris) "SHACL PropertyShape exception - sh:lessThan")) - (is (= [{:id :ex/alice, - :rdf/type [:ex/User], - :schema/name "Alice" - :ex/p1 [11 17] - :ex/p2 [18 19]}] + (is (= [{"id" "ex:alice", + "rdf:type" ["ex:User"], + "schema:name" "Alice" + "ex:p1" [11 17] + "ex:p2" [18 19]}] @(fluree/query db-ok1 user-query))) - (is (= [{:id :ex/alice, - :rdf/type [:ex/User], - :schema/name "Alice" - :ex/p1 [11 17] - :ex/p2 18}] + (is (= [{"id" "ex:alice", + "rdf:type" ["ex:User"], + "schema:name" "Alice" + "ex:p1" [11 17] + "ex:p2" 18}] @(fluree/query db-ok2 user-query))))) (testing "lessThanOrEquals" - (let [db @(fluree/stage - (fluree/db ledger) - {:id :ex/LessThanOrEqualsShape - :type [:sh/NodeShape], - :sh/targetClass :ex/User - :sh/property [{:sh/path :ex/p1 - :sh/lessThanOrEquals :ex/p2}]}) - db-ok1 @(fluree/stage - db - {:id :ex/alice, - :type [:ex/User], - :schema/name "Alice" - :ex/p1 [11 17] - :ex/p2 [17 19]}) + (let [db @(fluree/stage + (fluree/db ledger) + {"id" "ex:LessThanOrEqualsShape" + "type" ["sh:NodeShape"], + "sh:targetClass" {"id" "ex:User"} + "sh:property" [{"sh:path" {"id" "ex:p1"} + "sh:lessThanOrEquals" {"id" "ex:p2"}}]}) + db-ok1 @(fluree/stage + db + {"id" "ex:alice", + "type" ["ex:User"], + "schema:name" "Alice" + "ex:p1" [11 17] + "ex:p2" [17 19]}) + + + db-ok2 @(fluree/stage + db + {"id" "ex:alice", + "type" ["ex:User"], + "schema:name" "Alice" + "ex:p1" [11 17] + "ex:p2" 17}) + + db-fail1 (try + @(fluree/stage + db + {"id" "ex:alice", + "type" ["ex:User"], + "schema:name" "Alice" + "ex:p1" [11 17] + "ex:p2" 10}) + (catch Exception e e)) + db-fail2 (try + @(fluree/stage + db + {"id" "ex:alice", + "type" ["ex:User"], + "schema:name" "Alice" + "ex:p1" [11 17] + "ex:p2" ["17" "19"]}) + (catch Exception e e)) - db-ok2 @(fluree/stage - db - {:id :ex/alice, - :type [:ex/User], - :schema/name "Alice" - :ex/p1 [11 17] - :ex/p2 17}) - - db-fail1 (try - @(fluree/stage - db - {:id :ex/alice, - :type [:ex/User], - :schema/name "Alice" - :ex/p1 [11 17] - :ex/p2 10}) - (catch Exception e e)) - - db-fail2 (try - @(fluree/stage - db - {:id :ex/alice, - :type [:ex/User], - :schema/name "Alice" - :ex/p1 [11 17] - :ex/p2 ["17" "19"]}) - (catch Exception e e)) - - db-fail3 (try - @(fluree/stage - db - {:id :ex/alice, - :type [:ex/User], - :schema/name "Alice" - :ex/p1 [12 17] - :ex/p2 [10 17]}) - (catch Exception e e)) - - db-fail4 (try - @(fluree/stage - db - {:id :ex/alice, - :type [:ex/User], - :schema/name "Alice" - :ex/p1 [11 17] - :ex/p2 [12 16]}) - (catch Exception e e))] + db-fail3 (try + @(fluree/stage + db + {"id" "ex:alice", + "type" ["ex:User"], + "schema:name" "Alice" + "ex:p1" [12 17] + "ex:p2" [10 17]}) + (catch Exception e e)) + + db-fail4 (try + @(fluree/stage + db + {"id" "ex:alice", + "type" ["ex:User"], + "schema:name" "Alice" + "ex:p1" [11 17] + "ex:p2" [12 16]}) + (catch Exception e e))] (is (util/exception? db-fail1) - "Exception, because :ex/p1 is not less than or equal to :ex/p2") + "Exception, because ex:p1 is not less than or equal to ex:p2") (is (str/starts-with? (ex-message db-fail1) "SHACL PropertyShape exception - sh:lessThanOrEquals")) (is (util/exception? db-fail2) - "Exception, because :ex/p1 is not less than or equal to :ex/p2") + "Exception, because ex:p1 is not less than or equal to ex:p2") (is (str/starts-with? (ex-message db-fail2) "SHACL PropertyShape exception - sh:lessThanOrEquals")) (is (util/exception? db-fail3) - "Exception, because :ex/p1 is not less than or equal to :ex/p2") + "Exception, because ex:p1 is not less than or equal to ex:p2") (is (str/starts-with? (ex-message db-fail3) "SHACL PropertyShape exception - sh:lessThanOrEquals")) (is (util/exception? db-fail4) - "Exception, because :ex/p1 is not less than or equal to :ex/p2") + "Exception, because ex:p1 is not less than or equal to ex:p2") (is (str/starts-with? (ex-message db-fail4) "SHACL PropertyShape exception - sh:lessThanOrEquals")) - (is (= [{:id :ex/alice, - :rdf/type [:ex/User], - :schema/name "Alice" - :ex/p1 [11 17] - :ex/p2 [17 19]}] + (is (= [{"id" "ex:alice", + "rdf:type" ["ex:User"], + "schema:name" "Alice" + "ex:p1" [11 17] + "ex:p2" [17 19]}] @(fluree/query db-ok1 user-query))) - (is (= [{:id :ex/alice, - :rdf/type [:ex/User], - :schema/name "Alice" - :ex/p1 [11 17] - :ex/p2 17}] + (is (= [{"id" "ex:alice", + "rdf:type" ["ex:User"], + "schema:name" "Alice" + "ex:p1" [11 17] + "ex:p2" 17}] @(fluree/query db-ok2 user-query)))))))) (deftest ^:integration shacl-value-range (testing "shacl value range constraints" - (let [conn (test-utils/create-conn) - ledger @(fluree/create conn "shacl/value-range" {:defaultContext ["" {:ex "http://example.org/ns/"}]}) - user-query {:select {'?s [:*]} - :where [['?s :rdf/type :ex/User]]}] + (let [conn (test-utils/create-conn) + ledger @(fluree/create conn "shacl/value-range" + {"defaults" + {"@context" + ["" {"ex" "http://example.org/ns/"}]}}) + user-query {"select" {'?s ["*"]} + "where" [['?s "rdf:type" "ex:User"]]}] (testing "exclusive constraints" - (let [db @(fluree/stage - (fluree/db ledger) - {:id :ex/ExclusiveNumRangeShape - :type [:sh/NodeShape], - :sh/targetClass :ex/User - :sh/property [{:sh/path :schema/age - :sh/minExclusive 1 - :sh/maxExclusive 100}]}) - db-ok @(fluree/stage - db - {:id :ex/john, - :type [:ex/User], - :schema/age 2}) - db-too-low (try @(fluree/stage - db - {:id :ex/john, - :type [:ex/User], - :schema/age 1}) - (catch Exception e e)) - db-too-high (try @(fluree/stage - db - {:id :ex/john, - :type [:ex/User], - :schema/age 100}) - (catch Exception e e))] + (let [db @(fluree/stage + (fluree/db ledger) + {"id" "ex:ExclusiveNumRangeShape" + "type" ["sh:NodeShape"], + "sh:targetClass" {"id" "ex:User"} + "sh:property" [{"sh:path" {"id" "schema:age"} + "sh:minExclusive" 1 + "sh:maxExclusive" 100}]}) + db-ok @(fluree/stage + db + {"id" "ex:john", + "type" ["ex:User"], + "schema:age" 2}) + db-too-low (try @(fluree/stage + db + {"id" "ex:john", + "type" ["ex:User"], + "schema:age" 1}) + (catch Exception e e)) + db-too-high (try @(fluree/stage + db + {"id" "ex:john", + "type" ["ex:User"], + "schema:age" 100}) + (catch Exception e e))] (is (util/exception? db-too-low) - "Exception, because :schema/age is below the minimum") + "Exception, because schema:age is below the minimum") (is (str/starts-with? (ex-message db-too-low) "SHACL PropertyShape exception - sh:minExclusive: value 1")) (is (util/exception? db-too-high) - "Exception, because :schema/age is above the maximum") + "Exception, because schema:age is above the maximum") (is (str/starts-with? (ex-message db-too-high) "SHACL PropertyShape exception - sh:maxExclusive: value 100")) - (is (= @(fluree/query db-ok user-query) - [{:id :ex/john, - :rdf/type [:ex/User], - :schema/age 2}])))) + (is (= [{"id" "ex:john", + "rdf:type" ["ex:User"], + "schema:age" 2}] + @(fluree/query db-ok user-query))))) + (testing "inclusive constraints" - (let [db @(fluree/stage - (fluree/db ledger) - {:id :ex/InclusiveNumRangeShape - :type [:sh/NodeShape], - :sh/targetClass :ex/User - :sh/property [{:sh/path :schema/age - :sh/minInclusive 1 - :sh/maxInclusive 100}]}) - db-ok @(fluree/stage - db - {:id :ex/brian - :type [:ex/User], - :schema/age 1}) - db-ok2 @(fluree/stage - db-ok - {:id :ex/alice, - :type [:ex/User], - :schema/age 100}) - db-too-low @(fluree/stage - db - {:id :ex/alice, - :type [:ex/User], - :schema/age 0}) - db-too-high @(fluree/stage - db - {:id :ex/alice, - :type [:ex/User], - :schema/age 101})] + (let [db @(fluree/stage + (fluree/db ledger) + {"id" "ex:InclusiveNumRangeShape" + "type" ["sh:NodeShape"], + "sh:targetClass" {"id" "ex:User"} + "sh:property" [{"sh:path" {"id" "schema:age"} + "sh:minInclusive" 1 + "sh:maxInclusive" 100}]}) + db-ok @(fluree/stage + db + {"id" "ex:brian" + "type" ["ex:User"], + "schema:age" 1}) + db-ok2 @(fluree/stage + db-ok + {"id" "ex:alice", + "type" ["ex:User"], + "schema:age" 100}) + db-too-low @(fluree/stage + db + {"id" "ex:alice", + "type" ["ex:User"], + "schema:age" 0}) + db-too-high @(fluree/stage + db + {"id" "ex:alice", + "type" ["ex:User"], + "schema:age" 101})] (is (util/exception? db-too-low) - "Exception, because :schema/age is below the minimum") + "Exception, because schema:age is below the minimum") (is (str/starts-with? (ex-message db-too-low) "SHACL PropertyShape exception - sh:minInclusive: value 0")) (is (util/exception? db-too-high) - "Exception, because :schema/age is above the maximum") + "Exception, because schema:age is above the maximum") (is (str/starts-with? (ex-message db-too-high) "SHACL PropertyShape exception - sh:maxInclusive: value 101")) - (is (= @(fluree/query db-ok2 user-query) - [{:id :ex/alice - :rdf/type [:ex/User] - :schema/age 100} - {:id :ex/brian - :rdf/type [:ex/User] - :schema/age 1}])))) + (is (= [{"id" "ex:alice" + "rdf:type" ["ex:User"] + "schema:age" 100} + {"id" "ex:brian" + "rdf:type" ["ex:User"] + "schema:age" 1}] + @(fluree/query db-ok2 user-query))))) + (testing "non-numeric values" - (let [db @(fluree/stage - (fluree/db ledger) - {:id :ex/NumRangeShape - :type [:sh/NodeShape], - :sh/targetClass :ex/User - :sh/property [{:sh/path :schema/age - :sh/minExclusive 0}]}) - db-subj-id (try @(fluree/stage - db - {:id :ex/alice, - :type [:ex/User], - :schema/age :ex/brian}) - (catch Exception e e)) - db-string (try @(fluree/stage - db - {:id :ex/alice, - :type [:ex/User], - :schema/age "10"}) - (catch Exception e e))] + (let [db @(fluree/stage + (fluree/db ledger) + {"id" "ex:NumRangeShape" + "type" ["sh:NodeShape"], + "sh:targetClass" {"id" "ex:User"} + "sh:property" [{"sh:path" {"id" "schema:age"} + "sh:minExclusive" 0}]}) + db-subj-id (try @(fluree/stage + db + {"id" "ex:alice", + "type" ["ex:User"], + "schema:age" {"id" "ex:brian"}}) + (catch Exception e e)) + db-string (try @(fluree/stage + db + {"id" "ex:alice", + "type" ["ex:User"], + "schema:age" "10"}) + (catch Exception e e))] (is (util/exception? db-subj-id) - "Exception, because :schema/age is not a number") + "Exception, because schema:age is not a number") (is (str/starts-with? (ex-message db-string) "SHACL PropertyShape exception - sh:minExclusive")) (is (util/exception? db-string) - "Exception, because :schema/age is not a number") + "Exception, because schema:age is not a number") (is (str/starts-with? (ex-message db-string) "SHACL PropertyShape exception - sh:minExclusive: value 10"))))))) From 6d38c581c970c5060893beb02a400a9aacc8710a Mon Sep 17 00:00:00 2001 From: Daniel Petranek Date: Mon, 17 Apr 2023 14:55:33 -0500 Subject: [PATCH 31/50] Fix result formatting The result formatter uses a parsed context which transforms the "@container" keyword into :container. It also transforms "@list" into :list, but it transforms "@set" into "set", so I just included both the strings and the keywords for now. --- src/fluree/db/query/json_ld/response.cljc | 2 +- test/fluree/db/query/misc_queries_test.clj | 27 +++++++++------------- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/src/fluree/db/query/json_ld/response.cljc b/src/fluree/db/query/json_ld/response.cljc index f10eb2908..a407d0fb8 100644 --- a/src/fluree/db/query/json_ld/response.cljc +++ b/src/fluree/db/query/json_ld/response.cljc @@ -139,7 +139,7 @@ (flake/o f))] (recur r (conj acc res))) (if (and (= 1 (count acc)) - (not (#{"list" "set"} (-> context (get p-iri) (get "@container"))))) + (not (#{"list" "set" :list :set} (-> context (get p-iri) (get :container))))) (first acc) acc))))] (if v diff --git a/test/fluree/db/query/misc_queries_test.clj b/test/fluree/db/query/misc_queries_test.clj index a20a07b90..3c2d60d94 100644 --- a/test/fluree/db/query/misc_queries_test.clj +++ b/test/fluree/db/query/misc_queries_test.clj @@ -36,29 +36,24 @@ (deftest ^:integration result-formatting (let [conn (test-utils/create-conn) - ledger @(fluree/create conn "query-context" - {"defaults" - {"@context" ["" {"ex" "http://example.org/ns/"}]}}) + ledger @(fluree/create conn "query-context" {"defaults" {"@context" ["" {"ex" "http://example.org/ns/"}]}}) db @(test-utils/transact ledger [{"id" "ex:dan" "ex:x" 1}])] - (is (= [{"id" "foo:dan" - "foo:x" 1}] - @(fluree/query db {"@context" ["" {"foo" "http://example.org/ns/"}] - "where" [['?s "id" "foo:dan"]] + (is (= [{"id" "ex:dan" + "ex:x" 1}] + @(fluree/query db {"where" [['?s "id" "ex:dan"]] "select" {'?s ["*"]}})) "default unwrapped objects") - (is (= [{"id" "foo:dan" - "foo:x" [1]}] - @(fluree/query db {"@context" ["" {"foo" "http://example.org/ns/" - "foo:x" {"@container" "set"}}] - "where" [['?s "id" "foo:dan"]] + (is (= [{"id" "ex:dan" + "ex:x" [1]}] + @(fluree/query db {"@context" ["" {"ex:x" {"@container" "set"}}] + "where" [['?s "id" "ex:dan"]] "select" {'?s ["*"]}})) "override unwrapping with :set") (is (= [{"id" "ex:dan" - "foo:x" [1]}] - @(fluree/query db {"@context" ["" {"foo" "http://example.org/ns/" - "foo:x" {"@container" "@list"}}] - "where" [['?s "@id" "foo:dan"]] + "ex:x" [1]}] + @(fluree/query db {"@context" ["" {"ex:x" {"@container" "@list"}}] + "where" [['?s "@id" "ex:dan"]] "select" {'?s ["*"]}})) "override unwrapping with @list"))) From 79df9e857a11080cb7508f3e97226beddfab9c50 Mon Sep 17 00:00:00 2001 From: Daniel Petranek Date: Mon, 17 Apr 2023 15:22:42 -0500 Subject: [PATCH 32/50] Convert transact test to use strings api --- test/fluree/db/transact/transact_test.clj | 50 +++++++++++------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/test/fluree/db/transact/transact_test.clj b/test/fluree/db/transact/transact_test.clj index 26a56fada..7bd1fc6c7 100644 --- a/test/fluree/db/transact/transact_test.clj +++ b/test/fluree/db/transact/transact_test.clj @@ -8,12 +8,12 @@ (deftest ^:integration staging-data (testing "Disallow staging invalid transactions" (let [conn (test-utils/create-conn ) - ledger @(fluree/create conn "tx/disallow" {:defaultContext ["" {:ex "http://example.org/ns/"}]}) + ledger @(fluree/create conn "tx/disallow" {"defaults" {"@context" ["" {"ex" "http://example.org/ns/"}]}}) stage-id-only (try @(fluree/stage (fluree/db ledger) - {:id :ex/alice}) + {"id" "ex:alice"}) (catch Exception e e)) stage-empty-txn (try @(fluree/stage @@ -23,14 +23,14 @@ stage-empty-node (try @(fluree/stage (fluree/db ledger) - [{:id :ex/alice - :schema/age 42} + [{"id" "ex:alice" + "schema:age" 42} {}]) (catch Exception e e)) db-ok @(fluree/stage (fluree/db ledger) - {:id :ex/alice - :schema/age 42})] + {"id" "ex:alice" + "schema:age" 42})] (is (util/exception? stage-id-only)) (is (str/starts-with? (ex-message stage-id-only) "Invalid transaction, transaction node contains no properties for @id:" )) @@ -40,26 +40,26 @@ (is (util/exception? stage-empty-node)) (is (= (ex-message stage-empty-node) "Invalid transaction, transaction node contains no properties." )) - (is (= [[:ex/alice :id "http://example.org/ns/alice"] - [:ex/alice :schema/age 42] - [:schema/age :id "http://schema.org/age"] - [:rdfs/Class :id "http://www.w3.org/2000/01/rdf-schema#Class"] - [:rdf/type :id "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"] - [:id :id "@id"]] - @(fluree/query db-ok '{:select [?s ?p ?o] - :where [[?s ?p ?o]]}))))) + (is (= [["ex:alice" "id" "http://example.org/ns/alice"] + ["ex:alice" "schema:age" 42] + ["schema:age" "id" "http://schema.org/age"] + ["rdfs:Class" "id" "http://www.w3.org/2000/01/rdf-schema#Class"] + ["rdf:type" "id" "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"] + ["id" "id" "@id"]] + @(fluree/query db-ok '{"select" [?s ?p ?o] + "where" [[?s ?p ?o]]}))))) (testing "Allow transacting `false` values" (let [conn (test-utils/create-conn) - ledger @(fluree/create conn "tx/bools" {:defaultContext ["" {:ex "http://example.org/ns/"}]}) + ledger @(fluree/create conn "tx/bools" {"defaults" {"@context" ["" {"ex" "http://example.org/ns/"}]}}) db-bool @(fluree/stage (fluree/db ledger) - {:id :ex/alice - :ex/isCool false})] - (is (= [[:ex/alice :id "http://example.org/ns/alice"] - [:ex/alice :ex/isCool false] - [:ex/isCool :id "http://example.org/ns/isCool"] - [:rdfs/Class :id "http://www.w3.org/2000/01/rdf-schema#Class"] - [:rdf/type :id "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"] - [:id :id "@id"]] - @(fluree/query db-bool '{:select [?s ?p ?o] - :where [[?s ?p ?o]]})))))) + {"id" "ex:alice" + "ex:isCool" false})] + (is (= [["ex:alice" "id" "http://example.org/ns/alice"] + ["ex:alice" "ex:isCool" false] + ["ex:isCool" "id" "http://example.org/ns/isCool"] + ["rdfs:Class" "id" "http://www.w3.org/2000/01/rdf-schema#Class"] + ["rdf:type" "id" "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"] + ["id" "id" "@id"]] + @(fluree/query db-bool '{"select" [?s ?p ?o] + "where" [[?s ?p ?o]]})))))) From 1cc45b211a68e1e5a5f696e4cdb3757d1b43be16 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Mon, 17 Apr 2023 14:58:03 -0600 Subject: [PATCH 33/50] Remove unneeded ExceptionInfo import --- src/fluree/db/query/fql/syntax.cljc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/fluree/db/query/fql/syntax.cljc b/src/fluree/db/query/fql/syntax.cljc index 6b9e4a384..7f991f6fd 100644 --- a/src/fluree/db/query/fql/syntax.cljc +++ b/src/fluree/db/query/fql/syntax.cljc @@ -2,8 +2,7 @@ (:require [fluree.db.util.core :as util :refer [pred-ident?]] [fluree.db.constants :as const] [malli.core :as m] - [fluree.db.util.validation :as v]) - (:import (clojure.lang ExceptionInfo))) + [fluree.db.util.validation :as v])) #?(:clj (set! *warn-on-reflection* true)) From cd7b20c2ff51a25ca158529592268f6d74e0a173 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Tue, 18 Apr 2023 06:59:10 -0600 Subject: [PATCH 34/50] Change "context" to "@context" in JS tests --- test/nodejs/flureenjs.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/nodejs/flureenjs.test.js b/test/nodejs/flureenjs.test.js index c003116d6..6ae3f5336 100644 --- a/test/nodejs/flureenjs.test.js +++ b/test/nodejs/flureenjs.test.js @@ -23,7 +23,7 @@ test("expect conn, ledger, stage, commit, and query to work", async () => { const conn = await flureenjs.connect({ method: "memory", defaults: { - context: { + "@context": { id: "@id", type: "@type", schema: "http://schema.org/", From e3e68bbde0c632eff9beff731834278305b876bf Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Tue, 18 Apr 2023 09:14:44 -0600 Subject: [PATCH 35/50] Use constants for a couple of things in f.d.db.json-ld --- src/fluree/db/db/json_ld.cljc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/fluree/db/db/json_ld.cljc b/src/fluree/db/db/json_ld.cljc index 8ee244a77..32968a79e 100644 --- a/src/fluree/db/db/json_ld.cljc +++ b/src/fluree/db/db/json_ld.cljc @@ -111,7 +111,10 @@ "Returns the iri for a given subject ID" [db subject-id compact-fn] (go-try - (when-let [flake (first ( db + (query-range/index-range :spot = + [subject-id const/$xsd:anyURI]) + flake flake/o compact-fn)))) ;; ================ GraphDB record support fns ================================ @@ -140,7 +143,7 @@ return value if it starts with the given predicate name" ([this tag-id] (go-try - (let [tag-pred-id 30] + (let [tag-pred-id const/$_tag:id] (some-> ( Date: Tue, 18 Apr 2023 14:05:59 -0600 Subject: [PATCH 36/50] Use keyword keys internally for queries --- src/fluree/db/json_ld/transact.cljc | 5 +- src/fluree/db/query/exec.cljc | 6 +- src/fluree/db/query/exec/group.cljc | 2 +- src/fluree/db/query/exec/having.cljc | 2 +- src/fluree/db/query/exec/order.cljc | 2 +- src/fluree/db/query/exec/select.cljc | 10 +- src/fluree/db/query/exec/where.cljc | 4 +- src/fluree/db/query/fql/parse.cljc | 81 +++++----- src/fluree/db/query/fql/syntax.cljc | 34 ++++- .../db/query/subject_crawl/reparse.cljc | 20 ++- test/fluree/db/query/fql_parse_test.clj | 143 ++++++++++-------- .../db/query/subject_crawl_reparse_test.clj | 59 +++++--- test/fluree/db/transact/delete_test.clj | 2 - 13 files changed, 202 insertions(+), 168 deletions(-) diff --git a/src/fluree/db/json_ld/transact.cljc b/src/fluree/db/json_ld/transact.cljc index c02337728..50e5f4b21 100644 --- a/src/fluree/db/json_ld/transact.cljc +++ b/src/fluree/db/json_ld/transact.cljc @@ -386,13 +386,14 @@ "Executes a delete statement" [db max-fuel json-ld {:keys [t] :as _tx-state}] (go-try - (let [{:strs [delete] :as parsed-query} + (let [{:keys [delete] :as parsed-query} (-> json-ld syntax/validate-query + syntax/encode-internal-query (q-parse/parse-delete db)) [s p o] delete - parsed-query (assoc parsed-query "delete" [s p o]) + parsed-query (assoc parsed-query :delete [s p o]) error-ch (async/chan) flake-ch (async/chan) where-ch (where/search db parsed-query error-ch)] diff --git a/src/fluree/db/query/exec.cljc b/src/fluree/db/query/exec.cljc index f455bd8a1..c2d356bf2 100644 --- a/src/fluree/db/query/exec.cljc +++ b/src/fluree/db/query/exec.cljc @@ -14,7 +14,7 @@ "Returns a channel containing the stream of solutions from `solution-ch` after the `offset` specified by the supplied query. Returns the original `solution-ch` if no offset is specified." - [{:strs [offset]} solution-ch] + [{:keys [offset]} solution-ch] (if offset (async/pipe solution-ch (async/chan 2 (drop offset))) @@ -24,7 +24,7 @@ "Returns a channel that contains at most the specified `:limit` of the supplied query solutions from `solution-ch`, if the supplied query has a limit. Returns the original `solution-ch` if the supplied query has no specified limit." - [{:strs [limit]} solution-ch] + [{:keys [limit]} solution-ch] (if limit (async/take limit solution-ch) solution-ch)) @@ -38,7 +38,7 @@ containing a single result to the output channel instead of the single result alone." [q result-ch] - (if (get q "select-one") + (if (:select-one q) (async/take 1 result-ch) (async/into [] result-ch))) diff --git a/src/fluree/db/query/exec/group.cljc b/src/fluree/db/query/exec/group.cljc index e9ef13ae0..e2966cf17 100644 --- a/src/fluree/db/query/exec/group.cljc +++ b/src/fluree/db/query/exec/group.cljc @@ -64,7 +64,7 @@ (defn combine "Returns a channel of solutions from `solution-ch` collected into groups defined by the `:group-by` clause specified in the supplied query." - [{:strs [group-by select]} solution-ch] + [{:keys [group-by select]} solution-ch] (if-let [grouping (or group-by (implicit-grouping select))] (-> (async/transduce (map (partial split-solution-by grouping)) diff --git a/src/fluree/db/query/exec/having.cljc b/src/fluree/db/query/exec/having.cljc index 63217d3db..8bb80a373 100644 --- a/src/fluree/db/query/exec/having.cljc +++ b/src/fluree/db/query/exec/having.cljc @@ -6,7 +6,7 @@ (defn filter [q error-ch solution-ch] - (if-let [filter-fn (get q "having")] + (if-let [filter-fn (:having q)] (let [filtered-ch (async/chan)] (async/pipeline-async 2 filtered-ch diff --git a/src/fluree/db/query/exec/order.cljc b/src/fluree/db/query/exec/order.cljc index 0274d16c8..ae517d80e 100644 --- a/src/fluree/db/query/exec/order.cljc +++ b/src/fluree/db/query/exec/order.cljc @@ -39,7 +39,7 @@ ordering specified by the `:order-by` clause of the supplied parsed query. Note that all solutions from `solution-ch` are first loaded into memory before they are sorted in place and placed individually on the output channel." - [{:strs [order-by]} solution-ch] + [{:keys [order-by]} solution-ch] (if order-by (let [comparator (partial compare-solutions order-by) coll-ch (async/into [] solution-ch) diff --git a/src/fluree/db/query/exec/select.cljc b/src/fluree/db/query/exec/select.cljc index e2a0a2771..9b18e4793 100644 --- a/src/fluree/db/query/exec/select.cljc +++ b/src/fluree/db/query/exec/select.cljc @@ -116,14 +116,14 @@ "Formats each solution within the stream of solutions in `solution-ch` according to the selectors within the select clause of the supplied parsed query `q`." [db q error-ch solution-ch] - (let [context (get q "@context") + (let [context (:context q) _ (log/debug "format context:" context) compact (json-ld/compact-fn context) - selectors (or (get q "select") - (get q "select-one") - (get q "select-distinct")) + selectors (or (:select q) + (:select-one q) + (:select-distinct q)) iri-cache (volatile! {}) - format-ch (if (contains? q "select-distinct") + format-ch (if (contains? q :select-distinct) (chan 1 (distinct)) (chan))] (async/pipeline-async 1 diff --git a/src/fluree/db/query/exec/where.cljc b/src/fluree/db/query/exec/where.cljc index b1320aab7..cb5c2c2c4 100644 --- a/src/fluree/db/query/exec/where.cljc +++ b/src/fluree/db/query/exec/where.cljc @@ -394,9 +394,9 @@ (defn search [db q error-ch] - (let [where-clause (get q "where") + (let [where-clause (:where q) initial-solutions (-> q - (get "values") + :values not-empty (or [blank-solution])) out-ch (async/chan)] diff --git a/src/fluree/db/query/fql/parse.cljc b/src/fluree/db/query/fql/parse.cljc index 208a8996d..b76027484 100644 --- a/src/fluree/db/query/fql/parse.cljc +++ b/src/fluree/db/query/fql/parse.cljc @@ -20,7 +20,7 @@ (defn parse-context [q db] - (dbproto/-context db (get q "@context"))) + (dbproto/-context db (:context q))) (defn parse-var-name "Returns a `x` as a symbol if `x` is a valid '?variable'." @@ -43,7 +43,7 @@ (defn parse-values [q] - (when-let [values (get q "values")] + (when-let [values (:values q)] (let [[vars vals] values vars* (util/sequential vars) vals* (mapv util/sequential vals) @@ -295,12 +295,12 @@ (where/anonymous-value o-pat))) (defmulti parse-pattern - (fn [pattern _vars _db _context] - (log/debug "parse-pattern pattern:" pattern) - (cond - (map? pattern) (->> pattern keys first keyword) - (map-entry? pattern) :binding - :else :triple))) + (fn [pattern _vars _db _context] + (log/debug "parse-pattern pattern:" pattern) + (cond + (map? pattern) (->> pattern keys first keyword) + (map-entry? pattern) :binding + :else :triple))) (defn type-pattern? [typ x] @@ -365,14 +365,14 @@ (parse-triple triple db context)) (defmethod parse-pattern :union - [{:strs [union]} vars db context] + [{:keys [union]} vars db context] (let [parsed (mapv (fn [clause] (parse-where-clause clause vars db context)) union)] (where/->pattern :union parsed))) (defmethod parse-pattern :optional - [{:strs [optional]} vars db context] + [{:keys [optional]} vars db context] (let [clause (if (coll? (first optional)) optional [optional]) @@ -380,7 +380,7 @@ (where/->pattern :optional parsed))) (defmethod parse-pattern :bind - [{:strs [bind]} _vars _db _context] + [{:keys [bind]} _vars _db _context] (let [parsed (parse-bind-map bind) _ (log/debug "parsed bind map:" parsed) pattern (where/->pattern :bind parsed)] @@ -394,7 +394,7 @@ (defn parse-where [q vars db context] - (when-let [where (get q "where")] + (when-let [where (:where q)] (parse-where-clause where vars db context))) (defn parse-selector @@ -415,26 +415,14 @@ (defn parse-select [q db context] - (let [depth (or (get q "depth") 0) + (let [depth (get q :depth 0) select-key (some (fn [k] (when (contains? q k) k)) - ["select" "selectOne" "select-one" - "selectDistinct" "select-distinct"]) + [:select :select-one :select-distinct]) select (-> q (get select-key) (parse-select-clause db context depth))] - (case select-key - ("select" - "select-one" - "select-distinct") (assoc q select-key select) - - "selectOne" (-> q - (dissoc "selectOne") - (assoc "select-one" select)) - - "selectDistinct" (-> q - (dissoc "selectDistinct") - (assoc "select-distinct" select))))) + (assoc q select-key select))) (defn ensure-vector [x] @@ -444,15 +432,15 @@ (defn parse-grouping [q] - (some->> (or (get q "groupBy") - (get q "group-by")) + (some->> q + :group-by ensure-vector (mapv parse-var-name))) (defn parse-ordering [q] - (some->> (or (get q "order-by") - (get q "orderBy")) + (some->> q + :order-by ensure-vector (mapv (fn [ord] (if-let [v (parse-var-name ord)] @@ -465,8 +453,8 @@ (defn parse-having [q] - (if-let [code (some-> q (get "having") parse-code)] - (assoc q "having" (eval/compile code)) + (if-let [code (some-> q :having parse-code)] + (assoc q :having (eval/compile code)) q)) (defn parse-analytical-query* @@ -478,11 +466,11 @@ grouping (parse-grouping q) ordering (parse-ordering q)] (-> q - (assoc "@context" context - "where" where) - (cond-> (seq values) (assoc "values" values) - grouping (assoc "group-by" grouping) - ordering (assoc "order-by" ordering)) + (assoc :context context + :where where) + (cond-> (seq values) (assoc :values values) + grouping (assoc :group-by grouping) + ordering (assoc :order-by ordering)) parse-having (parse-select db context)))) @@ -496,17 +484,20 @@ (defn parse [q db] - (syntax/validate-query q) - (parse-analytical-query q db)) + (-> q + syntax/validate-query + syntax/encode-internal-query + (log/debug->val "encoded query:") + (parse-analytical-query db))) (defn parse-delete [q db] - (when (get q "delete") + (when (:delete q) (let [context (parse-context q db) [vars values] (parse-values q) where (parse-where q vars db context)] (-> q - (assoc "@context" context - "where" where) - (cond-> (seq values) (assoc "values" values)) - (update "delete" parse-triple db context))))) + (assoc :context context + :where where) + (cond-> (seq values) (assoc :values values)) + (update :delete parse-triple db context))))) diff --git a/src/fluree/db/query/fql/syntax.cljc b/src/fluree/db/query/fql/syntax.cljc index 7f991f6fd..7207666fd 100644 --- a/src/fluree/db/query/fql/syntax.cljc +++ b/src/fluree/db/query/fql/syntax.cljc @@ -1,8 +1,11 @@ (ns fluree.db.query.fql.syntax - (:require [fluree.db.util.core :as util :refer [pred-ident?]] + (:require [clojure.string :as str] + [fluree.db.util.core :as util :refer [pred-ident?]] + [fluree.db.util.log :as log] [fluree.db.constants :as const] [malli.core :as m] - [fluree.db.util.validation :as v])) + [fluree.db.util.validation :as v] + [malli.transform :as mt])) #?(:clj (set! *warn-on-reflection* true)) @@ -49,6 +52,25 @@ (when (map? x) (-> x first key))) +(defn ->kebab-case + [s] + (str/replace s #"([a-z])?([A-Z])" + (fn [[_ m1 m2]] + (if m1 + (str m1 "-" (str/lower-case m2)) + (str/lower-case m2))))) + +(defn encode-query-key + [k] + (if (string? k) + (do + (log/debug "encoding query key:" k) + (let [kebab-k (->kebab-case k)] + (if (str/starts-with? kebab-k "@") + (-> kebab-k (subs 1) keyword) + (keyword kebab-k)))) + k)) + (def registry (merge (m/predicate-schemas) @@ -111,6 +133,7 @@ [:desc [:fn desc?]]] ::ordering [:orn [:scalar ::var] + [:fn [:fn fn-list?]] [:vector [:catn [:direction ::direction] [:dimension ::var]]]] @@ -124,7 +147,8 @@ ::group-by ::groupBy ::where-op [:enum "filter" "optional" "union" "bind"] ::where-map [:and - [:map-of {:max 1} ::where-op :any] + [:map-of {:max 1} + ::where-op :any] [:multi {:dispatch where-op} ["filter" [:map ["filter" [:ref ::filter]]]] ["optional" [:map ["optional" [:ref ::optional]]]] @@ -219,6 +243,10 @@ [:select-one ::result-map]] ::multi-query-results [:map-of :string ::analytical-query-results]})) +(def encode-internal-query + (m/encoder ::query {:registry registry} + (mt/key-transformer {:encode encode-query-key}))) + (def valid-query? (m/validator ::query {:registry registry})) diff --git a/src/fluree/db/query/subject_crawl/reparse.cljc b/src/fluree/db/query/subject_crawl/reparse.cljc index 084c8933f..5b9d02e02 100644 --- a/src/fluree/db/query/subject_crawl/reparse.cljc +++ b/src/fluree/db/query/subject_crawl/reparse.cljc @@ -117,7 +117,7 @@ "Revises where clause for simple-subject-crawl query to optimize processing. If where does not end up meeting simple-subject-crawl criteria, returns nil so other strategies can be tried." - [{:strs [where vars] :as parsed-query}] + [{:keys [where vars] :as parsed-query}] (log/debug "simple-subject-merge-where parsed-query:" parsed-query) (let [{::where/keys [patterns]} where [first-pattern & rest-patterns] patterns @@ -127,17 +127,17 @@ (log/debug "simple-subject-merge-where first-s:" first-s) (if (empty? rest-patterns) (assoc parsed-query - "where" [reparsed-first-clause] + :where [reparsed-first-clause] :strategy :simple-subject-crawl) (if-let [subj-filter-map (merge-wheres-to-filter first-s rest-patterns vars)] - (assoc parsed-query "where" [reparsed-first-clause - {:s-filter subj-filter-map}] + (assoc parsed-query :where [reparsed-first-clause + {:s-filter subj-filter-map}] :strategy :simple-subject-crawl)))))) (defn simple-subject-crawl? "Simple subject crawl is where the same variable is used in the leading position of each where statement." - [{:strs [where select vars] :as _parsed-query}] + [{:keys [where select vars] :as _parsed-query}] (and (instance? SubgraphSelector select) ;;TODO, filtering not supported yet (empty? (::where/filters where)) @@ -157,14 +157,12 @@ (defn re-parse-as-simple-subj-crawl "Returns true if query contains a single subject crawl. e.g. - {\"select\" {?subjects ['*']} - \"where\" [...]}" - [{:strs [order-by group-by] :as parsed-query}] + {:select {?subjects ['*']} + :where [...]}" + [{:keys [order-by group-by] :as parsed-query}] (log/debug "re-parse-as-simple-subj-crawl parsed-query:" parsed-query) (when (and (not group-by) (not order-by) (simple-subject-crawl? parsed-query)) ;; following will return nil if parts of where clause exclude it from being a simple-subject-crawl - (when-let [ssmw (simple-subject-merge-where parsed-query)] - (util/assoc-from-str-opts ssmw #{"select" "where" {"@context" :context} "opts"} - (dissoc ssmw "select" "where" "@context" "opts"))))) + (simple-subject-merge-where parsed-query))) diff --git a/test/fluree/db/query/fql_parse_test.clj b/test/fluree/db/query/fql_parse_test.clj index 732fbf012..6d52fadff 100644 --- a/test/fluree/db/query/fql_parse_test.clj +++ b/test/fluree/db/query/fql_parse_test.clj @@ -1,10 +1,11 @@ (ns fluree.db.query.fql-parse-test (:require - [clojure.test :refer :all] - [fluree.db.query.exec.where :as where] - [fluree.db.test-utils :as test-utils] - [fluree.db.json-ld.api :as fluree] - [fluree.db.query.fql.parse :as parse])) + [clojure.test :refer :all] + [fluree.db.query.exec.where :as where] + [fluree.db.query.fql.syntax :as syntax] + [fluree.db.test-utils :as test-utils] + [fluree.db.json-ld.api :as fluree] + [fluree.db.query.fql.parse :as parse])) (defn de-recordify-select "Select statements are parsed into records. @@ -20,31 +21,32 @@ {"default-context" ["" {"ex" "http://example.org/ns/"}]}) db @(fluree/stage - (fluree/db ledger) - [{"id" "ex:brian", - "type" "ex:User", - "schema:name" "Brian" - "schema:email" "brian@example.org" - "schema:age" 50 - "ex:favNums" 7} - {"id" "ex:alice", - "type" "ex:User", - "ex:favColor" "Green" - "schema:name" "Alice" - "schema:email" "alice@example.org" - "schema:age" 50 - "ex:favNums" [42, 76, 9]} - {"id" "ex:cam", - "type" "ex:User", - "schema:name" "Cam" - "ex:email" "cam@example.org" - "schema:age" 34 - "ex:favNums" [5, 10] - "ex:friend" ["ex:brian" "ex:alice"]}])] + (fluree/db ledger) + [{"id" "ex:brian", + "type" "ex:User", + "schema:name" "Brian" + "schema:email" "brian@example.org" + "schema:age" 50 + "ex:favNums" 7} + {"id" "ex:alice", + "type" "ex:User", + "ex:favColor" "Green" + "schema:name" "Alice" + "schema:email" "alice@example.org" + "schema:age" 50 + "ex:favNums" [42, 76, 9]} + {"id" "ex:cam", + "type" "ex:User", + "schema:name" "Cam" + "ex:email" "cam@example.org" + "schema:age" 34 + "ex:favNums" [5, 10] + "ex:friend" ["ex:brian" "ex:alice"]}])] (testing "parse-analytical-query" - (let [ssc {"select" {"?s" ["*"]} - "where" [["?s" "schema:name" "Alice"]]} - {:strs [select where]} (parse/parse-analytical-query* ssc db) + (let [ssc (syntax/encode-internal-query + {"select" {"?s" ["*"]} + "where" [["?s" "schema:name" "Alice"]]}) + {:keys [select where]} (parse/parse-analytical-query* ssc db) {::where/keys [patterns]} where] (is (= {:var '?s :selection ["*"] @@ -56,17 +58,18 @@ {::where/val "Alice" ::where/datatype 1}]] patterns))) - (let [values-query '{"select" {"?s" ["*"]} - "where" [["?s" "schema:name" ?name]] - "values" [?name ["Alice"]]} + (let [values-query (syntax/encode-internal-query + '{"select" {"?s" ["*"]} + "where" [["?s" "schema:name" ?name]] + "values" [?name ["Alice"]]}) - {:strs [select where values]} + {:keys [select where values]} (parse/parse-analytical-query* values-query db) {::where/keys [patterns]} where] (is (= '[{?name - {::where/var ?name - ::where/val "Alice" + {::where/var ?name + ::where/val "Alice" ::where/datatype 1}}] values)) (is (= {:var '?s @@ -79,13 +82,14 @@ ::where/datatype 8} {::where/var '?name}]] patterns))) - (let [query {"select" ['?name '?age '?email] - "where" [['?s "schema:name" "Cam"] - ['?s "ex:friend" '?f] - ['?f "schema:name" '?name] - ['?f "schema:age" '?age] - ['?f "ex:email" '?email]]} - {:strs [select where]} (parse/parse-analytical-query query db) + (let [query (syntax/encode-internal-query + {"select" ['?name '?age '?email] + "where" [['?s "schema:name" "Cam"] + ['?s "ex:friend" '?f] + ['?f "schema:name" '?name] + ['?f "schema:age" '?age] + ['?f "ex:email" '?email]]}) + {:keys [select where]} (parse/parse-analytical-query query db) {::where/keys [patterns]} where] (is (= [{:var '?name} {:var '?age} @@ -114,17 +118,19 @@ {::where/var '?email}]] patterns))) (testing "not a `:class` pattern if obj is a var" - (let [query {"select" ['?class] - "where" [["ex:cam" "rdf:type" '?class]]} - {:strs [where]} (parse/parse-analytical-query query db) + (let [query (syntax/encode-internal-query + {"select" ['?class] + "where" [["ex:cam" "rdf:type" '?class]]}) + {:keys [where]} (parse/parse-analytical-query query db) {::where/keys [patterns]} where] (is (= :tuple (where/pattern-type (first patterns)))))) (testing "class, optional" - (let [optional-q {"select" ['?name '?favColor] - "where" [['?s "rdf:type" "ex:User"] - ['?s "schema:name" '?name] - {"optional" ['?s "ex:favColor" '?favColor]}]} - {:strs [select where]} (parse/parse-analytical-query optional-q db) + (let [optional-q (syntax/encode-internal-query + {"select" ['?name '?favColor] + "where" [['?s "rdf:type" "ex:User"] + ['?s "schema:name" '?name] + {"optional" ['?s "ex:favColor" '?favColor]}]}) + {:keys [select where]} (parse/parse-analytical-query optional-q db) {::where/keys [patterns]} where] (is (= [{:var '?name} {:var '?favColor}] (mapv #(into {} %) select))) @@ -146,11 +152,12 @@ {::where/var '?favColor}]]}]] patterns)))) (testing "class, union" - (let [union-q {"select" ['?s '?email1 '?email2] - "where" [['?s "rdf:type" "ex:User"] - {"union" [[['?s "ex:email" '?email1]] - [['?s "schema:email" '?email2]]]}]} - {:strs [select where]} (parse/parse-analytical-query union-q db) + (let [union-q (syntax/encode-internal-query + {"select" ['?s '?email1 '?email2] + "where" [['?s "rdf:type" "ex:User"] + {"union" [[['?s "ex:email" '?email1]] + [['?s "schema:email" '?email2]]]}]}) + {:keys [select where]} (parse/parse-analytical-query union-q db) {::where/keys [patterns]} where] (is (= [{:var '?s} {:var '?email1} {:var '?email2}] (de-recordify-select select))) @@ -173,12 +180,13 @@ {::where/var '?email2}]]}]]] patterns)))) (testing "class, filters" - (let [filter-q {"select" ['?name '?age] - "where" [['?s "rdf:type" "ex:User"] - ['?s "schema:age" '?age] - ['?s "schema:name" '?name] - {"filter" ["(> ?age 45)", "(< ?age 50)"]}]} - {:strs [select where]} (parse/parse-analytical-query filter-q db) + (let [filter-q (syntax/encode-internal-query + {"select" ['?name '?age] + "where" [['?s "rdf:type" "ex:User"] + ['?s "schema:age" '?age] + ['?s "schema:name" '?name] + {"filter" ["(> ?age 45)", "(< ?age 50)"]}]}) + {:keys [select where]} (parse/parse-analytical-query filter-q db) {::where/keys [patterns]} where] (is (= [{:var '?name} {:var '?age}] (de-recordify-select select))) @@ -198,12 +206,13 @@ {::where/var '?name}]] patterns)))) (testing "group-by, order-by" - (let [query {"select" ['?name '?favNums] - "where" [['?s "schema:name" '?name] - ['?s "ex:favNums" '?favNums]] - "group-by" '?name - "order-by" '?name} - {:strs [group-by order-by]} (parse/parse-analytical-query query db)] + (let [query (syntax/encode-internal-query + {"select" ['?name '?favNums] + "where" [['?s "schema:name" '?name] + ['?s "ex:favNums" '?favNums]] + "group-by" '?name + "order-by" '?name}) + {:keys [group-by order-by]} (parse/parse-analytical-query query db)] (is (= ['?name] group-by)) (is (= [['?name :asc]] diff --git a/test/fluree/db/query/subject_crawl_reparse_test.clj b/test/fluree/db/query/subject_crawl_reparse_test.clj index 80b540a0c..ec75844ac 100644 --- a/test/fluree/db/query/subject_crawl_reparse_test.clj +++ b/test/fluree/db/query/subject_crawl_reparse_test.clj @@ -1,6 +1,7 @@ (ns fluree.db.query.subject-crawl-reparse-test (:require [clojure.test :refer :all] + [fluree.db.query.fql.syntax :as syntax] [fluree.db.test-utils :as test-utils] [fluree.db.json-ld.api :as fluree] [fluree.db.query.fql.parse :as parse] @@ -37,47 +38,55 @@ "ex:friend" [{"id" "ex:brian"} {"id" "ex:alice"}]}]) ssc-q1-parsed (parse/parse-analytical-query* - {"select" {"?s" ["*"]} - "where" [["?s" "schema:name" "Alice"]]} + (syntax/encode-internal-query + {"select" {"?s" ["*"]} + "where" [["?s" "schema:name" "Alice"]]}) db) ssc-q2-parsed (parse/parse-analytical-query* - {"select" {"?s" ["*"]} - "where" [["?s" "schema:age" 50] - ["?s" "ex:favColor" "Blue"]]} - db) + (syntax/encode-internal-query + {"select" {"?s" ["*"]} + "where" [["?s" "schema:age" 50] + ["?s" "ex:favColor" "Blue"]]}) + db) not-ssc-parsed (parse/parse-analytical-query* - {"select" ['?name '?age '?email] - "where" [['?s "schema:name" "Cam"] + (syntax/encode-internal-query + {"select" ['?name '?age '?email] + "where" [['?s "schema:name" "Cam"] ['?s "ex:friend" '?f] ['?f "schema:name" '?name] ['?f "schema:age" '?age] - ['?f "schema:email" '?email]]} + ['?f "schema:email" '?email]]}) db) order-group-parsed (parse/parse-analytical-query* - {"select" ['?name '?favNums] - "where" [['?s "schema:name" '?name] - ['?s "ex:favNums" '?favNums]] - "group-by" '?name - "order-by" '?name} + (syntax/encode-internal-query + {"select" ['?name '?favNums] + "where" [['?s "schema:name" '?name] + ['?s "ex:favNums" '?favNums]] + "group-by" '?name + "order-by" '?name}) db) vars-query-parsed (parse/parse-analytical-query* - {"select" {"?s" ["*"]} - "where" [["?s" "schema:name" '?name]] - "vars" {'?name "Alice"}} + (syntax/encode-internal-query + {"select" {"?s" ["*"]} + "where" [["?s" "schema:name" '?name]] + "vars" {'?name "Alice"}}) db) s+p+o-parsed (parse/parse-analytical-query - {"select" {"?s" [:*]} - "where" [["?s" "?p" "?o"]]} + (syntax/encode-internal-query + {"select" {"?s" [:*]} + "where" [["?s" "?p" "?o"]]}) db) s+p+o2-parsed (parse/parse-analytical-query - {"select" {'?s ["*"]} - "where" [['?s "schema:age" 50] - ['?s '?p '?o]]} + (syntax/encode-internal-query + {"select" {'?s ["*"]} + "where" [['?s "schema:age" 50] + ['?s '?p '?o]]}) db) s+p+o3-parsed (parse/parse-analytical-query - {"select" {'?s ["*"]} - "where" [['?s '?p '?o] - ['?s "schema:age" 50]]} + (syntax/encode-internal-query + {"select" {'?s ["*"]} + "where" [['?s '?p '?o] + ['?s "schema:age" 50]]}) db)] (testing "simple-subject-crawl?" (is (= true diff --git a/test/fluree/db/transact/delete_test.clj b/test/fluree/db/transact/delete_test.clj index 50fb6c07e..e04832322 100644 --- a/test/fluree/db/transact/delete_test.clj +++ b/test/fluree/db/transact/delete_test.clj @@ -54,7 +54,6 @@ @(fluree/query db-subj-delete '{"select" ?name "where" [[?s "schema:name" ?name]]})) - "Only Jane and Bob should be left in the db.") (is (= {"id" "ex:bob", @@ -63,7 +62,6 @@ @(fluree/query db-subj-pred-del '{"selectOne" {?s ["*"]} "where" [[?s "id" "ex:bob"]]})) - "Bob should no longer have an age property.") (is (= ["Bob"] From 378cf8daa60e7a3c1f413a95af5a2bd2b8abe7a0 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Tue, 18 Apr 2023 14:08:59 -0600 Subject: [PATCH 37/50] Update deps to latest compatible versions --- deps.edn | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/deps.edn b/deps.edn index becde2377..b03337f0f 100644 --- a/deps.edn +++ b/deps.edn @@ -9,14 +9,14 @@ byte-streams/byte-streams {:mvn/version "0.2.4"} cheshire/cheshire {:mvn/version "5.11.0"} instaparse/instaparse {:mvn/version "1.4.12"} - metosin/malli {:mvn/version "0.10.4"} + metosin/malli {:mvn/version "0.11.0"} com.fluree/json-ld {:git/url "https://github.com/fluree/json-ld.git" - :sha "a909330e33196504ef8a5411aaa0409ab72aaa35"} + :sha "a570e8f5898a59d28c775cf3ce72938da95d1541"} ;; logging org.clojure/tools.logging {:mvn/version "1.2.4"} - ch.qos.logback/logback-classic {:mvn/version "1.4.5"} - org.slf4j/slf4j-api {:mvn/version "2.0.6"} + ch.qos.logback/logback-classic {:mvn/version "1.4.6"} + org.slf4j/slf4j-api {:mvn/version "2.0.7"} ;; Lucene clucie/clucie {:mvn/version "0.4.2"} @@ -45,20 +45,20 @@ {:build {:deps {io.github.clojure/tools.build {:git/tag "v0.9.4" :git/sha "b0ee58b"} - slipset/deps-deploy {:mvn/version "0.2.0"}} + slipset/deps-deploy {:mvn/version "0.2.1"}} :ns-default build} :dev {:extra-paths ["dev" "test" "dev-resources" "src-cljs" "src-nodejs" "src-docs"] - :extra-deps {org.clojure/tools.namespace {:mvn/version "1.4.2"} + :extra-deps {org.clojure/tools.namespace {:mvn/version "1.4.4"} criterium/criterium {:mvn/version "0.4.6"} figwheel-sidecar/figwheel-sidecar {:mvn/version "0.5.20"} - thheller/shadow-cljs {:mvn/version "2.22.0"} + thheller/shadow-cljs {:mvn/version "2.22.10"} com.magnars/test-with-files {:mvn/version "2021-02-17"}}} :cljtest {:extra-paths ["test" "dev-resources"] - :extra-deps {lambdaisland/kaocha {:mvn/version "1.80.1274"} + :extra-deps {lambdaisland/kaocha {:mvn/version "1.82.1306"} org.clojure/test.check {:mvn/version "1.1.1"} io.github.cap10morgan/test-with-files {:git/tag "v1.0.0" :git/sha "9181a2e"}} @@ -78,7 +78,7 @@ :output-path "docs"}} :deploy - {:replace-deps {slipset/deps-deploy {:mvn/version "0.2.0"}} + {:replace-deps {slipset/deps-deploy {:mvn/version "0.2.1"}} :main-opts ["-m" "deps-deploy.deps-deploy" "deploy" "target/fluree-db.jar"]} @@ -91,7 +91,7 @@ :main-opts ["-m" "cloverage.coverage" "-p" "src" "-s" "test" "--output" "scanning_results/coverage"]} :eastwood - {:extra-deps {jonase/eastwood {:mvn/version "1.3.0"}} + {:extra-deps {jonase/eastwood {:mvn/version "1.4.0"}} :main-opts ["-m" "eastwood.lint" {:source-paths ["src" "src-docs"] :test-paths ["test"] @@ -106,5 +106,5 @@ :main-opts ["-m" "antq.core"]} :clj-kondo - {:extra-deps {clj-kondo/clj-kondo {:mvn/version "2023.02.17"}} + {:extra-deps {clj-kondo/clj-kondo {:mvn/version "2023.04.14"}} :main-opts ["-m" "clj-kondo.main" "--lint" "src" "--config" ".clj-kondo/config.edn"]}}} From 9e9a6e4e23a4e0476cbd7961dd4eb26e6426507f Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Tue, 18 Apr 2023 14:11:46 -0600 Subject: [PATCH 38/50] Correct bad defmulti indentation ...that my editor keeps insisting on applying for some reason. --- src/fluree/db/query/exec/where.cljc | 4 ++-- src/fluree/db/query/fql/parse.cljc | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/fluree/db/query/exec/where.cljc b/src/fluree/db/query/exec/where.cljc index cb5c2c2c4..9f69b5ae2 100644 --- a/src/fluree/db/query/exec/where.cljc +++ b/src/fluree/db/query/exec/where.cljc @@ -154,8 +154,8 @@ (defmulti match-pattern "Return a channel that will contain all pattern match solutions from flakes in - `db` that are compatible with the initial solution `solution` and matches the - additional where-clause pattern `pattern`." + `db` that are compatible with the initial solution `solution` and matches the + additional where-clause pattern `pattern`." (fn [_db _solution pattern _filters _error-ch] (pattern-type pattern))) diff --git a/src/fluree/db/query/fql/parse.cljc b/src/fluree/db/query/fql/parse.cljc index b76027484..f4fe7e01d 100644 --- a/src/fluree/db/query/fql/parse.cljc +++ b/src/fluree/db/query/fql/parse.cljc @@ -295,12 +295,12 @@ (where/anonymous-value o-pat))) (defmulti parse-pattern - (fn [pattern _vars _db _context] - (log/debug "parse-pattern pattern:" pattern) - (cond - (map? pattern) (->> pattern keys first keyword) - (map-entry? pattern) :binding - :else :triple))) + (fn [pattern _vars _db _context] + (log/debug "parse-pattern pattern:" pattern) + (cond + (map? pattern) (->> pattern keys first keyword) + (map-entry? pattern) :binding + :else :triple))) (defn type-pattern? [typ x] From 6ae337ffac8e712e9d57716e2dacd729cfc4bd87 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Thu, 20 Apr 2023 07:03:56 -0600 Subject: [PATCH 39/50] Fix json-ld.api-test indentation --- test/fluree/db/json_ld/api_test.cljc | 140 +++++++++++++-------------- 1 file changed, 68 insertions(+), 72 deletions(-) diff --git a/test/fluree/db/json_ld/api_test.cljc b/test/fluree/db/json_ld/api_test.cljc index 844dd60b9..154083af1 100644 --- a/test/fluree/db/json_ld/api_test.cljc +++ b/test/fluree/db/json_ld/api_test.cljc @@ -268,79 +268,75 @@ "ex:friends" [{"id" "ex:john"} {"id" "ex:cam"}]}} (set @(fluree/query loaded-db '{"select" {?s ["*"]} - "where" [[?s "rdf:type" "ex:User"]]})) - ;; TODO: Delete me and uncomment above - #_(set - @(fluree/query db '{"select" {?s ["*"]} - "where" [[?s "rdf:type" "ex:User"]]}))))))) + "where" [[?s "rdf:type" "ex:User"]]})))))))) - (testing "can load with policies" - (with-tmp-dir storage-path - (let [conn @(fluree/connect - {"method" "file" - "storage-path" storage-path - "defaults" - {"@context" (merge test-utils/default-context - {"ex" "http://example.org/ns/"})}}) - ledger-alias "load-policy-test" - ledger @(fluree/create conn ledger-alias) - db @(fluree/stage - (fluree/db ledger) - [{"id" "ex:alice", - "type" "ex:User", - "schema:name" "Alice" - "schema:ssn" "111-11-1111" - "ex:friend" {"id" "ex:john"}} - {"id" "ex:john", - "schema:name" "John" - "type" "ex:User", - "schema:ssn" "888-88-8888"} - {"id" "did:fluree:123" - "ex:user" {"id" "ex:alice"} - "f:role" {"id" "ex:userRole"}}]) - db+policy @(fluree/stage - db - [{"id" "ex:UserPolicy" - "type" ["f:Policy"] - "f:targetClass" {"id" "ex:User"} - "f:allow" [{"id" "ex:globalViewAllow" - "f:targetRole" {"id" "ex:userRole"} - "f:action" [{"id" "f:view"}]}] - "f:property" - [{"f:path" {"id" "schema:ssn"} - "f:allow" - [{"id" "ex:ssnViewRule" - "f:targetRole" {"id" "ex:userRole"} - "f:action" [{"id" "f:view"}] - "f:equals" {"@list" [{"id" "f:$identity"} - {"id" "ex:user"}]}}]}]}]) - db+policy @(fluree/commit! ledger db+policy) - loaded (test-utils/retry-load conn ledger-alias 100) - loaded-db (fluree/db loaded)] - (is (= (:t db) (:t loaded-db))) - (testing "query returns expected policy" - (is (= [{"id" "ex:UserPolicy", - "rdf:type" ["f:Policy"], - "f:allow" - {"id" "ex:globalViewAllow", - "f:action" {"id" "f:view"}, - "f:targetRole" {"_id" 211106232532995}}, - "f:property" - {"id" "_:f211106232532999", - "f:allow" - {"id" "ex:ssnViewRule", - "f:action" {"id" "f:view"}, - "f:targetRole" {"_id" 211106232532995}, - "f:equals" [{"id" "f:$identity"} {"id" "ex:user"}]}, - "f:path" {"id" "schema:ssn"}}, - "f:targetClass" {"id" "ex:User"}}] - @(fluree/query - loaded-db - '{"select" {?s ["*" - {"rdf:type" ["_id"]} - {"f:allow" ["*" {"f:targetRole" ["_id"]}]} - {"f:property" ["*" {"f:allow" ["*" {"f:targetRole" ["_id"]}]}]}]} - "where" [[?s "rdf:type" "f:Policy"]]})))))))))) + (testing "can load with policies" + (with-tmp-dir storage-path + (let [conn @(fluree/connect + {"method" "file" + "storage-path" storage-path + "defaults" + {"@context" (merge test-utils/default-context + {"ex" "http://example.org/ns/"})}}) + ledger-alias "load-policy-test" + ledger @(fluree/create conn ledger-alias) + db @(fluree/stage + (fluree/db ledger) + [{"id" "ex:alice", + "type" "ex:User", + "schema:name" "Alice" + "schema:ssn" "111-11-1111" + "ex:friend" {"id" "ex:john"}} + {"id" "ex:john", + "schema:name" "John" + "type" "ex:User", + "schema:ssn" "888-88-8888"} + {"id" "did:fluree:123" + "ex:user" {"id" "ex:alice"} + "f:role" {"id" "ex:userRole"}}]) + db+policy @(fluree/stage + db + [{"id" "ex:UserPolicy" + "type" ["f:Policy"] + "f:targetClass" {"id" "ex:User"} + "f:allow" [{"id" "ex:globalViewAllow" + "f:targetRole" {"id" "ex:userRole"} + "f:action" [{"id" "f:view"}]}] + "f:property" + [{"f:path" {"id" "schema:ssn"} + "f:allow" + [{"id" "ex:ssnViewRule" + "f:targetRole" {"id" "ex:userRole"} + "f:action" [{"id" "f:view"}] + "f:equals" {"@list" [{"id" "f:$identity"} + {"id" "ex:user"}]}}]}]}]) + db+policy @(fluree/commit! ledger db+policy) + loaded (test-utils/retry-load conn ledger-alias 100) + loaded-db (fluree/db loaded)] + (is (= (:t db) (:t loaded-db))) + (testing "query returns expected policy" + (is (= [{"id" "ex:UserPolicy", + "rdf:type" ["f:Policy"], + "f:allow" + {"id" "ex:globalViewAllow", + "f:action" {"id" "f:view"}, + "f:targetRole" {"_id" 211106232532995}}, + "f:property" + {"id" "_:f211106232532999", + "f:allow" + {"id" "ex:ssnViewRule", + "f:action" {"id" "f:view"}, + "f:targetRole" {"_id" 211106232532995}, + "f:equals" [{"id" "f:$identity"} {"id" "ex:user"}]}, + "f:path" {"id" "schema:ssn"}}, + "f:targetClass" {"id" "ex:User"}}] + @(fluree/query + loaded-db + '{"select" {?s ["*" + {"rdf:type" ["_id"]} + {"f:allow" ["*" {"f:targetRole" ["_id"]}]} + {"f:property" ["*" {"f:allow" ["*" {"f:targetRole" ["_id"]}]}]}]} + "where" [[?s "rdf:type" "f:Policy"]]}))))))))) #?(:clj (deftest load-from-memory-test From f0d42bbbf34c177cfd569b15a36462fb659149d6 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Thu, 20 Apr 2023 09:05:59 -0600 Subject: [PATCH 40/50] Add Clojure SDK basics & tests Currently supports connect, create, load, stage, commit!, and query w/ keyword compact IRIs & contexts both in requests and responses. --- src/fluree/db/db/json_ld.cljc | 1 + src/fluree/db/json_ld/api.cljc | 2 +- src/fluree/db/json_ld/transact.cljc | 308 ++++++++++--------- src/fluree/db/query/fql/syntax.cljc | 49 ++- src/fluree/db/util/validation.cljc | 70 ++++- src/fluree/sdk/clojure.clj | 102 +++++++ test/fluree/db/json_ld/api_test.cljc | 22 +- test/fluree/sdk/clojure_test.clj | 430 +++++++++++++++++++++++++++ 8 files changed, 798 insertions(+), 186 deletions(-) create mode 100644 src/fluree/sdk/clojure.clj create mode 100644 test/fluree/sdk/clojure_test.clj diff --git a/src/fluree/db/db/json_ld.cljc b/src/fluree/db/db/json_ld.cljc index 32968a79e..bf1040fa0 100644 --- a/src/fluree/db/db/json_ld.cljc +++ b/src/fluree/db/db/json_ld.cljc @@ -247,6 +247,7 @@ (-iri [this subject-id compact-fn] (iri this subject-id compact-fn)) (-search [this fparts] (query-range/search this fparts)) (-query [this query-map] (fql/query this query-map)) + (-query [this query-map opts] (fql/query this query-map opts)) (-stage [db json-ld] (jld-transact/stage db json-ld nil)) (-stage [db json-ld opts] (jld-transact/stage db json-ld opts)) (-index-update [db commit-index] (index-update db commit-index)) diff --git a/src/fluree/db/json_ld/api.cljc b/src/fluree/db/json_ld/api.cljc index 012aca643..fe1ee626c 100644 --- a/src/fluree/db/json_ld/api.cljc +++ b/src/fluree/db/json_ld/api.cljc @@ -51,7 +51,7 @@ - defaults: - did - (optional) DiD information to use, if storing blocks as verifiable credentials, or issuing queries against a permissioned database. - - context - (optional) Default @context map to use for ledgers formed with this connection." + - @context - (optional) Default @context map to use for ledgers formed with this connection." [{:strs [method parallelism] :as opts}] ;; TODO - do some validation (promise-wrap diff --git a/src/fluree/db/json_ld/transact.cljc b/src/fluree/db/json_ld/transact.cljc index 50e5f4b21..66ece16a7 100644 --- a/src/fluree/db/json_ld/transact.cljc +++ b/src/fluree/db/json_ld/transact.cljc @@ -32,32 +32,30 @@ (def registry (merge - (m/base-schemas) - (m/type-schemas) - v/registry - {::iri ::v/iri - ::val ::v/val - ::context ::v/context - ::txn-val [:orn - [:multiple [:sequential - [:or [:ref ::txn-map] [:ref ::txn-val]]]] - [:node [:ref ::txn-map]] - [:iri ::iri] - [:val ::val]] - ::txn-leaf-map [:map - ["@context" {:optional true} ::context] - [::m/default [:map-of ::iri ::txn-val]]] - ::retract-key [:and ::iri [:re retract-key-re]] - ::txn-map [:orn - [:assert ::txn-leaf-map] - [:retract - [:map - ["@context" {:optional true} ::context] - [::m/default [:map-of ::retract-key ::txn-leaf-map]]]]] - ::txn [:orn - [:single-amp ::txn-map] - [:sequence-of-maps [:sequential ::txn-map]]] - ::opts [map?]})) + (m/base-schemas) + (m/type-schemas) + v/registry + {::iri ::v/iri + ::val ::v/val + ::context ::v/context + ::txn-val [:orn + [:multiple [:sequential + [:or [:ref ::txn-map] [:ref ::txn-val]]]] + [:node [:ref ::txn-map]] + [:iri ::iri] + [:val ::val]] + ::txn-leaf-map [:map-of ::iri ::txn-val] + ::retract-key [:and ::iri [:re retract-key-re]] + ::txn-map [:orn + [:assert ::txn-leaf-map] + [:retract [:map-of ::retract-key ::txn-leaf-map]]] + ::txn [:orn + [:single-map ::txn-map] + [:sequence-of-maps [:sequential ::txn-map]]] + ::opts [map?]})) + +(def coerce-txn + (m/coercer ::txn v/fluree-transformer {:registry registry})) (declare json-ld-node->flakes) @@ -67,22 +65,22 @@ flakes if they didn't already exist." [class-iris {:keys [t next-pid ^clojure.lang.Volatile iris db-before] :as _tx-state}] (go-try - (loop [[class-iri & r] (util/sequential class-iris) - class-sids #{} - class-flakes #{}] - (if class-iri - (if-let [existing (> ( v-map :idx last)}) - flakes (cond - ;; a new node's data is contained, process as another node then link to this one - (jld-reify/node? v-map) - (let [[node-sid node-flakes] (flakes v-map tx-state pid))] - (conj node-flakes (flake/create sid pid node-sid const/$xsd:anyURI t true m))) - - ;; a literal value - (and (some? value) (not= shacl-dt const/$xsd:anyURI)) - (let [[value* dt] (datatype/from-expanded v-map shacl-dt)] - (when validate-fn - (or (validate-fn value*) - (throw (ex-info (str "Value did not pass SHACL validation: " value) - {:status 400 :error :db/shacl-validation})))) - [(flake/create sid pid value* dt t true m)]) - - :else - (throw (ex-info (str "JSON-LD value must be a node or a value, instead found ambiguous value: " v-map) - {:status 400 :error :db/invalid-transaction})))] - (into flakes retractions)))) + (let [retractions (when check-retracts? ;; don't need to check if generated pid during this transaction + (->> ( v-map :idx last)}) + flakes (cond + ;; a new node's data is contained, process as another node then link to this one + (jld-reify/node? v-map) + (let [[node-sid node-flakes] (flakes v-map tx-state pid))] + (conj node-flakes (flake/create sid pid node-sid const/$xsd:anyURI t true m))) + + ;; a literal value + (and (some? value) (not= shacl-dt const/$xsd:anyURI)) + (let [[value* dt] (datatype/from-expanded v-map shacl-dt)] + (when validate-fn + (or (validate-fn value*) + (throw (ex-info (str "Value did not pass SHACL validation: " value) + {:status 400 :error :db/shacl-validation})))) + [(flake/create sid pid value* dt t true m)]) + + :else + (throw (ex-info (str "JSON-LD value must be a node or a value, instead found ambiguous value: " v-map) + {:status 400 :error :db/invalid-transaction})))] + (into flakes retractions)))) (defn list-value? "returns true if json-ld value is a list object." @@ -133,11 +131,11 @@ new-types are a set of newly created types in the transaction." [db sid added-classes] (go-try - (let [type-sids (->> (> ( (db-after staged-map tx-state) - vocab-flakes vocab/refresh-schema - vocab-flakes (db-after staged-map tx-state) + vocab-flakes vocab/refresh-schema + vocab-flakes json-ld - syntax/validate-query - syntax/encode-internal-query - (q-parse/parse-delete db)) - - [s p o] delete - parsed-query (assoc parsed-query :delete [s p o]) - error-ch (async/chan) - flake-ch (async/chan) - where-ch (where/search db parsed-query error-ch)] - (async/pipeline-async 1 - flake-ch - (fn [solution ch] - (let [s* (if (::where/val s) - s - (get solution (::where/var s))) - p* (if (::where/val p) - p - (get solution (::where/var p))) - o* (if (::where/val o) - o - (get solution (::where/var o)))] - (async/pipe - (where/resolve-flake-range db error-ch [s* p* o*]) - ch))) - where-ch) - (let [delete-ch (async/transduce (comp cat - (map (fn [f] - (flake/flip-flake f t)))) - (completing conj) - (flake/sorted-set-by flake/cmp-flakes-spot) - flake-ch) - flakes (async/alt! - error-ch ([e] - (throw e)) - delete-ch ([flakes] - flakes))] - flakes)))) + (let [{:keys [delete] :as parsed-query} + (-> json-ld + syntax/validate-query + syntax/encode-internal-query + (q-parse/parse-delete db)) + + [s p o] delete + parsed-query (assoc parsed-query :delete [s p o]) + error-ch (async/chan) + flake-ch (async/chan) + where-ch (where/search db parsed-query error-ch)] + (async/pipeline-async 1 + flake-ch + (fn [solution ch] + (let [s* (if (::where/val s) + s + (get solution (::where/var s))) + p* (if (::where/val p) + p + (get solution (::where/var p))) + o* (if (::where/val o) + o + (get solution (::where/var o)))] + (async/pipe + (where/resolve-flake-range db error-ch [s* p* o*]) + ch))) + where-ch) + (let [delete-ch (async/transduce (comp cat + (map (fn [f] + (flake/flip-flake f t)))) + (completing conj) + (flake/sorted-set-by flake/cmp-flakes-spot) + flake-ch) + flakes (async/alt! + error-ch ([e] + (throw e)) + delete-ch ([flakes] + flakes))] + flakes)))) (defn flakes->final-db "Takes final set of proposed staged flakes and turns them into a new db value along with performing any final validation and policy enforcement." [tx-state flakes] (go-try - (-> flakes - (final-db tx-state) - flakes + (final-db tx-state) + tx-state db* (assoc opts :issuer issuer)) - flakes (if (and (contains? tx "delete") - (contains? tx "where")) - (final-db tx-state flakes))))) + (let [{tx :subject issuer :issuer} (or (tx-state db* (assoc opts :issuer issuer)) + flakes (if (and (contains? tx "delete") + (contains? tx "where")) + (final-db tx-state flakes))))) diff --git a/src/fluree/db/query/fql/syntax.cljc b/src/fluree/db/query/fql/syntax.cljc index 7207666fd..b9db9fb89 100644 --- a/src/fluree/db/query/fql/syntax.cljc +++ b/src/fluree/db/query/fql/syntax.cljc @@ -1,8 +1,10 @@ (ns fluree.db.query.fql.syntax (:require [clojure.string :as str] + [clojure.walk :as walk] [fluree.db.util.core :as util :refer [pred-ident?]] [fluree.db.util.log :as log] [fluree.db.constants :as const] + [fluree.json-ld :as json-ld] [malli.core :as m] [fluree.db.util.validation :as v] [malli.transform :as mt])) @@ -63,14 +65,38 @@ (defn encode-query-key [k] (if (string? k) - (do - (log/debug "encoding query key:" k) - (let [kebab-k (->kebab-case k)] - (if (str/starts-with? kebab-k "@") - (-> kebab-k (subs 1) keyword) - (keyword kebab-k)))) + (let [kebab-k (->kebab-case k)] + (if (str/starts-with? kebab-k "@") + (-> kebab-k (subs 1) keyword) + (keyword kebab-k))) k)) +(defn decode-query + [q] + (log/debug "decoding query:" q) + ;; TODO: Replace this with a :map-with schema once this lands upstream: + ;; https://github.com/metosin/malli/issues/881 + (walk/postwalk + (fn [v] + (cond + (qualified-keyword? v) (str (namespace v) ":" (name v)) + (= v :context) "@context" + (keyword? v) (name v) + :else v)) + q)) + +(defn analytical-query-results-transformer + [context] + ;; TODO: Replace this with a :map-with schema once this lands upstream: + ;; https://github.com/metosin/malli/issues/881 + (let [kw-context (util/keywordize-keys context)] + (mt/transformer + {:encoders + {:string (fn [s] + (if-let [expanded (json-ld/expand-iri s context)] + (json-ld/compact expanded kw-context) + s))}}))) + (def registry (merge (m/predicate-schemas) @@ -199,7 +225,7 @@ ["where" ::where] ["values" {:optional true} ::values]] ::context ::v/context - ::analytical-query [:map + ::analytical-query [:map {:decode/fluree decode-query} ["where" ::where] ["t" {:optional true} ::t] ["@context" {:optional true} ::context] @@ -247,6 +273,15 @@ (m/encoder ::query {:registry registry} (mt/key-transformer {:encode encode-query-key}))) +(def coerce-analytical-query + (m/coercer ::analytical-query v/fluree-transformer {:registry registry})) + +(defn analytical-query-results-encoder + [context] + (m/encoder ::analytical-query-results + {:registry registry} + (analytical-query-results-transformer context))) + (def valid-query? (m/validator ::query {:registry registry})) diff --git a/src/fluree/db/util/validation.cljc b/src/fluree/db/util/validation.cljc index e99b827c2..6a471c7aa 100644 --- a/src/fluree/db/util/validation.cljc +++ b/src/fluree/db/util/validation.cljc @@ -1,18 +1,64 @@ (ns fluree.db.util.validation - (:require [malli.core :as m])) + (:require [malli.core :as m] + [malli.transform :as mt])) (def value? (complement coll?)) +(defn decode-iri + [v] + (cond + (qualified-keyword? v) (str (namespace v) ":" (name v)) + (keyword? v) (name v) + :else v)) + (def registry (merge - (m/base-schemas) - (m/type-schemas) - (m/comparator-schemas) - (m/predicate-schemas) - {::iri :string - ::val [:fn value?] - ::context [:orn - [:sequence [:sequential [:orn - [:string :string] - [:map map?]]]] - [:map map?]]})) + (m/base-schemas) + (m/type-schemas) + (m/comparator-schemas) + (m/predicate-schemas) + {::iri [:string {:decode/fluree decode-iri}] + ::val [:fn value?] + ::string-key [:string {:decode/fluree name + :encode/fluree keyword}] + ::context-map [:map-of ::iri ::iri] + ::context [:orn + [:sequence [:sequential [:orn + [:string :string] + [:map ::context-map]]]] + [:map ::context-map]] + ::context-key [:= {:decode/fluree #(if (= % :context) + "@context" %)} + "@context"] + ::did [:orn + [:id :string] + [:map [:and + [:map-of ::string-key :any] + [:map + ["id" :string] + ["public" :string] + ["private" :string]]]]] + ::connect-defaults [:map + [:did {:optional true} ::did] + [::m/default [:map-of {:max 1} ::context-key ::context]]] + ::connect-opts [:and + [:map-of ::string-key :any] + [:map + ["method" {:decode/fluree name} :string] + ["defaults" {:optional true} ::connect-defaults]]] + ::create-opts [:maybe + [:and + [:map-of ::string-key :any] + [:map + ["defaults" {:optional true} + [:map-of {:max 1} ::context-key ::context]]]]] + ::create-response [:map-of ::string-key :any]})) + +(def fluree-transformer + (mt/transformer {:name :fluree})) + +(def coerce-connect-opts + (m/coercer ::connect-opts fluree-transformer {:registry registry})) + +(def coerce-create-opts + (m/coercer ::create-opts fluree-transformer {:registry registry})) diff --git a/src/fluree/sdk/clojure.clj b/src/fluree/sdk/clojure.clj new file mode 100644 index 000000000..85573d516 --- /dev/null +++ b/src/fluree/sdk/clojure.clj @@ -0,0 +1,102 @@ +(ns fluree.sdk.clojure + (:require [fluree.db.json-ld.api :as api] + [fluree.db.json-ld.transact :as ftx] + [fluree.db.util.log :as log] + [fluree.db.util.validation :as v] + [fluree.db.query.fql.syntax :as fql] + [fluree.db.query.fql.parse :as fqp] + [fluree.json-ld :as json-ld]) + (:refer-clojure :exclude [load])) + +(defn connect + "Forms connection to ledger, enabling automatic pulls of new updates, event + services, index service. + + Multiple connections to same endpoint will share underlying network connection. + + Options include: + - :defaults - (optional) with any of the following values: + - :did - (optional) DiD information to use, if storing blocks as verifiable + credentials, or issuing queries against a permissioned database. + - :context - (optional) Default @context map to use for ledgers formed with + this connection." + [opts] + (let [opts* (v/coerce-connect-opts opts)] + (log/debug "connect opts:" opts*) + (api/connect opts*))) + +(defn create + "Creates a new json-ld ledger. A connection (conn) must always be supplied. + + Ledger-alias (optional) is a friendly name that is used for: + - When publishing to a naming service that allows multiple pointers for the + same namespace (e.g. IPNS), this becomes a sub-directory off the namespace. + For multiple directories deep, use '/' for a + e.g. the ledgers movies/popular, books/authors, books/best-sellers could + use the same IPNS id (in this example using IPNS DNSLink): + fluree:ipns://my.dns.com/books/authors + fluree:ipns://my.dns.com/books/best-sellers + fluree:ipns://my.dns.com/movies/top-rated + - When combining multiple ledgers, each ledger becomes an individual named + graph which can be referenced by name. + + Options map (opts) can include: + - :defaults + - :did - DiD information to use, if storing blocks as verifiable credentials + - :context - Default @context map to use for ledgers formed with this connection" + ([conn] (create conn nil nil)) + ([conn ledger-alias] (create conn ledger-alias nil)) + ([conn ledger-alias opts] + (let [opts* (v/coerce-create-opts opts)] + (api/create conn ledger-alias opts*)))) + +(defn load + "Loads an existing ledger by its alias (which will be converted to a + connection-specific address first)." + [conn ledger-alias] + (api/load conn ledger-alias)) + +(defn exists? + "Returns a promise with true if the ledger alias or address exists, false + otherwise." + [conn ledger-alias-or-address] + (api/exists? conn ledger-alias-or-address)) + +(defn stage + "Performs a transaction and queues change if valid (does not commit)" + ([db json-ld] (stage db json-ld nil)) + ([db json-ld opts] + (let [json-ld* (ftx/coerce-txn json-ld)] + (api/stage db json-ld* opts)))) + +(defn commit! + ([ledger db] (commit! ledger db nil)) + ([ledger db opts] (api/commit! ledger db opts))) + +(defn db + "Retrieves latest db, or optionally a db at a moment in time + and/or permissioned to a specific identity." + ([ledger] (db ledger nil)) + ([ledger opts] (api/db ledger opts))) + +(defn query + [db query] + (let [context (json-ld/parse-context (fqp/parse-context query db)) + results-encoder (fql/analytical-query-results-encoder context)] + (future + (->> query + fql/coerce-analytical-query + (api/query db) + deref + (log/debug->>val "pre-encoded query results:") + results-encoder)))) + +(comment + ;; TODO: Finish these + (defn multi-query + [db query] + (api/multi-query db query)) + + (defn history + [ledger query] + (api/history ledger query))) diff --git a/test/fluree/db/json_ld/api_test.cljc b/test/fluree/db/json_ld/api_test.cljc index 154083af1..0e50b041f 100644 --- a/test/fluree/db/json_ld/api_test.cljc +++ b/test/fluree/db/json_ld/api_test.cljc @@ -573,20 +573,20 @@ loaded-db (fluree/db loaded)] (is (= (:t db) (:t loaded-db))) (testing "query returns expected policy" - (is (= [{"id" "ex:UserPolicy", - "rdf:type" ["f:Policy"], + (is (= [{"id" "ex:UserPolicy" + "rdf:type" ["f:Policy"] "f:allow" - {"id" "ex:globalViewAllow", - "f:action" {"id" "f:view"}, - "f:targetRole" {"_id" 211106232532995}}, + {"id" "ex:globalViewAllow" + "f:action" {"id" "f:view"} + "f:targetRole" {"_id" 211106232532995}} "f:property" - {"id" "_:f211106232532999", + {"id" "_:f211106232532999" "f:allow" - {"id" "ex:ssnViewRule", - "f:action" {"id" "f:view"}, - "f:targetRole" {"_id" 211106232532995}, - "f:equals" [{"id" "f:$identity"} {"id" "ex:user"}]}, - "f:path" {"id" "schema:ssn"}}, + {"id" "ex:ssnViewRule" + "f:action" {"id" "f:view"} + "f:targetRole" {"_id" 211106232532995} + "f:equals" [{"id" "f:$identity"} {"id" "ex:user"}]} + "f:path" {"id" "schema:ssn"}} "f:targetClass" {"id" "ex:User"}}] @(fluree/query loaded-db '{"select" {?s ["*" diff --git a/test/fluree/sdk/clojure_test.clj b/test/fluree/sdk/clojure_test.clj new file mode 100644 index 000000000..60d861fa0 --- /dev/null +++ b/test/fluree/sdk/clojure_test.clj @@ -0,0 +1,430 @@ +(ns fluree.sdk.clojure-test + (:require [clojure.test :refer [deftest is testing]] + [fluree.db.dbproto :as dbproto] + [fluree.sdk.clojure :as fluree] + [fluree.db.test-utils :as test-utils] + [fluree.db.util.core :as util] + [test-with-files.tools :refer [with-tmp-dir] :as twf])) + +(deftest exists?-test + (testing "returns false before committing data to a ledger" + (let [conn (test-utils/create-conn) + ledger-alias "testledger" + check1 @(fluree/exists? conn ledger-alias) + ledger @(fluree/create conn ledger-alias) + check2 @(fluree/exists? conn ledger-alias) + _ @(fluree/stage (fluree/db ledger) + [{:id :f/me + :type :schema/Person + :schema/fname "Me"}]) + check3 @(fluree/exists? conn ledger-alias)] + (is (every? false? [check1 check2 check3])))) + (testing "returns true after committing data to a ledger" + (let [conn (test-utils/create-conn) + ledger-alias "testledger" + ledger @(fluree/create conn ledger-alias) + db @(fluree/stage (fluree/db ledger) + [{:id :f/me + :type :schema/Person + :schema/fname "Me"}])] + @(fluree/commit! ledger db) + (is (test-utils/retry-exists? conn ledger-alias 100)) + (is (not @(fluree/exists? conn "notaledger")))))) + +(deftest create-test + (testing "string ledger context gets correctly merged with keyword conn context" + (let [conn (test-utils/create-conn) + ledger-alias "testledger" + ledger-context {:ex "http://example.com/" + :foo "http://foobar.com/"} + ledger @(fluree/create conn ledger-alias + {:defaults + {:context ["" ledger-context]}}) + merged-context (merge test-utils/default-context + (util/stringify-keys ledger-context))] + (is (= merged-context (dbproto/-default-context (fluree/db ledger))) + (str "merged context is: " (pr-str merged-context)))))) + +(deftest load-from-file-test + (testing "can load a file ledger with single cardinality predicates" + (with-tmp-dir storage-path + (let [conn @(fluree/connect + {:method :file, :storage-path storage-path + :defaults + {:context test-utils/default-context}}) + ledger-alias "load-from-file-test-single-card" + ledger @(fluree/create conn ledger-alias + {:defaults + {:context + ["" {:ex "http://example.org/ns/"}]}}) + db @(fluree/stage + (fluree/db ledger) + [{:id :ex/brian + :type :ex/User + :schema/name "Brian" + :schema/email "brian@example.org" + :schema/age 50 + :ex/favNums 7} + + {:id :ex/cam + :type :ex/User + :schema/name "Cam" + :schema/email "cam@example.org" + :schema/age 34 + :ex/favNums 5 + :ex/friend :ex/brian}]) + db @(fluree/commit! ledger db) + db @(fluree/stage + db + ;; test a retraction + {:f/retract {:id :ex/brian + :ex/favNums 7}}) + _ @(fluree/commit! ledger db) + ;; TODO: Replace this w/ :syncTo equivalent once we have it + loaded (test-utils/retry-load conn ledger-alias 100) + loaded-db (fluree/db loaded)] + (is (= (:t db) (:t loaded-db))) + (is (= (:context ledger) (:context loaded)))))) + + (testing "can load a file ledger with multi-cardinality predicates" + (with-tmp-dir storage-path + (let [conn @(fluree/connect + {:method :file, :storage-path storage-path + :defaults + {:context test-utils/default-context}}) + ledger-alias "load-from-file-test-multi-card" + ledger @(fluree/create conn ledger-alias) + db @(fluree/stage + (fluree/db ledger) + [{:context {:ex "http://example.org/ns/"} + :id :ex/brian + :type :ex/User + :schema/name "Brian" + :schema/email "brian@example.org" + :schema/age 50 + :ex/favNums 7} + + {:context {:ex "http://example.org/ns/"} + :id :ex/alice + :type :ex/User + :schema/name "Alice" + :schema/email "alice@example.org" + :schema/age 50 + :ex/favNums [42 76 9]} + + {:context {:ex "http://example.org/ns/"} + :id :ex/cam + :type :ex/User + :schema/name "Cam" + :schema/email "cam@example.org" + :schema/age 34 + :ex/favNums [5 10] + :ex/friend [:ex/brian :ex/alice]}]) + db @(fluree/commit! ledger db) + db @(fluree/stage + db + ;; test a multi-cardinality retraction + [{:context {:ex "http://example.org/ns/"} + :f/retract {:id :ex/alice + :ex/favNums [42 76 9]}}]) + _ @(fluree/commit! ledger db) + ;; TODO: Replace this w/ :syncTo equivalent once we have it + loaded (test-utils/retry-load conn ledger-alias 100) + loaded-db (fluree/db loaded)] + (is (= (:t db) (:t loaded-db))) + (is (= (:context ledger) (:context loaded)))))) + + (testing "can load a file ledger with its own context" + (with-tmp-dir storage-path #_{::twf/delete-dir false} + #_(println "storage path:" storage-path) + (let [conn-context {:id "@id", :type "@type" + :xsd "http://www.w3.org/2001/XMLSchema#"} + ledger-context {:ex "http://example.com/" + :schema "http://schema.org/"} + conn @(fluree/connect + {:method :file :storage-path storage-path + :defaults {:context conn-context}}) + ledger-alias "load-from-file-with-context" + ledger @(fluree/create conn ledger-alias + {:defaults {:context + ["" ledger-context]}}) + db @(fluree/stage + (fluree/db ledger) + [{:id :ex/wes + :type :ex/User + :schema/name "Wes" + :schema/email "wes@example.org" + :schema/age 42 + :schema/favNums [1 2 3] + :ex/friend {:id :ex/jake + :type :ex/User + :schema/name "Jake" + :schema/email "jake@example.org"}}]) + db @(fluree/commit! ledger db) + loaded (test-utils/retry-load conn ledger-alias 100) + loaded-db (fluree/db loaded) + merged-ctx (merge (util/stringify-keys conn-context) + (util/stringify-keys ledger-context)) + query {:where '[[?p :schema/email "wes@example.org"]] + :select '{?p [:*]}} + results @(fluree/query loaded-db query) + full-type-url "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"] + (is (= (:t db) (:t loaded-db))) + (is (= merged-ctx (dbproto/-default-context (fluree/db loaded)))) + (is (= [{full-type-url [:ex/User] + :id :ex/wes + :schema/age 42 + :schema/email "wes@example.org" + :schema/favNums [1 2 3] + :schema/name "Wes" + :ex/friend {:id :ex/jake}}] + results))))) + + (testing "query returns the correct results from a loaded ledger" + (with-tmp-dir storage-path + (let [conn-context {:id "@id", :type "@type"} + ledger-context {:ex "http://example.com/" + :schema "http://schema.org/"} + conn @(fluree/connect + {:method :file :storage-path storage-path + :defaults {:context conn-context}}) + ledger-alias "load-from-file-query" + ledger @(fluree/create conn ledger-alias + {:defaults {:context + ["" ledger-context]}}) + db @(fluree/stage + (fluree/db ledger) + [{:id :ex/Andrew + :type :schema/Person + :schema/name "Andrew" + :ex/friend {:id :ex/Jonathan + :type :schema/Person + :schema/name "Jonathan"}}]) + query {:select '{?s [:*]} + :where '[[?s :id :ex/Andrew]]} + res1 @(fluree/query db query) + _ @(fluree/commit! ledger db) + loaded (test-utils/retry-load conn ledger-alias 100) + loaded-db (fluree/db loaded) + res2 @(fluree/query loaded-db query)] + (is (= res1 res2))))) + + (testing "can load a ledger with `list` values" + (with-tmp-dir storage-path + (let [conn @(fluree/connect + {:method :file + :storage-path storage-path + :defaults + {:context (merge (util/keywordize-keys + test-utils/default-context) + {:ex "http://example.org/ns/" + :list "@list"})}}) + ledger-alias "load-lists-test" + ledger @(fluree/create conn ledger-alias) + db @(fluree/stage + (fluree/db ledger) + [{:id :ex/alice + :type :ex/User + :ex/friends {:list [:ex/john :ex/cam]}} + {:id :ex/cam + :type :ex/User + :ex/numList {:list [7 8 9 10]}} + {:id :ex/john + :type :ex/User}]) + db @(fluree/commit! ledger db) + loaded (test-utils/retry-load conn ledger-alias 100) + loaded-db (fluree/db loaded)] + (is (= (:t db) (:t loaded-db))) + (testing "query returns expected `list` values" + (is (= #{{:id :ex/cam + :rdf/type [:ex/User] + :ex/numList [7 8 9 10]} + {:id :ex/john :rdf/type [:ex/User]} + {:id :ex/alice + :rdf/type [:ex/User] + :ex/friends [:ex/john :ex/cam]}} + (set + @(fluree/query loaded-db '{:select {?s [:*]} + :where [[?s :rdf/type :ex/User]]}))))))) + + (testing "can load with policies" + (with-tmp-dir storage-path + (let [conn @(fluree/connect + {:method :file + :storage-path storage-path + :defaults + {:context (merge (util/keywordize-keys + test-utils/default-context) + {:ex "http://example.org/ns/" + :list "@list"})}}) + ledger-alias "load-policy-test" + ledger @(fluree/create conn ledger-alias) + db @(fluree/stage + (fluree/db ledger) + [{:id :ex/alice + :type :ex/User + :schema/name "Alice" + :schema/ssn "111-11-1111" + :ex/friend {:id :ex/john}} + {:id :ex/john + :schema/name "John" + :type :ex/User + :schema/ssn "888-88-8888"} + {:id "did:fluree:123" + :ex/user {:id :ex/alice} + :f/role {:id :ex/userRole}}]) + db+policy @(fluree/stage + db + [{:id :ex/UserPolicy + :type [:f/Policy] + :f/targetClass {:id :ex/User} + :f/allow [{:id :ex/globalViewAllow + :f/targetRole {:id :ex/userRole} + :f/action [{:id :f/view}]}] + :f/property + [{:f/path {:id :schema/ssn} + :f/allow + [{:id :ex/ssnViewRule + :f/targetRole {:id :ex/userRole} + :f/action [{:id :f/view}] + :f/equals {:list [{:id :f/$identity} + {:id :ex/user}]}}]}]}]) + db+policy @(fluree/commit! ledger db+policy) + loaded (test-utils/retry-load conn ledger-alias 100) + loaded-db (fluree/db loaded)] + (is (= (:t db) (:t loaded-db))) + (testing "query returns expected policy" + (is (= [{:id :ex/UserPolicy + :rdf/type [:f/Policy] + :f/allow + {:id :ex/globalViewAllow + :f/action {:id :f/view} + ;; TODO: We can likely make "_id" come back as :_id instead + ;; but need to think through where this should happen + :f/targetRole {"_id" 211106232532995}} + :f/property + {:id "_:f211106232532999" + :f/allow + {:id :ex/ssnViewRule + :f/action {:id :f/view} + :f/targetRole {"_id" 211106232532995} + :f/equals [{:id :f/$identity} {:id :ex/user}]} + :f/path {:id :schema/ssn}}, + :f/targetClass {:id :ex/User}}] + @(fluree/query + loaded-db + '{:select {?s [:* + {:rdf/type ["_id"]} + {:f/allow [:* {:f/targetRole ["_id"]}]} + {:f/property [:* {:f/allow [:* {:f/targetRole ["_id"]}]}]}]} + :where [[?s :rdf/type :f/Policy]]})))))))) + + (testing "can load a ledger with `list` values" + (with-tmp-dir storage-path + (let [conn @(fluree/connect + {:method :file + :storage-path storage-path + :defaults + {:context (merge (util/keywordize-keys + test-utils/default-context) + {:ex "http://example.org/ns/" + :list "@list"})}}) + ledger-alias "load-lists-test" + ledger @(fluree/create conn ledger-alias) + db @(fluree/stage + (fluree/db ledger) + [{:id :ex/alice + :type :ex/User + ;; TODO: We can likely add support of inference of + ;; nodes w/ keywords here if we want to + :ex/friends {:list [{:id :ex/john} {:id :ex/cam}]}} + {:id :ex/cam + :type :ex/User + :ex/numList {:list [7 8 9 10]}} + {:id :ex/john + :type :ex/User}]) + db @(fluree/commit! ledger db) + loaded (test-utils/retry-load conn ledger-alias 100) + loaded-db (fluree/db loaded)] + (is (= (:t db) (:t loaded-db))) + (testing "query returns expected `list` values" + (is (= #{{:id :ex/cam + :rdf/type [:ex/User] + :ex/numList [7 8 9 10]} + {:id :ex/john, :rdf/type [:ex/User]} + {:id :ex/alice + :rdf/type [:ex/User] + :ex/friends [{:id :ex/john} {:id :ex/cam}]}} + (set + @(fluree/query loaded-db '{:select {?s [:*]} + :where [[?s :rdf/type :ex/User]]}))))))) + + (testing "can load with policies" + (with-tmp-dir storage-path + (let [conn @(fluree/connect + {:method :file + :storage-path storage-path + :defaults + {:context (merge (util/keywordize-keys + test-utils/default-context) + {:ex "http://example.org/ns/" + :list "@list"})}}) + ledger-alias "load-policy-test" + ledger @(fluree/create conn ledger-alias) + db @(fluree/stage + (fluree/db ledger) + [{:id :ex/alice + :type :ex/User + :schema/name "Alice" + :schema/ssn "111-11-1111" + :ex/friend {:id :ex/john}} + {:id :ex/john + :schema/name "John" + :type :ex/User + :schema/ssn "888-88-8888"} + {:id "did:fluree:123" + :ex/user {:id :ex/alice} + :f/role {:id :ex/userRole}}]) + db+policy @(fluree/stage + db + [{:id :ex/UserPolicy + :type [:f/Policy] + :f/targetClass {:id :ex/User} + :f/allow [{:id :ex/globalViewAllow + :f/targetRole {:id :ex/userRole} + :f/action [{:id :f/view}]}] + :f/property + [{:f/path {:id :schema/ssn} + :f/allow + [{:id :ex/ssnViewRule + :f/targetRole {:id :ex/userRole} + :f/action [{:id :f/view}] + :f/equals {:list [{:id :f/$identity} + {:id :ex/user}]}}]}]}]) + db+policy @(fluree/commit! ledger db+policy) + loaded (test-utils/retry-load conn ledger-alias 100) + loaded-db (fluree/db loaded)] + (is (= (:t db) (:t loaded-db))) + (testing "query returns expected policy" + (is (= [{:id :ex/UserPolicy + :rdf/type [:f/Policy] + :f/allow + {:id :ex/globalViewAllow + :f/action {:id :f/view} + :f/targetRole {"_id" 211106232532995}} + :f/property + {:id "_:f211106232532999" + :f/allow + {:id :ex/ssnViewRule + :f/action {:id :f/view} + :f/targetRole {"_id" 211106232532995} + :f/equals [{:id :f/$identity} {:id :ex/user}]} + :f/path {:id :schema/ssn}} + :f/targetClass {:id :ex/User}}] + @(fluree/query + loaded-db + '{:select {?s [:* + {:rdf/type ["_id"]} + {:f/allow [:* {:f/targetRole ["_id"]}]} + {:f/property [:* {:f/allow [:* {:f/targetRole ["_id"]}]}]}]} + :where [[?s :rdf/type :f/Policy]]}))))))))) From 2418301bcf51362c59438f56daca9d957f37ede7 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Thu, 20 Apr 2023 09:07:06 -0600 Subject: [PATCH 41/50] Remove old comment in test --- test/fluree/db/json_ld/api_test.cljc | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/test/fluree/db/json_ld/api_test.cljc b/test/fluree/db/json_ld/api_test.cljc index 844dd60b9..248a222db 100644 --- a/test/fluree/db/json_ld/api_test.cljc +++ b/test/fluree/db/json_ld/api_test.cljc @@ -268,11 +268,7 @@ "ex:friends" [{"id" "ex:john"} {"id" "ex:cam"}]}} (set @(fluree/query loaded-db '{"select" {?s ["*"]} - "where" [[?s "rdf:type" "ex:User"]]})) - ;; TODO: Delete me and uncomment above - #_(set - @(fluree/query db '{"select" {?s ["*"]} - "where" [[?s "rdf:type" "ex:User"]]}))))))) + "where" [[?s "rdf:type" "ex:User"]]}))))))) (testing "can load with policies" (with-tmp-dir storage-path From e5aa705d99198809c00ce759e6bf033037595963 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Thu, 20 Apr 2023 09:58:49 -0600 Subject: [PATCH 42/50] Humanize query validation errors --- src/fluree/db/query/fql/syntax.cljc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/fluree/db/query/fql/syntax.cljc b/src/fluree/db/query/fql/syntax.cljc index b9db9fb89..61635946e 100644 --- a/src/fluree/db/query/fql/syntax.cljc +++ b/src/fluree/db/query/fql/syntax.cljc @@ -7,6 +7,7 @@ [fluree.json-ld :as json-ld] [malli.core :as m] [fluree.db.util.validation :as v] + [malli.error :as me] [malli.transform :as mt])) #?(:clj (set! *warn-on-reflection* true)) @@ -295,5 +296,7 @@ (throw (ex-info "Invalid Query" {:status 400 :error :db/invalid-query - :reasons (m/explain ::query qry - {:registry registry})})))) + :query qry + :reasons (me/humanize + (m/explain ::query qry + {:registry registry}))})))) From 507e5482d803a296407021bbcc324722d55d7361 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Thu, 20 Apr 2023 09:59:10 -0600 Subject: [PATCH 43/50] Allow map values in contexts --- src/fluree/db/util/validation.cljc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/fluree/db/util/validation.cljc b/src/fluree/db/util/validation.cljc index 6a471c7aa..9a77ade6b 100644 --- a/src/fluree/db/util/validation.cljc +++ b/src/fluree/db/util/validation.cljc @@ -21,7 +21,9 @@ ::val [:fn value?] ::string-key [:string {:decode/fluree name :encode/fluree keyword}] - ::context-map [:map-of ::iri ::iri] + ::context-map [:map-of ::iri [:orn + [:iri ::iri] + [:map [:ref ::context-map]]]] ::context [:orn [:sequence [:sequential [:orn [:string :string] From a8a0ffccfe89504b567bf273e6c90161c0a98359 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Thu, 20 Apr 2023 09:59:42 -0600 Subject: [PATCH 44/50] Use @set instead of set in container specifier in a test --- test/fluree/db/query/misc_queries_test.clj | 12 ++++++++---- test/fluree/db/query/reverse_query_test.clj | 7 ++++--- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/test/fluree/db/query/misc_queries_test.clj b/test/fluree/db/query/misc_queries_test.clj index 3c2d60d94..c51ab9787 100644 --- a/test/fluree/db/query/misc_queries_test.clj +++ b/test/fluree/db/query/misc_queries_test.clj @@ -36,20 +36,24 @@ (deftest ^:integration result-formatting (let [conn (test-utils/create-conn) - ledger @(fluree/create conn "query-context" {"defaults" {"@context" ["" {"ex" "http://example.org/ns/"}]}}) + ledger @(fluree/create conn "query-context" + {"defaults" + {"@context" + ["" {"ex" "http://example.org/ns/"}]}}) db @(test-utils/transact ledger [{"id" "ex:dan" "ex:x" 1}])] - (is (= [{"id" "ex:dan" + + (is (= [{"id" "ex:dan" "ex:x" 1}] @(fluree/query db {"where" [['?s "id" "ex:dan"]] "select" {'?s ["*"]}})) "default unwrapped objects") (is (= [{"id" "ex:dan" "ex:x" [1]}] - @(fluree/query db {"@context" ["" {"ex:x" {"@container" "set"}}] + @(fluree/query db {"@context" ["" {"ex:x" {"@container" "@set"}}] "where" [['?s "id" "ex:dan"]] "select" {'?s ["*"]}})) - "override unwrapping with :set") + "override unwrapping with @set") (is (= [{"id" "ex:dan" "ex:x" [1]}] @(fluree/query db {"@context" ["" {"ex:x" {"@container" "@list"}}] diff --git a/test/fluree/db/query/reverse_query_test.clj b/test/fluree/db/query/reverse_query_test.clj index 369406ff0..021062ecc 100644 --- a/test/fluree/db/query/reverse_query_test.clj +++ b/test/fluree/db/query/reverse_query_test.clj @@ -7,9 +7,10 @@ (deftest ^:integration context-reverse-test (testing "Test that the @reverse context values pulls select values back correctly." (let [conn (test-utils/create-conn) - ledger @(fluree/create conn "query/reverse" {"defaults" - {"@context" - ["" {"ex" "http://example.org/ns/"}]}}) + ledger @(fluree/create conn "query/reverse" + {"defaults" + {"@context" + ["" {"ex" "http://example.org/ns/"}]}}) db @(fluree/stage (fluree/db ledger) [{"id" "ex:brian", From e4f29a7d12343d2b63b8365bc2d46a9a51461a24 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Thu, 20 Apr 2023 10:10:11 -0600 Subject: [PATCH 45/50] Reconcile query arities between db protocol & API --- src/fluree/db/db/json_ld.cljc | 1 - src/fluree/db/dbproto.cljc | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/fluree/db/db/json_ld.cljc b/src/fluree/db/db/json_ld.cljc index bf1040fa0..32968a79e 100644 --- a/src/fluree/db/db/json_ld.cljc +++ b/src/fluree/db/db/json_ld.cljc @@ -247,7 +247,6 @@ (-iri [this subject-id compact-fn] (iri this subject-id compact-fn)) (-search [this fparts] (query-range/search this fparts)) (-query [this query-map] (fql/query this query-map)) - (-query [this query-map opts] (fql/query this query-map opts)) (-stage [db json-ld] (jld-transact/stage db json-ld nil)) (-stage [db json-ld opts] (jld-transact/stage db json-ld opts)) (-index-update [db commit-index] (index-update db commit-index)) diff --git a/src/fluree/db/dbproto.cljc b/src/fluree/db/dbproto.cljc index 6a175d880..20425a8b8 100644 --- a/src/fluree/db/dbproto.cljc +++ b/src/fluree/db/dbproto.cljc @@ -14,7 +14,7 @@ (-class-ids [db subject-id] "For the provided subject-id (long int), returns a list of class subject ids it is a member of (long ints)") (-iri [db subject-id] [db ident compact-fn] "Returns the IRI for the requested subject ID (json-ld only)") (-search [db fparts] "Performs a slice, but determines best index to use.") - (-query [db query] [db query opts] "Performs a query.") + (-query [db query] "Performs a query.") (-stage [db tx] [db tx opts] "Stages a database transaction.") (-index-update [db commit-index] "Updates db to reflect a new index point described by commit-index metadata") (-context [db] [db context] "Returns parsed context given supplied context. If no context is supplied, returns default context.") From b19e9451a22095409e2ab42bcf6b55a16d29b57e Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Thu, 20 Apr 2023 13:30:47 -0600 Subject: [PATCH 46/50] Add support for multi-queries in CLJ SDK --- src/fluree/db/query/fql/syntax.cljc | 11 +++++++---- src/fluree/sdk/clojure.clj | 25 +++++++++++++++++-------- test/fluree/db/query/fql_test.clj | 4 ++-- test/fluree/db/test_utils.cljc | 2 +- test/fluree/sdk/clojure_test.clj | 27 +++++++++++++++++++++++++++ 5 files changed, 54 insertions(+), 15 deletions(-) diff --git a/src/fluree/db/query/fql/syntax.cljc b/src/fluree/db/query/fql/syntax.cljc index 61635946e..a8e82728a 100644 --- a/src/fluree/db/query/fql/syntax.cljc +++ b/src/fluree/db/query/fql/syntax.cljc @@ -86,7 +86,7 @@ :else v)) q)) -(defn analytical-query-results-transformer +(defn keyword-compact-iri-transformer [context] ;; TODO: Replace this with a :map-with schema once this lands upstream: ;; https://github.com/metosin/malli/issues/881 @@ -98,6 +98,7 @@ (json-ld/compact expanded kw-context) s))}}))) + (def registry (merge (m/predicate-schemas) @@ -277,11 +278,13 @@ (def coerce-analytical-query (m/coercer ::analytical-query v/fluree-transformer {:registry registry})) +(def coerce-multi-query + (m/coercer ::multi-query v/fluree-transformer {:registry registry})) + (defn analytical-query-results-encoder [context] - (m/encoder ::analytical-query-results - {:registry registry} - (analytical-query-results-transformer context))) + (m/encoder ::analytical-query-results {:registry registry} + (keyword-compact-iri-transformer context))) (def valid-query? (m/validator ::query {:registry registry})) diff --git a/src/fluree/sdk/clojure.clj b/src/fluree/sdk/clojure.clj index 85573d516..b9b5344b3 100644 --- a/src/fluree/sdk/clojure.clj +++ b/src/fluree/sdk/clojure.clj @@ -81,21 +81,30 @@ (defn query [db query] - (let [context (json-ld/parse-context (fqp/parse-context query db)) - results-encoder (fql/analytical-query-results-encoder context)] + (let [context (json-ld/parse-context (fqp/parse-context query db)) + encode-results (fql/analytical-query-results-encoder context)] (future (->> query fql/coerce-analytical-query (api/query db) deref (log/debug->>val "pre-encoded query results:") - results-encoder)))) + encode-results)))) -(comment - ;; TODO: Finish these - (defn multi-query - [db query] - (api/multi-query db query)) +(defn multi-query + [db query] + (let [context (json-ld/parse-context (fqp/parse-context query db)) + encode-result (fql/analytical-query-results-encoder context) + encode-results #(reduce-kv (fn [rs k r] + (assoc rs k (encode-result r))) + {} %)] + (future + (->> query + fql/coerce-multi-query + (api/multi-query db) + deref + (log/debug->>val "pre-encoded multi query results:") + encode-results)))) (defn history [ledger query] diff --git a/test/fluree/db/query/fql_test.clj b/test/fluree/db/query/fql_test.clj index 280b3f4b5..4b23faca0 100644 --- a/test/fluree/db/query/fql_test.clj +++ b/test/fluree/db/query/fql_test.clj @@ -191,9 +191,9 @@ people (test-utils/load-people conn) db (fluree/db people)] (testing "multi queries" - (let [q '{"alice" {"select" {?s [:*]} + (let [q '{"alice" {"select" {?s ["*"]} "where" [[?s "schema:email" "alice@example.org"]]} - "brian" {"select" {?s [:*]} + "brian" {"select" {?s ["*"]} "where" [[?s "schema:email" "brian@example.org"]]}} subject @(fluree/multi-query db q)] (is (= {"alice" [{"id" "ex:alice" diff --git a/test/fluree/db/test_utils.cljc b/test/fluree/db/test_utils.cljc index 4dc6018dd..70e8df4b7 100644 --- a/test/fluree/db/test_utils.cljc +++ b/test/fluree/db/test_utils.cljc @@ -83,7 +83,7 @@ (defn create-conn ([] (create-conn {})) - ([{:strs [context did] + ([{:keys [context did] :or {context default-context did (did/private->did-map default-private-key false)}}] (let [conn-p (fluree/connect-memory {"defaults" {"@context" context diff --git a/test/fluree/sdk/clojure_test.clj b/test/fluree/sdk/clojure_test.clj index 60d861fa0..ceb8f898a 100644 --- a/test/fluree/sdk/clojure_test.clj +++ b/test/fluree/sdk/clojure_test.clj @@ -428,3 +428,30 @@ {:f/allow [:* {:f/targetRole ["_id"]}]} {:f/property [:* {:f/allow [:* {:f/targetRole ["_id"]}]}]}]} :where [[?s :rdf/type :f/Policy]]}))))))))) + +(deftest ^:integration multi-query-test + (let [conn (test-utils/create-conn + {:context (merge test-utils/default-context + {"ex" "http://example.org/ns/"})}) + people (test-utils/load-people conn) + db (fluree/db people)] + (testing "multi queries" + (let [q '{:alice {:select {?s [:*]} + :where [[?s :schema/email "alice@example.org"]]} + :brian {:select {?s [:*]} + :where [[?s :schema/email "brian@example.org"]]}} + subject @(fluree/multi-query db q)] + (is (= {:alice [{:id :ex/alice + :rdf/type [:ex/User] + :ex/favNums [9 42 76] + :schema/age 50 + :schema/email "alice@example.org" + :schema/name "Alice"}] + :brian [{:id :ex/brian + :rdf/type [:ex/User] + :ex/favNums 7 + :schema/age 50 + :schema/email "brian@example.org" + :schema/name "Brian"}]} + subject) + "returns all results in a map keyed by alias."))))) From caffde5a1c5c979d208706f55cb882ecd1dcb28a Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Thu, 20 Apr 2023 15:33:42 -0600 Subject: [PATCH 47/50] Add history queries to CLJ SDK & tests --- src/fluree/db/query/fql/syntax.cljc | 1 - src/fluree/db/query/history.cljc | 41 ++++-- src/fluree/sdk/clojure.clj | 16 ++- test/fluree/db/test_utils.cljc | 10 +- test/fluree/sdk/clojure_test.clj | 192 +++++++++++++++++++++++++++- 5 files changed, 241 insertions(+), 19 deletions(-) diff --git a/src/fluree/db/query/fql/syntax.cljc b/src/fluree/db/query/fql/syntax.cljc index a8e82728a..752b4055b 100644 --- a/src/fluree/db/query/fql/syntax.cljc +++ b/src/fluree/db/query/fql/syntax.cljc @@ -98,7 +98,6 @@ (json-ld/compact expanded kw-context) s))}}))) - (def registry (merge (m/predicate-schemas) diff --git a/src/fluree/db/query/history.cljc b/src/fluree/db/query/history.cljc index 3216ab302..7ae750c45 100644 --- a/src/fluree/db/query/history.cljc +++ b/src/fluree/db/query/history.cljc @@ -1,6 +1,7 @@ (ns fluree.db.query.history (:require [clojure.core.async :as async] + [fluree.db.query.fql.syntax :as fql] [malli.core :as m] [fluree.json-ld :as json-ld] [fluree.db.constants :as const] @@ -60,7 +61,7 @@ pos-int? [:re datatype/iso8601-datetime-re]] ::history-query [:and - [:map + [:map {:decode/fluree fql/decode-query} ["history" {:optional true} [:orn [:subject ::iri] @@ -96,9 +97,23 @@ [:fn {:error/message "Must supply either a \"history\" or \"commit-details\" key."} (fn [{:strs [history commit-details]}] (or history commit-details))]] - ::history-query-result [:fn history-query-result?] + ::results-map [:map-of :string [:or + :string + [:sequential [:ref ::results-map]] + :any]] + ::history-query-result [:and + ::results-map + [:fn history-query-result?]] ::history-query-results [:sequential ::history-query-result]})) +(def coerce-query + (m/coercer ::history-query v/fluree-transformer {:registry registry})) + +(defn results-encoder + [context] + (m/encoder ::history-query-results {:registry registry} + (fql/keyword-compact-iri-transformer context))) + (def history-query-validator (m/validator ::history-query {:registry registry})) @@ -134,16 +149,16 @@ [db cache context compact fuel error-ch s-flakes] (async/go (try* - (let [json-chan (json-ld-resp/flakes->res db cache context compact fuel 1000000 - {:wildcard? true, :depth 0} - 0 s-flakes)] - (-> ( s-flakes first flake/s (->> (dbproto/-iri db)) ! error-ch e))))) + (let [json-chan (json-ld-resp/flakes->res db cache context compact fuel 1000000 + {:wildcard? true, :depth 0} + 0 s-flakes)] + (-> ( s-flakes first flake/s (->> (dbproto/-iri db)) ! error-ch e))))) (defn t-flakes->json-ld "Build a collection of subject maps out of a set of flakes with the same t. @@ -156,7 +171,7 @@ (vals) (async/to-chan!)) - s-out-ch (async/chan)] + s-out-ch (async/chan)] (async/pipeline-async 2 s-out-ch (fn [assert-flakes ch] diff --git a/src/fluree/sdk/clojure.clj b/src/fluree/sdk/clojure.clj index b9b5344b3..1711b6276 100644 --- a/src/fluree/sdk/clojure.clj +++ b/src/fluree/sdk/clojure.clj @@ -5,6 +5,7 @@ [fluree.db.util.validation :as v] [fluree.db.query.fql.syntax :as fql] [fluree.db.query.fql.parse :as fqp] + [fluree.db.query.history :as fqh] [fluree.json-ld :as json-ld]) (:refer-clojure :exclude [load])) @@ -106,6 +107,15 @@ (log/debug->>val "pre-encoded multi query results:") encode-results)))) - (defn history - [ledger query] - (api/history ledger query))) +(defn history + [ledger query] + (let [latest-db (db ledger) + context (json-ld/parse-context (fqp/parse-context query latest-db)) + encode-results (fqh/results-encoder context)] + (future + (->> query + fqh/coerce-query + (api/history ledger) + deref + (log/debug->>val "pre-encoded history query results:") + encode-results)))) diff --git a/test/fluree/db/test_utils.cljc b/test/fluree/db/test_utils.cljc index 70e8df4b7..09bca1038 100644 --- a/test/fluree/db/test_utils.cljc +++ b/test/fluree/db/test_utils.cljc @@ -2,7 +2,8 @@ (:require [fluree.db.did :as did] [fluree.db.json-ld.api :as fluree] [fluree.db.util.core :as util :refer [try* catch*]] - #?@(:cljs [[clojure.core.async :refer [go go-loop]] + #?@(:clj [[fluree.sdk.clojure :as fluree-clj]] + :cljs [[clojure.core.async :refer [go go-loop]] [clojure.core.async.interop :refer [ db2 :commit :time) + db3 @(test-utils/transact-clj ledger {:id :ex/dan + :ex/x "foo-3" + :ex/y "bar-3"}) + + ts3 (-> db3 :commit :time) + db4 @(test-utils/transact-clj ledger [{:id :ex/cat + :ex/x "foo-cat" + :ex/y "bar-cat"} + {:id :ex/dog + :ex/x "foo-dog" + :ex/y "bar-dog"}]) + db5 @(test-utils/transact-clj ledger {:id :ex/dan + :ex/x "foo-cat" + :ex/y "bar-cat"})] + (testing "subject history" + (is (= [{:f/t 1 + :f/assert [{:id :ex/dan :ex/x "foo-1" :ex/y "bar-1"}] + :f/retract []} + {:f/t 2 + :f/assert [{:id :ex/dan :ex/x "foo-2" :ex/y "bar-2"}] + :f/retract [{:id :ex/dan :ex/x "foo-1" :ex/y "bar-1"}]} + {:f/t 3 + :f/assert [{:id :ex/dan :ex/x "foo-3" :ex/y "bar-3"}] + :f/retract [{:id :ex/dan :ex/x "foo-2" :ex/y "bar-2"}]} + {:f/t 5 + :f/assert [{:id :ex/dan :ex/x "foo-cat" :ex/y "bar-cat"}] + :f/retract [{:id :ex/dan :ex/x "foo-3" :ex/y "bar-3"}]}] + @(fluree/history ledger {:history :ex/dan, :t {:from 1}})))) + (testing "one-tuple flake history" + (is (= [{:f/t 1 + :f/assert [{:id :ex/dan :ex/x "foo-1" :ex/y "bar-1"}] + :f/retract []} + {:f/t 2 + :f/assert [{:ex/x "foo-2" :ex/y "bar-2" :id :ex/dan}] + :f/retract [{:ex/x "foo-1" :ex/y "bar-1" :id :ex/dan}]} + {:f/t 3 + :f/assert [{:ex/x "foo-3" :ex/y "bar-3" :id :ex/dan}] + :f/retract [{:ex/x "foo-2" :ex/y "bar-2" :id :ex/dan}]} + {:f/t 5 + :f/assert [{:ex/x "foo-cat" :ex/y "bar-cat" :id :ex/dan}] + :f/retract [{:ex/x "foo-3" :ex/y "bar-3" :id :ex/dan}]}] + @(fluree/history ledger {:history [:ex/dan] :t {:from 1}})))) + (testing "two-tuple flake history" + (is (= [{:f/t 1 :f/assert [{:ex/x "foo-1" :id :ex/dan}] :f/retract []} + {:f/t 2 + :f/assert [{:ex/x "foo-2" :id :ex/dan}] + :f/retract [{:ex/x "foo-1" :id :ex/dan}]} + {:f/t 3 + :f/assert [{:ex/x "foo-3" :id :ex/dan}] + :f/retract [{:ex/x "foo-2" :id :ex/dan}]} + {:f/t 5 + :f/assert [{:ex/x "foo-cat" :id :ex/dan}] + :f/retract [{:ex/x "foo-3" :id :ex/dan}]}] + @(fluree/history ledger {:history [:ex/dan :ex/x] :t {:from 1}}))) + + (is (= [{:f/t 1 :f/assert [{:ex/x "foo-1" :id :ex/dog} + {:ex/x "foo-1" :id :ex/cat} + {:ex/x "foo-1" :id :ex/dan}] + :f/retract []} + {:f/t 2 + :f/assert [{:ex/x "foo-2" :id :ex/dan}] + :f/retract [{:ex/x "foo-1" :id :ex/dan}]} + {:f/t 3 + :f/assert [{:ex/x "foo-3" :id :ex/dan}] + :f/retract [{:ex/x "foo-2" :id :ex/dan}]} + {:f/t 4 + :f/assert [{:ex/x "foo-dog" :id :ex/dog} + {:ex/x "foo-cat" :id :ex/cat}] + :f/retract [{:ex/x "foo-1" :id :ex/dog} + {:ex/x "foo-1" :id :ex/cat}]} + {:f/t 5 + :f/assert [{:ex/x "foo-cat" :id :ex/dan}] + :f/retract [{:ex/x "foo-3" :id :ex/dan}]}] + @(fluree/history ledger {:history [nil :ex/x] :t {:from 1}})))) + (testing "three-tuple flake history" + (is (= [{:f/t 4 :f/assert [{:ex/x "foo-cat" :id :ex/cat}] :f/retract []} + {:f/t 5 :f/assert [{:ex/x "foo-cat" :id :ex/dan}] :f/retract []}] + @(fluree/history ledger {:history [nil :ex/x "foo-cat"] :t {:from 1}}))) + (is (= [{:f/t 2 + :f/assert [{:ex/x "foo-2" :id :ex/dan}] + :f/retract []} + {:f/t 3 + :f/assert [] + :f/retract [{:ex/x "foo-2" :id :ex/dan}]}] + @(fluree/history ledger {:history [nil :ex/x "foo-2"] :t {:from 1}}))) + (is (= [{:f/t 5 :f/assert [{:ex/x "foo-cat" :id :ex/dan}] :f/retract []}] + @(fluree/history ledger {:history [:ex/dan :ex/x "foo-cat"] :t {:from 1}})))) + + (testing "at-t" + (let [expected [{:f/t 3 + :f/assert [{:ex/x "foo-3" :id :ex/dan}] + :f/retract [{:ex/x "foo-2" :id :ex/dan}]}]] + (is (= expected + @(fluree/history ledger {:history [:ex/dan :ex/x] :t {:from 3 :to 3}}))) + (is (= expected + @(fluree/history ledger {:history [:ex/dan :ex/x] :t {:at 3}}))))) + (testing "from-t" + (is (= [{:f/t 3 + :f/assert [{:ex/x "foo-3" :id :ex/dan}] + :f/retract [{:ex/x "foo-2" :id :ex/dan}]} + {:f/t 5 + :f/assert [{:ex/x "foo-cat" :id :ex/dan}] + :f/retract [{:ex/x "foo-3" :id :ex/dan}]}] + @(fluree/history ledger {:history [:ex/dan :ex/x] :t {:from 3}})))) + (testing "to-t" + (is (= [{:f/t 1 + :f/assert [{:ex/x "foo-1" :id :ex/dan}] + :f/retract []} + {:f/t 2 + :f/assert [{:ex/x "foo-2" :id :ex/dan}] + :f/retract [{:ex/x "foo-1" :id :ex/dan}]} + {:f/t 3 + :f/assert [{:ex/x "foo-3" :id :ex/dan}] + :f/retract [{:ex/x "foo-2" :id :ex/dan}]}] + @(fluree/history ledger {:history [:ex/dan :ex/x] :t {:to 3}})))) + (testing "t-range" + (is (= [{:f/t 2 + :f/assert [{:ex/x "foo-2" :id :ex/dan}] + :f/retract [{:ex/x "foo-1" :id :ex/dan}]} + {:f/t 3 + :f/assert [{:ex/x "foo-3" :id :ex/dan}] + :f/retract [{:ex/x "foo-2" :id :ex/dan}]} + {:f/t 4 + :f/assert [{:ex/x "foo-dog" :id :ex/dog} {:ex/x "foo-cat" :id :ex/cat}] + :f/retract [{:ex/x "foo-1" :id :ex/dog} {:ex/x "foo-1" :id :ex/cat}]}] + @(fluree/history ledger {:history [nil :ex/x] :t {:from 2 :to 4}})))) + (testing "datetime-t" + (is (= [{:f/t 2 + :f/assert [{:ex/x "foo-2" :id :ex/dan}] + :f/retract [{:ex/x "foo-1" :id :ex/dan}]} + {:f/t 3 + :f/assert [{:ex/x "foo-3" :id :ex/dan}] + :f/retract [{:ex/x "foo-2" :id :ex/dan}]}] + @(fluree/history ledger {:history [nil :ex/x] :t {:from ts2 :to ts3}})) + "does not include t 1 4 or 5") + (is (= [{:f/t 5 + :f/assert [{:ex/x "foo-cat" :id :ex/dan}] + :f/retract [{:ex/x "foo-3" :id :ex/dan}]}] + @(fluree/history ledger {:history [:ex/dan :ex/x] :t {:from (util/current-time-iso)}})) + "timestamp translates to first t before ts") + + (is (= (str "There is no data as of " ts-primeval) + (-> @(fluree/history ledger {:history [:ex/dan :ex/x] :t {:from ts-primeval}}) + (Throwable->map) + :cause)))) + + (testing "invalid query" + (is (thrown? Exception @(fluree/history ledger {:history []})))) + + (testing "small cache" + (let [conn (test-utils/create-conn) + ledger @(fluree/create conn "historycachetest" + {:defaults + {:context + ["" {:ex "http://example.org/ns/"}]}}) + + db1 @(test-utils/transact-clj ledger [{:id :ex/dan + :ex/x "foo-1" + :ex/y "bar-1"}]) + db2 @(test-utils/transact-clj ledger {:id :ex/dan + :ex/x "foo-2" + :ex/y "bar-2"})] + (testing "no t-range cache collision" + (is (= [{:f/t 2 + :f/assert [{:ex/x "foo-2" :ex/y "bar-2" :id :ex/dan}] + :f/retract [{:ex/x "foo-1" :ex/y "bar-1" :id :ex/dan}]}] + @(fluree/history ledger {:history [:ex/dan] :t {:from 2}})))))))) From ea1588b69bf20976733fa04221947426f760bd64 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Fri, 21 Apr 2023 09:23:59 -0600 Subject: [PATCH 48/50] Stringify new policy subj-flakes test --- test/fluree/db/policy/subj_flakes_test.clj | 161 +++++++++++---------- 1 file changed, 82 insertions(+), 79 deletions(-) diff --git a/test/fluree/db/policy/subj_flakes_test.clj b/test/fluree/db/policy/subj_flakes_test.clj index fd34f073f..676f42862 100644 --- a/test/fluree/db/policy/subj_flakes_test.clj +++ b/test/fluree/db/policy/subj_flakes_test.clj @@ -1,11 +1,12 @@ (ns fluree.db.policy.subj-flakes-test (:require - [clojure.test :refer :all] - [fluree.db.test-utils :as test-utils] - [fluree.db.json-ld.api :as fluree] - [fluree.db.did :as did] - [fluree.db.permissions-validate :as policy-enforce] - [clojure.core.async :as async])) + [clojure.test :refer :all] + [fluree.db.test-utils :as test-utils] + [fluree.db.json-ld.api :as fluree] + [fluree.db.did :as did] + [fluree.db.permissions-validate :as policy-enforce] + [clojure.core.async :as async] + [fluree.db.util.log :as log])) ;; tests for the optimized policy filtering for groups of flakes of the same subject ;; (used for simple subject crawl) @@ -18,57 +19,57 @@ root-did (:id (did/private->did-map "8ce4eca704d653dec594703c81a84c403c39f262e54ed014ed857438933a2e1c")) alice-did (:id (did/private->did-map "c0459840c334ca9f20c257bed971da88bd9b1b5d4fca69d4e3f4b8504f981c07")) db @(fluree/stage - (fluree/db ledger) - [{"id" "ex:alice", - "type" "ex:User", - "schema:name" "Alice" - "schema:email" "alice@flur.ee" - "schema:birthDate" "2022-08-17" - "schema:ssn" "111-11-1111" - "ex:location" {"ex:state" "NC" - "ex:country" "USA"}} - {"id" "ex:john", - "type" "ex:User", - "schema:name" "John" - "schema:email" "john@flur.ee" - "schema:birthDate" "2021-08-17" - "schema:ssn" "888-88-8888"} - {"id" "ex:widget", - "type" "ex:Product", - "schema:name" "Widget" - "schema:price" 99.99 - "schema:priceCurrency" "USD"} - ;; assign root-did to "ex:rootRole" - {"id" root-did - "f:role" {"id" "ex:rootRole"}} - ;; assign alice-did to "ex:userRole" and also link the did to "ex:alice" via "ex:user" - {"id" alice-did - "ex:user" {"id" "ex:alice"} - "f:role" {"id" "ex:userRole"}}]) + (fluree/db ledger) + [{"id" "ex:alice", + "type" "ex:User", + "schema:name" "Alice" + "schema:email" "alice@flur.ee" + "schema:birthDate" "2022-08-17" + "schema:ssn" "111-11-1111" + "ex:location" {"ex:state" "NC" + "ex:country" "USA"}} + {"id" "ex:john", + "type" "ex:User", + "schema:name" "John" + "schema:email" "john@flur.ee" + "schema:birthDate" "2021-08-17" + "schema:ssn" "888-88-8888"} + {"id" "ex:widget", + "type" "ex:Product", + "schema:name" "Widget" + "schema:price" 99.99 + "schema:priceCurrency" "USD"} + ;; assign root-did to "ex:rootRole" + {"id" root-did + "f:role" {"id" "ex:rootRole"}} + ;; assign alice-did to "ex:userRole" and also link the did to "ex:alice" via "ex:user" + {"id" alice-did + "ex:user" {"id" "ex:alice"} + "f:role" {"id" "ex:userRole"}}]) db+policy @(fluree/stage db ;; add policy targeting "ex:rootRole" that can view and modify everything - [{"id" "ex:rootPolicy", - "type" ["f:Policy"], ;; must be of type "f:Policy", else it won't be treated as a policy - "f:targetNode" {"id" "f:allNodes"} ;; "f:allNodes" special keyword meaning every node (everything) - "f:allow" [{"id" "ex:rootAccessAllow" - "f:targetRole" {"id" "ex:rootRole"} ;; our name for global / root role - "f:action" [{"id" "f:view"} - {"id" "f:modify"}]}]} - ;; add a policy targeting "ex:userRole" that can see all users, but only SSN if belonging to themselves - {"id" "ex:UserPolicy", - "type" ["f:Policy"], - "f:targetClass" {"id" "ex:User"} - "f:allow" [{"id" "ex:globalViewAllow" - "f:targetRole" {"id" "ex:userRole"} ;; our assigned name for standard user's role (given to Alice above) - "f:action" [{"id" "f:view"}]}] - "f:property" [{"f:path" {"id" "schema:ssn"} - "f:allow" [{"id" "ex:ssnViewRule" - "f:targetRole" {"id" "ex:userRole"} - "f:action" [{"id" "f:view"}] - "f:equals" {"@list" [{"id" "f:$identity"} - {"id" "ex:user"}]}}]}]}]) + [{"id" "ex:rootPolicy", + "type" ["f:Policy"], ;; must be of type "f:Policy", else it won't be treated as a policy + "f:targetNode" {"id" "f:allNodes"} ;; "f:allNodes" special keyword meaning every node (everything) + "f:allow" [{"id" "ex:rootAccessAllow" + "f:targetRole" {"id" "ex:rootRole"} ;; our name for global / root role + "f:action" [{"id" "f:view"} + {"id" "f:modify"}]}]} + ;; add a policy targeting "ex:userRole" that can see all users, but only SSN if belonging to themselves + {"id" "ex:UserPolicy", + "type" ["f:Policy"], + "f:targetClass" {"id" "ex:User"} + "f:allow" [{"id" "ex:globalViewAllow" + "f:targetRole" {"id" "ex:userRole"} ;; our assigned name for standard user's role (given to Alice above) + "f:action" [{"id" "f:view"}]}] + "f:property" [{"f:path" {"id" "schema:ssn"} + "f:allow" [{"id" "ex:ssnViewRule" + "f:targetRole" {"id" "ex:userRole"} + "f:action" [{"id" "f:view"}] + "f:equals" {"@list" [{"id" "f:$identity"} + {"id" "ex:user"}]}}]}]}]) ;; get a group of flakes that we know will have different permissions for different users. john-flakes @(fluree/range db+policy :spot = ["ex:john"]) alice-flakes @(fluree/range db+policy :spot = [(fluree/expand-iri db+policy "ex:alice")]) @@ -115,37 +116,39 @@ (deftest ^:integration no-subject-flakes-test (testing "If no subject flakes return from a select, policy code should not be run to avoid error." (let [conn (test-utils/create-conn) - ledger @(fluree/create conn "policy/b" {:defaultContext ["" {:ex "http://example.org/ns/"}]}) + ledger @(fluree/create conn "policy/b" + {"defaults" + {"@context" ["" {"ex" "http://example.org/ns/"}]}}) alice-did (:id (did/private->did-map "c0459840c334ca9f20c257bed971da88bd9b1b5d4fca69d4e3f4b8504f981c07")) db @(fluree/stage - (fluree/db ledger) - [{:id :ex/alice, - :type :ex/User, - :schema/name "alice"} - {:id :ex/john, - :type :ex/User, - :schema/name "john"} - {:id :ex/widget, - :type :ex/Product, - :schema/price 99.99} - {:id alice-did - :ex/user :ex/alice - :f/role :ex/userRole}]) + (fluree/db ledger) + [{"id" "ex:alice" + "type" "ex:User" + "schema:name" "alice"} + {"id" "ex:john" + "type" "ex:User" + "schema:name" "john"} + {"id" "ex:widget", + "type" "ex:Product", + "schema:price" 99.99} + {"id" alice-did + "ex:user" {"id" "ex:alice"} + "f:role" {"id" "ex:userRole"}}]) db+policy @(fluree/stage - db - ;; add policy targeting :ex/rootRole that can view and modify everything - [{:id :ex/userPolicy, - :type [:f/Policy], - :f/targetClass :ex/User - :f/allow [{:id :ex/globalViewAllow - :f/targetRole :ex/userRole - :f/action [:f/view]}]}]) + db + ;; add policy targeting :ex/rootRole that can view and modify everything + [{"id" "ex:userPolicy", + "type" ["f:Policy"], + "f:targetClass" {"id" "ex:User"} + "f:allow" [{"id" "ex:globalViewAllow" + "f:targetRole" {"id" "ex:userRole"} + "f:action" [{"id" "f:view"}]}]}]) - alice-db @(fluree/wrap-policy db+policy {:did alice-did - :role :ex/userRole})] + alice-db @(fluree/wrap-policy db+policy {"did" alice-did + "role" "ex:userRole"})] (is (= [] - @(fluree/query alice-db '{:select {?s [:*]} - :where [[?s :schema/price 99.99]]})) + @(fluree/query alice-db '{"select" {?s ["*"]} + "where" [[?s "schema:price" 99.99]]})) "There shouldn't be an exception, just empty vector.")))) From 54e972f3ba3283d475a75c490652d94406599a32 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Fri, 21 Apr 2023 09:35:52 -0600 Subject: [PATCH 49/50] Fix default context comparison in a test --- test/fluree/sdk/clojure_test.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/fluree/sdk/clojure_test.clj b/test/fluree/sdk/clojure_test.clj index dfe73a160..e95a9b696 100644 --- a/test/fluree/sdk/clojure_test.clj +++ b/test/fluree/sdk/clojure_test.clj @@ -84,7 +84,7 @@ loaded (test-utils/retry-load conn ledger-alias 100) loaded-db (fluree/db loaded)] (is (= (:t db) (:t loaded-db))) - (is (= (:context ledger) (:context loaded)))))) + (is (= (dbproto/-default-context db) (dbproto/-default-context loaded-db)))))) (testing "can load a file ledger with multi-cardinality predicates" (with-tmp-dir storage-path From 7c502a1ee22b6c4d494509f2f366b8d9d99e19e3 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Fri, 21 Apr 2023 09:36:54 -0600 Subject: [PATCH 50/50] Fix default context comparison in another test --- test/fluree/sdk/clojure_test.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/fluree/sdk/clojure_test.clj b/test/fluree/sdk/clojure_test.clj index e95a9b696..158a10e48 100644 --- a/test/fluree/sdk/clojure_test.clj +++ b/test/fluree/sdk/clojure_test.clj @@ -132,7 +132,7 @@ loaded (test-utils/retry-load conn ledger-alias 100) loaded-db (fluree/db loaded)] (is (= (:t db) (:t loaded-db))) - (is (= (:context ledger) (:context loaded)))))) + (is (= (dbproto/-default-context db) (dbproto/-default-context loaded-db)))))) (testing "can load a file ledger with its own context" (with-tmp-dir storage-path #_{::twf/delete-dir false}