diff --git a/src/fluree/db/query/exec/select.cljc b/src/fluree/db/query/exec/select.cljc index f3e3564cc..4ab6a6632 100644 --- a/src/fluree/db/query/exec/select.cljc +++ b/src/fluree/db/query/exec/select.cljc @@ -41,12 +41,12 @@ (defprotocol ValueSelector "Format a where search solution (map of pattern matches) by extracting and displaying relevant pattern matches." - (format-value [fmt db iri-cache compact error-ch solution])) + (format-value [fmt db iri-cache context compact error-ch solution])) (defrecord VariableSelector [var] ValueSelector (format-value - [_ db iri-cache compact error-ch solution] + [_ db iri-cache context compact error-ch solution] (-> solution (get var) (display db iri-cache compact error-ch)))) @@ -60,7 +60,7 @@ (defrecord AggregateSelector [agg-fn] ValueSelector (format-value - [_ db iri-cache compact error-ch solution] + [_ db iri-cache context compact error-ch solution] (go (try* (agg-fn solution) (catch* e (log/error e "Error applying aggregate selector") @@ -77,7 +77,7 @@ (defrecord SubgraphSelector [var selection depth spec] ValueSelector (format-value - [_ db iri-cache compact error-ch solution] + [_ db iri-cache context compact error-ch solution] (go (let [sid (-> solution (get var) @@ -85,7 +85,7 @@ (try* (let [flakes (res db iri-cache compact nil nil spec 0 flakes))) + (res db iri-cache context compact nil nil spec 0 flakes))) (catch* e (log/error e "Error formatting subgraph for subject:" sid) (>! error-ch e))))))) @@ -101,22 +101,23 @@ (defn format-values "Formats the values from the specified where search solution `solution` according to the selector or collection of selectors specified by `selectors`" - [selectors db iri-cache compact error-ch solution] + [selectors db iri-cache context compact error-ch solution] (if (sequential? selectors) (go-loop [selectors selectors values []] (if-let [selector (first selectors)] - (let [value (> q :context json-ld/compact-fn) + (let [context (:context q) + compact (json-ld/compact-fn context) selectors (or (:select q) (:select-one q) (:select-distinct q)) @@ -126,7 +127,7 @@ format-ch (fn [solution ch] (log/debug "select/format solution:" solution) - (-> (format-values selectors db iri-cache compact error-ch solution) + (-> (format-values selectors db iri-cache context compact error-ch solution) (async/pipe ch))) solution-ch) format-ch)) diff --git a/src/fluree/db/query/fql/resp.cljc b/src/fluree/db/query/fql/resp.cljc deleted file mode 100644 index 2e36b744f..000000000 --- a/src/fluree/db/query/fql/resp.cljc +++ /dev/null @@ -1,437 +0,0 @@ -(ns fluree.db.query.fql.resp - (:require [fluree.db.dbproto :as dbproto] - [fluree.db.util.async :refer [go-try res) - -(defn p->pred-config - [db p compact?] - (let [name (dbproto/-p-prop db :name p)] - {:p p - :limit nil - :name name - :as (if (and compact? name) - (second (re-find #"/(.+)" name)) - (or name (str p))) - :multi? (dbproto/-p-prop db :multi p) - :component? (dbproto/-p-prop db :component p) - :tag? (= :tag (dbproto/-p-prop db :type p)) - :ref? (dbproto/-p-prop db :ref? p)})) - - -(defn- build-predicate-map - "For a flake selection, build out parts of the - base set of predicates so we don't need to look them up - each time... like multi, component, etc." - [db pred-name] - (when-let [p (dbproto/-p-prop db :id pred-name)] - (p->pred-config db p false))) - - -(defn ns-lookup-pred-spec - "Given an predicate spec produced by the parsed select statement, - when an predicate does not have a namespace we will default it to - utilize the namespace of the subject. - - This fills out the predicate spec that couldn't be done earlier because - we did not know the collection." - [db collection-id ns-lookup-spec-map] - (let [collection-name (dbproto/-c-prop db :name collection-id)] - (reduce-kv - (fn [acc k v] - (let [pred (str collection-name "/" k)] - (if-let [p-map (build-predicate-map db pred)] - (assoc acc (:p p-map) (merge p-map v)) - acc))) - nil ns-lookup-spec-map))) - -(defn- has-ns-lookups? - "Returns true if the predicate spec has a sub-selection that requires a namespace lookup." - [select-spec] - (get-in select-spec [:select :ns-lookup])) - - -(defn full-select-spec - "Resolves a full predicate select spec in case there are - any namespace lookups (:ns-lookup) in the map that - need to be resolved for this given subject." - [db cache base-pred-spec subject-id] - (let [coll-id (flake/sid->cid subject-id)] - (or (get @cache [coll-id base-pred-spec]) - (let [lookup-specs (ns-lookup-pred-spec db coll-id (get-in base-pred-spec [:select :ns-lookup])) - updated-spec (update base-pred-spec :select (fn [sel] (-> sel - (assoc :pred-id (merge lookup-specs (:pred-id sel))) - (dissoc :ns-lookup))))] - (vswap! cache assoc [coll-id base-pred-spec] updated-spec) - updated-spec)))) - - -(defn select-spec->reverse-pred-specs - [select-spec] - (reduce (fn [acc spec] - (let [key-spec (key spec) - val-spec (if (nil? (:componentFollow? (val spec))) - (assoc (val spec) :componentFollow? (:componentFollow? select-spec)) - (val spec))] - (assoc acc key-spec val-spec))) - {} (get-in select-spec [:select :reverse]))) - - -(defn add-fuel - "Adds a n amount of fuel and will throw if max fuel exceeded." - [fuel n max-fuel] - (vswap! fuel + n) - (when (and max-fuel (> @fuel max-fuel)) - (throw (ex-info (str "Maximum query cost of " max-fuel " exceeded.") - {:status 400 :error :db/exceeded-cost})))) - -(defn resolve-reverse-refs - "Resolves all reverse references into a result map." - [db cache fuel max-fuel subject-id opts reverse-refs-specs] - (go-try - (loop [[n & r] reverse-refs-specs ;; loop through reverse refs - acc nil] - (if-not n - acc - (let [[pred-id pred-spec] n - {:keys [offset limit as name p]} pred-spec - sub-ids (->> (= n limit))) - acc' - - (and offset (< n offset)) - (recur r' (inc n) acc') - - :else - (let [sub-flakes (res db cache fuel max-fuel sub-pred-spec - opts sub-flakes)))))] - (recur r' (inc n) acc'*))))] - (recur r (assoc acc (or as name p) sub-result))))))) - -(defn component-follow? - [pred-spec select-spec] - (cond (contains? pred-spec :componentFollow?) - (:componentFollow? pred-spec) - - (not (nil? (:componentFollow? select-spec))) - (:componentFollow? select-spec) - - (or (:component? pred-spec) (:wildcard? select-spec)) - true)) - -(defn fuel-flake-transducer - "Can sit in a flake pipeline and accumulate a count of 'fuel-per' for every flake pulled - or item touched. 'fuel-per' defaults to 1 fuel per item. - - Inputs are: - - fuel - volatile! that holds fuel counter - - max-fuel - throw exception if @fuel ever exceeds this number - - To get final count, just deref fuel volatile when when where is complete." - ([fuel max-fuel] (fuel-flake-transducer fuel max-fuel 1)) - ([fuel max-fuel fuel-per] - (fn [xf] - (fn - ([] (xf)) ;; transducer start - ([result] (xf result)) ;; transducer stop - ([result flake] - (vswap! fuel + fuel-per) - (when (and max-fuel (> @fuel max-fuel)) - (throw (ex-info (str "Maximum query cost of " max-fuel " exceeded.") - {:status 400 :error :db/exceeded-cost}))) - (xf result flake)))))) - -(defn- recur-select-spec - "For recursion, takes current select-spec and nests the recur predicate as a child, updating - recur-depth and recur-seen values. Uses flake as the recursion flake being operated on." - [select-spec flake] - (let [recur-subject (flake/o flake) - recur-pred (flake/p flake) - {:keys [recur-seen recur-depth]} select-spec] - (-> select-spec - (assoc-in [:select :pred-id recur-pred] select-spec) ;; move current pred-spec to child in :select key for next recursion round - (assoc-in [:select :pred-id recur-pred :recur-depth] (inc recur-depth)) - (assoc-in [:select :pred-id recur-pred :recur-seen] (conj recur-seen recur-subject)) - ;; only need inherited keys - (select-keys [:select :componentFollow? :compact?])))) - -(defn flake->recur - "Performs recursion on a select spec graph crawl when specified. flakes input is list - of flakes all with the same subject and predicate values." - [db flakes select-spec results fuel max-fuel cache opts] - (go-try - (let [{:keys [multi? as recur-seen recur-depth limit]} select-spec ;; recur contains # with requested recursion depth - max-depth? (> recur-depth (:recur select-spec))] - (if max-depth? - results - (loop [[flake & r] flakes - i 0 - acc []] - (if (or (not flake) (and limit (< i limit))) - (cond (empty? acc) results - multi? (assoc results as acc) - :else (assoc results as (first acc))) - (let [recur-subject (flake/o flake) ;; ref, so recur subject is the object of the incoming flake - seen? (contains? recur-seen recur-subject) ;; subject has been seen before, stop recursion - sub-flakes (cond->> (res db cache fuel max-fuel select-spec* opts sub-flakes)))))))))))) - - -(defn wildcard-pred-spec - "Just uses query cache to avoid constant lookups." - [db cache p compact?] - (or (get-in @cache [p compact?]) - (let [p-map (p->pred-config db p compact?)] - (vswap! cache assoc-in [p compact?] p-map) - p-map))) - - -(defn- add-pred - "Adds a predicate to a select spec graph crawl. flakes input is a list of flakes - all with the same subject and predicate values." - [db cache fuel max-fuel acc pred-spec flakes componentFollow? recur? offset-map opts] - (go-try - (let [compact? (:compact? pred-spec) ;retain original value - pred-spec (if (and (:wildcard? pred-spec) (nil? (:as pred-spec))) - ;; nested 'refs' can be wildcard, but also have a pred-spec... so only get a default wildcard spec if we have no other spec - (wildcard-pred-spec db cache (-> flakes first :p) (:compact? pred-spec)) - pred-spec) - pred-spec' (cond-> pred-spec - (not (contains? pred-spec :componentFollow?)) (assoc :componentFollow? componentFollow?) - (not (contains? pred-spec :compact?)) (assoc :compact? compact?)) - ;; TODO - I think we can eliminate the check below for fallbacks and ensure we always have an 'as' in every spec - k (or (:as pred-spec') (:name pred-spec') (:p pred-spec')) ;; use :as, then full pred name, then just p-id as backup - {:keys [multi? ref? limit orderBy offset p]} pred-spec' - [k-val offset-map] (cond - (and multi? - offset - (not= 0 offset) - (not= 0 (get offset-map p))) - [nil - (if (get offset-map p) - (update offset-map p dec) - (assoc offset-map p (dec offset)))] - - ;; check if have hit limit of predicate spec - (and multi? - (not orderBy) - limit - (>= (count (get acc k)) limit)) - [nil offset-map] - - ;; have a sub-selection - (and (not recur?) - (or (:select pred-spec') (:wildcard? pred-spec'))) - (let [nested-select-spec (select-keys pred-spec' [:wildcard? :compact? :select])] - [(loop [[flake & r] flakes - acc []] - (if flake - (let [sub-sel (res db cache fuel max-fuel nested-select-spec opts - sub-sel)))] - (when fuel (vswap! fuel + (count sub-sel))) - (recur r (if (seq res) - (conj acc res) - acc))) - acc)) - offset-map]) - - ;; resolve tag - (:tag? pred-spec') - [(loop [[flake & r] flakes - acc []] - (if flake - (let [res (or (get @cache [(flake/o flake) (:name pred-spec')]) - (let [res (res db cache fuel max-fuel - {:wildcard? true :compact? compact?} - opts children))))] - (when fuel (vswap! fuel + (count children))) - (recur r acc*)) - acc)) - offset-map] - - ;; if a ref, put out an {:_id ...} - ref? - (if (true? (-> db :policy :f/view :root?)) - [(mapv #(hash-map :_id (flake/o %)) flakes) offset-map] - (loop [[f & r] flakes - acc []] - (if f - (if (seq (> res - sortPred (sort-by #(get % sortPred) compare-fn) - (= "DESC" sortOrder) reverse - offset (drop offset) - limit (take limit)) - res)) - - -(defn flakes->res - "Takes a sequence of flakes of the same subject and - composes them into a map result based on the 'select' spec - provided. Optionally, also follows components or recurs." - [db cache fuel max-fuel base-select-spec {:keys [parse-json?] :as opts} flakes] - (go-try - (when (not-empty flakes) - (log/debug "flakes->res flakes:" flakes) - (let [top-level-subject (try* - (flake/s (first flakes)) - (catch* e - (log/error e) - (throw e))) - _ (log/debug "flakes->res top-level-subject:" top-level-subject) - select-spec (if (has-ns-lookups? base-select-spec) - (full-select-spec db cache base-select-spec top-level-subject) - base-select-spec) - _ (log/debug "flakes->res select-spec:" select-spec) - base-acc (if (or (:wildcard? select-spec) (:id? select-spec)) - {:_id top-level-subject} - {}) - _ (log/debug "flakes->res base-acc:" base-acc) - acc+refs (if (get-in select-spec [:select :reverse]) - (->> select-spec - select-spec->reverse-pred-specs - (resolve-reverse-refs db cache fuel max-fuel (flake/s (first flakes)) opts) - res acc+refs:" acc+refs) - result (loop [p-flakes (partition-by :p flakes) - acc acc+refs - offset-map {}] - (if (empty? p-flakes) - acc - (let [flakes (first p-flakes) - _ (log/debug "flakes->res loop flakes:" flakes) - deserialized-flakes (if parse-json? - (json/parse-json-flakes db flakes) - flakes) - pred-spec (get-in select-spec [:select :pred-id - (-> deserialized-flakes first :p)]) - _ (log/debug "flakes->res pred-spec:" pred-spec) - componentFollow? (component-follow? pred-spec select-spec) - [acc flakes' offset-map'] (cond - (:recur pred-spec) - [(recur db deserialized-flakes pred-spec - acc fuel max-fuel cache opts)) - (rest p-flakes) offset-map] - - pred-spec - (let [[acc offset-map] ( deserialized-flakes first :s)} - (rest p-flakes) offset-map] - - :else - [acc (rest p-flakes) offset-map]) - acc* (assoc acc :_id (-> deserialized-flakes first :s))] - (recur flakes' acc* offset-map')))) - _ (log/debug "flakes->res result:" result) - sort-preds (reduce (fn [acc spec] - (log/debug "flakes->res sort-preds acc:" acc) - (if (or (and (:multi? spec) (:orderBy spec)) - (and (:reverse? spec) (:orderBy spec))) - (conj acc [(:as spec) - (-> spec :orderBy :order) - (-> spec :orderBy :predicate) - (:limit spec)]) - acc)) - [] (concat (-> select-spec :select :pred-id vals) - (-> select-spec :select :reverse vals)))] - (reduce (fn [acc [select-pred sort-order sort-pred limit]] - (log/debug "flakes->res return acc:" acc) - (->> select-pred - (get acc) - (sort-offset-and-limit-res sort-pred sort-order 0 limit) - (assoc acc select-pred))) - result sort-preds))))) diff --git a/src/fluree/db/query/history.cljc b/src/fluree/db/query/history.cljc index b63dd8dbd..1a5a8e0e8 100644 --- a/src/fluree/db/query/history.cljc +++ b/src/fluree/db/query/history.cljc @@ -9,7 +9,6 @@ [fluree.db.flake :as flake] [fluree.db.query.json-ld.response :as json-ld-resp] [fluree.db.query.fql.parse :as fql-parse] - [fluree.db.query.fql.resp :refer [flakes->res]] [fluree.db.util.async :refer [res db cache compact fuel 1000000 + (let [json-chan (json-ld-resp/flakes->res db cache context compact fuel 1000000 {:wildcard? true, :depth 0} 0 s-flakes)] (-> (> t-flakes (group-by flake/s) (vals) @@ -126,7 +125,7 @@ (async/pipeline-async 2 s-out-ch (fn [assert-flakes ch] - (-> (s-flakes->json-ld db cache compact fuel error-ch assert-flakes) + (-> (s-flakes->json-ld db cache context compact fuel error-ch assert-flakes) (async/pipe ch))) s-flakes-ch) s-out-ch)) @@ -164,11 +163,11 @@ t (- (flake/t (first t-flakes))) - asserts (->> (t-flakes->json-ld db compact cache fuel error-ch assert-flakes) + asserts (->> (t-flakes->json-ld db context compact cache fuel error-ch assert-flakes) (async/into []) (async/> (t-flakes->json-ld db compact cache fuel error-ch retract-flakes) + retracts (->> (t-flakes->json-ld db context compact cache fuel error-ch retract-flakes) (async/into []) (async/json-ld "Build a commit maps given a set of all flakes with the same t." - [db compact cache fuel error-ch t-flakes] + [db context compact cache fuel error-ch t-flakes] (async/go (try* (let [{commit-wrapper-flakes :commit-wrapper @@ -252,21 +251,21 @@ (and (not (flake/op f)) (not (extra-data-flake? f))) :retract-flakes :else :ignore-flakes)) t-flakes) - commit-wrapper-chan (json-ld-resp/flakes->res db cache compact fuel 1000000 + commit-wrapper-chan (json-ld-resp/flakes->res db cache context compact fuel 1000000 {:wildcard? true, :depth 0} 0 commit-wrapper-flakes) - commit-meta-chan (json-ld-resp/flakes->res db cache compact fuel 1000000 + commit-meta-chan (json-ld-resp/flakes->res db cache context compact fuel 1000000 {:wildcard? true, :depth 0} 0 commit-meta-flakes) commit-wrapper (> (t-flakes->json-ld db compact cache fuel error-ch assert-flakes) + asserts (->> (t-flakes->json-ld db context compact cache fuel error-ch assert-flakes) (async/into []) (async/> (t-flakes->json-ld db compact cache fuel error-ch retract-flakes) + retracts (->> (t-flakes->json-ld db context compact cache fuel error-ch retract-flakes) (async/into []) (async/ (commit-t-flakes->json-ld db compact cache fuel error-ch t-flakes) + (-> (commit-t-flakes->json-ld db context compact cache fuel error-ch t-flakes) (async/pipe ch))) t-flakes-ch) out-ch)) diff --git a/src/fluree/db/query/json_ld/response.cljc b/src/fluree/db/query/json_ld/response.cljc index 03ff6ceeb..87a7e5764 100644 --- a/src/fluree/db/query/json_ld/response.cljc +++ b/src/fluree/db/query/json_ld/response.cljc @@ -22,22 +22,6 @@ (vswap! cache assoc pid p-spec) p-spec))) -(defn p-values - [flakes] - (let [ff (first flakes)] - (cond - (= 1 (count flakes)) - (flake/o ff) ;; TODO - if @container = @list in the query context, should always return a vector even if just one result - - ;; flakes are an @list, return in sorted order - (:i (flake/m ff)) - (->> flakes - (sort-by #(:i (flake/m %))) - (mapv flake/o)) - - :else - (mapv flake/o flakes)))) - (defn iri? [pid] (= const/$iri pid)) @@ -58,15 +42,15 @@ iri))) (defn crawl-ref-item - [db compact-fn flake-sid sub-select cache fuel-vol max-fuel depth-i] + [db context compact-fn flake-sid sub-select cache fuel-vol max-fuel depth-i] (go-try (let [sub-flakes (res db cache compact-fn fuel-vol max-fuel sub-select depth-i sub-flakes))))) + (res db cache context compact-fn fuel-vol max-fuel sub-select depth-i sub-flakes))))) (defn add-reverse-specs "When @reverse variables are present, crawl for the reverse specs." - [db cache compact-fn fuel-vol max-fuel {:keys [reverse] :as select-spec} depth-i flakes] + [db cache context compact-fn fuel-vol max-fuel {:keys [reverse] :as select-spec} depth-i flakes] (go-try (let [sid (flake/s (first flakes))] (loop [[reverse-item & r] (vals reverse) @@ -79,7 +63,7 @@ (if ref-sid (let [result (if spec ;; have a sub-selection - (iri db cache compact-fn ref-sid))))] @@ -94,12 +78,12 @@ (defn flakes->res "depth-i param is the depth of the graph crawl. Each successive 'ref' increases the graph depth, up to the requested depth within the select-spec" - [db cache compact-fn fuel-vol max-fuel {:keys [wildcard? _id? depth reverse] :as select-spec} depth-i flakes] + [db cache context compact-fn fuel-vol max-fuel {:keys [wildcard? _id? depth reverse] :as select-spec} depth-i s-flakes] (go-try - (when (not-empty flakes) - (loop [[p-flakes & r] (partition-by flake/p flakes) + (when (not-empty s-flakes) + (loop [[p-flakes & r] (partition-by flake/p s-flakes) acc (if _id? - {:_id (flake/s (first flakes))} + {:_id (flake/s (first s-flakes))} {})] (if p-flakes (let [ff (first p-flakes) @@ -108,6 +92,7 @@ spec (or (get select-spec p) (when wildcard? (wildcard-spec db cache compact-fn p))) + p-iri (:as spec) v (cond (nil? spec) nil @@ -136,11 +121,11 @@ (cond ;; have a specified sub-selection (graph crawl) (:spec spec) - (} for each ref iri :else @@ -150,12 +135,13 @@ {id-key c-iri})) (flake/o f))] (recur r (conj acc res))) - (if (= 1 (count acc)) + (if (and (= 1 (count acc)) + (not (#{:list :set} (-> context (get p-iri) :container)))) (first acc) acc))))] (if v - (recur r (assoc acc (:as spec) v)) + (recur r (assoc acc p-iri v)) (recur r acc))) (if reverse - (merge acc (> parsed-query :context json-ld/compact-fn) - result-fn (partial json-ld-resp/flakes->res db cache compact-fn fuel-vol fuel select-spec 0) + context (:context parsed-query) + compact-fn (json-ld/compact-fn context) + result-fn (partial json-ld-resp/flakes->res db cache context compact-fn fuel-vol fuel select-spec 0) finish-fn (build-finishing-fn parsed-query) opts {:rdf-type? rdf-type? :db db diff --git a/test/fluree/db/query/misc_queries_test.clj b/test/fluree/db/query/misc_queries_test.clj index 18d0846fc..41faf5ece 100644 --- a/test/fluree/db/query/misc_queries_test.clj +++ b/test/fluree/db/query/misc_queries_test.clj @@ -31,6 +31,32 @@ @(fluree/query db {:select {'?s [:_id :* {:ex/favArtist [:_id :schema/name]}]} :where [['?s :type :ex/User]]})))))) +(deftest ^:integration result-formatting + (let [conn (test-utils/create-conn) + ledger @(fluree/create conn "query-context" {:context {:ex "http://example.org/ns/"}}) + db @(test-utils/transact ledger [{:id :ex/dan + :ex/x 1}])] + (is (= [{:id :foo/dan + :foo/x 1}] + @(fluree/query db {"@context" {:foo "http://example.org/ns/"} + :where [['?s :id :foo/dan]] + :select {'?s [:*]}})) + "default unwrapped objects") + (is (= [{:id :foo/dan + :foo/x [1]}] + @(fluree/query db {"@context" {:foo "http://example.org/ns/" + :foo/x {:container :set}} + :where [['?s :id :foo/dan]] + :select {'?s [:*]}})) + "override unwrapping with :set") + (is (= [{:id :ex/dan + "foo:x" [1]}] + @(fluree/query db {"@context" {"foo" "http://example.org/ns/" + "foo:x" {"@container" "@list"}} + :where [['?s "@id" "foo:dan"]] + :select {'?s ["*"]}})) + "override unwrapping with @list"))) + (deftest ^:integration s+p+o-full-db-queries (testing "Query that pulls entire database." (with-redefs [fluree.db.util.core/current-time-iso (fn [] "1970-01-01T00:12:00.00000Z")]