Skip to content

Commit

Permalink
Merge pull request #561 from fluree/feature/transact-graph-id-syntax
Browse files Browse the repository at this point in the history 
Update transact! syntax to be json-ld
  • Loading branch information
@mpoffald
mpoffald authored Aug 22, 2023
2 parents 2746c2e + c04b441 commit e3f36f6
Show file tree
Hide file tree
Showing 7 changed files with 248 additions and 90 deletions.
88 changes: 69 additions & 19 deletions src/fluree/db/api/transact.cljc
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,14 @@
(ns fluree.db.api.transact
(:require [fluree.db.fuel :as fuel]
(:require [clojure.walk :refer [keywordize-keys]]
[fluree.db.constants :as const]
[fluree.db.fuel :as fuel]
[fluree.db.util.core :as util :refer [try* catch*]]
[fluree.db.util.async :as async-util :refer [<? go-try]]
[fluree.db.json-ld.transact :as tx]
[fluree.db.dbproto :as dbproto]))
[fluree.db.ledger.json-ld :as jld-ledger]
[fluree.db.conn.proto :as conn-proto]
[fluree.db.dbproto :as dbproto]
[fluree.json-ld :as json-ld]))

(defn stage
[db json-ld opts]
Expand All @@ -25,22 +30,67 @@
:fuel (fuel/tally fuel-tracker))))))))
(<? (dbproto/-stage db json-ld opts)))))

(defn- parse-json-ld-txn
"Expands top-level keys and parses any opts in json-ld transaction document,
for use by `transact!`.
Throws if required keys @id or @graph are absent."
[json-ld]
(let [context-key (cond
(contains? json-ld "@context") "@context"
(contains? json-ld :context) :context)
context (get json-ld context-key)]
(let [parsed-context (json-ld/parse-context context)
{id "@id" graph "@graph" :as parsed-txn}
(into {}
(map (fn [[k v]]
(let [k* (if (= context-key k)
"@context"
(json-ld/expand-iri k parsed-context))
v* (if (= const/iri-opts k*)
(keywordize-keys v)
v)]
[k* v*])))
json-ld)]
(if-not (and id graph)
(throw (ex-info (str "Invalid transaction, missing required keys:"
(when (nil? id)
" @id")
(when (nil? graph)
" @graph")
".")
{:status 400 :error :db/invalid-transaction}))
parsed-txn))))

(defn transact!
[ledger json-ld opts]
[conn json-ld opts]
(go-try
(if (:meta opts)
(let [start-time #?(:clj (System/nanoTime)
:cljs (util/current-time-millis))
fuel-tracker (fuel/tracker)]
(try* (let [tx-result (<? (tx/transact! ledger fuel-tracker json-ld opts))]
{:status 200
:result tx-result
:time (util/response-time-formatted start-time)
:fuel (fuel/tally fuel-tracker)})
(catch* e
(throw (ex-info "Error updating ledger"
(-> e
ex-data
(assoc :time (util/response-time-formatted start-time)
:fuel (fuel/tally fuel-tracker))))))))
(<? (tx/transact! ledger json-ld opts)))))
(let [{txn-context "@context"
txn "@graph"
ledger-id "@id"
txn-opts const/iri-opts
default-context const/iri-default-context} (parse-json-ld-txn json-ld)
address (<? (conn-proto/-address conn ledger-id nil))]
(if-not (<? (conn-proto/-exists? conn address))
(throw (ex-info "Ledger does not exist" {:ledger address}))
(let [ledger (<? (jld-ledger/load conn address))
opts* (cond-> opts
txn-opts (merge txn-opts)
txn-context (assoc :txn-context txn-context)
default-context (assoc :defaultContext default-context))]
(if (:meta opts*)
(let [start-time #?(:clj (System/nanoTime)
:cljs (util/current-time-millis))
fuel-tracker (fuel/tracker)]
(try* (let [tx-result (<? (tx/transact! ledger fuel-tracker txn opts*))]
{:status 200
:result tx-result
:time (util/response-time-formatted start-time)
:fuel (fuel/tally fuel-tracker)})
(catch* e
(throw (ex-info "Error updating ledger"
(-> e
ex-data
(assoc :time (util/response-time-formatted start-time)
:fuel (fuel/tally fuel-tracker))))))))
(<? (tx/transact! ledger txn opts*))))))))
1 change: 1 addition & 0 deletions src/fluree/db/constants.cljc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
(def ^:const iri-target-objects-of "https://ns.flur.ee/ledger#targetObjectsOf")
(def ^:const iri-property "https://ns.flur.ee/ledger#property")
(def ^:const iri-policy "https://ns.flur.ee/ledger#Policy")
(def ^:const iri-opts "https://ns.flur.ee/ledger#opts")
(def ^:const iri-path "https://ns.flur.ee/ledger#path")
(def ^:const iri-action "https://ns.flur.ee/ledger#action")
(def ^:const iri-all-nodes "https://ns.flur.ee/ledger#allNodes")
Expand Down
16 changes: 13 additions & 3 deletions src/fluree/db/json_ld/api.cljc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
[fluree.db.ledger.proto :as ledger-proto]
[fluree.db.util.log :as log]
[fluree.db.query.range :as query-range]
[fluree.json-ld :as json-ld]
[fluree.db.json-ld.policy :as perm])
(:refer-clojure :exclude [merge load range exists?]))

Expand Down Expand Up @@ -238,10 +239,19 @@
(ledger-proto/-commit! ledger db opts))))

(defn transact!
"Stages and commits the transaction `json-ld` to the specified `ledger`"
[ledger json-ld opts]
"Expects a conn and json-ld document containing at least the following keys:
`@id`: the id of the ledger to transact to
`@graph`: the data to be transacted
Loads the specified ledger and peforms stage and commit! operations.
Returns the new db.
Note: Loading the ledger results in a new ledger object, so references to existing
ledger objects will be rendered stale. To obtain a ledger with the new changes,
call `load` on the ledger alias."
[conn json-ld opts]
(promise-wrap
(transact-api/transact! ledger json-ld opts)))
(transact-api/transact! conn json-ld opts)))

(defn status
"Returns current status of ledger branch."
Expand Down
61 changes: 36 additions & 25 deletions src/fluree/db/json_ld/transact.cljc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -253,7 +253,7 @@
[sid (into subj-flakes property-flakes)])))))

(defn ->tx-state
[db {:keys [bootstrap? did context-type] :as _opts}]
[db {:keys [bootstrap? did context-type txn-context] :as _opts}]
(let [{:keys [schema branch ledger policy], db-t :t} db
last-pid (volatile! (jld-ledger/last-pid db))
last-sid (volatile! (jld-ledger/last-sid db))
Expand All @@ -276,6 +276,7 @@
:next-sid (fn [] (vswap! last-sid inc))
:subj-mods (atom {}) ;; holds map of subj ids (keys) for modified flakes map with shacl shape and classes
:iris (volatile! {})
:txn-context txn-context
:shacl-target-objects-of? (shacl/has-target-objects-of-rule? db-before)}))

(defn final-ecount
Expand Down Expand Up @@ -349,25 +350,42 @@
[db]
(-> db :t zero?))

(defn stage-flakes
[{:keys [t] :as db} fuel-tracker tx-state nodes]
(defn validate-node
"Throws if node is invalid, otherwise returns node."
[node]
(if (empty? (dissoc node :idx :id))
(throw (ex-info (str "Invalid transaction, transaction node contains no properties"
(some->> (:id node)
(str " for @id: "))
".")
{:status 400 :error :db/invalid-transaction}))
node))

(defn insert
"Performs insert transaction. Returns async chan with resulting flakes."
[{:keys [t] :as db} fuel-tracker json-ld {:keys [default-ctx txn-context] :as tx-state}]
(go-try
(let [track-fuel (when fuel-tracker
(fuel/track fuel-tracker))
flakeset (cond-> (flake/sorted-set-by flake/cmp-flakes-spot)
(init-db? db) (track-into track-fuel (base-flakes t)))]
(loop [[node & r] nodes
(loop [[node & r] (util/sequential json-ld)
flakes flakeset]
(if node
(if (empty? (dissoc node :idx :id))
(throw (ex-info (str "Invalid transaction, transaction node contains no properties"
(some->> (:id node)
(str " for @id: "))
".")
{:status 400 :error :db/invalid-transaction}))
(let [[_node-sid node-flakes] (<? (json-ld-node->flakes node tx-state nil))
flakes* (track-into flakes track-fuel node-flakes)]
(recur r flakes*)))
(let [node* {"@context" txn-context
"@graph" [node]}
[expanded] (json-ld/expand node* default-ctx)
flakes* (if (map? expanded)
(let [[_sid node-flakes] (<? (json-ld-node->flakes (validate-node expanded) tx-state nil))]
(track-into flakes track-fuel node-flakes))
;;node expanded to a list of child nodes
(loop [[child & children] expanded
all-flakes flakes]
(if child
(let [[_sid child-flakes] (<? (json-ld-node->flakes (validate-node child) tx-state nil))]
(recur children (track-into all-flakes track-fuel child-flakes)))
all-flakes)))]
(recur r flakes*))
flakes)))))

(defn validate-rules
Expand Down Expand Up @@ -406,15 +424,6 @@
(vocab/reset-shapes (:schema db-after)))
staged-map)))))))

(defn insert
"Performs insert transaction. Returns async chan with resulting flakes."
[db fuel-tracker json-ld {:keys [default-ctx] :as tx-state}]
(log/trace "insert default-ctx:" default-ctx)
(let [nodes (-> json-ld
(json-ld/expand default-ctx)
util/sequential)]
(stage-flakes db fuel-tracker tx-state nodes)))

(defn into-flakeset
[fuel-tracker flake-ch]
(let [flakeset (flake/sorted-set-by flake/cmp-flakes-spot)]
Expand Down Expand Up @@ -476,9 +485,11 @@
([ledger json-ld opts]
(stage-ledger ledger nil json-ld opts))
([ledger fuel-tracker json-ld opts]
(-> ledger
ledger-proto/-db
(stage fuel-tracker json-ld opts))))
(let [{:keys [defaultContext]} opts
db (cond-> (ledger-proto/-db ledger)
defaultContext (dbproto/-default-context-update
defaultContext))]
(stage db fuel-tracker json-ld opts))))

(defn transact!
([ledger json-ld opts]
Expand Down
26 changes: 12 additions & 14 deletions test/fluree/db/query/fql_test.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,20 +208,18 @@
"rdfs" "http://www.w3.org/2000/01/rdf-schema#",
"schema" "http://schema.org/",
"xsd" "http://www.w3.org/2001/XMLSchema#"}}})
love (let [ledger @(fluree/create conn "test/love")]
@(fluree/transact! ledger
[{"@id" "ex:fluree",
"@type" "schema:Organization",
"schema:description" "We ❤️ Data"}
{"@id" "ex:w3c",
"@type" "schema:Organization",
"schema:description" "We ❤️ Internet"}
{"@id" "ex:mosquitos",
"@type" "ex:Monster",
"schema:description" "We ❤️ Human Blood"}]
{})
ledger)
db (fluree/db love)]
ledger @(fluree/create conn "test/love")
db @(fluree/stage (fluree/db ledger)
[{"@id" "ex:fluree",
"@type" "schema:Organization",
"schema:description" "We ❤️ Data"}
{"@id" "ex:w3c",
"@type" "schema:Organization",
"schema:description" "We ❤️ Internet"}
{"@id" "ex:mosquitos",
"@type" "ex:Monster",
"schema:description" "We ❤️ Human Blood"}]
{})]
(testing "subject-object scans"
(let [q '{:select [?s ?p ?o]
:where [[?s "schema:description" ?o]
Expand Down
84 changes: 84 additions & 0 deletions test/fluree/db/transact/transact_test.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -203,3 +203,87 @@
(is (every? (set results)
["Interstellar" "Wreck-It Ralph" "The Jungle Book" "WALL·E"
"Iron Man" "Avatar"]))))))

(deftest ^:integration transact-api-test
(let [conn (test-utils/create-conn)
ledger-name "example-ledger"
ledger @(fluree/create conn ledger-name {:defaultContext ["" {:ex "http://example.org/ns/"}]})
;; can't `transact!` until ledger can be loaded (ie has at least one commit)
db @(fluree/stage (fluree/db ledger)
{:id :ex/firstTransaction
:type :ex/Nothing})
_ @(fluree/commit! ledger db)
user-query '{:select {?s [:*]}
:where [[?s :type :ex/User]]}]
(testing "Top-level context is used for transaction nodes"
(let [txn {:id ledger-name
:context {:foo "http://foo.com/"
:id "@id"
:graph "@graph"}
:graph [{:id :ex/alice
:type :ex/User
:foo/bar "foo"
:schema/name "Alice"}
{:id :ex/bob
:type :ex/User
:foo/baz "baz"
:schema/name "Bob"}]}
db @(fluree/transact! conn txn {})]
(is (= [{:id :ex/bob,
:type :ex/User,
:schema/name "Bob",
:foo/baz "baz"}
{:id :ex/alice,
:type :ex/User,
:foo/bar "foo",
:schema/name "Alice"}]
@(fluree/query db (assoc user-query
:context ["" {:foo "http://foo.com/"}]))))))
(testing "Aliased @id, @graph are correctly identified"
(let [txn {:context {:id-alias "@id"
:graph-alias "@graph"}
:id-alias ledger-name
:graph-alias {:id-alias :ex/alice
:schema/givenName "Alicia"}}
db @(fluree/transact! conn txn {})]
(is (= [{:id :ex/bob,
:type :ex/User,
:schema/name "Bob",
:foo/baz "baz"}
{:id :ex/alice,
:type :ex/User,
:schema/name "Alice",
:foo/bar "foo",
:schema/givenName "Alicia"}]
@(fluree/query db (assoc user-query
:context ["" {:foo "http://foo.com/"
:bar "http://bar.com/"}]))))))
(testing "@context inside node is correctly handled"
(let [txn {"@id" ledger-name
"@graph" [{:context {:quux "http://quux.com/"}
:id :ex/alice
:quux/corge "grault"}]}
db @(fluree/transact! conn txn {})]
(is (= [{:id :ex/bob,
:type :ex/User,
:schema/name "Bob",
:foo/baz "baz"}
{:id :ex/alice,
:type :ex/User,
:schema/name "Alice",
:schema/givenName "Alicia"
:quux/corge "grault"
:foo/bar "foo",}]
@(fluree/query db (assoc user-query
:context ["" {:foo "http://foo.com/"
:bar "http://bar.com/"
:quux "http://quux.com/"}]))))))
(testing "Throws on invalid txn"
(let [txn {"@graph" [{:context {:quux "http://quux.com/"}
:id :ex/cam
:quux/corge "grault"}]}
db (try @(fluree/transact! conn txn {})
(catch Exception e e))]
(is (util/exception? db))
(is (str/starts-with? (ex-message db)
"Invalid transaction, missing required keys: @id"))))))
Loading

0 comments on commit e3f36f6

Please sign in to comment.