Merge pull request #373 from fluree/fix/context-bug

deps.edn

@@ -60,7 +60,8 @@
   {:extra-paths ["test" "dev-resources"]
    :extra-deps  {lambdaisland/kaocha         {:mvn/version "1.71.1119"}
                  org.clojure/test.check      {:mvn/version "1.1.1"}
-                 com.magnars/test-with-files {:mvn/version "2021-02-17"}}
+                 io.github.cap10morgan/test-with-files {:git/tag "v1.0.0"
+                                                        :git/sha "9181a2e"}}
    :exec-fn     kaocha.runner/exec-fn
    :exec-args   {}}
 

dev/logback.xml

@@ -3,7 +3,7 @@
 
     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
         <encoder>
-            <pattern>%highlight(%-5level) %white(%logger{24}) - %msg%n</pattern>
+            <pattern>%highlight(%-5level) %cyan(%logger{24}) - %msg%n</pattern>
         </encoder>
     </appender>
 

src/fluree/db/conn/file.cljc

@@ -33,7 +33,7 @@
 
 (defn local-path
   [{:keys [storage-path] :as _conn}]
-  (let [abs-path? #?(:clj  (.isAbsolute (io/file storage-path))
+  (let [abs-path? #?(:clj (.isAbsolute (io/file storage-path))
                      :cljs (path/isAbsolute storage-path))
         abs-root  (if abs-path?
                     ""
@@ -141,28 +141,33 @@
   #?(:clj  (.getBytes ^String s)
      :cljs (js/Uint8Array. (js/Buffer.from s "utf8"))))
 
-(defn commit
-  ([conn data] (commit conn nil data))
-  ([conn db data]
-   (let [ledger      (:ledger db)
-         alias       (ledger-proto/-alias ledger)
-         branch      (name (:name (ledger-proto/-branch ledger)))
+(defn- write-data
+  [conn ledger data-type data]
+  (let [alias      (ledger-proto/-alias ledger)
+        branch     (name (:name (ledger-proto/-branch ledger)))
+        json       (json-ld/normalize-data data)
+        bytes      (->bytes json)
+        hash       (crypto/sha2-256 bytes :hex)
+        type-dir   (-> data-type name (str "s"))
+        path       (str alias
+                        (when branch (str "/" branch))
+                        (str "/" type-dir "/")
+                        hash ".json")
+        write-path (str (local-path conn) "/" path)]
+    (log/debug (str "Writing " (name data-type) " at " write-path))
+    (write-file write-path bytes)
+    {:name    hash
+     :hash    hash
+     :size    (count json)
+     :address (file-address path)}))
 
-         json        (json-ld/normalize-data data)
-         bytes       (->bytes json)
-         hash        (crypto/sha2-256 bytes :hex)
+(defn write-commit
+  [conn ledger commit-data]
+  (write-data conn ledger :commit commit-data))
 
-         commit-path (str alias
-                          (when branch (str "/" branch))
-                          "/commits/"
-                          hash ".json")
-         write-path  (str (local-path conn) "/" commit-path)]
-     (log/debug (str "Writing commit at " write-path))
-     (write-file write-path bytes)
-     {:name    hash
-      :hash    hash
-      :size    (count json)
-      :address (file-address commit-path)})))
+(defn write-context
+  [conn ledger context-data]
+  (write-data conn ledger :context context-data))
 
 (defn push
   "Just write to a different directory?"
@@ -186,13 +191,20 @@
   (let [[_ ledger & r] (str/split k #"_")]
     (str (local-path store) "/" ledger "/" "indexes" "/" (str/join "/" r))))
 
-(defrecord FileConnection [id memory state ledger-defaults push commit
-                           parallelism msg-in-ch msg-out-ch lru-cache-atom]
+(defn read-context
+  [conn context-key]
+  (json/parse (read-address conn context-key) true))
+
+(defrecord FileConnection [id memory state ledger-defaults parallelism msg-in-ch
+                           msg-out-ch lru-cache-atom]
 
   conn-proto/iStorage
   (-c-read [conn commit-key] (go (read-commit conn commit-key)))
-  (-c-write [conn commit-data] (go (commit conn commit-data)))
-  (-c-write [conn db commit-data] (go (commit conn db commit-data)))
+  (-c-write [conn ledger commit-data] (go (write-commit conn ledger
+                                                        commit-data)))
+  (-ctx-read [conn context-key] (go (read-context conn context-key)))
+  (-ctx-write [conn ledger context-data] (go (write-context conn ledger
+                                                            context-data)))
 
   conn-proto/iNameService
   (-pull [conn ledger] (throw (ex-info "Unsupported FileConnection op: pull" {})))
@@ -286,7 +298,7 @@
 
 (defn connect
   "Create a new file system connection."
-  [{:keys [defaults parallelism storage-path lru-cache-atom memory] :as opts}]
+  [{:keys [defaults parallelism storage-path lru-cache-atom memory] :as _opts}]
   (go
     (let [storage-path (trim-last-slash storage-path)
           conn-id      (str (random-uuid))
@@ -300,8 +312,6 @@
                             :ledger-defaults (ledger-defaults defaults)
                             :serializer      #?(:clj  (avro-serde/avro-serde)
                                                 :cljs (json-serde/json-serde))
-                            :commit          commit
-                            :push            push
                             :parallelism     parallelism
                             :msg-in-ch       (async/chan)
                             :msg-out-ch      (async/chan)

src/fluree/db/conn/ipfs.cljc

@@ -83,12 +83,12 @@
   conn-proto/iStorage
   (-c-read [_ commit-key]
     (ipfs/read ipfs-endpoint commit-key))
-
-  (-c-write [_ commit-data]
-    (ipfs/commit ipfs-endpoint commit-data))
-
   (-c-write [_ _ commit-data]
-    (ipfs/commit ipfs-endpoint commit-data))
+    (ipfs/write ipfs-endpoint commit-data))
+  (-ctx-read [_ context-key]
+    (ipfs/read ipfs-endpoint context-key))
+  (-ctx-write [_ _ context-data]
+    (ipfs/write ipfs-endpoint context-data))
 
   conn-proto/iNameService
   (-push [_ address ledger-data]
@@ -129,7 +129,7 @@
   (read [_ k]
     (ipfs/read ipfs-endpoint k true))
   (write [_ k data]
-    (ipfs/commit ipfs-endpoint data))
+    (ipfs/write ipfs-endpoint data))
   (exists? [conn k]
     (storage/read conn k))
   (rename [_ old-key new-key]

src/fluree/db/conn/memory.cljc

@@ -31,38 +31,61 @@
     (throw (ex-info (str "Incorrectly formatted Fluree memory db address: " address)
                     {:status 500 :error :db/invalid-db}))))
 
-(defn c-write!
-  [data-atom commit-data]
+(defn- write-data!
+  [data-atom data-type data]
   (go-try
-    (let [json (json-ld/normalize-data commit-data)
-          hash (crypto/sha2-256-normalize json)]
+    (let [json (json-ld/normalize-data data)
+          hash (crypto/sha2-256-normalize json)
+          path (if (= data-type :context)
+                 (str "/contexts/" hash)
+                 hash)]
       #?(:cljs (when platform/BROWSER
                  (.setItem js/localStorage hash json)))
-      (swap! data-atom assoc hash commit-data)
+      (swap! data-atom assoc hash data)
       {:name    hash
        :hash    hash
        :size    (count json)
-       :address (memory-address hash)})))
+       :address (memory-address path)})))
+
+(defn write-commit!
+  [data-atom commit-data]
+  (write-data! data-atom :commit commit-data))
 
-(defn read-address
+(defn- read-address
   [data-atom address]
-  #?(:clj (get @data-atom (address-path address))
-     :cljs (or (get @data-atom (address-path address))
-               (and platform/BROWSER (.getItem js/localStorage (address-path address))))))
+  (let [addr-path (address-path address)]
+    #?(:clj  (get @data-atom addr-path)
+       :cljs (or (get @data-atom addr-path)
+                 (and platform/BROWSER (.getItem js/localStorage addr-path))))))
+
+(defn- read-data
+  [data-atom data-type address]
+  (let [addr (if (= data-type :context)
+               (str "/contexts/" address)
+               address)
+        data (read-address data-atom addr)]
+    #?(:cljs (if (and platform/BROWSER (string? data))
+               (js->clj (.parse js/JSON data))
+               data)
+       :clj  data)))
 
 (defn read-commit
   [data-atom address]
-  (let [commit (read-address data-atom address)]
-    #?(:cljs (if (and platform/BROWSER (string? commit))
-               (js->clj (.parse js/JSON commit))
-               commit)
-       :clj commit)))
+  (read-data data-atom :commit address))
+
+(defn write-context!
+  [data-atom context-data]
+  (write-data! data-atom :context context-data))
+
+(defn read-context
+  [data-atom context-key]
+  (read-data data-atom :context context-key))
 
 (defn push!
   [data-atom publish-address ledger-data]
   (let [commit-address (:address ledger-data)
-        commit-path (address-path commit-address)
-        address-path (address-path publish-address)]
+        commit-path    (address-path commit-address)
+        address-path   (address-path publish-address)]
     (swap! data-atom
            (fn [state]
              (let [commit (get state commit-path)]
@@ -80,8 +103,9 @@
 
   conn-proto/iStorage
   (-c-read [_ commit-key] (go (read-commit data-atom commit-key)))
-  (-c-write [_ commit-data] (c-write! data-atom commit-data))
-  (-c-write [_ db commit-data] (c-write! data-atom commit-data))
+  (-c-write [_ _ledger commit-data] (write-commit! data-atom commit-data))
+  (-ctx-write [_ _ledger context-data] (write-context! data-atom context-data))
+  (-ctx-read [_ context-key] (go (read-context data-atom context-key)))
 
   conn-proto/iNameService
   (-pull [this ledger] :TODO)
@@ -124,7 +148,7 @@
   (-parallelism [_] parallelism)
   (-id [_] id)
   (-context [_] (:context ledger-defaults))
-  (-new-indexer [_ opts] (idx-default/create opts))         ;; default new ledger indexer
+  (-new-indexer [_ opts] (idx-default/create opts)) ;; default new ledger indexer
   (-did [_] (:did ledger-defaults))
   (-msg-in [_ msg] (go-try
                      ;; TODO - push into state machine
@@ -185,7 +209,8 @@
           state           (state-machine/blank-state)
 
           cache-size     (conn-cache/memory->cache-size memory)
-          lru-cache-atom (or lru-cache-atom (atom (conn-cache/create-lru-cache cache-size)))]
+          lru-cache-atom (or lru-cache-atom (atom (conn-cache/create-lru-cache
+                                                    cache-size)))]
       (map->MemoryConnection {:id              conn-id
                               :ledger-defaults ledger-defaults
                               :data-atom       data-atom

src/fluree/db/conn/proto.cljc

@@ -16,10 +16,11 @@
   (-msg-out [conn msg] "Pushes outgoing messages/commands to connection service")
   (-state [conn] [conn ledger] "Returns internal state-machine information for connection, or specific ledger"))
 
-
 (defprotocol iStorage
   (-c-read [conn commit-key] "Reads a commit from storage")
-  (-c-write [conn commit-data] [conn db commit-data] "Writes a commit to storage"))
+  (-c-write [conn ledger commit-data] "Writes a commit to storage")
+  (-ctx-write [conn ledger context-data] "Writes a context to storage and returns the key. Expects string keys.")
+  (-ctx-read [conn context-key] "Reads a context from storage"))
 
 (defprotocol iNameService
   (-push [conn address commit-data] "Pushes ledger metadata to all name service destinations")

src/fluree/db/db/json_ld.cljc

@@ -423,8 +423,7 @@
 ;; TODO - conn is included here because current index-range query looks for conn on the db
 ;; TODO - this can likely be excluded once index-range is changed to get 'conn' from (:conn ledger) where it also exists
 (defrecord JsonLdDb [ledger conn method alias branch commit block t tt-id stats
-                     spot psot post opst tspo
-                     schema comparators novelty
+                     spot psot post opst tspo schema comparators novelty
                      policy ecount]
   dbproto/IFlureeDb
   (-latest-db [this] (graphdb-latest-db this))

src/fluree/db/json_ld/bootstrap.cljc

@@ -1,6 +1,8 @@
 (ns fluree.db.json-ld.bootstrap
   (:require [clojure.string :as str]
+            [clojure.core.async :refer [go]]
             [fluree.crypto :as crypto]
+            [fluree.db.ledger.proto :as ledger-proto]
             [fluree.db.util.core :as util #?(:clj :refer :cljs :refer-macros) [try* catch*]]
             [fluree.db.util.json :as json]
             [fluree.json-ld :as json-ld]
@@ -133,27 +135,15 @@
                          "Must be a valid JSON context, or a valid context map or array/vector. Provided: " default-ctx)
                     {:status 400 :error :db/invalid-context}))))
 
-
-(defn bootstrap-tx
-  [default-ctx]
-  (let [ctx    (when default-ctx
-                 (let [default-ctx* (normalize-default-ctx default-ctx)]
-                   {"@id"     "fluree-default-context"
-                    "@type"   ["Context"]
-                    "context" default-ctx*}))
-        graph (cond-> []
-                      ctx (conj ctx))]
-    (when (seq graph)
-      {"@context" "https://ns.flur.ee/ledger/v1"
-       "@graph"   graph})))
-
 (defn bootstrap
   "Bootstraps a permissioned JSON-LD db. Returns async channel."
-  [blank-db default-ctx]
-  (let [tx (bootstrap-tx default-ctx)]
-    (if tx
-      (db-proto/-stage blank-db tx {:bootstrap? true})
-      blank-db)))
+  ([blank-db] (bootstrap blank-db nil))
+  ([blank-db initial-tx]
+   (if-let [tx (when initial-tx
+                 {"@context" "https://ns.flur.ee/ledger/v1"
+                  "@graph"   initial-tx})]
+     (db-proto/-stage blank-db tx {:bootstrap? true})
+     (go blank-db))))
 
 (defn blank-db
   "When not bootstrapping with a transaction, bootstraps initial base set of flakes required for a db."

src/fluree/db/json_ld/branch.cljc

@@ -133,15 +133,11 @@
 
 ;; TODO
 #_(defn branch
-  "Creates, or changes, a ledger's branch"
-  [ledger branch]
-  (let [{:keys [state]} ledger
-        {:keys [branches branch]} @state
-        [branch-t [branch-current branch-commit]] branch
-        branch*     (util/str->keyword branch)
-        new?        (contains? branches branch*)
-        is-current? (= branch)]
-
-    )
-
-  )
+    "Creates, or changes, a ledger's branch"
+    [ledger branch]
+    (let [{:keys [state]} ledger
+          {:keys [branches branch]} @state
+          [branch-t [branch-current branch-commit]] branch
+          branch*     (util/str->keyword branch)
+          new?        (contains? branches branch*)
+          is-current? (= branch)]))