Make load work for memory conns

deps.edn

@@ -11,7 +11,7 @@
          instaparse/instaparse           {:mvn/version "1.4.12"}
          metosin/malli                   {:mvn/version "0.9.2"}
          com.fluree/json-ld              {:git/url "https://github.com/fluree/json-ld.git"
-                                          :sha     "0613d03a5657294a5575556f8eac68ab9f12705a"}
+                                          :sha     "a909330e33196504ef8a5411aaa0409ab72aaa35"}
 
          ;; logging
          org.clojure/tools.logging       {:mvn/version "1.2.4"}

shadow-cljs.edn

@@ -61,6 +61,8 @@
    :output-to       "out/browser-test/browser-tests.js"
    :closure-defines {fluree.db.platform/BROWSER true
                      cljs.core/*global*         "window"}
+   :compiler-options {:pretty-print true
+                      :pseudo-names true}
    :js-options      {:resolve {"fs"      false
                                "path"    false
                                "process" false

src/fluree/db/conn/memory.cljc

@@ -33,24 +33,21 @@
                     {:status 500 :error :db/invalid-db}))))
 
 (defn- write-data!
-  [data-atom data-type data]
-  (go-try
-    (let [json (json-ld/normalize-data data)
-          hash (crypto/sha2-256-normalize json)
-          path (if (= data-type :context)
-                 (str "/contexts/" hash)
-                 hash)]
-      #?(:cljs (when platform/BROWSER
-                 (.setItem js/localStorage hash json)))
-      (swap! data-atom assoc hash data)
-      {:name    hash
-       :hash    hash
-       :size    (count json)
-       :address (memory-address path)})))
+  [data-atom data]
+  (let [json (json-ld/normalize-data data)
+        hash (crypto/sha2-256-normalize json)
+        path hash]
+    #?(:cljs (when platform/BROWSER
+               (.setItem js/localStorage hash json)))
+    (swap! data-atom assoc hash data)
+    {:name    hash
+     :hash    hash
+     :size    (count json)
+     :address (memory-address path)}))
 
 (defn write-commit!
   [data-atom commit-data]
-  (write-data! data-atom :commit commit-data))
+  (write-data! data-atom commit-data))
 
 (defn- read-address
   [data-atom address]
@@ -60,52 +57,49 @@
                  (and platform/BROWSER (.getItem js/localStorage addr-path))))))
 
 (defn- read-data
-  [data-atom data-type address]
-  (let [addr (if (= data-type :context)
-               (str "/contexts/" address)
-               address)
-        data (read-address data-atom addr)]
+  [data-atom address]
+  (let [data (read-address data-atom address)]
     #?(:cljs (if (and platform/BROWSER (string? data))
                (js->clj (.parse js/JSON data))
                data)
        :clj  data)))
 
 (defn read-commit
   [data-atom address]
-  (read-data data-atom :commit address))
+  (read-data data-atom address))
 
 (defn write-context!
   [data-atom context-data]
-  (write-data! data-atom :context context-data))
+  (write-data! data-atom context-data))
 
 (defn read-context
   [data-atom context-key]
-  (read-data data-atom :context context-key))
+  (read-data data-atom context-key))
 
 (defn push!
-  [data-atom publish-address ledger-data]
-  (let [commit-address (:address ledger-data)
-        commit-path    (address-path commit-address)
-        address-path   (address-path publish-address)]
+  [data-atom publish-address {commit-address :address :as ledger-data}]
+  (let [commit-path (address-path commit-address)
+        head-path   (address-path publish-address)]
     (swap! data-atom
            (fn [state]
              (let [commit (get state commit-path)]
                (when-not commit
                  (throw (ex-info (str "Unable to locate commit in memory, cannot push!: " commit-address)
                                  {:status 500 :error :db/invalid-db})))
                (log/debug "pushing:" publish-address "referencing commit:" commit-address)
-               (assoc state address-path commit))))
-    #?(:cljs (and platform/BROWSER (.setItem js/localStorage address-path commit-path))))
-  ledger-data)
+               (let [commit (assoc commit "address" commit-address)]
+                 (assoc state head-path commit)))))
+    #?(:cljs (and platform/BROWSER (.setItem js/localStorage address-path commit-path)))
+    ledger-data))
 
 
 (defrecord MemoryConnection [id memory state ledger-defaults lru-cache-atom
                              parallelism msg-in-ch msg-out-ch data-atom]
 
   conn-proto/iStorage
   (-c-read [_ commit-key] (go (read-commit data-atom commit-key)))
-  (-c-write [_ _ledger commit-data] (write-commit! data-atom commit-data))
-  (-ctx-write [_ _ledger context-data] (write-context! data-atom context-data))
+  (-c-write [_ _ledger commit-data] (go (write-commit! data-atom commit-data)))
+  (-ctx-write [_ _ledger context-data] (go (write-context! data-atom context-data)))
   (-ctx-read [_ context-key] (go (read-context data-atom context-key)))
 
   conn-proto/iNameService
@@ -120,7 +114,7 @@
   (-lookup [this head-commit-address]
     (go #?(:clj
            (if-let [head-commit (read-address data-atom head-commit-address)]
-             (-> head-commit (get "credentialSubject") (get "data") (get "address"))
+             (-> head-commit (get "address"))
              (throw (ex-info (str "Unable to lookup ledger address from conn: "
                                   head-commit-address)
                              {:status 500 :error :db/missing-head})))
@@ -209,9 +203,9 @@
           data-atom       (atom {})
           state           (state-machine/blank-state)
 
-          cache-size     (conn-cache/memory->cache-size memory)
-          lru-cache-atom (or lru-cache-atom (atom (conn-cache/create-lru-cache
-                                                    cache-size)))]
+          cache-size      (conn-cache/memory->cache-size memory)
+          lru-cache-atom  (or lru-cache-atom (atom (conn-cache/create-lru-cache
+                                                     cache-size)))]
       (map->MemoryConnection {:id              conn-id
                               :ledger-defaults ledger-defaults
                               :data-atom       data-atom

src/fluree/db/json_ld/commit.cljc

@@ -357,10 +357,7 @@
   [{:keys [conn] :as ledger} commit]
   (go-try
     (let [context (get commit (keyword const/iri-default-context))
-          stringify? (-> context keys first keyword?) ; (too?) simple check if we need to stringify the keys before storing
-          context-str (if stringify?
-                        (util/stringify-keys context)
-                        context)
+          context-str (util/stringify-keys context)
           {:keys [address]} (<? (conn-proto/-ctx-write conn ledger context-str))]
       (assoc commit (keyword const/iri-default-context) address))))
 
@@ -379,7 +376,7 @@
                                    new-commit*)
           [new-commit** jld-commit] (commit-data/commit-jsonld new-commit*)
           signed-commit (if did
-                          (cred/generate jld-commit private (:id did))
+                          (<? (cred/generate jld-commit private (:id did)))
                           jld-commit)
           commit-res    (<? (conn-proto/-c-write conn ledger signed-commit)) ;; write commit credential
           new-commit*** (commit-data/update-commit-address new-commit** (:address commit-res))

src/fluree/db/json_ld/credential.cljc

@@ -58,23 +58,26 @@
 
 (defn generate
   "Generate a VerifiableCredential given a subject and some issuer opts."
-  ([credential-subject private] (generate credential-subject private (did/private->did-map private)))
+  ([credential-subject private] (generate credential-subject private
+                                          (did/private->did-map private)))
   ([credential-subject private did]
    (go-try
-     {"@context"          "https://www.w3.org/2018/credentials/v1"
-      "id"                ""
-      "type"              ["VerifiableCredential" "CommitProof"]
-      "issuer"            (:id did)
-      "issuanceDate"      (util/current-time-iso)
-      "credentialSubject" credential-subject
-      "proof"             #?(:clj (create-proof (-> (jld-processor/canonize credential-subject)
-                                                    (crypto/sha2-256))
-                                                (did/encode-did-key (:public did))
-                                                private)
-                             :cljs (let [canonicalized (<p! (jld-processor/canonize credential-subject))]
-                                     (create-proof (crypto/sha2-256 canonicalized)
-                                                   (did/encode-did-key (:public did))
-                                                   private)))})))
+     (let [proof #?(:clj (create-proof (-> credential-subject
+                                           jld-processor/canonize
+                                           crypto/sha2-256)
+                                       (did/encode-did-key (:public did))
+                                       private)
+                    :cljs (let [canonicalized (<p! (jld-processor/canonize credential-subject))]
+                            (create-proof (crypto/sha2-256 canonicalized)
+                                          (did/encode-did-key (:public did))
+                                          private)))]
+       {"@context"          "https://www.w3.org/2018/credentials/v1"
+        "id"                ""
+        "type"              ["VerifiableCredential" "CommitProof"]
+        "issuer"            (:id did)
+        "issuanceDate"      (util/current-time-iso)
+        "credentialSubject" credential-subject
+        "proof" proof}))))
 
 (defn verify
   "Takes a credential and returns the credential subject and issuer id if it verifies. If

test/fluree/db/json_ld/api_test.cljc

@@ -34,7 +34,7 @@
                                            :type         :schema/Person
                                            :schema/fname "Me"}])]
          @(fluree/commit! ledger db)
-         (is (test-utils/retry-exists? conn ledger-alias 10))
+         (is (test-utils/retry-exists? conn ledger-alias 100))
          (is (not @(fluree/exists? conn "notaledger"))))
 
        :cljs
@@ -48,7 +48,7 @@
                                                    :type         :schema/Person
                                                    :schema/fname "Me"}]))]
              (<p! (fluree/commit! ledger db))
-             (is (<p! (fluree/exists? conn ledger-alias)))
+             (is (test-utils/retry-exists? conn ledger-alias 100))
              (is (not (<p! (fluree/exists? conn "notaledger"))))
              (done)))))))
 
@@ -64,7 +64,6 @@
                                              :context      ledger-context})
              merged-context (merge test-utils/default-context
                                    (util/keywordize-keys ledger-context))]
-         (println "ledger context:" (pr-str (:context ledger)))
          (is (= merged-context (:context ledger)))))))
 
 #?(:clj

test/fluree/db/test_utils.cljc

@@ -1,7 +1,8 @@
 (ns fluree.db.test-utils
   (:require [fluree.db.did :as did]
             [fluree.db.json-ld.api :as fluree]
-            #?@(:cljs [[clojure.core.async :refer [go]]
+            [fluree.db.util.core :refer [try* catch*]]
+            #?@(:cljs [[clojure.core.async :refer [go go-loop]]
                        [clojure.core.async.interop :refer [<p!]]])))
 
 (def default-context
@@ -109,22 +110,27 @@
 
 (defn retry-promise-wrapped
   "Retries a fn that when deref'd might return a Throwable. Intended for
-  retrying promise-wrapped API fns. Do not deref the return value, this will do
-  it for you."
-  [pwrapped max-attempts]
-  (loop [attempt 0]
-    (let [final (try
-                  (let [res @(pwrapped)]
-                    (if (instance? Throwable res)
-                      (throw res)
-                      res))
-                  (catch Throwable t
-                    (when (= (inc attempt) max-attempts)
-                      (throw t)
-                      (Thread/sleep 100))))]
-      (if final
-        final
-        (recur (inc attempt))))))
+  retrying promise-wrapped API fns. Do not deref the return value, this will
+  do it for you. In CLJS it will not retry and will return a core.async chan."
+  [pwrapped max-attempts & [retry-on-false?]]
+  (#?(:clj loop :cljs go-loop) [attempt 0]
+    (let [error? #(instance? #?(:clj Throwable :cljs js/Error) %)
+          res' (try*
+                 (let [res (#?(:clj deref :cljs <p!) (pwrapped))]
+                   (if (error? res)
+                     (throw res)
+                     res))
+                 (catch* e e))]
+      (if (= (inc attempt) max-attempts)
+        (if (error? res')
+          (throw res')
+          res')
+        (if (or (error? res')
+                (and retry-on-false? (false? res')))
+          (do
+            #?(:clj (Thread/sleep 100))
+            (recur (inc attempt)))
+          res')))))
 
 (defn retry-load
   "Retry loading a ledger until it loads or max-attempts. Hopefully not needed
@@ -135,4 +141,4 @@
 (defn retry-exists?
   "Retry calling exists? until it returns true or max-attempts."
   [conn ledger-alias max-atttemts]
-  (retry-promise-wrapped #(fluree/exists? conn ledger-alias) max-atttemts))
+  (retry-promise-wrapped #(fluree/exists? conn ledger-alias) max-atttemts true))