From 2c31c33fd9951f130f0561ed7a42f24b9c7405b8 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Thu, 16 Feb 2023 09:58:42 -0700 Subject: [PATCH 1/9] Fix load API fn on memory conn Primarily for testing purposes --- src/fluree/db/conn/memory.cljc | 42 +++++++++++++++------------------- 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/src/fluree/db/conn/memory.cljc b/src/fluree/db/conn/memory.cljc index 3ea0001bd..becaa514a 100644 --- a/src/fluree/db/conn/memory.cljc +++ b/src/fluree/db/conn/memory.cljc @@ -33,13 +33,11 @@ {:status 500 :error :db/invalid-db})))) (defn- write-data! - [data-atom data-type data] + [data-atom data] (go-try (let [json (json-ld/normalize-data data) hash (crypto/sha2-256-normalize json) - path (if (= data-type :context) - (str "/contexts/" hash) - hash)] + path hash] #?(:cljs (when platform/BROWSER (.setItem js/localStorage hash json))) (swap! data-atom assoc hash data) @@ -50,7 +48,7 @@ (defn write-commit! [data-atom commit-data] - (write-data! data-atom :commit commit-data)) + (write-data! data-atom commit-data)) (defn- read-address [data-atom address] @@ -60,11 +58,8 @@ (and platform/BROWSER (.getItem js/localStorage addr-path)))))) (defn- read-data - [data-atom data-type address] - (let [addr (if (= data-type :context) - (str "/contexts/" address) - address) - data (read-address data-atom addr)] + [data-atom address] + (let [data (read-address data-atom address)] #?(:cljs (if (and platform/BROWSER (string? data)) (js->clj (.parse js/JSON data)) data) @@ -72,21 +67,20 @@ (defn read-commit [data-atom address] - (read-data data-atom :commit address)) + (read-data data-atom address)) (defn write-context! [data-atom context-data] - (write-data! data-atom :context context-data)) + (write-data! data-atom context-data)) (defn read-context [data-atom context-key] - (read-data data-atom :context context-key)) + (read-data data-atom context-key)) (defn push! - [data-atom publish-address ledger-data] - (let [commit-address (:address ledger-data) - commit-path (address-path commit-address) - address-path (address-path publish-address)] + [data-atom publish-address {commit-address :address :as ledger-data}] + (let [commit-path (address-path commit-address) + head-path (address-path publish-address)] (swap! data-atom (fn [state] (let [commit (get state commit-path)] @@ -94,9 +88,9 @@ (throw (ex-info (str "Unable to locate commit in memory, cannot push!: " commit-address) {:status 500 :error :db/invalid-db}))) (log/debug "pushing:" publish-address "referencing commit:" commit-address) - (assoc state address-path commit)))) - #?(:cljs (and platform/BROWSER (.setItem js/localStorage address-path commit-path)))) - ledger-data) + (assoc state head-path (assoc commit "address" commit-address))))) + #?(:cljs (and platform/BROWSER (.setItem js/localStorage address-path commit-path))) + ledger-data)) (defrecord MemoryConnection [id memory state ledger-defaults lru-cache-atom @@ -120,7 +114,7 @@ (-lookup [this head-commit-address] (go #?(:clj (if-let [head-commit (read-address data-atom head-commit-address)] - (-> head-commit (get "credentialSubject") (get "data") (get "address")) + (-> head-commit (get "address")) (throw (ex-info (str "Unable to lookup ledger address from conn: " head-commit-address) {:status 500 :error :db/missing-head}))) @@ -209,9 +203,9 @@ data-atom (atom {}) state (state-machine/blank-state) - cache-size (conn-cache/memory->cache-size memory) - lru-cache-atom (or lru-cache-atom (atom (conn-cache/create-lru-cache - cache-size)))] + cache-size (conn-cache/memory->cache-size memory) + lru-cache-atom (or lru-cache-atom (atom (conn-cache/create-lru-cache + cache-size)))] (map->MemoryConnection {:id conn-id :ledger-defaults ledger-defaults :data-atom data-atom From 631031d8917a3c181d469dc75b3d9b6c921eef5f Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Thu, 16 Feb 2023 09:59:33 -0700 Subject: [PATCH 2/9] Remove stringify? check in link-context-to-commit It was indeed too simple. http-api-gateway was sneaking mixed keyword / string contexts in w/ memory conns. So let's just always stringify instead. --- src/fluree/db/json_ld/commit.cljc | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/fluree/db/json_ld/commit.cljc b/src/fluree/db/json_ld/commit.cljc index a86f98f75..de0058597 100644 --- a/src/fluree/db/json_ld/commit.cljc +++ b/src/fluree/db/json_ld/commit.cljc @@ -357,10 +357,7 @@ [{:keys [conn] :as ledger} commit] (go-try (let [context (get commit (keyword const/iri-default-context)) - stringify? (-> context keys first keyword?) ; (too?) simple check if we need to stringify the keys before storing - context-str (if stringify? - (util/stringify-keys context) - context) + context-str (util/stringify-keys context) {:keys [address]} ( Date: Thu, 16 Feb 2023 15:41:53 -0700 Subject: [PATCH 3/9] Use latest json-ld lib sha Fixes a bug in processor/canonize that I ran into --- deps.edn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps.edn b/deps.edn index 308c50f12..70ee329f8 100644 --- a/deps.edn +++ b/deps.edn @@ -11,7 +11,7 @@ instaparse/instaparse {:mvn/version "1.4.12"} metosin/malli {:mvn/version "0.9.2"} com.fluree/json-ld {:git/url "https://github.com/fluree/json-ld.git" - :sha "0613d03a5657294a5575556f8eac68ab9f12705a"} + :sha "a909330e33196504ef8a5411aaa0409ab72aaa35"} ;; logging org.clojure/tools.logging {:mvn/version "1.2.4"} From 1ace3a41132398682e0827e9d5a293290c551378 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Thu, 16 Feb 2023 15:42:23 -0700 Subject: [PATCH 4/9] Pretty-print browser test JS w/ pseudo-names Much easier debugging when things go wrong --- shadow-cljs.edn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shadow-cljs.edn b/shadow-cljs.edn index 44d8fe64d..0e254cdd1 100644 --- a/shadow-cljs.edn +++ b/shadow-cljs.edn @@ -61,6 +61,8 @@ :output-to "out/browser-test/browser-tests.js" :closure-defines {fluree.db.platform/BROWSER true cljs.core/*global* "window"} + :compiler-options {:pretty-print true + :pseudo-names true} :js-options {:resolve {"fs" false "path" false "process" false From 6b20f5bbd51d15a2456fa1934c4c99b9ad6d35d1 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Thu, 16 Feb 2023 15:46:10 -0700 Subject: [PATCH 5/9] Move go to the protocol layer in mem conn write fns Nothing in the write-data! fn needs to be inside a go-try --- src/fluree/db/conn/memory.cljc | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/fluree/db/conn/memory.cljc b/src/fluree/db/conn/memory.cljc index becaa514a..94e2e089c 100644 --- a/src/fluree/db/conn/memory.cljc +++ b/src/fluree/db/conn/memory.cljc @@ -34,17 +34,16 @@ (defn- write-data! [data-atom data] - (go-try - (let [json (json-ld/normalize-data data) - hash (crypto/sha2-256-normalize json) - path hash] - #?(:cljs (when platform/BROWSER - (.setItem js/localStorage hash json))) - (swap! data-atom assoc hash data) - {:name hash - :hash hash - :size (count json) - :address (memory-address path)}))) + (let [json (json-ld/normalize-data data) + hash (crypto/sha2-256-normalize json) + path hash] + #?(:cljs (when platform/BROWSER + (.setItem js/localStorage hash json))) + (swap! data-atom assoc hash data) + {:name hash + :hash hash + :size (count json) + :address (memory-address path)})) (defn write-commit! [data-atom commit-data] @@ -88,7 +87,8 @@ (throw (ex-info (str "Unable to locate commit in memory, cannot push!: " commit-address) {:status 500 :error :db/invalid-db}))) (log/debug "pushing:" publish-address "referencing commit:" commit-address) - (assoc state head-path (assoc commit "address" commit-address))))) + (let [commit (assoc commit "address" commit-address)] + (assoc state head-path commit))))) #?(:cljs (and platform/BROWSER (.setItem js/localStorage address-path commit-path))) ledger-data)) @@ -98,8 +98,8 @@ conn-proto/iStorage (-c-read [_ commit-key] (go (read-commit data-atom commit-key))) - (-c-write [_ _ledger commit-data] (write-commit! data-atom commit-data)) - (-ctx-write [_ _ledger context-data] (write-context! data-atom context-data)) + (-c-write [_ _ledger commit-data] (go (write-commit! data-atom commit-data))) + (-ctx-write [_ _ledger context-data] (go (write-context! data-atom context-data))) (-ctx-read [_ context-key] (go (read-context data-atom context-key))) conn-proto/iNameService From d7bb508e7a322bfda5084365aa13d19908d55c0c Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Thu, 16 Feb 2023 15:48:33 -0700 Subject: [PATCH 6/9] Take from cred/generate chan in do-commit+push Not sure why this wasn't causing more problems before now, but for whatever reason testing w/ mem conn load revealed it. Might be b/c only the tests set a default did usually. --- src/fluree/db/json_ld/commit.cljc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fluree/db/json_ld/commit.cljc b/src/fluree/db/json_ld/commit.cljc index de0058597..0a984d74c 100644 --- a/src/fluree/db/json_ld/commit.cljc +++ b/src/fluree/db/json_ld/commit.cljc @@ -376,7 +376,7 @@ new-commit*) [new-commit** jld-commit] (commit-data/commit-jsonld new-commit*) signed-commit (if did - (cred/generate jld-commit private (:id did)) + ( Date: Thu, 16 Feb 2023 15:49:37 -0700 Subject: [PATCH 7/9] Rearrange f.d.json-ld.credential/generate a little --- src/fluree/db/json_ld/credential.cljc | 33 +++++++++++++++------------ 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/src/fluree/db/json_ld/credential.cljc b/src/fluree/db/json_ld/credential.cljc index 7b3ccd229..f156da567 100644 --- a/src/fluree/db/json_ld/credential.cljc +++ b/src/fluree/db/json_ld/credential.cljc @@ -58,23 +58,26 @@ (defn generate "Generate a VerifiableCredential given a subject and some issuer opts." - ([credential-subject private] (generate credential-subject private (did/private->did-map private))) + ([credential-subject private] (generate credential-subject private + (did/private->did-map private))) ([credential-subject private did] (go-try - {"@context" "https://www.w3.org/2018/credentials/v1" - "id" "" - "type" ["VerifiableCredential" "CommitProof"] - "issuer" (:id did) - "issuanceDate" (util/current-time-iso) - "credentialSubject" credential-subject - "proof" #?(:clj (create-proof (-> (jld-processor/canonize credential-subject) - (crypto/sha2-256)) - (did/encode-did-key (:public did)) - private) - :cljs (let [canonicalized ( credential-subject + jld-processor/canonize + crypto/sha2-256) + (did/encode-did-key (:public did)) + private) + :cljs (let [canonicalized ( Date: Thu, 16 Feb 2023 15:50:05 -0700 Subject: [PATCH 8/9] Fix retry-promise-wrapped in test-utils ...and implement it for CLJS too --- test/fluree/db/json_ld/api_test.cljc | 5 +-- test/fluree/db/test_utils.cljc | 58 +++++++++++++++++++--------- 2 files changed, 42 insertions(+), 21 deletions(-) diff --git a/test/fluree/db/json_ld/api_test.cljc b/test/fluree/db/json_ld/api_test.cljc index 0f247d2ff..14c6f964e 100644 --- a/test/fluree/db/json_ld/api_test.cljc +++ b/test/fluree/db/json_ld/api_test.cljc @@ -34,7 +34,7 @@ :type :schema/Person :schema/fname "Me"}])] @(fluree/commit! ledger db) - (is (test-utils/retry-exists? conn ledger-alias 10)) + (is (test-utils/retry-exists? conn ledger-alias 100)) (is (not @(fluree/exists? conn "notaledger")))) :cljs @@ -48,7 +48,7 @@ :type :schema/Person :schema/fname "Me"}]))] ( Date: Thu, 16 Feb 2023 16:20:48 -0700 Subject: [PATCH 9/9] Consolidate retry-promise-wrapped CLJ / CLJS code It had more in common than not --- test/fluree/db/test_utils.cljc | 52 ++++++++++++---------------------- 1 file changed, 18 insertions(+), 34 deletions(-) diff --git a/test/fluree/db/test_utils.cljc b/test/fluree/db/test_utils.cljc index f0bfbf134..c45afabdb 100644 --- a/test/fluree/db/test_utils.cljc +++ b/test/fluree/db/test_utils.cljc @@ -113,40 +113,24 @@ retrying promise-wrapped API fns. Do not deref the return value, this will do it for you. In CLJS it will not retry and will return a core.async chan." [pwrapped max-attempts & [retry-on-false?]] - #?(:clj - (loop [attempt 0] - (let [res' (try - (let [res @(pwrapped)] - (if (instance? Throwable res) - (throw res) - res)) - (catch Throwable t t))] - (if (= (inc attempt) max-attempts) - (if (instance? Throwable res') - (throw res') - res') - (if (or (instance? Throwable res') - (and retry-on-false? (false? res'))) - (do - (Thread/sleep 100) - (recur (inc attempt))) - res')))) - :cljs - (go-loop [attempt 0] - (let [res' (try - (let [res (