From cd8b29c9257fddff3c295279d8d13bdb8aaed0d7 Mon Sep 17 00:00:00 2001 From: khaled basbous Date: Tue, 3 Sep 2024 15:28:04 +0200 Subject: [PATCH 1/7] feat(Std-crud): Support json patch edit format --- code/project.clj | 3 +- .../server/resources/common/std_crud.clj | 48 ++++++++++++++----- .../nuvlabox_status_2_lifecycle_test.clj | 44 +++++++++++++++++ 3 files changed, 81 insertions(+), 14 deletions(-) diff --git a/code/project.clj b/code/project.clj index c0076f6fe..ff151b5e3 100644 --- a/code/project.clj +++ b/code/project.clj @@ -80,7 +80,8 @@ [com.github.kenglxn.qrgen/javase] [com.google.zxing/javase]]] [funcool/promesa "11.0.678"] - [nrepl "1.1.1"]] + [nrepl "1.1.1"] + [by.borge/clj-json-pointer "1.0.0"]] :profiles {:provided {:dependencies [[org.clojure/clojure "1.11.2"] diff --git a/code/src/com/sixsq/nuvla/server/resources/common/std_crud.clj b/code/src/com/sixsq/nuvla/server/resources/common/std_crud.clj index 25ac65aef..38b5f3e39 100644 --- a/code/src/com/sixsq/nuvla/server/resources/common/std_crud.clj +++ b/code/src/com/sixsq/nuvla/server/resources/common/std_crud.clj @@ -7,6 +7,7 @@ [clojure.walk :as w] [com.sixsq.nuvla.auth.acl-resource :as a] [com.sixsq.nuvla.auth.utils :as auth] + [clj-json-pointer.core :as jp] [com.sixsq.nuvla.db.impl :as db] [com.sixsq.nuvla.server.middleware.cimi-params.impl :as impl] [com.sixsq.nuvla.server.resources.common.crud :as crud] @@ -62,6 +63,25 @@ (catch Exception e (or (ex-data e) (throw e)))))) +(defn json-safe-patch + [obj patches] + (try + (jp/patch obj patches) + (catch Exception e + (throw (r/ex-bad-request (str "Json patch exception: " (ex-message e))))))) + +(defn- json-patch-request? [request] + (= (get-in request [:headers "content-type"]) "application/json-patch+json")) + +(defn json-patch + [resource {:keys [json-patch body] :as request}] + (if (json-patch-request? request) + (->> (w/stringify-keys body) + (json-safe-patch (w/stringify-keys resource)) + w/keywordize-keys + (assoc request :body)) + request)) + (defn edit-fn [resource-name & {:keys [pre-validate-hook @@ -73,19 +93,21 @@ immutable-keys []}}] (fn [{{uuid :uuid} :params :as request}] (try - (-> (str resource-name "/" uuid) - db/retrieve - (a/throw-cannot-edit request) - (sm/throw-can-not-do-action request) - (pre-delete-attrs-hook request) - (u/delete-attributes request immutable-keys) - u/update-timestamps - (u/set-updated-by request) - (pre-validate-hook request) - (dissoc :operations) - crud/validate - (db/edit options) - (update :body crud/set-operations request)) + (let [resource (-> (str resource-name "/" uuid) + db/retrieve + (a/throw-cannot-edit request) + (sm/throw-can-not-do-action request)) + request (json-patch resource request)] + (-> resource + (pre-delete-attrs-hook request) + (u/delete-attributes request immutable-keys) + u/update-timestamps + (u/set-updated-by request) + (pre-validate-hook request) + (dissoc :operations) + crud/validate + (db/edit options) + (update :body crud/set-operations request))) (catch Exception e (or (ex-data e) (throw e)))))) diff --git a/code/test/com/sixsq/nuvla/server/resources/nuvlabox_status_2_lifecycle_test.clj b/code/test/com/sixsq/nuvla/server/resources/nuvlabox_status_2_lifecycle_test.clj index a3f5a4343..8fb27f2b2 100644 --- a/code/test/com/sixsq/nuvla/server/resources/nuvlabox_status_2_lifecycle_test.clj +++ b/code/test/com/sixsq/nuvla/server/resources/nuvlabox_status_2_lifecycle_test.clj @@ -264,6 +264,50 @@ (doseq [k nb-status/blacklist-response-keys] (is (not (contains? r k))))))) + (let [new-tags ["foo"]] + (testing "nuvlabox user is able to patch update nuvlabox-status" + (-> session-nb + (request status-url + :content-type "application/json-patch+json" + :request-method :put + :body (json/write-str [{"op" "add" "path" "/tags" "value" new-tags} + {"op" "test" "path" "/tags" "value" new-tags}])) + (ltu/body->edn) + (ltu/is-status 200)) + (testing "spec error are returned to the user" + (-> session-nb + (request status-url + :content-type "application/json-patch+json" + :request-method :put + :body (json/write-str [{"op" "add" "path" "/foo" "value" "x"}])) + (ltu/body->edn) + (ltu/is-status 400))) + (testing "patch error are returned to the user" + (-> session-nb + (request status-url + :content-type "application/json-patch+json" + :request-method :put + :body (json/write-str [{"op" "test" "path" "/tags" "value" []}])) + (ltu/body->edn) + (ltu/is-status 400) + (ltu/message-matches "Json patch exception: test failure for path /tags and value []")) + (-> session-nb + (request status-url + :content-type "application/json-patch+json" + :request-method :put + :body (json/write-str [{"op" "remove" "path" "/wrong/1" "value" "x"}])) + (ltu/body->edn) + (ltu/is-status 400) + (ltu/message-matches "Json patch exception: can't traverse past non-existent node: wrong")) + (-> session-nb + (request status-url + :content-type "application/json-patch+json" + :request-method :put + :body (json/write-str "plain text")) + (ltu/body->edn) + (ltu/is-status 400) + (ltu/message-matches "Json patch exception: missing op attribute"))))) + (testing "nuvlabox identity cannot delete the state" (-> session-nb (request status-url From 38135031a996ebdaa5f4f815f5d7aeac250654da Mon Sep 17 00:00:00 2001 From: khaled basbous Date: Tue, 3 Sep 2024 16:51:40 +0200 Subject: [PATCH 2/7] fix(Nuvlabox Status): Spec support for docker free map not indexed entry --- .../nuvla/server/resources/spec/nuvlabox_status_2.cljc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/code/src/com/sixsq/nuvla/server/resources/spec/nuvlabox_status_2.cljc b/code/src/com/sixsq/nuvla/server/resources/spec/nuvlabox_status_2.cljc index 14c3e9510..d90aab7a1 100644 --- a/code/src/com/sixsq/nuvla/server/resources/spec/nuvlabox_status_2.cljc +++ b/code/src/com/sixsq/nuvla/server/resources/spec/nuvlabox_status_2.cljc @@ -217,6 +217,12 @@ :json-schema/order 88))) +(s/def ::docker + (assoc (st/spec map?) + :name "docker" + :json-schema/type "map" + :json-schema/description "Docker resources info")) + (s/def ::schema (su/only-keys-maps common/common-attrs @@ -260,5 +266,6 @@ ::orchestrator ::nb-status-0/temperatures ::components - ::network]})) + ::network + ::docker]})) From 1b380ce97505d30bc55ae6c753f50200a9685707 Mon Sep 17 00:00:00 2001 From: khaled basbous Date: Wed, 4 Sep 2024 14:33:57 +0200 Subject: [PATCH 3/7] fix(Nuvlabox Status): Spec support for coe-resources free map not indexed entry replace json patch buggy library --- code/project.clj | 2 +- .../com/sixsq/nuvla/server/resources/common/std_crud.clj | 8 +++++--- .../com/sixsq/nuvla/server/resources/nuvlabox_status.clj | 5 ++++- .../nuvla/server/resources/spec/nuvlabox_status_2.cljc | 8 ++++---- .../server/resources/nuvlabox_status_2_lifecycle_test.clj | 6 +++--- .../server/resources/spec/nuvlabox_status_2_test.cljc | 5 +++-- 6 files changed, 20 insertions(+), 14 deletions(-) diff --git a/code/project.clj b/code/project.clj index ff151b5e3..49aaaab81 100644 --- a/code/project.clj +++ b/code/project.clj @@ -81,7 +81,7 @@ [com.google.zxing/javase]]] [funcool/promesa "11.0.678"] [nrepl "1.1.1"] - [by.borge/clj-json-pointer "1.0.0"]] + [clj-json-patch "0.1.7"]] :profiles {:provided {:dependencies [[org.clojure/clojure "1.11.2"] diff --git a/code/src/com/sixsq/nuvla/server/resources/common/std_crud.clj b/code/src/com/sixsq/nuvla/server/resources/common/std_crud.clj index 38b5f3e39..2d67a8296 100644 --- a/code/src/com/sixsq/nuvla/server/resources/common/std_crud.clj +++ b/code/src/com/sixsq/nuvla/server/resources/common/std_crud.clj @@ -7,7 +7,7 @@ [clojure.walk :as w] [com.sixsq.nuvla.auth.acl-resource :as a] [com.sixsq.nuvla.auth.utils :as auth] - [clj-json-pointer.core :as jp] + [clj-json-patch.core :as json-patch] [com.sixsq.nuvla.db.impl :as db] [com.sixsq.nuvla.server.middleware.cimi-params.impl :as impl] [com.sixsq.nuvla.server.resources.common.crud :as crud] @@ -66,7 +66,9 @@ (defn json-safe-patch [obj patches] (try - (jp/patch obj patches) + (if-let [result (json-patch/patch obj patches)] + result + (throw (Exception. "Patch interpretation failed!"))) (catch Exception e (throw (r/ex-bad-request (str "Json patch exception: " (ex-message e))))))) @@ -74,7 +76,7 @@ (= (get-in request [:headers "content-type"]) "application/json-patch+json")) (defn json-patch - [resource {:keys [json-patch body] :as request}] + [resource {:keys [body] :as request}] (if (json-patch-request? request) (->> (w/stringify-keys body) (json-safe-patch (w/stringify-keys resource)) diff --git a/code/src/com/sixsq/nuvla/server/resources/nuvlabox_status.clj b/code/src/com/sixsq/nuvla/server/resources/nuvlabox_status.clj index 36d8f48fd..08ce9c54c 100644 --- a/code/src/com/sixsq/nuvla/server/resources/nuvlabox_status.clj +++ b/code/src/com/sixsq/nuvla/server/resources/nuvlabox_status.clj @@ -123,7 +123,10 @@ Versioned subclasses define the attributes for a particular NuvlaBox release. ex))] (if exception (do - (crud/edit (dissoc request :body)) + (-> request + (assoc-in [:headers "content-type"] "application/json") + (dissoc request :body) + crud/edit) (throw exception)) resource))) diff --git a/code/src/com/sixsq/nuvla/server/resources/spec/nuvlabox_status_2.cljc b/code/src/com/sixsq/nuvla/server/resources/spec/nuvlabox_status_2.cljc index d90aab7a1..c747926a6 100644 --- a/code/src/com/sixsq/nuvla/server/resources/spec/nuvlabox_status_2.cljc +++ b/code/src/com/sixsq/nuvla/server/resources/spec/nuvlabox_status_2.cljc @@ -217,11 +217,11 @@ :json-schema/order 88))) -(s/def ::docker +(s/def ::coe-resources (assoc (st/spec map?) - :name "docker" + :name "coe-resources" :json-schema/type "map" - :json-schema/description "Docker resources info")) + :json-schema/description "coe resources info")) (s/def ::schema @@ -267,5 +267,5 @@ ::nb-status-0/temperatures ::components ::network - ::docker]})) + ::coe-resources]})) diff --git a/code/test/com/sixsq/nuvla/server/resources/nuvlabox_status_2_lifecycle_test.clj b/code/test/com/sixsq/nuvla/server/resources/nuvlabox_status_2_lifecycle_test.clj index 8fb27f2b2..349d9b900 100644 --- a/code/test/com/sixsq/nuvla/server/resources/nuvlabox_status_2_lifecycle_test.clj +++ b/code/test/com/sixsq/nuvla/server/resources/nuvlabox_status_2_lifecycle_test.clj @@ -290,7 +290,7 @@ :body (json/write-str [{"op" "test" "path" "/tags" "value" []}])) (ltu/body->edn) (ltu/is-status 400) - (ltu/message-matches "Json patch exception: test failure for path /tags and value []")) + (ltu/message-matches "Json patch exception: The test failed. [] is not found at /tags")) (-> session-nb (request status-url :content-type "application/json-patch+json" @@ -298,7 +298,7 @@ :body (json/write-str [{"op" "remove" "path" "/wrong/1" "value" "x"}])) (ltu/body->edn) (ltu/is-status 400) - (ltu/message-matches "Json patch exception: can't traverse past non-existent node: wrong")) + (ltu/message-matches "Json patch exception: There is no value at '/wrong/1' to remove.")) (-> session-nb (request status-url :content-type "application/json-patch+json" @@ -306,7 +306,7 @@ :body (json/write-str "plain text")) (ltu/body->edn) (ltu/is-status 400) - (ltu/message-matches "Json patch exception: missing op attribute"))))) + (ltu/message-matches "Json patch exception: Patch interpretation failed!"))))) (testing "nuvlabox identity cannot delete the state" (-> session-nb diff --git a/code/test/com/sixsq/nuvla/server/resources/spec/nuvlabox_status_2_test.cljc b/code/test/com/sixsq/nuvla/server/resources/spec/nuvlabox_status_2_test.cljc index 6a350ca34..cc295cb82 100644 --- a/code/test/com/sixsq/nuvla/server/resources/spec/nuvlabox_status_2_test.cljc +++ b/code/test/com/sixsq/nuvla/server/resources/spec/nuvlabox_status_2_test.cljc @@ -128,7 +128,8 @@ :ips [{:address "3.4.5.6"}]} {:interface "enp3s0" :ips []} - {:interface "abc"}]}}) + {:interface "abc"}]} + :coe-resources {:docker {}}}) (deftest check-nuvlabox-status @@ -149,5 +150,5 @@ :power-consumption ::jobs :swarm-node-cert-expiry-date :online :host-user-home :cluster-id :cluster-node-labels :cluster-node-role :status-notes :cluster-nodes :cluster-managers :orchestrator :cluster-join-address :temperatures :components - :network}] + :network :coe-resources}] (stu/is-valid ::nb-status-2/schema (dissoc state attr)))) From d1668b7adb6fbf357a554ee2d15ab24667492999 Mon Sep 17 00:00:00 2001 From: khaled basbous Date: Fri, 6 Sep 2024 14:27:49 +0200 Subject: [PATCH 4/7] nuvlabox status spec add not indexed annotation for coe-resources --- .../com/sixsq/nuvla/server/resources/spec/nuvlabox_status_2.cljc | 1 + 1 file changed, 1 insertion(+) diff --git a/code/src/com/sixsq/nuvla/server/resources/spec/nuvlabox_status_2.cljc b/code/src/com/sixsq/nuvla/server/resources/spec/nuvlabox_status_2.cljc index c747926a6..78d9a04c4 100644 --- a/code/src/com/sixsq/nuvla/server/resources/spec/nuvlabox_status_2.cljc +++ b/code/src/com/sixsq/nuvla/server/resources/spec/nuvlabox_status_2.cljc @@ -221,6 +221,7 @@ (assoc (st/spec map?) :name "coe-resources" :json-schema/type "map" + :json-schema/indexed false :json-schema/description "coe resources info")) From 91c943091620fa570cb015c60fb89a8583f642ae Mon Sep 17 00:00:00 2001 From: khaled basbous Date: Tue, 17 Sep 2024 07:22:54 +0200 Subject: [PATCH 5/7] add log error of all elements for json patch --- code/src/com/sixsq/nuvla/server/resources/common/std_crud.clj | 1 + 1 file changed, 1 insertion(+) diff --git a/code/src/com/sixsq/nuvla/server/resources/common/std_crud.clj b/code/src/com/sixsq/nuvla/server/resources/common/std_crud.clj index 2d67a8296..fb508b5d3 100644 --- a/code/src/com/sixsq/nuvla/server/resources/common/std_crud.clj +++ b/code/src/com/sixsq/nuvla/server/resources/common/std_crud.clj @@ -70,6 +70,7 @@ result (throw (Exception. "Patch interpretation failed!"))) (catch Exception e + (log/error "ex-message:" (ex-message e) "ex-data:" (ex-data e) "exception:" e "resource:" obj "patches:" patches) (throw (r/ex-bad-request (str "Json patch exception: " (ex-message e))))))) (defn- json-patch-request? [request] From 0abd3a5b371aaf52fa55a67ba8d5b8704b6db505 Mon Sep 17 00:00:00 2001 From: khaled basbous Date: Tue, 17 Sep 2024 13:50:38 +0200 Subject: [PATCH 6/7] prn-str to better debug --- code/src/com/sixsq/nuvla/server/resources/common/std_crud.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/src/com/sixsq/nuvla/server/resources/common/std_crud.clj b/code/src/com/sixsq/nuvla/server/resources/common/std_crud.clj index fb508b5d3..7537a22cf 100644 --- a/code/src/com/sixsq/nuvla/server/resources/common/std_crud.clj +++ b/code/src/com/sixsq/nuvla/server/resources/common/std_crud.clj @@ -70,7 +70,7 @@ result (throw (Exception. "Patch interpretation failed!"))) (catch Exception e - (log/error "ex-message:" (ex-message e) "ex-data:" (ex-data e) "exception:" e "resource:" obj "patches:" patches) + (log/error "ex-message:" (ex-message e) "ex-data:" (ex-data e) "exception:" e "resource:" (prn-str obj) "patches:" (prn-str (vec patches))) (throw (r/ex-bad-request (str "Json patch exception: " (ex-message e))))))) (defn- json-patch-request? [request] From ca0a579092394fb56121458a6ebf66715cf4d8d0 Mon Sep 17 00:00:00 2001 From: khaled basbous Date: Mon, 23 Sep 2024 12:00:04 +0200 Subject: [PATCH 7/7] Change patch json from error to debug --- code/src/com/sixsq/nuvla/server/resources/common/std_crud.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/src/com/sixsq/nuvla/server/resources/common/std_crud.clj b/code/src/com/sixsq/nuvla/server/resources/common/std_crud.clj index 7537a22cf..537f7dbd8 100644 --- a/code/src/com/sixsq/nuvla/server/resources/common/std_crud.clj +++ b/code/src/com/sixsq/nuvla/server/resources/common/std_crud.clj @@ -70,7 +70,7 @@ result (throw (Exception. "Patch interpretation failed!"))) (catch Exception e - (log/error "ex-message:" (ex-message e) "ex-data:" (ex-data e) "exception:" e "resource:" (prn-str obj) "patches:" (prn-str (vec patches))) + (log/debug "Json patch exception - ex-message:" (ex-message e) "ex-data:" (ex-data e) "exception:" e "resource:" (prn-str obj) "patches:" (prn-str (vec patches))) (throw (r/ex-bad-request (str "Json patch exception: " (ex-message e))))))) (defn- json-patch-request? [request]