wip

threatgrid · Sep 12, 2024 · 78b338e · 78b338e
1 parent c8c4011
commit 78b338e
Show file tree

Hide file tree

Showing 6 changed files with 28 additions and 30 deletions.
diff --git a/test/ctia/entity/incident_test.clj b/test/ctia/entity/incident_test.clj
@@ -185,8 +185,8 @@
 (deftest sort-scores-test
   (es-helpers/for-each-es-version
     "Can sort by multiple scores"
-    [5 7]
-    #(ductile.index/delete! % "ctia_*")
+    [7]
+    #(es-helpers/clean-es-state! % "ctia_*")
     (helpers/with-properties (-> ["ctia.auth.type" "allow-all"]
                                  (into es-helpers/basic-auth-properties)
                                  (conj "ctia.http.incident.score-types" "asset,ttp"
@@ -305,7 +305,7 @@
   (es-helpers/for-each-es-version
     "severity sorts like #'ctim-severity-order"
     [7]
-    #(ductile.index/delete! % "ctia_*")
+    #(es-helpers/clean-es-state! % "ctia_*")
     (helpers/with-properties (into ["ctia.auth.type" "allow-all"]
                                    es-helpers/basic-auth-properties)
       (helpers/fixture-ctia-with-app
@@ -354,7 +354,7 @@
   (es-helpers/for-each-es-version
     "sort by tactics"
     [7]
-    #(ductile.index/delete! % "ctia_*")
+    #(es-helpers/clean-es-state! % "ctia_*")
     (helpers/with-properties (into ["ctia.auth.type" "allow-all"]
                                    es-helpers/basic-auth-properties)
       (helpers/fixture-ctia-with-app
@@ -423,7 +423,7 @@
    (es-helpers/for-each-es-version
      "severity sorts like #'ctim-severity-order"
      [7]
-     #(ductile.index/delete! % "ctia_*")
+     #(es-helpers/clean-es-state! % "ctia_*")
      (helpers/with-properties (into ["ctia.auth.type" "allow-all"]
                                     es-helpers/basic-auth-properties)
        (helpers/fixture-ctia-with-app
@@ -573,7 +573,7 @@
   (es-helpers/for-each-es-version
     "sort by tactics"
     [7]
-    #(ductile.index/delete! % "ctia_*")
+    #(es-helpers/clean-es-state! % "ctia_*")
     (helpers/with-properties (into ["ctia.auth.type" "allow-all"]
                                    es-helpers/basic-auth-properties)
       (helpers/fixture-ctia-with-app
@@ -610,7 +610,7 @@
   (es-helpers/for-each-es-version
     "filter by scores"
     [7]
-    #(ductile.index/delete! % "ctia_*")
+    #(es-helpers/clean-es-state! % "ctia_*")
     (helpers/with-properties (-> ["ctia.auth.type" "allow-all"]
                                  (into es-helpers/basic-auth-properties)
                                  (conj "ctia.http.incident.score-types" "asset,ttp"))

diff --git a/test/ctia/http/generative/fulltext_search_test.clj b/test/ctia/http/generative/fulltext_search_test.clj
@@ -283,7 +283,7 @@
   (es-helpers/for-each-es-version
    "Extended Fulltext query search"
    [7]
-   #(es-index/delete! % "ctia_*")
+   #(es-helpers/clean-es-state! % "ctia_*")
    (helpers/fixture-ctia-with-app
     (fn [app]
       (helpers/set-capabilities! app "foouser" ["foogroup"] "user" (capabilities/all-capabilities))

diff --git a/test/ctia/stores/es/init_test.clj b/test/ctia/stores/es/init_test.clj
@@ -115,7 +115,7 @@
             (is (= "1" number_of_replicas))
             (is (= "2s" refresh_interval))))
         (finally
-          (index/delete! conn (str indexname "*"))
+          (es-helpers/clean-es-state! conn (str indexname "*"))
           (conn/close conn))))))
 
 (defn exceptional-system-exit [] (throw (ex-info (str `exceptional-system-exit) {::exceptional-system-exit true})))
@@ -179,17 +179,14 @@
   (helpers/with-config-transformer
     #(assoc-in % [:ctia :task :ctia.task.update-index-state] true)
     (let [indexname (gen-indexname)
-          clean-template #(index/delete-template! % indexname)
-          clean-index #(index/delete! % (str indexname "*"))
-          clean-all #(do (clean-index %) (clean-template %))
           prepare-props (fn [props version]
                           (assoc props
                                  :version version
                                  :port (+ 9200 version)))]
       (for-each-es-version
         "get-existing-indices should retrieve existing indices if any."
         [7]
-        clean-index
+        #(es-helpers/clean-es-state! % (str indexname "*"))
         (testing "update mapping should allow adding fields or identical mapping"
           (let [services (->ESConnServices)
                 props (prepare-props (mk-props indexname) version)
@@ -212,7 +209,7 @@
                                   (is (= expected-successful? (not (exceptional-system-exit? output)))))
                                 (finally
                                   ;; reset state
-                                  (clean-all conn)
+                                  (es-helpers/clean-es-state! conn)
                                   (conn/close conn))))))]
             (test-fn "update mapping should not fail on unchanged mapping"
                      true nil nil)

diff --git a/test/ctia/task/migration/migrate_es_stores_test.clj b/test/ctia/task/migration/migrate_es_stores_test.clj
@@ -63,7 +63,7 @@
       (t)
       (finally
         (doto (es-conn get-in-config)
-          (es-index/delete! "v0.0.0*")
+          (es-helpers/clean-es-state! es-conn "v0.0.0*")
           (es-index/delete! (str (migration-index get-in-config) "*")))))))
 
 (s/defn with-each-fixtures*

diff --git a/test/ctia/task/migration/store_test.clj b/test/ctia/task/migration/store_test.clj
@@ -256,7 +256,7 @@
   (es-helpers/for-each-es-version
    "rollover should refresh write index and trigger rollover when index size is strictly bigger than max-docs"
    [7]
-   #(ductile.index/delete! % "ctia_*")
+   #(es-helpers/clean-es-state! % "ctia_*")
    (helpers/with-properties*
      ["ctia.store.es.default.port" es-port
       "ctia.store.es.default.version" version
@@ -330,7 +330,7 @@
   (es-helpers/for-each-es-version
    "Sliced-queries should properly decompose a store into time window queries for given time interval"
    [7]
-   #(ductile.index/delete! % "*ctia_relationship*")
+   #(es-helpers/clean-es-state! % "*ctia_relationship*")
    (helpers/with-properties ;; simple way to have a proper store initialization
      ["ctia.store.es.default.port" es-port
       "ctia.store.es.default.version" version]
@@ -493,7 +493,7 @@
   (es-helpers/for-each-es-version
    "bulk-metas prepares ES bulk data for given document ids"
    [7]
-   #(ductile.index/delete! % "ctia_*")
+   #(es-helpers/clean-es-state! % "ctia_*")
    (helpers/with-properties
      ["ctia.store.es.default.port" es-port
       "ctia.store.es.default.version" version
@@ -559,7 +559,7 @@
   (es-helpers/for-each-es-version
    "prepare-docs properly generates meta data for bulk ops for new and modified documents"
    [7]
-   #(ductile.index/delete! % "ctia_*")
+   #(es-helpers/clean-es-state! % "ctia_*")
    (helpers/with-properties
      ["ctia.store.es.default.port" es-port
       "ctia.store.es.default.version" version
@@ -627,8 +627,9 @@
   (es-helpers/for-each-es-version
    "store-batch should properly write data in given store"
    [7]
-   #(do (ductile.index/delete! % "test_index*")
-        (ductile.index/delete! % "ctia_*"))
+   #(do
+      (es-helpers/clean-es-state! % "ctia_*")
+      (es-helpers/clean-es-state! % "test_index*"))
    (helpers/with-properties
      ["ctia.store.es.default.port" es-port
       "ctia.store.es.default.version" version
@@ -664,7 +665,7 @@
                (is (= (+ nb-docs-1 nb-docs-2) (sut/store-size store))
                    "store size shall return the proper number of documents after second refresh"))
              (finally
-               (ductile.index/delete! conn indexname)))))))))
+               (es-helpers/clean-es-state! % (str indexname "*"))))))))))
 
 (defn test-query-fetch-batch-events
   [{:keys [version] :as conn} services]
@@ -801,10 +802,10 @@
    "query-fetch should properly fetch and sort data"
    [7]
    (fn [conn]
-     (ductile.index/delete! conn "ctia_*")
-     (ductile.index/delete! conn "event_index*")
-     (ductile.index/delete! conn "tool_index*")
-     (ductile.index/delete! conn "malware_index*"))
+     (es-helpers/clean-es-state! conn "ctia_*")
+     (es-helpers/clean-es-state! conn "event_index*")
+     (es-helpers/clean-es-state! conn "tool_index*")
+     (es-helpers/clean-es-state! conn "malware_index*"))
    (helpers/with-properties
      ["ctia.store.es.default.port" es-port
       "ctia.store.es.default.version" version
@@ -819,7 +820,7 @@
   (es-helpers/for-each-es-version
    "fetch-deletes should be properly configured to fetch deletes in source store"
    [7]
-   #(ductile.index/delete! % "ctia_*")
+   #(es-helpers/clean-es-state! % "ctia_*")
    (helpers/with-properties
      ["ctia.store.es.default.port" es-port
       "ctia.store.es.default.version" version
@@ -895,8 +896,8 @@
    "init-migration should properly create new migration state from selected types."
    [7]
    (fn [c]
-     (ductile.index/delete! c "ctia_*")
-     (ductile.index/delete! c "v0.0.0*"))
+     (es-helpers/clean-es-state! c "ctia_*")
+     (es-helpers/clean-es-state! c "v0.0.0*"))
    (helpers/with-properties
      ["ctia.migration.store.es.default.port" es-port
       "ctia.migration.store.es.default.version" version

diff --git a/test/ctia/task/update_index_state_test.clj b/test/ctia/task/update_index_state_test.clj
@@ -12,7 +12,7 @@
   (es-helpers/for-each-es-version
     "update-index-state task"
     [7]
-    #(ductile.index/delete! % "ctia_*")
+    #(es-helpers/clean-es-state! % "ctia_*")
     (es-helpers/fixture-properties:es-store
       (fn []
         (is (= 0 (sut/do-task (h/build-transformed-init-config))))