Do not init disabled stores (#1379)

<!-- Specify linked issues and REMOVE THE UNUSED LINES -->

Close advthreat/iroh#8042

<!--

Describe your PR for reviewers.
Don't forget to set correct labels (User Facing / Beta / Feature Flag)
If there is UI change please add a screen capture.
-->

<!-- UNCOMMENT THIS SECTION IF NEEDED
<a name="iroh-services-clients">[§](#iroh-services-clients)</a> IROH Services Clients
=====================================================================================

Put all informations that need to be communicated to IROH Services Clients.
Typically IROH-UI, ATS Integration, Orbital, etc...
 -->

<a name="qa">[§](#qa)</a> QA
============================

<!--
Describe the steps to test your PR.

1.
2.
3.

Or if no QA is needed keep it as is.
 -->
No QA is needed.

<!-- UNCOMMENT THIS SECTION IF NEEDED
<a name="ops">[§](#ops)</a> Ops
===============================

  Specify Ops related issues and documentation
- Config change needed: threatgrid/tenzin#
- Migration needed: threatgrid/tenzin#
- How to enable/disable that feature: (ex remove service from `bootstrap.cfg`, add scope to org)
-->

<!-- UNCOMMENT THIS SECTION IF NEEDED
<a name="documentation">[§](#documentation)</a> Documentation
=============================================================

  Public Facing documentation section;
- Public documentation updated needed: threatgrid/iroh-ui#
  See internal [doc file](./services/iroh-auth/doc/public-doc.org)
 -->

<a name="release-notes">[§](#release-notes)</a> Release Notes
=============================================================

<!-- REMOVE UNUSED LINES -->

```
intern: Do not init disabled stores
```

<a name="squashed-commits">[§](#squashed-commits)</a> Squashed Commits
======================================================================

src/ctia/graphql_named_type_registry_service_core.clj

@@ -5,4 +5,4 @@
   (helpers/get-or-update-named-type-registry type-registry nme f))
 
 (defn start [context]
-  (assoc context :type-registry (atom {})))
+  (assoc context :type-registry (helpers/create-named-type-registry)))

src/ctia/properties.clj

@@ -54,7 +54,7 @@
 
 (s/defschema StorePropertiesSchema
   "All entity store properties for every implementation"
-  (let [configurable-stores (map name (keys store/empty-stores))
+  (let [configurable-stores (map name store/known-stores)
         store-names (conj configurable-stores "default")]
     (st/optional-keys
      (reduce merge {}

src/ctia/schemas/services.clj

@@ -13,4 +13,5 @@
                          [[s/Any] s/Any])})
 
 (s/defschema FeaturesServiceFns
-  {:flag-value (s/=> s/Any)})
+  {:entity-enabled? (s/=> s/Bool s/Keyword)
+   :flag-value (s/=> s/Any)})

src/ctia/store.clj

@@ -39,30 +39,30 @@
   `fetch-page-fn` using `init-page-params` for the first call."
   (paginate [this fetch-page-fn] [this fetch-page-fn init-page-params]))
 
-(def empty-stores
-  {:actor []
-   :asset []
-   :asset-mapping []
-   :asset-properties []
-   :attack-pattern []
-   :campaign []
-   :casebook []
-   :coa []
-   :data-table []
-   :event []
-   :feed []
-   :feedback []
-   :identity []
-   :identity-assertion []
-   :incident []
-   :indicator []
-   :investigation []
-   :judgement []
-   :malware []
-   :note []
-   :relationship []
-   :sighting []
-   :target-record []
-   :tool []
-   :vulnerability []
-   :weakness []})
+(def known-stores
+  #{:actor
+    :asset
+    :asset-mapping
+    :asset-properties
+    :attack-pattern
+    :campaign
+    :casebook
+    :coa
+    :data-table
+    :event
+    :feed
+    :feedback
+    :identity
+    :identity-assertion
+    :incident
+    :indicator
+    :investigation
+    :judgement
+    :malware
+    :note
+    :relationship
+    :sighting
+    :target-record
+    :tool
+    :vulnerability
+    :weakness})

src/ctia/store_service.clj

@@ -15,14 +15,11 @@
   "A service to manage the central storage area for all stores."
   StoreService
   [[:ConfigService get-in-config]
-   [:FeaturesService flag-value]]
-  (init [this context]
-        (core/init context))
-  (start [this context]
+   FeaturesService]
+  (start [this _]
          (core/start
           {:ConfigService {:get-in-config get-in-config}
-           :FeaturesService {:flag-value flag-value}}
-          context))
+           :FeaturesService (select-keys FeaturesService #{:entity-enabled? :flag-value})}))
   (stop [this context]
         (core/stop context))
 

src/ctia/store_service/schemas.clj

@@ -22,10 +22,6 @@
   "ctia.store-service/all-stores in the service graph."
   (s/=> Stores))
 
-(s/defschema StoresAtom
-  "An atom containing a sequence of stores."
-  (s/atom Stores))
-
 (s/defschema StoreServiceCtx
   "The service-context for StoreService."
-  {:stores-atom StoresAtom})
+  {:stores Stores})

src/ctia/store_service_core.clj

@@ -1,19 +1,14 @@
 (ns ctia.store-service-core
   (:require [clojure.string :as str]
-            [ctia.store :refer [empty-stores close]]
-            [ctia.store-service.schemas :refer [Store Stores StoresAtom StoreID StoreServiceCtx]]
+            [ctia.store :refer [known-stores close]]
+            [ctia.store-service.schemas :refer [Store Stores StoreID StoreServiceCtx]]
             [ctia.stores.es.init :as es-init]
             [schema.core :as s]
             [schema-tools.core :as st]))
 
-(s/defn init :- StoreServiceCtx
-  [context :- (st/optional-keys StoreServiceCtx)]
-  (assoc context
-         :stores-atom (atom empty-stores)))
-
 (s/defn all-stores :- Stores
-  [{:keys [stores-atom]} :- StoreServiceCtx]
-  @stores-atom)
+  [{:keys [stores]} :- StoreServiceCtx]
+  stores)
 
 (s/defn get-store :- Store
   [ctx :- StoreServiceCtx
@@ -38,25 +33,19 @@
   (case store-type
     "es" (es-init/init-store! services store-kw)))
 
-(s/defn ^:private init-store-service!
-  [services
-   stores-atom :- StoresAtom]
-  (reset! stores-atom
-          (->> (keys empty-stores)
-               (map (fn [store-kw]
-                      [store-kw (keep (partial build-store store-kw services)
-                                      (get-store-types store-kw services))]))
-               (into {})
-               (merge-with into empty-stores))))
-
 (s/defn start :- StoreServiceCtx
-  [services
-   {:keys [stores-atom] :as context} :- StoreServiceCtx]
-  (init-store-service! services stores-atom)
-  context)
+  [{{:keys [entity-enabled?]} :FeaturesService
+    :as services}]
+  {:stores (reduce (fn [stores store-kw]
+                     (cond-> stores
+                       (entity-enabled? store-kw)
+                       (assoc store-kw (into [] (keep #(build-store store-kw services %))
+                                             (get-store-types store-kw services)))))
+                   {} known-stores)})
 
 (s/defn stop :- (st/optional-keys StoreServiceCtx)
   [ctx :- StoreServiceCtx]
-  (doseq [[kw [s]] (all-stores ctx)]
-    (close s))
-  ctx)
+  (doseq [stores (vals (all-stores ctx))
+          store stores]
+    (close store))
+  {})

src/ctia/stores/es/schemas.clj

@@ -9,7 +9,7 @@
   {:ConfigService (-> external-svc-fns/ConfigServiceFns
                       (csu/select-all-keys #{:get-in-config}))
    :FeaturesService (-> external-svc-fns/FeaturesServiceFns
-                      (csu/select-all-keys #{:flag-value}))})
+                        (csu/select-all-keys #{:entity-enabled? :flag-value}))})
 
 (s/defschema ESConnState
   (st/merge

src/ctia/task/migration/store.clj

@@ -107,9 +107,7 @@
     {:id em/token
      :timestamp em/ts
      :stores {:type "object"
-              :properties (->> (keys store/empty-stores)
-                               (map store-mapping)
-                               (into {}))}}}})
+              :properties (into {} (map store-mapping) store/known-stores)}}}})
 
 (s/defn migration-store-properties [{{:keys [get-in-config]} :ConfigService} :- MigrationStoreServices]
   (into (target-store-properties nil :migration get-in-config)

src/ctia/task/settings.clj

@@ -7,7 +7,7 @@
             [ctia.stores.es.schemas :refer [ESConnServices]]
             [ctia.init :refer [log-properties]]
             [ctia.properties :as p]
-            [ctia.store :refer [empty-stores]]
+            [ctia.store :refer [known-stores]]
             [schema.core :as s]))
 
 (s/defn update-stores!
@@ -22,7 +22,7 @@
 (def cli-options
   [["-h" "--help"]
    ["-s" "--stores STORES" "comma separated list of store names"
-    :default (set (keys empty-stores))
+    :default known-stores
     :parse-fn #(map keyword (str/split % #","))]])
 
 (defn -main [& args]

src/ctia/task/update_mapping.clj

@@ -30,7 +30,7 @@
 (def cli-options
   [["-h" "--help"]
    ["-s" "--stores STORES" "comma separated list of store names"
-    :default (set (keys store/empty-stores))
+    :default store/known-stores
     :parse-fn #(map keyword (str/split % #","))]])
 
 (defn -main [& args]

test/ctia/http/generative/fulltext_search_test.clj

@@ -424,11 +424,10 @@
   store-service/StoreService
   [[:ConfigService get-in-config]
    [:FeaturesService flag-value]]
-  (init [this context] (store-svc-core/init context))
-  (start [this context] (store-svc-core/start
-                         {:ConfigService {:get-in-config get-in-config}
-                          :FeaturesService {:flag-value flag-value}}
-                         context))
+  (start [this _] (store-svc-core/start
+                    {:ConfigService {:get-in-config get-in-config}
+                     :FeaturesService {:entity-enabled? (constantly true)
+                                       :flag-value flag-value}}))
   (stop [this context] (store-svc-core/stop context))
   (all-stores [this]
               (store-svc-core/all-stores (tk-svcs/service-context this)))
@@ -444,7 +443,8 @@
          (let [res (es.query/enforce-search-fields
                     {:props {:entity :incident}
                      :searchable-fields #{:foo :bar :zap}
-                     :services {:FeaturesService {:flag-value (constantly "true")}}}
+                     :services {:FeaturesService {:entity-enabled? (constantly true)
+                                                  :flag-value (constantly "true")}}}
                     fields)]
            (is (= expected-search-fields res)))
       [] ["zap" "bar" "foo"]

test/ctia/store_service_test.clj

@@ -1,7 +1,32 @@
 (ns ctia.store-service-test
-  (:require [ctia.store-service :as sut]))
+  (:require [clj-momo.test-helpers.core :as mth]
+            [clojure.test :refer [deftest is testing use-fixtures]]
+            [ctia.store-service :as sut]
+            [ctia.stores.es.init :as es-init]
+            [ctia.test-helpers.core :as th]
+            [ctia.test-helpers.es :as es-th]
+            [ctia.test-helpers.store :refer [test-for-each-store-with-app]]
+            [ductile.conn :as es-conn]
+            [ductile.index :as index]
+            [puppetlabs.trapperkeeper.app :as app]))
+
+(use-fixtures :once mth/fixture-schema-validation)
 
 (defn store-service-map
   "Service map for #'sut/store-service"
   []
   {:StoreService sut/store-service})
+
+(deftest disabled-initialization-test
+  (doseq [disable? [true false]]
+    (testing disable?
+      (th/with-properties ["ctia.features.disable" (if disable? "asset" "")]
+        (test-for-each-store-with-app
+          (fn [app]
+            (let [conn (es-conn/connect (es-init/get-store-properties
+                                          ::no-store
+                                          (get-in (app/service-graph app) [:ConfigService :get-in-config])))]
+              (try
+                (is (= disable? (not (index/get conn (es-th/get-indexname app :asset)))))
+                (finally
+                  (es-conn/close conn))))))))))

test/ctia/stores/es/crud_test.clj

@@ -24,7 +24,8 @@
 
 (deftest refine-full-text-query-parts-test
   (let [es-conn-state {:props {:entity :incident}
-                       :services {:FeaturesService {:flag-value (constantly nil)}}}
+                       :services {:FeaturesService {:entity-enabled? (constantly true)
+                                                    :flag-value (constantly nil)}}}
         with-def-op (assoc-in es-conn-state [:props :default_operator] "and")]
     (testing "simple queries"
       (are [queries exp]

test/ctia/test_helpers/core.clj

@@ -291,7 +291,7 @@
   [app]
   (let [{{:keys [get-config]} :ConfigService
          {:keys [all-stores]} :StoreService
-         {:keys [flag-value]} :FeaturesService} (app/service-graph app)
+         :keys [FeaturesService]} (app/service-graph app)
         ;; simulate the current output of these functions before we stop or restart
         ;; the app
         get-in-config (partial get-in (get-config))
@@ -300,7 +300,7 @@
     (@purge-indices-and-templates
      all-stores
      {:ConfigService {:get-in-config get-in-config}
-      :FeaturesService {:flag-value flag-value}})))
+      :FeaturesService (select-keys FeaturesService #{:entity-enabled? :flag-value})})))
 
 (s/defschema WithAppOptions
   (st/optional-keys

test/ctia/test_helpers/es.clj

@@ -21,7 +21,7 @@
   (let [get-in-config (h/current-get-in-config-fn app)]
     {:ConfigService {:get-in-config get-in-config}
      :FeaturesService (-> (h/get-service-map app :FeaturesService)
-                          (select-keys [:flag-value]))}))
+                          (select-keys [:entity-enabled? :flag-value]))}))
 
 (s/defn ->ESConnServices
   :- ESConnServices

test/ctia/test_helpers/migration.clj

@@ -10,4 +10,4 @@
                       (select-keys [:get-config
                                     :get-in-config]))
    :FeaturesService (-> (helpers/get-service-map app :FeaturesService)
-                        (select-keys [:flag-value]))})
+                        (select-keys [:entity-enabled? :flag-value]))})