POC to preindex relatinships types

src/ctia/entity/relationship.clj

@@ -18,6 +18,11 @@
    [schema-tools.core :as st]
    [schema.core :as s]))
 
+(def node-type
+  (assoc em/token
+         :fields {:type {:type "text"
+                         :analyzer "type_analyzer"}}))
+
 (def relationship-mapping
   {"relationship"
    {:dynamic false
@@ -28,8 +33,8 @@
      em/sourcable-entity-mapping
      em/stored-entity-mapping
      {:relationship_type em/token
-      :source_ref        em/token
-      :target_ref        em/token})}})
+      :source_ref        node-type
+      :target_ref        node-type})}})
 
 (def-es-store RelationshipStore
   :relationship

src/ctia/stores/es/init.clj

@@ -47,8 +47,9 @@
         settings {:refresh_interval refresh_interval
                   :number_of_shards shards
                   :number_of_replicas replicas}
-        mappings (cond-> (get-in entity-fields [entity :es-mapping] mappings)
-                   (< 5 version) (some-> first val))
+        mappings (some-> (get-in entity-fields [entity :es-mapping] mappings)
+                         first
+                         val)
         searchable-fields (get-in entity-fields [entity :searchable-fields])]
     {:index indexname
      :props (assoc props :write-index write-index)
@@ -89,21 +90,17 @@
 (s/defn update-mappings!
   [{:keys [conn index]
     {:keys [mappings]} :config} :- ESConnState]
-  (let [[entity-type type-mappings] (when (= (:version conn) 5)
-                                      (first mappings))
-        update-body (or type-mappings mappings)]
     (try
       (log/info "updating mapping: " index)
       (index/update-mappings! conn
                               index
-                              entity-type
-                              update-body)
+                              mappings)
       (catch clojure.lang.ExceptionInfo e
         (log/error "cannot update mapping. You probably tried to update the mapping of an existing field. It's only possible to add new field to existing mappings. If you need to modify the type of a field in an existing index, you must perform a migration"
                    (assoc (ex-data e)
                           :conn conn
                           :mappings mappings))
-        (system-exit-error)))))
+        (system-exit-error))))
 
 (s/defn refresh-mappings!
   [{:keys [conn index]

src/ctia/stores/es/mapping.clj

@@ -1,4 +1,5 @@
 (ns ctia.stores.es.mapping
+  (:require [clojure.string :as string])
   (:refer-clojure :exclude [identity]))
 
 ;; This provides a reasonable default mapping for all of our entities.
@@ -260,6 +261,34 @@
   {:properties {:type token
                 :text text}})
 
+(def type-simple-pattern
+  (->> [:actor
+        "asset([-](mapping|properties))?"
+        :attack-pattern
+        :campaign
+        :casebook
+        :coa
+        :data-table
+        :event
+        :feed
+        :feedback
+        :identity-assertion
+        :identity
+        :incident
+        :indicator
+        :investigation
+        :judgement
+        :malware
+        :note
+        :relationship
+        :sighting
+        :target-record
+        :tool
+        :vulnerability
+        :weakness]
+       (map (comp #(string/replace % "-" "\\-") name))
+       (string/join "|")))
+
 (def store-settings
   {:number_of_replicas 1
    :number_of_shards 1
@@ -284,6 +313,10 @@
      :english_stemmer {:type "stemmer"
                        :language "english"}}
     ;; when applying filters, order matters
+    :tokenizer
+    {:type_tokenizer
+     {:type "simple_pattern",
+      :pattern type-simple-pattern}}
     :analyzer
     {:default ;; same as text_analyzer
      {:type "custom"
@@ -298,6 +331,10 @@
       :filter ["lowercase"
                "ctia_stemmer"
                "english_stemmer"]}
+     :type_analyzer {
+        :tokenizer "type_tokenizer"
+        :filter [ "fingerprint"]
+     }
      :search_analyzer
      {:type "custom"
       :tokenizer "standard"

test/ctia/entity/relationship_test.clj

@@ -6,7 +6,7 @@
             [ctia.test-helpers
              [access-control :refer [access-control-test]]
              [auth :refer [all-capabilities]]
-             [core :as helpers :refer [POST]]
+             [core :as helpers :refer [POST GET]]
              [crud :refer [entity-crud-test]]
              [aggregate :refer [test-metric-routes]]
              [fake-whoami-service :as whoami-helpers]
@@ -43,6 +43,75 @@
         "http://ex.tld/ctia/relationship/relationship-456"])
       (dissoc :id)))
 
+(deftest test-relationship-source-target-type-search
+  (test-for-each-store-with-app
+   (fn [app]
+     (establish-user! app)
+     (testing "POST /ctia/relationship"
+       (let [new-relationship-1
+             (-> new-relationship-minimal
+                 (assoc
+                  :source_ref (str "http://example.com/ctia/judgement/judgement-"
+                                   "f9832ac2-ee90-4e18-9ce6-0c4e4ff61a7a")
+                  :target_ref (str "http://example.com/ctia/indicator/indicator-"
+                                   "8c94ca8d-fb2b-4556-8517-8e6923d8d3c7"))
+                 (dissoc :id))
+             new-relationship-2 (update new-relationship-1
+                                        :source_ref
+                                        str/replace
+                                        "judgement"
+                                        "sighting")
+             new-relationship-3 (-> new-relationship-1
+                                    (update :source_ref
+                                            str/replace
+                                            "judgement"
+                                            "asset")
+                                    (update :target_ref
+                                            str/replace
+                                            "indicator"
+                                            "asset-properties"))
+             {status-1 :status rel1 :parsed-body}
+             (POST app
+                   "ctia/relationship"
+                   :body new-relationship-1
+                   :headers {"Authorization" "45c1f5e3f05d0"})
+             {status-2 :status rel2 :parsed-body}
+             (POST app
+                   "ctia/relationship"
+                   :body new-relationship-2
+                   :headers {"Authorization" "45c1f5e3f05d0"})
+             {status-3 :status rel3 :parsed-body}
+             (POST app
+                   "ctia/relationship"
+                   :body new-relationship-3
+                   :headers {"Authorization" "45c1f5e3f05d0"})
+             test-plan [{:expected [rel1 rel2]
+                         :query "target_ref.type=indicator"}
+                        {:expected [rel1]
+                         :query "source_ref.type=judgement"}
+                        {:expected [rel2]
+                         :query "source_ref.type=sighting"}
+                        {:expected [rel3]
+                         :query "source_ref.type=asset"}
+                        {:expected [rel3]
+                         :query "target_ref.type=\"asset-properties\""}
+                        {:expected []
+                         :query "target_ref.type=asset"}
+                        {:expected []
+                         :query "source_ref.type=malware"}]]
+         (assert (= 201 status-1))
+         (assert (= 201 status-2))
+         (assert (= 201 status-3))
+         (doseq [{:keys [expected query]} test-plan]
+           (testing query
+             (is (= (set expected)
+                    (-> (GET app
+                            "ctia/relationship/search"
+                          :query-params {:query query}
+                          :headers {"Authorization" "45c1f5e3f05d0"})
+                        :parsed-body
+                        set))))))))))
+
 (deftest test-relationship-routes-bad-reference
   (test-for-each-store-with-app
    (fn [app]