Fix ILM template Migration (#1444)

src/ctia/stores/es/init.clj

@@ -108,9 +108,9 @@
     (index/delete-template! conn index))
   (log/info "Creating policy: " index)
   (index/create-policy! conn index (:policy config))
-  (log/info "Creating template: " index)
+  (log/info "Creating index template: " index)
   (index/create-index-template! conn index (:template config))
-  (log/infof "updated template: %s" index))
+  (log/infof "Updated index template: %s" index))
 
 (defn system-exit-error
   []
@@ -173,6 +173,7 @@
 
 (s/defn update-ilm-settings!
   [{:keys [conn index props] :as es-conn} :- ESConnState]
+  (upsert-index-template! es-conn)
   (let [existing-indices  (get-existing-indices conn index)
         write-index (:write-index props)
         real-index (filter (fn [[_ {:keys [aliases]}]] (get aliases (keyword write-index))) existing-indices)
@@ -190,13 +191,9 @@
         _ (when-not (= {:acknowledged true} update-alias-res)
             (throw (ex-info "Cannot update ilm settings: failed to update write alias." update-alias-res)))
         _ (log/infof "updated write alias: %s" write-index)
-        create-policy-res (index/create-policy! conn index policy)
-        _ (when-not (= {:acknowledged true} create-policy-res)
-            (throw (ex-info "Cannot update ilm settings: failed to create policy." create-policy-res)))
-        _ (log/infof "created policy: %s" index)
         update-settings-res (index/update-settings! conn index lifecycle-update)]
     (when-not (= {:acknowledged true} update-settings-res)
-      (throw (ex-info "Cannot update ilm settings: failed to create policy." create-policy-res)))
+      (throw (ex-info "Cannot update ilm settings." update-settings-res)))
     (log/infof "updated settings with lifecycle" index)
     true))
 

test/ctia/stores/es/init_test.clj

@@ -427,7 +427,11 @@
     (doseq [[_ real-index-updated] updated-indices]
       (is (= {:name index :rollover_alias write-index}
              (get-in real-index-updated [:settings :index :lifecycle]))
-          "lifecycle must be added to all indices"))))
+          "lifecycle must be added to all indices"))
+    (is (nil? (index/get-template conn index))
+        "Legacy template must be deleted")
+    (is (seq (index/get-index-template conn index))
+        "Index template must be created")))
 
 (deftest update-ilm-settings!-test
   (let [indexname "ctia_ilm_migration_sighting"
@@ -441,8 +445,9 @@
       #(es-helpers/clean-es-state! % (str indexname "*"))
       (let [services (->ESConnServices)
             props (prepare-props (mk-props indexname) version)
-            {:keys [conn index props] :as store-conn} (legacy-init-es-conn! props services)
-            _ (force-n-rollover! store-conn 4)
+            legacy-store-conn (legacy-init-es-conn! props services)
+            _ (force-n-rollover! legacy-store-conn 4)
+            store-conn (sut/init-es-conn! props services)
             _ (is (true? (sut/update-ilm-settings! store-conn)))]
         (check-ilm-migration store-conn)))))
 
@@ -470,25 +475,26 @@
               _ (assert (seq (index/get-template legacy-conn legacy-index)))
 
               ;; shorter lifecycle poll interval for test purpose
-              _ (clojure.pprint/pprint (es-helpers/update-cluster-lifecycle-poll_interval conn "1s"))
+              _ (es-helpers/update-cluster-lifecycle-poll_interval conn "1s")
 
               ;; restart CTIA with migrate-to-ilm
               ilm-migration-props (assoc base-props
                                          :migrate-to-ilm true
+                                         :update-mappings false
+                                         :update-settings false
+                                         :refresh-mappings false
                                          :rollover {:max_docs 1}
                                          )
               {:keys [conn index props] :as store-conn}
               (sut/init-es-conn! ilm-migration-props services)]
           (check-ilm-migration store-conn)
-          (is (nil? (index/get-template conn index)))
           ;; check that ILM properly rollover
           ;; max_docs is set to 1 and indices.lifecycle.poll_interval to 1s in ES container
           ;; we create 1 doc, wait long enough to let the rollover to be triggered and the index created
           (doc/create-doc conn (:write-index props) {:id "doc-1"} {:refresh "true"})
           (Thread/sleep 5000)
           (is (= 6 (count (index/get conn index))))
-          (es-helpers/update-cluster-lifecycle-poll_interval conn "10m")
-          )))))
+          (es-helpers/update-cluster-lifecycle-poll_interval conn "10m"))))))
 
 (deftest init-with-legacy-state-without-update
   ;; this whole test is about ensuring

test/ctia/test_helpers/es.clj

@@ -262,7 +262,8 @@
   (es-index/delete! conn index-pattern)
   (es-index/delete-template! conn index-pattern)
   (es-index/delete-index-template! conn index-pattern)
-  (es-index/delete-policy! conn index-pattern))
+  ;; delete policy does not work with wildcard, try real index
+  (es-index/delete-policy! conn (string/replace index-pattern "*" "")))
 
 (defmacro for-each-es-version
   "for each given ES version: