use sidecar for es graceful termination

Signed-off-by: Taehyun Kim <[email protected]>

stable/elasticsearch/templates/configmap.yaml

@@ -115,7 +115,7 @@ data:
     logger.searchguard.level = info
 {{- end }}
 {{- if .Values.data.hooks.drain.enabled }}
-  data-pre-stop-hook.sh: |-
+  pre-stop-hook.sh: |-
     #!/bin/bash
     exec &> >(tee -a "/var/log/elasticsearch-hooks.log")
     NODE_NAME=${HOSTNAME}
@@ -137,7 +137,7 @@ data:
       sleep 1
     done
     echo "Node ${NODE_NAME} is ready to shutdown"
-  data-post-start-hook.sh: |-
+  post-start-hook.sh: |-
     #!/bin/bash
     exec &> >(tee -a "/var/log/elasticsearch-hooks.log")
     NODE_NAME=${HOSTNAME}
@@ -152,35 +152,3 @@ data:
     fi
     echo "Node ${NODE_NAME} is ready to be used"
 {{- end }}
-
-{{- if hasPrefix "7." .Values.appVersion }}
-  master-pre-stop-hook.sh: |-
-    #!/usr/bin/env bash -e
-    # Exclude node from voting config to prevent https://github.com/elastic/helm-charts/issues/63
-    # reference: https://www.elastic.co/guide/en/elasticsearch/reference/7.0/modules-discovery-adding-removing-nodes.html#modules-discovery-removing-nodes
-    http () {
-        local path="${1}"
-        local method="${2:-GET}"
-        curl -X${method} -s --fail http://{{ template "elasticsearch.client.fullname" . }}:9200${path}
-    }
-
-    NODE_NAME=${HOSTNAME}
-    echo "Exclude the node ${NODE_NAME} from voting config"
-
-    http "/_cluster/voting_config_exclusions/${NODE_NAME}" "POST"
-
-    while true ; do
-      echo -e "Wait for new master node to be elected"
-      if [[ "$(http "/_cat/master")" != *"${NODE_NAME}"* ]]; then
-        break
-      fi
-      sleep 1
-    done
-
-    # Node won't be actually removed from cluster,
-    # so node should be deleted from voting config exclusions
-    echo "Node ${NODE_NAME} is now being deleted from voting config exclusions"
-    http "/_cluster/voting_config_exclusions?wait_for_removal=false" "DELETE"
-
-    echo "Node ${NODE_NAME} is ready to shutdown"
-{{- end }}

stable/elasticsearch/templates/data-statefulset.yaml

@@ -167,10 +167,10 @@ spec:
 {{- if .Values.data.hooks.drain.enabled }}
         - name: config
           mountPath: /pre-stop-hook.sh
-          subPath: data-pre-stop-hook.sh
+          subPath: pre-stop-hook.sh
         - name: config
           mountPath: /post-start-hook.sh
-          subPath: data-post-start-hook.sh
+          subPath: post-start-hook.sh
         lifecycle:
           preStop:
             exec:

stable/elasticsearch/templates/master-statefulset.yaml

@@ -168,15 +168,46 @@ spec:
           subPath: elasticsearch.keystore
           readOnly: true
 {{- end }}
-{{- if hasPrefix "7." .Values.appVersion }}
-        - name: config
-          mountPath: /pre-stop-hook.sh
-          subPath: master-pre-stop-hook.sh
-        lifecycle:
-          preStop:
-            exec:
-              command: ["/bin/bash","/pre-stop-hook.sh"]
-{{- end }}
+      - name: elasticsearch-graceful-termination-handler
+        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+        imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
+        command:
+        - "sh"
+        - -c
+        - |
+          #!/usr/bin/env bash
+          set -eo pipefail
+
+          http () {
+              local path="${1}"
+              if [ -n "${ELASTIC_USERNAME}" ] && [ -n "${ELASTIC_PASSWORD}" ]; then
+                BASIC_AUTH="-u ${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}"
+              else
+                BASIC_AUTH=''
+              fi
+              curl -XGET -s -k --fail ${BASIC_AUTH} http://{{ template "elasticsearch.client.fullname" . }}:9200${path}
+          }
+
+          cleanup () {
+            while true ; do
+              local master="$(http "/_cat/master")"
+              if [[ $master == *"{{ template "elasticsearch.master.fullname" . }}"* && $master != *"${HOSTNAME}"* ]]; then
+                echo "This node is not master."
+                break
+              fi
+              echo "This node is still master, waiting gracefully for it to step down"
+              sleep 1
+            done
+
+            exit 0
+          }
+
+          trap cleanup SIGTERM
+
+          while true; do
+            sleep 60 &
+            wait $!
+          done
 {{- if .Values.image.pullSecrets }}
       imagePullSecrets:
       {{- range $pullSecret := .Values.image.pullSecrets }}