[elasticsearch] SSL by default (#1519)

elasticsearch/README.md

@@ -113,6 +113,7 @@ support multiple versions with minimal changes.
 | `antiAffinity`                     | Setting this to hard enforces the [anti-affinity][] rules. If it is set to soft it will be done "best effort". Other values will be ignored                                                                                                                                                                       | `hard`                                           |
 | `clusterHealthCheckParams`         | The [Elasticsearch cluster health status params][] that will be used by readiness [probe][] command                                                                                                                                                                                                               | `wait_for_status=green&timeout=1s`               |
 | `clusterName`                      | This will be used as the Elasticsearch [cluster.name][] and should be unique per cluster in the namespace                                                                                                                                                                                                         | `elasticsearch`                                  |
+| `createCert`                       | This will automatically create the SSL certificates                                                                                                                                                                                                                                                               | `true`                                           |
 | `enableServiceLinks`               | Set to false to disabling service links, which can cause slow pod startup times when there are many services in the current namespace.                                                                                                                                                                            | `true`                                           |
 | `envFrom`                          | Templatable string to be passed to the [environment from variables][] which will be appended to the `envFrom:` definition for the container                                                                                                                                                                       | `[]`                                             |
 | `esConfig`                         | Allows you to add any config files in `/usr/share/elasticsearch/config/` such as `elasticsearch.yml` and `log4j2.properties`. See [values.yaml][] for an example of the formatting                                                                                                                                | `{}`                                             |
@@ -265,14 +266,13 @@ sufficient.
 ### How to deploy clusters with security (authentication and TLS) enabled?
 
 This Helm chart can generate a [Kubernetes Secret][] or use an existing one to
-setup Elastic credentials.
+setup Elastic credentials. 
 
 This Helm chart can use existing [Kubernetes Secret][] to setup Elastic
 certificates for example. These secrets should be created outside of this chart
 and accessed using [environment variables][] and volumes.
 
-An example of Elasticsearch cluster using security can be found in
-[examples/security][].
+This chart is setting TLS and creating a certificate by default, but you can also provide your own certs as a K8S secret. An example of configuration for providing existing certificates can be found in [examples/security][].
 
 ### How to migrate from helm/charts stable chart?
 

elasticsearch/examples/config/test/goss.yaml

@@ -1,18 +1,20 @@
 http:
-  http://localhost:9200/_cluster/health:
+  https://localhost:9200/_cluster/health:
     status: 200
     timeout: 2000
+    allow-insecure: true
     username: elastic
     password: "{{ .Env.ELASTIC_PASSWORD }}"
     body:
       - "green"
       - '"number_of_nodes":1'
       - '"number_of_data_nodes":1'
 
-  http://localhost:9200:
+  https://localhost:9200:
     status: 200
     timeout: 2000
     username: elastic
+    allow-insecure: true
     password: "{{ .Env.ELASTIC_PASSWORD }}"
     body:
       - '"cluster_name" : "config"'

elasticsearch/examples/default/test/goss.yaml

@@ -3,19 +3,21 @@ kernel-param:
     value: "262144"
 
 http:
-  http://elasticsearch-master:9200/_cluster/health:
+  https://elasticsearch-master:9200/_cluster/health:
     status: 200
     timeout: 2000
     username: elastic
+    allow-insecure: true
     password: "{{ .Env.ELASTIC_PASSWORD }}"
     body:
       - "green"
       - '"number_of_nodes":3'
       - '"number_of_data_nodes":3'
 
-  http://localhost:9200:
+  https://localhost:9200:
     status: 200
     timeout: 2000
+    allow-insecure: true
     username: elastic
     password: "{{ .Env.ELASTIC_PASSWORD }}"
     body:

elasticsearch/examples/multi/client.yaml

@@ -8,6 +8,26 @@ extraEnvs:
       secretKeyRef:
         name: multi-master-credentials
         key: password
+  - name: xpack.security.enabled 
+    value: "true" 
+  - name: xpack.security.transport.ssl.enabled 
+    value: "true" 
+  - name: xpack.security.http.ssl.enabled 
+    value: "true" 
+  - name: xpack.security.transport.ssl.verification_mode 
+    value: "certificate" 
+  - name: xpack.security.transport.ssl.key 
+    value: "/usr/share/elasticsearch/config/certs/tls.key" 
+  - name: xpack.security.transport.ssl.certificate 
+    value: "/usr/share/elasticsearch/config/certs/tls.crt"
+  - name: xpack.security.transport.ssl.certificate_authorities
+    value: "/usr/share/elasticsearch/config/certs/ca.crt"
+  - name: xpack.security.http.ssl.key
+    value: "/usr/share/elasticsearch/config/certs/tls.key"
+  - name: xpack.security.http.ssl.certificate
+    value: "/usr/share/elasticsearch/config/certs/tls.crt"
+  - name: xpack.security.http.ssl.certificate_authorities
+    value: "/usr/share/elasticsearch/config/certs/ca.crt"
 
 roles: []
 
@@ -22,3 +42,9 @@ esConfig:
 
 secret:
   enabled: false
+
+createCert: false
+secretMounts:
+  - name: elastic-certificates
+    secretName: multi-master-certs
+    path: /usr/share/elasticsearch/config/certs

elasticsearch/examples/multi/data.yaml

@@ -8,6 +8,26 @@ extraEnvs:
       secretKeyRef:
         name: multi-master-credentials
         key: password
+  - name: xpack.security.enabled 
+    value: "true" 
+  - name: xpack.security.transport.ssl.enabled 
+    value: "true" 
+  - name: xpack.security.http.ssl.enabled 
+    value: "true" 
+  - name: xpack.security.transport.ssl.verification_mode 
+    value: "certificate" 
+  - name: xpack.security.transport.ssl.key 
+    value: "/usr/share/elasticsearch/config/certs/tls.key" 
+  - name: xpack.security.transport.ssl.certificate 
+    value: "/usr/share/elasticsearch/config/certs/tls.crt" 
+  - name: xpack.security.transport.ssl.certificate_authorities 
+    value: "/usr/share/elasticsearch/config/certs/ca.crt" 
+  - name: xpack.security.http.ssl.key 
+    value: "/usr/share/elasticsearch/config/certs/tls.key" 
+  - name: xpack.security.http.ssl.certificate 
+    value: "/usr/share/elasticsearch/config/certs/tls.crt" 
+  - name: xpack.security.http.ssl.certificate_authorities 
+    value: "/usr/share/elasticsearch/config/certs/ca.crt" 
 
 roles:
   - data
@@ -20,3 +40,9 @@ roles:
 
 secret:
   enabled: false
+
+createCert: false
+secretMounts:
+  - name: elastic-certificates
+    secretName: multi-master-certs
+    path: /usr/share/elasticsearch/config/certs

elasticsearch/examples/multi/test/goss.yaml

@@ -1,7 +1,8 @@
 http:
-  http://localhost:9200/_cluster/health:
+  https://localhost:9200/_cluster/health:
     status: 200
     timeout: 2000
+    allow-insecure: true
     username: elastic
     password: "{{ .Env.ELASTIC_PASSWORD }}"
     body:

elasticsearch/examples/openshift/test/goss.yaml

@@ -1,5 +1,5 @@
 http:
-  http://localhost:9200/_cluster/health:
+  https://localhost:9200/_cluster/health:
     status: 200
     timeout: 2000
     username: elastic
@@ -9,7 +9,7 @@ http:
       - '"number_of_nodes":3'
       - '"number_of_data_nodes":3'
 
-  http://localhost:9200:
+  https://localhost:9200:
     status: 200
     timeout: 2000
     username: elastic

elasticsearch/examples/security/values.yaml

@@ -2,6 +2,8 @@
 clusterName: "security"
 nodeGroup: "master"
 
+createCert: false
+
 roles:
   - master
   - ingest

elasticsearch/examples/upgrade/Makefile

@@ -5,11 +5,13 @@ include ../../../helpers/examples.mk
 CHART := elasticsearch
 RELEASE := helm-es-upgrade
 FROM := 7.4.0	# versions before 7.4.O aren't compatible with Kubernetes >= 1.16.0
-TO := 7.13.0	# upgrade from 7.x to 8.0.0-SNAPSHOT currently doesn't work
 
 install:
-	../../../helpers/upgrade.sh --chart $(CHART) --release $(RELEASE) --from $(FROM) --to $(TO)
-	kubectl rollout status statefulset upgrade-master
+	../../../helpers/upgrade.sh --chart $(CHART) --release $(RELEASE) --from $(FROM)
+	# Rolling upgrade doesn't work when upgrading from clusters with security disabled.
+	# This is because nodes with security enabled can't join a cluster with security disabled.
+	# Every nodes need to be recreated at the same time so they can recreate a cluster with security enabled
+	kubectl delete pod --selector=app=upgrade-master
 
 test: install goss
 

elasticsearch/examples/upgrade/test/goss.yaml

@@ -1,16 +1,23 @@
 http:
-  http://localhost:9200/_cluster/health:
+  https://localhost:9200/_cluster/health:
     status: 200
+    username: elastic
+    password: "{{ .Env.ELASTIC_PASSWORD }}"
+    allow-insecure: true
     timeout: 2000
     body:
       - "green"
       - '"number_of_nodes":3'
       - '"number_of_data_nodes":3'
 
-  http://localhost:9200:
+  https://localhost:9200:
     status: 200
+    username: elastic
+    password: "{{ .Env.ELASTIC_PASSWORD }}"
+    allow-insecure: true
     timeout: 2000
+    allow-insecure: true
     body:
-      - '"number" : "7.13.0"'
+      - '"number" : "8.0.0-SNAPSHOT"'
       - '"cluster_name" : "upgrade"'
       - "You Know, for Search"

elasticsearch/examples/upgrade/values.yaml

@@ -1,2 +1,6 @@
 ---
 clusterName: upgrade
+# Rolling upgrade doesn't work when upgrading from clusters with security disabled.
+# This is because nodes with security enabled can't join a cluster with security disabled.
+# Every nodes need to be recreated at the same time so they can recreate a cluster with security enabled
+updateStrategy: OnDelete

elasticsearch/templates/_helpers.tpl

@@ -27,6 +27,18 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 {{- end -}}
 {{- end -}}
 
+{{/*
+Generate certificates 
+*/}}
+{{- define "elasticsearch.gen-certs" -}}
+{{- $altNames := list ( printf "%s.%s" (include "elasticsearch.name" .) .Release.Namespace ) ( printf "%s.%s.svc" (include "elasticsearch.name" .) .Release.Namespace ) -}}
+{{- $ca := genCA "elasticsearch-ca" 365 -}}
+{{- $cert := genSignedCert ( include "elasticsearch.name" . ) nil $altNames 365 $ca -}}
+tls.crt: {{ $cert.Cert | toString | b64enc }}
+tls.key: {{ $cert.Key | toString | b64enc }}
+ca.crt: {{ $ca.Cert | toString | b64enc }}
+{{- end -}}
+
 {{- define "elasticsearch.masterService" -}}
 {{- if empty .Values.masterService -}}
 {{- if empty .Values.fullnameOverride -}}

elasticsearch/templates/secret-cert.yaml

@@ -0,0 +1,17 @@
+{{- if .Values.createCert }}
+apiVersion: v1
+kind: Secret
+type: kubernetes.io/tls
+metadata:
+  name: {{ template "elasticsearch.uname" . }}-certs
+  labels:
+    app: {{ template "elasticsearch.uname" . }}
+    chart: "{{ .Chart.Name }}"
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+  annotations:
+    "helm.sh/hook": "pre-install,pre-upgrade"
+    "helm.sh/hook-delete-policy": "before-hook-creation"
+data:
+{{ ( include "elasticsearch.gen-certs" . ) | indent 2 }}
+{{- end }}

elasticsearch/templates/statefulset.yaml

@@ -136,6 +136,11 @@ spec:
           configMap:
             name: {{ template "elasticsearch.uname" . }}-jvm-options
         {{- end }}
+        {{- if .Values.createCert }}
+        - name: elasticsearch-certs
+          secret:
+           secretName: {{ template "elasticsearch.uname" . }}-certs
+        {{- end }}
 {{- if .Values.keystore }}
         - name: keystore
           emptyDir: {}
@@ -333,6 +338,28 @@ spec:
           - name: ES_JAVA_OPTS
             value: "{{ .Values.esJavaOpts }}"
           {{- end }}
+          {{- if .Values.createCert }}
+          - name: xpack.security.enabled
+            value: "true"
+          - name: xpack.security.transport.ssl.enabled
+            value: "true"
+          - name: xpack.security.http.ssl.enabled
+            value: "true"
+          - name: xpack.security.transport.ssl.verification_mode
+            value: "certificate"
+          - name: xpack.security.transport.ssl.key
+            value: "/usr/share/elasticsearch/config/certs/tls.key"
+          - name: xpack.security.transport.ssl.certificate
+            value: "/usr/share/elasticsearch/config/certs/tls.crt"
+          - name: xpack.security.transport.ssl.certificate_authorities
+            value: "/usr/share/elasticsearch/config/certs/ca.crt"
+          - name: xpack.security.http.ssl.key
+            value: "/usr/share/elasticsearch/config/certs/tls.key"
+          - name: xpack.security.http.ssl.certificate
+            value: "/usr/share/elasticsearch/config/certs/tls.crt"
+          - name: xpack.security.http.ssl.certificate_authorities
+            value: "/usr/share/elasticsearch/config/certs/ca.crt"
+          {{- end }}
 {{- if .Values.extraEnvs }}
 {{ toYaml .Values.extraEnvs | indent 10 }}
 {{- end }}
@@ -345,6 +372,11 @@ spec:
           - name: "{{ template "elasticsearch.uname" . }}"
             mountPath: /usr/share/elasticsearch/data
           {{- end }}
+          {{- if .Values.createCert }}
+          - name: elasticsearch-certs
+            mountPath: /usr/share/elasticsearch/config/certs
+            readOnly: true
+          {{- end }}
 {{ if .Values.keystore }}
           - name: keystore
             mountPath: /usr/share/elasticsearch/config/elasticsearch.keystore

elasticsearch/tests/elasticsearch_test.py

@@ -72,7 +72,7 @@ def test_defaults():
     assert c["readinessProbe"]["timeoutSeconds"] == 5
 
     assert "curl" in c["readinessProbe"]["exec"]["command"][-1]
-    assert "http://127.0.0.1:9200" in c["readinessProbe"]["exec"]["command"][-1]
+    assert "https://127.0.0.1:9200" in c["readinessProbe"]["exec"]["command"][-1]
 
     # Resources
     assert c["resources"] == {
@@ -451,9 +451,10 @@ def test_adding_a_secret_mount():
         "mountPath": "/usr/share/elasticsearch/config/certs",
         "name": "elastic-certificates",
     }
-    assert s["volumes"] == [
-        {"name": "elastic-certificates", "secret": {"secretName": "elastic-certs"}}
-    ]
+    assert {
+        "name": "elastic-certificates",
+        "secret": {"secretName": "elastic-certs"},
+    } in s["volumes"]
 
 
 def test_adding_a_secret_mount_with_subpath():
@@ -806,10 +807,10 @@ def test_dont_add_data_volume_when_persistance_is_disabled():
     r = helm_template(config)
     assert "volumeClaimTemplates" not in r["statefulset"][uname]["spec"]
     assert (
-        r["statefulset"][uname]["spec"]["template"]["spec"]["containers"][0][
+        {"name": "elasticsearch-master", "mountPath": "/usr/share/elasticsearch/data"}
+        not in r["statefulset"][uname]["spec"]["template"]["spec"]["containers"][0][
             "volumeMounts"
         ]
-        == None
     )
 
 
@@ -1117,13 +1118,6 @@ def test_adding_pod_labels():
 
 
 def test_keystore_enable():
-    config = ""
-
-    r = helm_template(config)
-    s = r["statefulset"][uname]["spec"]["template"]["spec"]
-
-    assert s["volumes"] == None
-
     config = """
 keystore:
   - secretName: test

elasticsearch/values.yaml

@@ -35,6 +35,8 @@ esConfig: {}
 #  log4j2.properties: |
 #    key = value
 
+createCert: true
+
 esJvmOptions: {}
 #  processors.options: |
 #    -XX:ActiveProcessorCount=3
@@ -185,7 +187,7 @@ podManagementPolicy: "Parallel"
 # If you experience slow pod startups you probably want to set this to `false`.
 enableServiceLinks: true
 
-protocol: http
+protocol: https
 httpPort: 9200
 transportPort: 9300