datadog-integration: updated consul-server agent telemetry-config.jso…

…n with dd specific items as well as additional missing VM based options, unit tests, dd unix socket integration, dd agent acl token generation, deployment override failsafes
hashicorp · Feb 9, 2024 · 8b6a9d8 · 8b6a9d8
1 parent 7221281
commit 8b6a9d8
Show file tree

Hide file tree

Showing 12 changed files with 859 additions and 12 deletions.
diff --git a/charts/consul/templates/_helpers.tpl b/charts/consul/templates/_helpers.tpl
@@ -145,7 +145,7 @@ substitution for HOST_IP/POD_IP/HOSTNAME. Useful for dogstats telemetry. The out
 is passed to consul as a -config-file param on command line.
 */}}
 {{- define "consul.extraconfig" -}}
-              cp /consul/tmp/extra-config/extra-from-values.json /consul/extra-config/extra-from-values.json
+              cp /consul/config/extra-from-values.json /consul/extra-config/extra-from-values.json
               [ -n "${HOST_IP}" ] && sed -Ei "s|HOST_IP|${HOST_IP?}|g" /consul/extra-config/extra-from-values.json
               [ -n "${POD_IP}" ] && sed -Ei "s|POD_IP|${POD_IP?}|g" /consul/extra-config/extra-from-values.json
               [ -n "${HOSTNAME}" ] && sed -Ei "s|HOSTNAME|${HOSTNAME?}|g" /consul/extra-config/extra-from-values.json
@@ -519,3 +519,92 @@ Usage: {{ template "consul.validateResourceAPIs" . }}
 {{fail "When the value global.experiments.resourceAPIs is set, apiGateway.enabled is currently unsupported."}}
 {{- end }}
 {{- end }}
+
+{{/*
+Validation for Consul Datadog Integration deployment:
+
+Fail if global.metrics.datadogIntegration.enabled=true and global.metrics.enabled or global.metrics.enableAgentMetrics are unset
+    - global.metrics.datadogIntegration.enabled=true
+    - global.metrics.enableAgentMetrics=false || global.metrics.enabled=false
+
+Fail if Consul OpenMetrics (Prometheus) and DogStatsD metrics are both enabled.
+    - global.metrics.datadogIntegration.dogstatsd.enabled (scrapes `/v1/agent/metrics?format=prometheus` via the `use_prometheus_endpoint` option)
+    - global.metrics.datadogIntegration.openMetricsPrometheus.enabled (scrapes `/v1/agent/metrics?format=prometheus`)
+    - see https://docs.datadoghq.com/integrations/consul/?tab=host#host for recommendation to not have both
+
+Usage: {{ template "consul.validateDatadogConfiguration" . }}
+
+*/}}
+
+{{- define "consul.validateDatadogConfiguration" -}}
+{{- if and .Values.global.metrics.datadogIntegration.enabled (or (not .Values.global.metrics.enableAgentMetrics) (not .Values.global.metrics.enabled) )}}
+{{fail "When enabling datadog metrics collection, the /v1/agent/metrics is required to be accessible, therefore global.metrics.enableAgentMetrics and global.metrics.enabled must be also be enabled."}}
+{{- end }}
+{{- if and .Values.global.metrics.datadogIntegration.dogstatsd.enabled .Values.global.metrics.datadogIntegration.openMetricsPrometheus.enabled }}
+{{fail "You must have one of DogStatsD (global.metrics.datadogIntegration.dogstatsd.enabled) or OpenMetrics (global.metrics.datadogIntegration.openMetricsPrometheus.enabled) enabled, not both as this is an unsupported configuration." }}
+{{- end }}
+{{- end -}}
+
+{{/*
+Sets the dogstatsd_addr field of the agent configuration dependent on the
+socket transport type being used:
+  - "UDS" (Unix Domain Socket): prefixes "unix://" to URL and appends path to socket (i.e., unix:///var/run/datadog/dsd.socket)
+  - "UDP" (User Datagram Protocol): adds no prefix and appends dogstatsd port number to hostname/IP (i.e., 172.20.180.10:8125)
+- global.metrics.enableDatadogIntegration.dogstatsd configuration
+
+Usage: {{ template "consul.dogstatsdAaddressInfo" . }}
+*/}}
+
+{{- define "consul.dogstatsdAaddressInfo" -}}
+{{- if (and .Values.global.metrics.datadogIntegration.enabled .Values.global.metrics.datadogIntegration.dogstatsd.enabled) }}
+        "dogstatsd_addr": "{{- if eq .Values.global.metrics.datadogIntegration.dogstatsd.socketTransportType "UDS" }}unix://{{ .Values.global.metrics.datadogIntegration.dogstatsd.dogstatsdAddr }}{{- else }}{{ .Values.global.metrics.datadogIntegration.dogstatsd.dogstatsdAddr | trimAll "\"" }}:{{ .Values.global.metrics.datadogIntegration.dogstatsd.dogstatsdPort | toString }}{{- end }}",{{- end }}
+{{- end -}}
+
+{{/*
+Configures the metrics prefixing that's required to either allow or dissallow certaing RPC or gRPC server calls:
+
+Usage: {{ template "consul.metricsPrefixFiltering" . }}
+*/}}
+{{- define "consul.metricsPrefixFiltering" -}}
+{{- $allowList := .Values.global.metrics.metricsPrefixFiltering.allowList }}
+{{- $blockList := .Values.global.metrics.metricsPrefixFiltering.blockList }}
+{{- if and (not (empty $allowList)) (not (empty $blockList)) }}
+        "prefix_filter": [{{- range $index, $value := concat $allowList $blockList -}}
+    "{{- if (has $value $allowList) }}{{ printf "+%s" ($value | trimAll "\"") }}{{- else }}{{ printf "-%s" ($value | trimAll "\"") }}{{- end }}"{{- if lt $index (sub (len (concat $allowList $blockList)) 1) -}},{{- end -}}
+  {{- end -}}],
+{{- else if not (empty $allowList) }}
+        "prefix_filter": [{{- range $index, $value := $allowList -}}
+    "{{ printf "+%s" ($value | trimAll "\"") }}"{{- if lt $index (sub (len $allowList) 1) -}},{{- end -}}
+  {{- end -}}],
+{{- else if not (empty $blockList) }}
+        "prefix_filter": [{{- range $index, $value := $blockList -}}
+    "{{ printf "-%s" ($value | trimAll "\"") }}"{{- if lt $index (sub (len $blockList) 1) -}},{{- end -}}
+  {{- end -}}],
+{{- end }}
+{{- end -}}
+
+{{/*
+Retrieves the global consul/consul-enterprise version string for use with labels or tags.
+Requirements for valid labels:
+ -  a valid label must be an empty string or consist of
+    =>  alphanumeric characters
+    =>  '-', '_' or '.'
+    =>  must start and end with an alphanumeric character
+        (e.g. 'MyValue',  or 'my_value',  or '12345', regex used for validation is
+        '(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?')
+
+Usage: {{ template "consul.versionInfo" }}
+*/}}
+{{- define "consul.versionInfo" -}}
+{{- $imageVersion := regexSplit ":" .Values.global.image -1 }}
+{{- $versionInfo := printf "%s" (index $imageVersion 1 ) | trimSuffix "\"" }}
+{{- $sanitizedVersion := "" }}
+{{- $pattern := "^([A-Za-z0-9][-A-Za-z0-9_.]*[A-Za-z0-9])?$" }}
+{{- if not (regexMatch $pattern $versionInfo) -}}
+    {{- $sanitizedVersion = regexReplaceAll "[^A-Za-z0-9-_.]|sha256" $versionInfo "" }}
+    {{- $sanitizedVersion = printf "%s" (trimSuffix "-" (trimPrefix "-" $sanitizedVersion)) -}}
+{{- else }}
+    {{- $sanitizedVersion = $versionInfo }}
+{{- end -}}
+{{- printf "%s" $sanitizedVersion | quote }}
+{{- end -}}
diff --git a/charts/consul/templates/server-acl-init-job.yaml b/charts/consul/templates/server-acl-init-job.yaml
@@ -2,6 +2,9 @@
 {{- if (and $serverEnabled .Values.externalServers.enabled) }}{{ fail "only one of server.enabled or externalServers.enabled can be set" }}{{ end -}}
 {{- if (or $serverEnabled .Values.externalServers.enabled) }}
 {{- if and .Values.global.acls.createReplicationToken (not .Values.global.acls.manageSystemACLs) }}{{ fail "if global.acls.createReplicationToken is true, global.acls.manageSystemACLs must be true" }}{{ end -}}
+{{- if .Values.global.metrics.enableDatadogIntegration }}
+{{- if and .Values.global.metrics.enableDatadogIntegration.createAgentToken (not .Values.global.acls.manageSystemACLs) }}{{ fail "if global.metrics.enableDatadogIntegration.createAgentToken is true, global.acls.manageSystemACLs must be true" }}{{ end -}}
+{{- end }}
 {{- if .Values.global.bootstrapACLs }}{{ fail "global.bootstrapACLs was removed, use global.acls.manageSystemACLs instead" }}{{ end -}}
 {{- if .Values.global.acls.manageSystemACLs }}
 {{- if or (and .Values.global.acls.bootstrapToken.secretName (not .Values.global.acls.bootstrapToken.secretKey))  (and .Values.global.acls.bootstrapToken.secretKey (not .Values.global.acls.bootstrapToken.secretName))}}{{ fail "both global.acls.bootstrapToken.secretKey and global.acls.bootstrapToken.secretName must be set if one of them is provided" }}{{ end -}}
@@ -273,6 +276,10 @@ spec:
             -create-enterprise-license-token=true \
             {{- end }}
 
+            {{- if (and .Values.global.metrics.datadogIntegration.enabled .Values.global.acls.manageSystemACLs) }}
+            -create-dd-agent-token=true \
+            {{- end }}
+
             {{- if .Values.server.snapshotAgent.enabled }}
             -snapshot-agent=true \
             {{- end }}

diff --git a/charts/consul/templates/server-config-configmap.yaml b/charts/consul/templates/server-config-configmap.yaml
@@ -192,8 +192,15 @@ data:
   telemetry-config.json: |-
     {
       "telemetry": {
-        "prometheus_retention_time": "{{ .Values.global.metrics.agentMetricsRetentionTime }}"
-      }
+        "prometheus_retention_time": "{{ .Values.global.metrics.agentMetricsRetentionTime }}",
+        "disable_hostname": {{ .Values.global.metrics.disableAgentHostName }},{{ template "consul.metricsPrefixFiltering" . }}
+        "enable_host_metrics": {{ .Values.global.metrics.enableHostMetrics }}{{- if .Values.global.metrics.datadogIntegration.dogstatsd.enabled }},{{ template "consul.dogstatsdAaddressInfo" . }}
+        {{- if .Values.global.metrics.datadogIntegration.dogstatsd.enabled }}
+        "dogstatsd_tags": {{ .Values.global.metrics.datadogIntegration.dogstatsd.dogstatsdTags | toJson }}
+        {{- end }}
+        {{- end }}
+      },
+      "enable_debug": {{ .Values.global.metrics.enableConsulAgentDebug }}
     }
   {{- end }}
   {{- if and .Values.server.auditLogs.enabled .Values.global.acls.manageSystemACLs }}

diff --git a/charts/consul/templates/server-statefulset.yaml b/charts/consul/templates/server-statefulset.yaml
@@ -19,6 +19,7 @@
 {{- end -}}
 {{ template "consul.validateRequiredCloudSecretsExist" . }}
 {{ template "consul.validateCloudSecretKeys" . }}
+{{ template "consul.validateDatadogConfiguration" . }}
 # StatefulSet to run the actual Consul server cluster.
 apiVersion: apps/v1
 kind: StatefulSet
@@ -62,6 +63,11 @@ spec:
         release: {{ .Release.Name }}
         component: server
         hasDNS: "true"
+        {{- if .Values.global.metrics.datadogIntegration.enabled }}
+        "tags.datadoghq.com/version": {{ template "consul.versionInfo" . }}
+        "tags.datadoghq.com/env": {{ template "consul.name" . }}
+        "tags.datadoghq.com/service": "consul-server"
+        {{- end }}
         {{- if .Values.server.extraLabels }}
           {{- toYaml .Values.server.extraLabels | nindent 8 }}
         {{- end }}
@@ -133,6 +139,64 @@ spec:
         "prometheus.io/port": "8500"
         "prometheus.io/scheme": "http"
         {{- end }}
+        {{- if .Values.global.metrics.datadogIntegration.enabled }}
+        "ad.datadoghq.com/tolerate-unready": "true"
+        "ad.datadoghq.com/consul.logs": {{ .Values.global.metrics.datadogIntegration.dogstatsd.dogstatsdTags | squote }}
+        {{- if .Values.global.metrics.datadogIntegration.openMetricsPrometheus.enabled }}
+        "ad.datadoghq.com/consul.checks": |
+          {
+            "openmetrics": {
+              "init_config": {},
+              "instances": [
+                {
+        {{- if .Values.global.tls.enabled }}
+                  "openmetrics_endpoint": "https://consul-server.{{ .Release.Namespace }}.svc:8501/v1/agent/metrics?format=prometheus",
+                  "tls_cert": "/etc/datadog-agent/conf.d/consul.d/certs/tls.crt",
+                  "tls_private_key": "/etc/datadog-agent/conf.d/consul.d/certs/tls.key",
+                  "tls_ca_cert": "/etc/datadog-agent/conf.d/consul.d/ca/tls.crt",
+        {{- else }}
+                  "openmetrics_endpoint": "http://%%host%%:8500/v1/agent/metrics?format=prometheus",
+        {{- end }}
+        {{- if ( .Values.global.acls.manageSystemACLs) }}
+                  "headers": {
+                    "X-Consul-Token": "ENC[k8s_secret@{{ .Release.Namespace }}/{{ .Release.Namespace }}-datadog-agent-metrics-acl-token/token]"
+                  },
+        {{- end }}
+                  "namespace": "{{ .Release.Namespace }}",
+                  "metrics": [ ".*" ]
+                }
+              ]
+            }
+          }
+        {{- else }}
+        "ad.datadoghq.com/consul.checks": |
+          {
+            "consul": {
+              "init_config": {},
+              "instances": [
+                {
+        {{- if .Values.global.tls.enabled }}
+                  "url": "https://consul-server.{{ .Release.Namespace }}.svc:8501",
+                  "tls_cert": "/etc/datadog-agent/conf.d/consul.d/certs/tls.crt",
+                  "tls_private_key": "/etc/datadog-agent/conf.d/consul.d/certs/tls.key",
+                  "tls_ca_cert": "/etc/datadog-agent/conf.d/consul.d/ca/tls.crt",
+        {{- else }}
+                  "url": "http://consul-server.consul.svc:8500",
+        {{- end }}
+                  "use_prometheus_endpoint": true,
+        {{- if ( .Values.global.acls.manageSystemACLs) }}
+                  "acl_token": "ENC[k8s_secret@{{ .Release.Namespace }}/{{ .Release.Namespace }}-datadog-agent-metrics-acl-token/token]",
+        {{- end }}
+                  "new_leader_checks": true,
+                  "network_latency_checks": true,
+                  "catalog_checks": true,
+                  "auth_type": "basic"
+                }
+              ]
+            }
+          }
+        {{- end }}
+        {{- end }}
         {{- end }}
     spec:
     {{- if .Values.server.affinity }}
@@ -219,6 +283,12 @@ spec:
           emptyDir:
             medium: "Memory"
         {{- end }}
+        {{- if and .Values.global.metrics.datadogIntegration.enabled .Values.global.metrics.datadogIntegration.dogstatsd.enabled (eq .Values.global.metrics.datadogIntegration.dogstatsd.socketTransportType "UDS" ) }}
+        - name: dsdsocket
+          hostPath:
+            path: /var/run/datadog
+            type: DirectoryOrCreate
+        {{- end }}
         {{- range .Values.server.extraVolumes }}
         - name: userconfig-{{ .name }}
           {{ .type }}:
@@ -257,7 +327,7 @@ spec:
         {{- include "consul.restrictedSecurityContext" . | nindent 8 }}
       containers:
         - name: consul
-          image: "{{ default .Values.global.image .Values.server.image }}"
+          image: "{{ default .Values.global.image .Values.server.image | trimPrefix "\"" | trimSuffix "\"" }}"
           imagePullPolicy: {{ .Values.global.imagePullPolicy }}
           env:
             - name: ADVERTISE_IP
@@ -464,6 +534,11 @@ spec:
               mountPath: /consul/license
               readOnly: true
             {{- end }}
+            {{- if and .Values.global.metrics.datadogIntegration.enabled .Values.global.metrics.datadogIntegration.dogstatsd.enabled (eq .Values.global.metrics.datadogIntegration.dogstatsd.socketTransportType "UDS" ) }}
+            - name: dsdsocket
+              mountPath: /var/run/datadog
+              readOnly: true
+            {{- end }}
             {{- range .Values.server.extraVolumes }}
             - name: userconfig-{{ .name }}
               readOnly: true

diff --git a/charts/consul/test/unit/server-acl-init-job.bats b/charts/consul/test/unit/server-acl-init-job.bats
@@ -2444,3 +2444,35 @@ load _helpers
     yq 'any(contains("-enable-resource-apis=true"))' | tee /dev/stderr)
   [ "${actual}" = "true" ]
 }
+
+#--------------------------------------------------------------------
+# global.metrics.datadogIntegration
+
+@test "serverACLInit/Job: -create-dd-agent-token not set when datadogIntegration=false and manageSystemACLs=true" {
+  cd `chart_dir`
+  local command=$(helm template \
+      -s templates/server-acl-init-job.yaml  \
+      --set 'global.acls.manageSystemACLs=true' \
+      . | tee /dev/stderr |
+      yq '.spec.template.spec.containers[0].command' | tee /dev/stderr)
+
+  local actual=$( echo "$command" |
+    yq 'any(contains("-create-dd-agent-token"))' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+}
+
+@test "serverACLInit/Job: -create-dd-agent-token set when global.metrics.datadogIntegration=true and global.acls.manageSystemACLs=true" {
+  cd `chart_dir`
+  local command=$(helm template \
+      -s templates/server-acl-init-job.yaml  \
+      --set 'global.metrics.enabled=true'  \
+      --set 'global.metrics.enableAgentMetrics=true'  \
+      --set 'global.metrics.datadogIntegration.enabled=true' \
+      --set 'global.acls.manageSystemACLs=true' \
+      . | tee /dev/stderr |
+      yq '.spec.template.spec.containers[0].command' | tee /dev/stderr)
+
+  local actual=$( echo "$command" |
+    yq 'any(contains("-create-dd-agent-token"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}