diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f4d60e80a..cec3581f34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ ## UNRELEASED +FEATURES: +* Helm Chart + * Add support for Consul services to utilize Consul DNS for service discovery. Set `dns.enableRedirection` to allow services to + use Consul DNS via the Consul DNS Service. [[GH-833](https://github.com/hashicorp/consul-k8s/pull/833)] +* Control Plane + * Connect: Allow services using Connect to utilize Consul DNS to perform service discovery. [[GH-833](https://github.com/hashicorp/consul-k8s/pull/833)] + BREAKING CHANGES: * Previously [UI metrics](https://www.consul.io/docs/connect/observability/ui-visualization) would be enabled when `global.metrics=false` and `ui.metrics.enabled=-`. If you are no longer seeing UI metrics, diff --git a/charts/consul/templates/_helpers.tpl b/charts/consul/templates/_helpers.tpl index 49676dc329..480c4b8895 100644 --- a/charts/consul/templates/_helpers.tpl +++ b/charts/consul/templates/_helpers.tpl @@ -28,6 +28,18 @@ is passed to consul as a -config-file param on command line. [ -n "${HOSTNAME}" ] && sed -Ei "s|HOSTNAME|${HOSTNAME?}|g" /consul/extra-config/extra-from-values.json {{- end -}} +{{/* +Sets up a list of recusor flags for Consul agents by iterating over the IPs of every nameserver +in /etc/resolv.conf and concatenating them into a string of arguments that can be passed directly +to the consul agent command. +*/}} +{{- define "consul.recursors" -}} + recursor_flags="" + for ip in $(cat /etc/resolv.conf | grep nameserver | cut -d' ' -f2) + do + recursor_flags="$recursor_flags -recursor=$ip" + done +{{- end -}} {{/* Create chart name and version as used by the chart label. diff --git a/charts/consul/templates/client-daemonset.yaml b/charts/consul/templates/client-daemonset.yaml index 7689b3ff8a..28c9512ce6 100644 --- a/charts/consul/templates/client-daemonset.yaml +++ b/charts/consul/templates/client-daemonset.yaml @@ -203,6 +203,10 @@ spec: - | CONSUL_FULLNAME="{{template "consul.fullname" . }}" + {{- if (and .Values.dns.enabled .Values.dns.enableRedirection) }} + {{ template "consul.recursors" }} + {{- end }} + {{ template "consul.extraconfig" }} exec /usr/local/bin/docker-entrypoint.sh consul agent \ @@ -276,6 +280,9 @@ spec: {{- range $value := .Values.global.recursors }} -recursor={{ quote $value }} \ {{- end }} + {{- if (and .Values.dns.enabled .Values.dns.enableRedirection) }} + $recursor_flags \ + {{- end }} -config-file=/consul/extra-config/extra-from-values.json \ -domain={{ .Values.global.domain }} volumeMounts: diff --git a/charts/consul/templates/connect-inject-deployment.yaml b/charts/consul/templates/connect-inject-deployment.yaml index d89635d059..11946b6235 100644 --- a/charts/consul/templates/connect-inject-deployment.yaml +++ b/charts/consul/templates/connect-inject-deployment.yaml @@ -83,8 +83,6 @@ spec: - "/bin/sh" - "-ec" - | - CONSUL_FULLNAME="{{template "consul.fullname" . }}" - consul-k8s-control-plane inject-connect \ -log-level={{ default .Values.global.logLevel .Values.connectInject.logLevel }} \ -log-json={{ .Values.global.logJSON }} \ @@ -108,6 +106,10 @@ spec: {{- else }} -transparent-proxy-default-overwrite-probes=false \ {{- end }} + -resource-prefix={{ template "consul.fullname" . }} \ + {{- if (and .Values.dns.enabled .Values.dns.enableRedirection) }} + -enable-consul-dns=true \ + {{- end }} {{- if .Values.global.openshift.enabled }} -enable-openshift \ {{- end }} diff --git a/charts/consul/templates/server-statefulset.yaml b/charts/consul/templates/server-statefulset.yaml index 2e332cb7ae..5e8cc89e76 100644 --- a/charts/consul/templates/server-statefulset.yaml +++ b/charts/consul/templates/server-statefulset.yaml @@ -192,6 +192,10 @@ spec: - | CONSUL_FULLNAME="{{template "consul.fullname" . }}" + {{- if (and .Values.dns.enabled .Values.dns.enableRedirection) }} + {{ template "consul.recursors" }} + {{- end }} + {{ template "consul.extraconfig" }} exec /usr/local/bin/docker-entrypoint.sh consul agent \ @@ -254,6 +258,9 @@ spec: {{- range $value := .Values.global.recursors }} -recursor={{ quote $value }} \ {{- end }} + {{- if (and .Values.dns.enabled .Values.dns.enableRedirection) }} + $recursor_flags \ + {{- end }} -config-file=/consul/extra-config/extra-from-values.json \ -server volumeMounts: diff --git a/charts/consul/test/unit/client-daemonset.bats b/charts/consul/test/unit/client-daemonset.bats index e986c2984a..a6f8a63b1a 100755 --- a/charts/consul/test/unit/client-daemonset.bats +++ b/charts/consul/test/unit/client-daemonset.bats @@ -1170,6 +1170,28 @@ load _helpers [ "${actual}" = "true" ] } +#-------------------------------------------------------------------- +# DNS + +@test "client/DaemonSet: recursor flags is not set by default" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/client-daemonset.yaml \ + . | tee /dev/stderr | + yq -c -r '.spec.template.spec.containers[0].command | join(" ") | contains("$recursor_flags")' | tee /dev/stderr) + [ "${actual}" = "false" ] +} + +@test "client/DaemonSet: add recursor flags if dns.enableRedirection is true" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/client-daemonset.yaml \ + --set 'dns.enableRedirection=true' \ + . | tee /dev/stderr | + yq -c -r '.spec.template.spec.containers[0].command | join(" ") | contains("$recursor_flags")' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + #-------------------------------------------------------------------- # hostNetwork diff --git a/charts/consul/test/unit/connect-inject-deployment.bats b/charts/consul/test/unit/connect-inject-deployment.bats index a8e23d0c53..1fb0dad743 100755 --- a/charts/consul/test/unit/connect-inject-deployment.bats +++ b/charts/consul/test/unit/connect-inject-deployment.bats @@ -519,6 +519,40 @@ EOF [ "${actual}" = "true" ] } +#-------------------------------------------------------------------- +# DNS + +@test "connectInject/Deployment: -enable-consul-dns unset by default" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/connect-inject-deployment.yaml \ + --set 'connectInject.enabled=true' \ + . | tee /dev/stderr | + yq -c -r '.spec.template.spec.containers[0].command | join(" ") | contains("-enable-consul-dns=true")' | tee /dev/stderr) + [ "${actual}" = "false" ] +} + +@test "connectInject/Deployment: -enable-consul-dns is true if dns.enabled=true and dns.enableRedirection=true" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/connect-inject-deployment.yaml \ + --set 'connectInject.enabled=true' \ + --set 'dns.enableRedirection=true' \ + . | tee /dev/stderr | + yq -c -r '.spec.template.spec.containers[0].command | join(" ") | contains("-enable-consul-dns=true")' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "connectInject/Deployment: -resource-prefix always set" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/connect-inject-deployment.yaml \ + --set 'connectInject.enabled=true' \ + . | tee /dev/stderr | + yq -c -r '.spec.template.spec.containers[0].command | join(" ") | contains("-resource-prefix=RELEASE-NAME-consul")' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + #-------------------------------------------------------------------- # global.tls.enabled diff --git a/charts/consul/test/unit/server-statefulset.bats b/charts/consul/test/unit/server-statefulset.bats index 647ad2994d..a7a31bb84d 100755 --- a/charts/consul/test/unit/server-statefulset.bats +++ b/charts/consul/test/unit/server-statefulset.bats @@ -580,6 +580,28 @@ load _helpers [ "${actualBaz}" = "qux" ] } +#-------------------------------------------------------------------- +# DNS + +@test "server/StatefulSet: recursor flags unset by default" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/server-statefulset.yaml \ + . | tee /dev/stderr | + yq -c -r '.spec.template.spec.containers[0].command | join(" ") | contains("$recursor_flags")' | tee /dev/stderr) + [ "${actual}" = "false" ] +} + +@test "server/StatefulSet: add recursor flags if dns.enableRedirection is true" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/server-statefulset.yaml \ + --set 'dns.enableRedirection=true' \ + . | tee /dev/stderr | + yq -c -r '.spec.template.spec.containers[0].command | join(" ") | contains("$recursor_flags")' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + #-------------------------------------------------------------------- # annotations diff --git a/charts/consul/values.yaml b/charts/consul/values.yaml index 68ffbb6630..9d5af13ac8 100644 --- a/charts/consul/values.yaml +++ b/charts/consul/values.yaml @@ -1122,6 +1122,12 @@ dns: # @type: boolean enabled: "-" + # If true, services using Consul Connect will use Consul DNS + # for default DNS resolution. The DNS lookups fall back to the nameserver IPs + # listed in /etc/resolv.conf if not found in Consul. + # @type: boolean + enableRedirection: false + # Used to control the type of service created. For # example, setting this to "LoadBalancer" will create an external load # balancer (for supported K8S installations) diff --git a/control-plane/connect-inject/annotations.go b/control-plane/connect-inject/annotations.go index 929c9e4c69..ccc9ab6341 100644 --- a/control-plane/connect-inject/annotations.go +++ b/control-plane/connect-inject/annotations.go @@ -90,6 +90,12 @@ const ( // annotationConsulNamespace is the Consul namespace the service is registered into. annotationConsulNamespace = "consul.hashicorp.com/consul-namespace" + // keyConsulDNS enables or disables Consul DNS for a given pod. It can also be set as a label + // on a namespace to define the default behaviour for connect-injected pods which do not otherwise override this setting + // with their own annotation. + // This annotation/label takes a boolean value (true/false). + keyConsulDNS = "consul.hashicorp.com/consul-dns" + // keyTransparentProxy enables or disables transparent proxy for a given pod. It can also be set as a label // on a namespace to define the default behaviour for connect-injected pods which do not otherwise override this setting // with their own annotation. diff --git a/control-plane/connect-inject/container_init.go b/control-plane/connect-inject/container_init.go index e69a91fbb1..18831fa57b 100644 --- a/control-plane/connect-inject/container_init.go +++ b/control-plane/connect-inject/container_init.go @@ -2,6 +2,8 @@ package connectinject import ( "bytes" + "fmt" + "os" "strconv" "strings" "text/template" @@ -16,6 +18,7 @@ const ( envoyUserAndGroupID = 5995 copyContainerUserAndGroupID = 5996 netAdminCapability = "NET_ADMIN" + dnsServiceHostEnvSuffix = "DNS_SERVICE_HOST" ) type initContainerCommandData struct { @@ -66,6 +69,9 @@ type initContainerCommandData struct { // TProxyExcludeUIDs is a list of additional user IDs to exclude from traffic redirection via // the consul connect redirect-traffic command. TProxyExcludeUIDs []string + + // ConsulDNSClusterIP is the IP of the Consul DNS Service. + ConsulDNSClusterIP string } // initCopyContainer returns the init container spec for the copy container which places @@ -107,6 +113,22 @@ func (h *Handler) containerInit(namespace corev1.Namespace, pod corev1.Pod) (cor return corev1.Container{}, err } + dnsEnabled, err := consulDNSEnabled(namespace, pod, h.EnableConsulDNS) + if err != nil { + return corev1.Container{}, err + } + + var consulDNSClusterIP string + if dnsEnabled { + // If Consul DNS is enabled, we find the environment variable that has the value + // of the ClusterIP of the Consul DNS Service. constructDNSServiceHostName returns + // the name of the env variable whose value is the ClusterIP of the Consul DNS Service. + consulDNSClusterIP = os.Getenv(h.constructDNSServiceHostName()) + if consulDNSClusterIP == "" { + return corev1.Container{}, fmt.Errorf("environment variable %s is not found", h.constructDNSServiceHostName()) + } + } + data := initContainerCommandData{ AuthMethod: h.AuthMethod, ConsulPartition: h.ConsulPartition, @@ -118,6 +140,7 @@ func (h *Handler) containerInit(namespace corev1.Namespace, pod corev1.Pod) (cor TProxyExcludeOutboundPorts: splitCommaSeparatedItemsFromAnnotation(annotationTProxyExcludeOutboundPorts, pod), TProxyExcludeOutboundCIDRs: splitCommaSeparatedItemsFromAnnotation(annotationTProxyExcludeOutboundCIDRs, pod), TProxyExcludeUIDs: splitCommaSeparatedItemsFromAnnotation(annotationTProxyExcludeUIDs, pod), + ConsulDNSClusterIP: consulDNSClusterIP, EnvoyUID: envoyUserAndGroupID, } @@ -223,6 +246,15 @@ func (h *Handler) containerInit(namespace corev1.Namespace, pod corev1.Pod) (cor return container, nil } +// constructDNSServiceHostName use the resource prefix and the DNS Service hostname suffix to construct the +// key of the env variable whose value is the cluster IP of the Consul DNS Service. +// It translates "resource-prefix" into "RESOURCE_PREFIX_DNS_SERVICE_HOST". +func (h *Handler) constructDNSServiceHostName() string { + upcaseResourcePrefix := strings.ToUpper(h.ResourcePrefix) + upcaseResourcePrefixWithUnderscores := strings.ReplaceAll(upcaseResourcePrefix, "-", "_") + return strings.Join([]string{upcaseResourcePrefixWithUnderscores, dnsServiceHostEnvSuffix}, "_") +} + // transparentProxyEnabled returns true if transparent proxy should be enabled for this pod. // It returns an error when the annotation value cannot be parsed by strconv.ParseBool or if we are unable // to read the pod's namespace label when it exists. @@ -239,6 +271,22 @@ func transparentProxyEnabled(namespace corev1.Namespace, pod corev1.Pod, globalE return globalEnabled, nil } +// consulDNSEnabled returns true if Consul DNS should be enabled for this pod. +// It returns an error when the annotation value cannot be parsed by strconv.ParseBool or if we are unable +// to read the pod's namespace label when it exists. +func consulDNSEnabled(namespace corev1.Namespace, pod corev1.Pod, globalEnabled bool) (bool, error) { + // First check to see if the pod annotation exists to override the namespace or global settings. + if raw, ok := pod.Annotations[keyConsulDNS]; ok { + return strconv.ParseBool(raw) + } + // Next see if the namespace has been defaulted. + if raw, ok := namespace.Labels[keyConsulDNS]; ok { + return strconv.ParseBool(raw) + } + // Else fall back to the global default. + return globalEnabled, nil +} + // pointerToInt64 takes an int64 and returns a pointer to it. func pointerToInt64(i int64) *int64 { return &i @@ -331,6 +379,9 @@ consul-k8s-control-plane connect-init -pod-name=${POD_NAME} -pod-namespace=${POD {{- if .ConsulNamespace }} -namespace="{{ .ConsulNamespace }}" \ {{- end }} + {{- if .ConsulDNSClusterIP }} + -consul-dns-ip="{{ .ConsulDNSClusterIP }}" \ + {{- end }} {{- range .TProxyExcludeInboundPorts }} -exclude-inbound-port="{{ . }}" \ {{- end }} diff --git a/control-plane/connect-inject/container_init_test.go b/control-plane/connect-inject/container_init_test.go index e131ba20f7..58cfe95d73 100644 --- a/control-plane/connect-inject/container_init_test.go +++ b/control-plane/connect-inject/container_init_test.go @@ -2,6 +2,7 @@ package connectinject import ( "fmt" + "os" "strings" "testing" @@ -310,6 +311,115 @@ func TestHandlerContainerInit_transparentProxy(t *testing.T) { } } +func TestHandlerContainerInit_consulDNS(t *testing.T) { + cases := map[string]struct { + globalEnabled bool + annotations map[string]string + expectedContainsCmd string + namespaceLabel map[string]string + }{ + "enabled globally, ns not set, annotation not provided": { + globalEnabled: true, + expectedContainsCmd: `/consul/connect-inject/consul connect redirect-traffic \ + -consul-dns-ip="10.0.34.16" \ + -proxy-id="$(cat /consul/connect-inject/proxyid)" \ + -proxy-uid=5995`, + }, + "enabled globally, ns not set, annotation is false": { + globalEnabled: true, + annotations: map[string]string{keyConsulDNS: "false"}, + expectedContainsCmd: `/consul/connect-inject/consul connect redirect-traffic \ + -proxy-id="$(cat /consul/connect-inject/proxyid)" \ + -proxy-uid=5995`, + }, + "enabled globally, ns not set, annotation is true": { + globalEnabled: true, + annotations: map[string]string{keyConsulDNS: "true"}, + expectedContainsCmd: `/consul/connect-inject/consul connect redirect-traffic \ + -consul-dns-ip="10.0.34.16" \ + -proxy-id="$(cat /consul/connect-inject/proxyid)" \ + -proxy-uid=5995`, + }, + "disabled globally, ns not set, annotation not provided": { + expectedContainsCmd: `/consul/connect-inject/consul connect redirect-traffic \ + -proxy-id="$(cat /consul/connect-inject/proxyid)" \ + -proxy-uid=5995`, + }, + "disabled globally, ns not set, annotation is false": { + annotations: map[string]string{keyConsulDNS: "false"}, + expectedContainsCmd: `/consul/connect-inject/consul connect redirect-traffic \ + -proxy-id="$(cat /consul/connect-inject/proxyid)" \ + -proxy-uid=5995`, + }, + "disabled globally, ns not set, annotation is true": { + annotations: map[string]string{keyConsulDNS: "true"}, + expectedContainsCmd: `/consul/connect-inject/consul connect redirect-traffic \ + -consul-dns-ip="10.0.34.16" \ + -proxy-id="$(cat /consul/connect-inject/proxyid)" \ + -proxy-uid=5995`, + }, + "disabled globally, ns enabled, annotation not set": { + expectedContainsCmd: `/consul/connect-inject/consul connect redirect-traffic \ + -consul-dns-ip="10.0.34.16" \ + -proxy-id="$(cat /consul/connect-inject/proxyid)" \ + -proxy-uid=5995`, + namespaceLabel: map[string]string{keyConsulDNS: "true"}, + }, + "enabled globally, ns disabled, annotation not set": { + globalEnabled: true, + expectedContainsCmd: `/consul/connect-inject/consul connect redirect-traffic \ + -proxy-id="$(cat /consul/connect-inject/proxyid)" \ + -proxy-uid=5995`, + namespaceLabel: map[string]string{keyConsulDNS: "false"}, + }, + } + for name, c := range cases { + t.Run(name, func(t *testing.T) { + h := Handler{EnableConsulDNS: c.globalEnabled, EnableTransparentProxy: true, ResourcePrefix: "consul-consul"} + os.Setenv("CONSUL_CONSUL_DNS_SERVICE_HOST", "10.0.34.16") + defer os.Unsetenv("CONSUL_CONSUL_DNS_SERVICE_HOST") + + pod := minimal() + pod.Annotations = c.annotations + + ns := testNS + ns.Labels = c.namespaceLabel + container, err := h.containerInit(ns, *pod) + require.NoError(t, err) + actualCmd := strings.Join(container.Command, " ") + + require.Contains(t, actualCmd, c.expectedContainsCmd) + }) + } +} + +func TestHandler_constructDNSServiceHostName(t *testing.T) { + cases := []struct { + prefix string + result string + }{ + { + prefix: "consul-consul", + result: "CONSUL_CONSUL_DNS_SERVICE_HOST", + }, + { + prefix: "release", + result: "RELEASE_DNS_SERVICE_HOST", + }, + { + prefix: "consul-dc1", + result: "CONSUL_DC1_DNS_SERVICE_HOST", + }, + } + + for _, c := range cases { + t.Run(c.prefix, func(t *testing.T) { + h := Handler{ResourcePrefix: c.prefix} + require.Equal(t, c.result, h.constructDNSServiceHostName()) + }) + } +} + func TestHandlerContainerInit_namespacesAndPartitionsEnabled(t *testing.T) { minimal := func() *corev1.Pod { return &corev1.Pod{ diff --git a/control-plane/connect-inject/handler.go b/control-plane/connect-inject/handler.go index aad0c4c53f..da2b4f681d 100644 --- a/control-plane/connect-inject/handler.go +++ b/control-plane/connect-inject/handler.go @@ -134,6 +134,14 @@ type Handler struct { // to point them to the Envoy proxy. TProxyOverwriteProbes bool + // EnableConsulDNS enables traffic redirection so that DNS requests are directed to Consul + // from mesh services. + EnableConsulDNS bool + + // ResourcePrefix is the prefix used for the installation which is used to determine the Service + // name of the Consul DNS service. + ResourcePrefix string + // EnableOpenShift indicates that when tproxy is enabled, the security context for the Envoy and init // containers should not be added because OpenShift sets a random user for those and will not allow // those containers to be created otherwise. diff --git a/control-plane/subcommand/inject-connect/command.go b/control-plane/subcommand/inject-connect/command.go index e93e07a48e..abebf69b81 100644 --- a/control-plane/subcommand/inject-connect/command.go +++ b/control-plane/subcommand/inject-connect/command.go @@ -92,6 +92,10 @@ type Command struct { flagDefaultEnableTransparentProxy bool flagTransparentProxyDefaultOverwriteProbes bool + // Consul DNS flags. + flagEnableConsulDNS bool + flagResourcePrefix string + flagEnableOpenShift bool flagSet *flag.FlagSet @@ -161,6 +165,10 @@ func (c *Command) init() { "Enable transparent proxy mode for all Consul service mesh applications by default.") c.flagSet.BoolVar(&c.flagTransparentProxyDefaultOverwriteProbes, "transparent-proxy-default-overwrite-probes", true, "Overwrite Kubernetes probes to point to Envoy by default when in Transparent Proxy mode.") + c.flagSet.BoolVar(&c.flagEnableConsulDNS, "enable-consul-dns", false, + "Enables Consul DNS lookup for services in the mesh.") + c.flagSet.StringVar(&c.flagResourcePrefix, "resource-prefix", "", + "Release prefix of the Consul installation used to determine Consul DNS Service name.") c.flagSet.BoolVar(&c.flagEnableOpenShift, "enable-openshift", false, "Indicates that the command runs in an OpenShift cluster.") c.flagSet.StringVar(&c.flagLogLevel, "log-level", zapcore.InfoLevel.String(), @@ -471,6 +479,8 @@ func (c *Command) Run(args []string) int { CrossNamespaceACLPolicy: c.flagCrossNamespaceACLPolicy, EnableTransparentProxy: c.flagDefaultEnableTransparentProxy, TProxyOverwriteProbes: c.flagTransparentProxyDefaultOverwriteProbes, + EnableConsulDNS: c.flagEnableConsulDNS, + ResourcePrefix: c.flagResourcePrefix, EnableOpenShift: c.flagEnableOpenShift, Log: ctrl.Log.WithName("handler").WithName("connect"), LogLevel: c.flagLogLevel,