From a19a035dd95f90dfd85eba5ea2073a72991179cb Mon Sep 17 00:00:00 2001 From: Curt Bushko Date: Wed, 28 Jun 2023 09:32:34 -0700 Subject: [PATCH] Backport of Proxy Lifecycle helm, connect-inject and acceptance tests into release/1.2.x (#2233) * Proxy Lifecycle helm, connect-inject and acceptance tests (#2233) Proxy Lifecycle helm, connect-inject and acceptance tests (#2233) Co-authored-by: Nitya Dhanushkodi * disable lifecycle test --------- Co-authored-by: Mike Morris Co-authored-by: Nitya Dhanushkodi Co-authored-by: Michael Wilkerson --- .changelog/2233.txt | 3 + acceptance/framework/config/config.go | 15 +- .../framework/connhelper/connect_helper.go | 34 +- acceptance/framework/flags/flags.go | 42 ++- .../connect/connect_proxy_lifecycle_test.go | 207 +++++++++++ .../templates/connect-inject-deployment.yaml | 13 + .../test/unit/connect-inject-deployment.bats | 154 +++++++- charts/consul/values.yaml | 34 +- cli/helm/values.go | 11 +- control-plane/connect-inject/common/common.go | 34 ++ .../connect-inject/common/common_test.go | 166 +++++++++ .../constants/annotations_and_labels.go | 7 + .../connect-inject/constants/constants.go | 6 + .../lifecycle/lifecycle_configuration.go | 95 +++++ .../lifecycle/lifecycle_configuration_test.go | 351 ++++++++++++++++++ .../metrics/metrics_configuration.go | 42 +-- .../metrics/metrics_configuration_test.go | 143 ------- .../webhook/consul_dataplane_sidecar.go | 39 ++ .../webhook/consul_dataplane_sidecar_test.go | 191 +++++++++- .../connect-inject/webhook/mesh_webhook.go | 8 + .../subcommand/inject-connect/command.go | 25 ++ 21 files changed, 1364 insertions(+), 256 deletions(-) create mode 100644 .changelog/2233.txt create mode 100644 acceptance/tests/connect/connect_proxy_lifecycle_test.go create mode 100644 control-plane/connect-inject/lifecycle/lifecycle_configuration.go create mode 100644 control-plane/connect-inject/lifecycle/lifecycle_configuration_test.go diff --git a/.changelog/2233.txt b/.changelog/2233.txt new file mode 100644 index 0000000000..bb929501c9 --- /dev/null +++ b/.changelog/2233.txt @@ -0,0 +1,3 @@ +```release-note:feature +Add support for configuring graceful shutdown proxy lifecycle management settings. +``` diff --git a/acceptance/framework/config/config.go b/acceptance/framework/config/config.go index 18673ca260..7151a75908 100644 --- a/acceptance/framework/config/config.go +++ b/acceptance/framework/config/config.go @@ -46,12 +46,14 @@ type TestConfig struct { DisablePeering bool - HelmChartVersion string - ConsulImage string - ConsulK8SImage string - ConsulVersion *version.Version - EnvoyImage string - ConsulCollectorImage string + HelmChartVersion string + ConsulImage string + ConsulK8SImage string + ConsulDataplaneImage string + ConsulVersion *version.Version + ConsulDataplaneVersion *version.Version + EnvoyImage string + ConsulCollectorImage string HCPResourceID string @@ -110,6 +112,7 @@ func (t *TestConfig) HelmValuesFromConfig() (map[string]string, error) { setIfNotEmpty(helmValues, "global.image", t.ConsulImage) setIfNotEmpty(helmValues, "global.imageK8S", t.ConsulK8SImage) setIfNotEmpty(helmValues, "global.imageEnvoy", t.EnvoyImage) + setIfNotEmpty(helmValues, "global.imageConsulDataplane", t.ConsulDataplaneImage) return helmValues, nil } diff --git a/acceptance/framework/connhelper/connect_helper.go b/acceptance/framework/connhelper/connect_helper.go index 8a7f4d3d53..cad6ba6885 100644 --- a/acceptance/framework/connhelper/connect_helper.go +++ b/acceptance/framework/connhelper/connect_helper.go @@ -50,8 +50,8 @@ type ConnectHelper struct { // consulCluster is the cluster to use for the test. consulCluster consul.Cluster - // consulClient is the client used to test service mesh connectivity. - consulClient *api.Client + // ConsulClient is the client used to test service mesh connectivity. + ConsulClient *api.Client } // Setup creates a new cluster using the New*Cluster function and assigns it @@ -69,14 +69,14 @@ func (c *ConnectHelper) Setup(t *testing.T) { func (c *ConnectHelper) Install(t *testing.T) { logger.Log(t, "Installing Consul cluster") c.consulCluster.Create(t) - c.consulClient, _ = c.consulCluster.SetupConsulClient(t, c.Secure) + c.ConsulClient, _ = c.consulCluster.SetupConsulClient(t, c.Secure) } // Upgrade uses the existing Consul cluster and upgrades it using Helm values // set by the Secure, AutoEncrypt, and HelmValues fields. func (c *ConnectHelper) Upgrade(t *testing.T) { require.NotNil(t, c.consulCluster, "consulCluster must be set before calling Upgrade (Call Install first).") - require.NotNil(t, c.consulClient, "consulClient must be set before calling Upgrade (Call Install first).") + require.NotNil(t, c.ConsulClient, "ConsulClient must be set before calling Upgrade (Call Install first).") logger.Log(t, "upgrading Consul cluster") c.consulCluster.Upgrade(t, c.helmValues()) @@ -94,9 +94,9 @@ func (c *ConnectHelper) DeployClientAndServer(t *testing.T) { // deployments because golang will execute them in reverse order // (i.e. the last registered cleanup function will be executed first). t.Cleanup(func() { - retrier := &retry.Timer{Timeout: 30 * time.Second, Wait: 100 * time.Millisecond} + retrier := &retry.Timer{Timeout: 60 * time.Second, Wait: 100 * time.Millisecond} retry.RunWith(retrier, t, func(r *retry.R) { - tokens, _, err := c.consulClient.ACL().TokenList(nil) + tokens, _, err := c.ConsulClient.ACL().TokenList(nil) require.NoError(r, err) for _, token := range tokens { require.NotContains(r, token.Description, StaticServerName) @@ -117,14 +117,18 @@ func (c *ConnectHelper) DeployClientAndServer(t *testing.T) { // Check that both static-server and static-client have been injected and // now have 2 containers. - for _, labelSelector := range []string{"app=static-server", "app=static-client"} { - podList, err := c.Ctx.KubernetesClient(t).CoreV1().Pods(c.Ctx.KubectlOptions(t).Namespace).List(context.Background(), metav1.ListOptions{ - LabelSelector: labelSelector, - }) - require.NoError(t, err) - require.Len(t, podList.Items, 1) - require.Len(t, podList.Items[0].Spec.Containers, 2) - } + + retrier := &retry.Timer{Timeout: 300 * time.Second, Wait: 100 * time.Millisecond} + retry.RunWith(retrier, t, func(r *retry.R) { + for _, labelSelector := range []string{"app=static-server", "app=static-client"} { + podList, err := c.Ctx.KubernetesClient(t).CoreV1().Pods(c.Ctx.KubectlOptions(t).Namespace).List(context.Background(), metav1.ListOptions{ + LabelSelector: labelSelector, + }) + require.NoError(t, err) + require.Len(t, podList.Items, 1) + require.Len(t, podList.Items[0].Spec.Containers, 2) + } + }) } // TestConnectionFailureWithoutIntention ensures the connection to the static @@ -142,7 +146,7 @@ func (c *ConnectHelper) TestConnectionFailureWithoutIntention(t *testing.T) { // the static-client pod. func (c *ConnectHelper) CreateIntention(t *testing.T) { logger.Log(t, "creating intention") - _, _, err := c.consulClient.ConfigEntries().Set(&api.ServiceIntentionsConfigEntry{ + _, _, err := c.ConsulClient.ConfigEntries().Set(&api.ServiceIntentionsConfigEntry{ Kind: api.ServiceIntentions, Name: StaticServerName, Sources: []*api.SourceIntention{ diff --git a/acceptance/framework/flags/flags.go b/acceptance/framework/flags/flags.go index fd1831c5e6..fb59ba39dc 100644 --- a/acceptance/framework/flags/flags.go +++ b/acceptance/framework/flags/flags.go @@ -34,14 +34,16 @@ type TestFlags struct { flagEnableTransparentProxy bool - flagHelmChartVersion string - flagConsulImage string - flagConsulK8sImage string - flagConsulVersion string - flagEnvoyImage string - flagConsulCollectorImage string - flagVaultHelmChartVersion string - flagVaultServerVersion string + flagHelmChartVersion string + flagConsulImage string + flagConsulK8sImage string + flagConsulVersion string + flagEnvoyImage string + flagConsulCollectorImage string + flagVaultHelmChartVersion string + flagVaultServerVersion string + flagConsulDataplaneImage string + flagConsulDataplaneVersion string flagHCPResourceID string @@ -75,7 +77,9 @@ func (t *TestFlags) init() { flag.StringVar(&t.flagConsulImage, "consul-image", "", "The Consul image to use for all tests.") flag.StringVar(&t.flagConsulK8sImage, "consul-k8s-image", "", "The consul-k8s image to use for all tests.") + flag.StringVar(&t.flagConsulDataplaneImage, "consul-dataplane-image", "", "The consul-dataplane image to use for all tests.") flag.StringVar(&t.flagConsulVersion, "consul-version", "", "The consul version used for all tests.") + flag.StringVar(&t.flagConsulDataplaneVersion, "consul-dataplane-version", "", "The consul-dataplane version used for all tests.") flag.StringVar(&t.flagHelmChartVersion, "helm-chart-version", config.HelmChartPath, "The helm chart used for all tests.") flag.StringVar(&t.flagEnvoyImage, "envoy-image", "", "The Envoy image to use for all tests.") flag.StringVar(&t.flagConsulCollectorImage, "consul-collector-image", "", "The consul collector image to use for all tests.") @@ -155,6 +159,7 @@ func (t *TestFlags) TestConfigFromFlags() *config.TestConfig { // if the Version is empty consulVersion will be nil consulVersion, _ := version.NewVersion(t.flagConsulVersion) + consulDataplaneVersion, _ := version.NewVersion(t.flagConsulDataplaneVersion) //vaultserverVersion, _ := version.NewVersion(t.flagVaultServerVersion) return &config.TestConfig{ @@ -180,16 +185,17 @@ func (t *TestFlags) TestConfigFromFlags() *config.TestConfig { DisablePeering: t.flagDisablePeering, - HelmChartVersion: t.flagHelmChartVersion, - ConsulImage: t.flagConsulImage, - ConsulK8SImage: t.flagConsulK8sImage, - ConsulVersion: consulVersion, - EnvoyImage: t.flagEnvoyImage, - ConsulCollectorImage: t.flagConsulCollectorImage, - VaultHelmChartVersion: t.flagVaultHelmChartVersion, - VaultServerVersion: t.flagVaultServerVersion, - - HCPResourceID: t.flagHCPResourceID, + HelmChartVersion: t.flagHelmChartVersion, + ConsulImage: t.flagConsulImage, + ConsulK8SImage: t.flagConsulK8sImage, + ConsulVersion: consulVersion, + EnvoyImage: t.flagEnvoyImage, + ConsulCollectorImage: t.flagConsulCollectorImage, + VaultHelmChartVersion: t.flagVaultHelmChartVersion, + VaultServerVersion: t.flagVaultServerVersion, + ConsulDataplaneImage: t.flagConsulDataplaneImage, + ConsulDataplaneVersion: consulDataplaneVersion, + HCPResourceID: t.flagHCPResourceID, NoCleanupOnFailure: t.flagNoCleanupOnFailure, DebugDirectory: tempDir, diff --git a/acceptance/tests/connect/connect_proxy_lifecycle_test.go b/acceptance/tests/connect/connect_proxy_lifecycle_test.go new file mode 100644 index 0000000000..3ec5d0a9b8 --- /dev/null +++ b/acceptance/tests/connect/connect_proxy_lifecycle_test.go @@ -0,0 +1,207 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package connect + +import ( + "context" + "fmt" + "strconv" + "strings" + "testing" + "time" + + "github.com/gruntwork-io/terratest/modules/k8s" + "github.com/hashicorp/consul-k8s/acceptance/framework/connhelper" + "github.com/hashicorp/consul-k8s/acceptance/framework/consul" + "github.com/hashicorp/consul-k8s/acceptance/framework/helpers" + "github.com/hashicorp/consul-k8s/acceptance/framework/logger" + "github.com/hashicorp/consul/sdk/testutil/retry" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type LifecycleShutdownConfig struct { + secure bool + helmValues map[string]string +} + +const ( + helmDrainListenersKey = "connectInject.sidecarProxy.lifecycle.defaultEnableShutdownDrainListeners" + helmGracePeriodSecondsKey = "connectInject.sidecarProxy.lifecycle.defaultShutdownGracePeriodSeconds" +) + +// Test the endpoints controller cleans up force-killed pods. +func TestConnectInject_ProxyLifecycleShutdown(t *testing.T) { + t.Skipf("skiping this test, will be re-added in a future commit") + cfg := suite.Config() + + for _, testCfg := range []LifecycleShutdownConfig{ + {secure: false, helmValues: map[string]string{ + helmDrainListenersKey: "true", + helmGracePeriodSecondsKey: "15", + }}, + {secure: true, helmValues: map[string]string{ + helmDrainListenersKey: "true", + helmGracePeriodSecondsKey: "15", + }}, + {secure: false, helmValues: map[string]string{ + helmDrainListenersKey: "false", + helmGracePeriodSecondsKey: "15", + }}, + {secure: true, helmValues: map[string]string{ + helmDrainListenersKey: "false", + helmGracePeriodSecondsKey: "15", + }}, + {secure: false, helmValues: map[string]string{ + helmDrainListenersKey: "false", + helmGracePeriodSecondsKey: "0", + }}, + {secure: true, helmValues: map[string]string{ + helmDrainListenersKey: "false", + helmGracePeriodSecondsKey: "0", + }}, + } { + // Determine if listeners should be expected to drain inbound connections + var drainListenersEnabled bool + var err error + val, ok := testCfg.helmValues[helmDrainListenersKey] + if ok { + drainListenersEnabled, err = strconv.ParseBool(val) + require.NoError(t, err) + } + + // Determine expected shutdown grace period + var gracePeriodSeconds int64 + val, ok = testCfg.helmValues[helmGracePeriodSecondsKey] + if ok { + gracePeriodSeconds, err = strconv.ParseInt(val, 10, 64) + require.NoError(t, err) + } else { + // Half of the helm default to speed tests up + gracePeriodSeconds = 15 + } + + name := fmt.Sprintf("secure: %t, drainListeners: %t, gracePeriodSeconds: %d", testCfg.secure, drainListenersEnabled, gracePeriodSeconds) + t.Run(name, func(t *testing.T) { + ctx := suite.Environment().DefaultContext(t) + releaseName := helpers.RandomName() + + connHelper := connhelper.ConnectHelper{ + ClusterKind: consul.Helm, + Secure: testCfg.secure, + ReleaseName: releaseName, + Ctx: ctx, + Cfg: cfg, + HelmValues: testCfg.helmValues, + } + + connHelper.Setup(t) + connHelper.Install(t) + connHelper.DeployClientAndServer(t) + + // TODO: should this move into connhelper.DeployClientAndServer? + logger.Log(t, "waiting for static-client and static-server to be registered with Consul") + retry.Run(t, func(r *retry.R) { + for _, name := range []string{ + "static-client", + "static-client-sidecar-proxy", + "static-server", + "static-server-sidecar-proxy", + } { + logger.Logf(t, "checking for %s service in Consul catalog", name) + instances, _, err := connHelper.ConsulClient.Catalog().Service(name, "", nil) + r.Check(err) + + if len(instances) != 1 { + r.Errorf("expected 1 instance of %s", name) + } + } + }) + + if testCfg.secure { + connHelper.TestConnectionFailureWithoutIntention(t) + connHelper.CreateIntention(t) + } + + connHelper.TestConnectionSuccess(t) + + // Get static-client pod name + ns := ctx.KubectlOptions(t).Namespace + pods, err := ctx.KubernetesClient(t).CoreV1().Pods(ns).List( + context.Background(), + metav1.ListOptions{ + LabelSelector: "app=static-client", + }, + ) + require.NoError(t, err) + require.Len(t, pods.Items, 1) + clientPodName := pods.Items[0].Name + + var terminationGracePeriod int64 = 60 + logger.Logf(t, "killing the %q pod with %dseconds termination grace period", clientPodName, terminationGracePeriod) + err = ctx.KubernetesClient(t).CoreV1().Pods(ns).Delete(context.Background(), clientPodName, metav1.DeleteOptions{GracePeriodSeconds: &terminationGracePeriod}) + require.NoError(t, err) + + // Exec into terminating pod, not just any static-client pod + args := []string{"exec", clientPodName, "-c", connhelper.StaticClientName, "--", "curl", "-vvvsSf"} + + if cfg.EnableTransparentProxy { + args = append(args, "http://static-server") + } else { + args = append(args, "http://localhost:1234") + } + + if gracePeriodSeconds > 0 { + // Ensure outbound requests are still successful during grace + // period. + retry.RunWith(&retry.Timer{Timeout: time.Duration(gracePeriodSeconds) * time.Second, Wait: 2 * time.Second}, t, func(r *retry.R) { + output, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), args...) + require.NoError(r, err) + require.Condition(r, func() bool { + exists := false + if strings.Contains(output, "curl: (7) Failed to connect") { + exists = true + } + return !exists + }) + }) + + // If listener draining is enabled, ensure inbound + // requests are rejected during grace period. + // connHelper.TestConnectionSuccess(t) + } else { + // Ensure outbound requests fail because proxy has terminated + retry.RunWith(&retry.Timer{Timeout: time.Duration(terminationGracePeriod) * time.Second, Wait: 2 * time.Second}, t, func(r *retry.R) { + output, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), args...) + require.Error(r, err) + require.Condition(r, func() bool { + exists := false + if strings.Contains(output, "curl: (7) Failed to connect") { + exists = true + } + return exists + }) + }) + } + + logger.Log(t, "ensuring pod is deregistered after termination") + retry.Run(t, func(r *retry.R) { + for _, name := range []string{ + "static-client", + "static-client-sidecar-proxy", + } { + logger.Logf(t, "checking for %s service in Consul catalog", name) + instances, _, err := connHelper.ConsulClient.Catalog().Service(name, "", nil) + r.Check(err) + + for _, instance := range instances { + if strings.Contains(instance.ServiceID, clientPodName) { + r.Errorf("%s is still registered", instance.ServiceID) + } + } + } + }) + }) + } +} diff --git a/charts/consul/templates/connect-inject-deployment.yaml b/charts/consul/templates/connect-inject-deployment.yaml index 479e05b25a..14c3961b4e 100644 --- a/charts/consul/templates/connect-inject-deployment.yaml +++ b/charts/consul/templates/connect-inject-deployment.yaml @@ -234,6 +234,19 @@ spec: -default-sidecar-proxy-cpu-request={{ $resources.requests.cpu }} \ {{- end }} -default-envoy-proxy-concurrency={{ .Values.connectInject.sidecarProxy.concurrency }} \ + {{- if .Values.connectInject.sidecarProxy.lifecycle.defaultEnabled }} + -default-enable-sidecar-proxy-lifecycle=true \ + {{- else }} + -default-enable-sidecar-proxy-lifecycle=false \ + {{- end }} + {{- if .Values.connectInject.sidecarProxy.lifecycle.defaultEnableShutdownDrainListeners }} + -default-enable-sidecar-proxy-lifecycle-shutdown-drain-listeners=true \ + {{- else }} + -default-enable-sidecar-proxy-lifecycle-shutdown-drain-listeners=false \ + {{- end }} + -default-sidecar-proxy-lifecycle-shutdown-grace-period-seconds={{ .Values.connectInject.sidecarProxy.lifecycle.defaultShutdownGracePeriodSeconds }} \ + -default-sidecar-proxy-lifecycle-graceful-port={{ .Values.connectInject.sidecarProxy.lifecycle.defaultGracefulPort }} \ + -default-sidecar-proxy-lifecycle-graceful-shutdown-path="{{ .Values.connectInject.sidecarProxy.lifecycle.defaultGracefulShutdownPath }}" \ {{- if .Values.connectInject.initContainer }} {{- $initResources := .Values.connectInject.initContainer.resources }} diff --git a/charts/consul/test/unit/connect-inject-deployment.bats b/charts/consul/test/unit/connect-inject-deployment.bats index c1bc63ffc3..6db96ce8b6 100755 --- a/charts/consul/test/unit/connect-inject-deployment.bats +++ b/charts/consul/test/unit/connect-inject-deployment.bats @@ -999,7 +999,7 @@ load _helpers local actual=$(echo "$cmd" | yq 'any(contains("-init-container-memory-limit=150Mi"))' | tee /dev/stderr) [ "${actual}" = "true" ] - + } @test "connectInject/Deployment: can set init container resources" { @@ -1231,6 +1231,144 @@ load _helpers [ "${actual}" = "true" ] } +#-------------------------------------------------------------------- +# sidecarProxy.lifecycle + +@test "connectInject/Deployment: by default sidecar proxy lifecycle management is enabled" { + cd `chart_dir` + local cmd=$(helm template \ + -s templates/connect-inject-deployment.yaml \ + --set 'connectInject.enabled=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].command' | tee /dev/stderr) + + local actual=$(echo "$cmd" | + yq 'any(contains("-default-enable-sidecar-proxy-lifecycle"))' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "connectInject/Deployment: sidecar proxy lifecycle management can be disabled" { + cd `chart_dir` + local cmd=$(helm template \ + -s templates/connect-inject-deployment.yaml \ + --set 'connectInject.enabled=true' \ + --set 'connectInject.sidecarProxy.lifecycle.defaultEnabled=false' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].command' | tee /dev/stderr) + + local actual=$(echo "$cmd" | + yq 'any(contains("-default-enable-sidecar-proxy-lifecycle=false"))' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "connectInject/Deployment: by default sidecar proxy lifecycle management shutdown listener draining is enabled" { + cd `chart_dir` + local cmd=$(helm template \ + -s templates/connect-inject-deployment.yaml \ + --set 'connectInject.enabled=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].command' | tee /dev/stderr) + + local actual=$(echo "$cmd" | + yq 'any(contains("-default-enable-sidecar-proxy-lifecycle-shutdown-drain-listeners"))' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "connectInject/Deployment: sidecar proxy lifecycle management shutdown listener draining can be disabled" { + cd `chart_dir` + local cmd=$(helm template \ + -s templates/connect-inject-deployment.yaml \ + --set 'connectInject.enabled=true' \ + --set 'connectInject.sidecarProxy.lifecycle.defaultEnableShutdownDrainListeners=false' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].command' | tee /dev/stderr) + + local actual=$(echo "$cmd" | + yq 'any(contains("-default-enable-sidecar-proxy-lifecycle-shutdown-drain-listeners=false"))' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "connectInject/Deployment: by default sidecar proxy lifecycle management shutdown grace period is set to 30 seconds" { + cd `chart_dir` + local cmd=$(helm template \ + -s templates/connect-inject-deployment.yaml \ + --set 'connectInject.enabled=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].command' | tee /dev/stderr) + + local actual=$(echo "$cmd" | + yq 'any(contains("-default-sidecar-proxy-lifecycle-shutdown-grace-period-seconds=30"))' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "connectInject/Deployment: sidecar proxy lifecycle management shutdown grace period can be set" { + cd `chart_dir` + local cmd=$(helm template \ + -s templates/connect-inject-deployment.yaml \ + --set 'connectInject.enabled=true' \ + --set 'connectInject.sidecarProxy.lifecycle.defaultShutdownGracePeriodSeconds=23' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].command' | tee /dev/stderr) + + local actual=$(echo "$cmd" | + yq 'any(contains("-default-sidecar-proxy-lifecycle-shutdown-grace-period-seconds=23"))' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "connectInject/Deployment: by default sidecar proxy lifecycle management port is set to 20600" { + cd `chart_dir` + local cmd=$(helm template \ + -s templates/connect-inject-deployment.yaml \ + --set 'connectInject.enabled=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].command' | tee /dev/stderr) + + local actual=$(echo "$cmd" | + yq 'any(contains("-default-sidecar-proxy-lifecycle-graceful-port=20600"))' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "connectInject/Deployment: sidecar proxy lifecycle management port can be set" { + cd `chart_dir` + local cmd=$(helm template \ + -s templates/connect-inject-deployment.yaml \ + --set 'connectInject.enabled=true' \ + --set 'connectInject.sidecarProxy.lifecycle.defaultGracefulPort=20307' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].command' | tee /dev/stderr) + + local actual=$(echo "$cmd" | + yq 'any(contains("-default-sidecar-proxy-lifecycle-graceful-port=20307"))' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "connectInject/Deployment: by default sidecar proxy lifecycle management graceful shutdown path is set to /graceful_shutdown" { + cd `chart_dir` + local cmd=$(helm template \ + -s templates/connect-inject-deployment.yaml \ + --set 'connectInject.enabled=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].command' | tee /dev/stderr) + + local actual=$(echo "$cmd" | + yq 'any(contains("-default-sidecar-proxy-lifecycle-graceful-shutdown-path=\"/graceful_shutdown\""))' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "connectInject/Deployment: sidecar proxy lifecycle management graceful shutdown path can be set" { + cd `chart_dir` + local cmd=$(helm template \ + -s templates/connect-inject-deployment.yaml \ + --set 'connectInject.enabled=true' \ + --set 'connectInject.sidecarProxy.lifecycle.defaultGracefulShutdownPath=/exit' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].command' | tee /dev/stderr) + + local actual=$(echo "$cmd" | + yq 'any(contains("-default-sidecar-proxy-lifecycle-graceful-shutdown-path=\"/exit\""))' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + #-------------------------------------------------------------------- # priorityClassName @@ -1418,7 +1556,7 @@ load _helpers } #-------------------------------------------------------------------- -# cni +# cni @test "connectInject/Deployment: cni is disabled by default" { cd `chart_dir` @@ -2300,7 +2438,7 @@ reservedNameTest() { --set 'global.cloud.authUrl.secretName=auth-url-name' \ . [ "$status" -eq 1 ] - + [[ "$output" =~ "When either global.cloud.authUrl.secretName or global.cloud.authUrl.secretKey is defined, both must be set." ]] } @@ -2321,7 +2459,7 @@ reservedNameTest() { --set 'global.cloud.authUrl.secretKey=auth-url-key' \ . [ "$status" -eq 1 ] - + [[ "$output" =~ "When either global.cloud.authUrl.secretName or global.cloud.authUrl.secretKey is defined, both must be set." ]] } @@ -2342,7 +2480,7 @@ reservedNameTest() { --set 'global.cloud.apiHost.secretName=auth-url-name' \ . [ "$status" -eq 1 ] - + [[ "$output" =~ "When either global.cloud.apiHost.secretName or global.cloud.apiHost.secretKey is defined, both must be set." ]] } @@ -2363,7 +2501,7 @@ reservedNameTest() { --set 'global.cloud.apiHost.secretKey=auth-url-key' \ . [ "$status" -eq 1 ] - + [[ "$output" =~ "When either global.cloud.apiHost.secretName or global.cloud.apiHost.secretKey is defined, both must be set." ]] } @@ -2384,7 +2522,7 @@ reservedNameTest() { --set 'global.cloud.scadaAddress.secretName=scada-address-name' \ . [ "$status" -eq 1 ] - + [[ "$output" =~ "When either global.cloud.scadaAddress.secretName or global.cloud.scadaAddress.secretKey is defined, both must be set." ]] } @@ -2405,7 +2543,7 @@ reservedNameTest() { --set 'global.cloud.scadaAddress.secretKey=scada-address-key' \ . [ "$status" -eq 1 ] - + [[ "$output" =~ "When either global.cloud.scadaAddress.secretName or global.cloud.scadaAddress.secretKey is defined, both must be set." ]] } diff --git a/charts/consul/values.yaml b/charts/consul/values.yaml index 2191e9c824..e416e68149 100644 --- a/charts/consul/values.yaml +++ b/charts/consul/values.yaml @@ -584,7 +584,7 @@ global: # The name (and tag) of the consul-dataplane Docker image used for the # connect-injected sidecar proxies and mesh, terminating, and ingress gateways. # @default: hashicorp/consul-dataplane: - imageConsulDataplane: "docker.mirror.hashicorp.services/hashicorppreview/consul-dataplane:1.2-dev" + imageConsulDataplane: docker.mirror.hashicorp.services/hashicorppreview/consul-dataplane:1.2-dev # Configuration for running this Helm chart on the Red Hat OpenShift platform. # This Helm chart currently supports OpenShift v4.x+. @@ -690,7 +690,7 @@ global: # ] # ``` # @type: array - trustedCAs: [ ] + trustedCAs: [] # Server, when enabled, configures a server cluster to run. This should # be disabled if you plan on connecting to a Consul cluster external to @@ -1217,7 +1217,7 @@ server: # global.acls.manageSystemACLs must be enabled to use this feature. enabled: false - # A single entry of the sink object provides configuration for the destination to which Consul + # A single entry of the sink object provides configuration for the destination to which Consul # will log auditing events. # # Example: @@ -1232,7 +1232,7 @@ server: # rotate_duration: 24h # rotate_max_files: 15 # rotate_bytes: 25165824 - # + # # ``` # # The sink object supports the following keys: @@ -2108,7 +2108,7 @@ connectInject: # @type: string nodeSelector: null - # Toleration settings for gateway pods created with the managed gateway class. + # Toleration settings for gateway pods created with the managed gateway class. # This should be a multi-line string matching the # [Tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) array in a Pod spec. # @@ -2134,7 +2134,7 @@ connectInject: service: null # This value defines the number of pods to deploy for each Gateway as well as a min and max number of pods for all Gateways - deployment: + deployment: defaultInstances: 1 maxInstances: 1 minInstances: 1 @@ -2554,6 +2554,26 @@ connectInject: # Recommended production default: 100m # @type: string cpu: null + # Set default lifecycle management configuration for sidecar proxy. + # These settings can be overridden on a per-pod basis via these annotations: + # + # - `consul.hashicorp.com/enable-sidecar-proxy-lifecycle` + # - `consul.hashicorp.com/enable-sidecar-proxy-shutdown-drain-listeners` + # - `consul.hashicorp.com/sidecar-proxy-lifecycle-shutdown-grace-period-seconds` + # - `consul.hashicorp.com/sidecar-proxy-lifecycle-graceful-port` + # - `consul.hashicorp.com/sidecar-proxy-lifecycle-graceful-shutdown-path` + # @type: map + lifecycle: + # @type: boolean + defaultEnabled: true + # @type: boolean + defaultEnableShutdownDrainListeners: true + # @type: integer + defaultShutdownGracePeriodSeconds: 30 + # @type: integer + defaultGracefulPort: 20600 + # @type: string + defaultGracefulShutdownPath: "/graceful_shutdown" # The resource settings for the Connect injected init container. If null, the resources # won't be set for the initContainer. The defaults are optimized for developer instances of @@ -3383,4 +3403,4 @@ telemetryCollector: # feature, in case kubernetes cluster is behind egress http proxies. Additionally, # it could be used to configure custom consul parameters. # @type: map - extraEnvironmentVars: { } + extraEnvironmentVars: {} diff --git a/cli/helm/values.go b/cli/helm/values.go index e6951074b1..06671382d1 100644 --- a/cli/helm/values.go +++ b/cli/helm/values.go @@ -411,7 +411,7 @@ type TransparentProxy struct { } type Metrics struct { - DefaultEnabled string `yaml:"defaultEnabled"` + DefaultEnabled bool `yaml:"defaultEnabled"` DefaultEnableMerging bool `yaml:"defaultEnableMerging"` DefaultMergedMetricsPort int `yaml:"defaultMergedMetricsPort"` DefaultPrometheusScrapePort int `yaml:"defaultPrometheusScrapePort"` @@ -425,12 +425,21 @@ type ACLInjectToken struct { type SidecarProxy struct { Resources Resources `yaml:"resources"` + Lifecycle Lifecycle `yaml:"lifecycle"` } type InitContainer struct { Resources Resources `yaml:"resources"` } +type Lifecycle struct { + DefaultEnabled bool `yaml:"defaultEnabled"` + DefaultEnableShutdownDrainListeners bool `yaml:"defaultEnableShutdownDrainListeners"` + DefaultShutdownGracePeriodSeconds int `yaml:"defaultShutdownGracePeriodSeconds"` + DefaultGracefulPort int `yaml:"defaultGracefulPort"` + DefaultGracefulShutdownPath string `yaml:"defaultGracefulShutdownPath"` +} + type ConnectInject struct { Enabled bool `yaml:"enabled"` Replicas int `yaml:"replicas"` diff --git a/control-plane/connect-inject/common/common.go b/control-plane/connect-inject/common/common.go index 67182e6d0a..a99d9fd12e 100644 --- a/control-plane/connect-inject/common/common.go +++ b/control-plane/connect-inject/common/common.go @@ -12,6 +12,40 @@ import ( corev1 "k8s.io/api/core/v1" ) +// DetermineAndValidatePort behaves as follows: +// If the annotation exists, validate the port and return it. +// If the annotation does not exist, return the default port. +// If the privileged flag is true, it will allow the port to be in the +// privileged port range of 1-1023. Otherwise, it will only allow ports in the +// unprivileged range of 1024-65535. +func DetermineAndValidatePort(pod corev1.Pod, annotation string, defaultPort string, privileged bool) (string, error) { + if raw, ok := pod.Annotations[annotation]; ok && raw != "" { + port, err := PortValue(pod, raw) + if err != nil { + return "", fmt.Errorf("%s annotation value of %s is not a valid integer", annotation, raw) + } + + if privileged && (port < 1 || port > 65535) { + return "", fmt.Errorf("%s annotation value of %d is not in the valid port range 1-65535", annotation, port) + } else if !privileged && (port < 1024 || port > 65535) { + return "", fmt.Errorf("%s annotation value of %d is not in the unprivileged port range 1024-65535", annotation, port) + } + + // If the annotation exists, return the validated port. + return fmt.Sprint(port), nil + } + + // If the annotation does not exist, return the default. + if defaultPort != "" { + port, err := PortValue(pod, defaultPort) + if err != nil { + return "", fmt.Errorf("%s is not a valid port on the pod %s", defaultPort, pod.Name) + } + return fmt.Sprint(port), nil + } + return "", nil +} + // PortValue returns the port of the container for the string value passed // in as an argument on the provided pod. func PortValue(pod corev1.Pod, value string) (int32, error) { diff --git a/control-plane/connect-inject/common/common_test.go b/control-plane/connect-inject/common/common_test.go index 3f995e2874..79a9294fe2 100644 --- a/control-plane/connect-inject/common/common_test.go +++ b/control-plane/connect-inject/common/common_test.go @@ -6,10 +6,153 @@ package common import ( "testing" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" + "github.com/hashicorp/consul-k8s/control-plane/namespaces" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +func TestCommonDetermineAndValidatePort(t *testing.T) { + cases := []struct { + Name string + Pod func(*corev1.Pod) *corev1.Pod + Annotation string + Privileged bool + DefaultPort string + Expected string + Err string + }{ + { + Name: "Valid annotation", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations["consul.hashicorp.com/test-annotation-port"] = "1234" + return pod + }, + Annotation: "consul.hashicorp.com/test-annotation-port", + Privileged: false, + Expected: "1234", + Err: "", + }, + { + Name: "Uses default when there's no annotation", + Pod: func(pod *corev1.Pod) *corev1.Pod { + return pod + }, + Annotation: "consul.hashicorp.com/test-annotation-port", + Privileged: false, + DefaultPort: "4321", + Expected: "4321", + Err: "", + }, + { + Name: "Gets the value of the named default port when there's no annotation", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Spec.Containers[0].Ports = []corev1.ContainerPort{ + { + Name: "web-port", + ContainerPort: 2222, + }, + } + return pod + }, + Annotation: "consul.hashicorp.com/test-annotation-port", + Privileged: false, + DefaultPort: "web-port", + Expected: "2222", + Err: "", + }, + { + Name: "Errors if the named default port doesn't exist on the pod", + Pod: func(pod *corev1.Pod) *corev1.Pod { + return pod + }, + Annotation: "consul.hashicorp.com/test-annotation-port", + Privileged: false, + DefaultPort: "web-port", + Expected: "", + Err: "web-port is not a valid port on the pod minimal", + }, + { + Name: "Gets the value of the named port", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations["consul.hashicorp.com/test-annotation-port"] = "web-port" + pod.Spec.Containers[0].Ports = []corev1.ContainerPort{ + { + Name: "web-port", + ContainerPort: 2222, + }, + } + return pod + }, + Annotation: "consul.hashicorp.com/test-annotation-port", + Privileged: false, + DefaultPort: "4321", + Expected: "2222", + Err: "", + }, + { + Name: "Invalid annotation (not an integer)", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations["consul.hashicorp.com/test-annotation-port"] = "not-an-int" + return pod + }, + Annotation: "consul.hashicorp.com/test-annotation-port", + Privileged: false, + Expected: "", + Err: "consul.hashicorp.com/test-annotation-port annotation value of not-an-int is not a valid integer", + }, + { + Name: "Invalid annotation (integer not in port range)", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations["consul.hashicorp.com/test-annotation-port"] = "100000" + return pod + }, + Annotation: "consul.hashicorp.com/test-annotation-port", + Privileged: true, + Expected: "", + Err: "consul.hashicorp.com/test-annotation-port annotation value of 100000 is not in the valid port range 1-65535", + }, + { + Name: "Invalid annotation (integer not in unprivileged port range)", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations["consul.hashicorp.com/test-annotation-port"] = "22" + return pod + }, + Annotation: "consul.hashicorp.com/test-annotation-port", + Privileged: false, + Expected: "", + Err: "consul.hashicorp.com/test-annotation-port annotation value of 22 is not in the unprivileged port range 1024-65535", + }, + { + Name: "Privileged ports allowed", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations["consul.hashicorp.com/test-annotation-port"] = "22" + return pod + }, + Annotation: "consul.hashicorp.com/test-annotation-port", + Privileged: true, + Expected: "22", + Err: "", + }, + } + + for _, tt := range cases { + t.Run(tt.Name, func(t *testing.T) { + require := require.New(t) + + actual, err := DetermineAndValidatePort(*tt.Pod(minimal()), tt.Annotation, tt.DefaultPort, tt.Privileged) + + if tt.Err == "" { + require.NoError(err) + require.Equal(tt.Expected, actual) + } else { + require.EqualError(err, tt.Err) + } + }) + } +} + func TestPortValue(t *testing.T) { cases := []struct { Name string @@ -93,3 +236,26 @@ func TestPortValue(t *testing.T) { }) } } + +func minimal() *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespaces.DefaultNamespace, + Name: "minimal", + Annotations: map[string]string{ + constants.AnnotationService: "foo", + }, + }, + + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + }, + { + Name: "web-side", + }, + }, + }, + } +} diff --git a/control-plane/connect-inject/constants/annotations_and_labels.go b/control-plane/connect-inject/constants/annotations_and_labels.go index fa5c7da26c..4efcc24c74 100644 --- a/control-plane/connect-inject/constants/annotations_and_labels.go +++ b/control-plane/connect-inject/constants/annotations_and_labels.go @@ -100,6 +100,13 @@ const ( AnnotationSidecarProxyMemoryLimit = "consul.hashicorp.com/sidecar-proxy-memory-limit" AnnotationSidecarProxyMemoryRequest = "consul.hashicorp.com/sidecar-proxy-memory-request" + // annotations for sidecar proxy lifecycle configuration. + AnnotationEnableSidecarProxyLifecycle = "consul.hashicorp.com/enable-sidecar-proxy-lifecycle" + AnnotationEnableSidecarProxyLifecycleShutdownDrainListeners = "consul.hashicorp.com/enable-sidecar-proxy-lifecycle-shutdown-drain-listeners" + AnnotationSidecarProxyLifecycleShutdownGracePeriodSeconds = "consul.hashicorp.com/sidecar-proxy-lifecycle-shutdown-grace-period-seconds" + AnnotationSidecarProxyLifecycleGracefulPort = "consul.hashicorp.com/sidecar-proxy-lifecycle-graceful-port" + AnnotationSidecarProxyLifecycleGracefulShutdownPath = "consul.hashicorp.com/sidecar-proxy-lifecycle-graceful-shutdown-path" + // annotations for sidecar volumes. AnnotationConsulSidecarUserVolume = "consul.hashicorp.com/consul-sidecar-user-volume" AnnotationConsulSidecarUserVolumeMount = "consul.hashicorp.com/consul-sidecar-user-volume-mount" diff --git a/control-plane/connect-inject/constants/constants.go b/control-plane/connect-inject/constants/constants.go index 0a341cd577..ca6fe23606 100644 --- a/control-plane/connect-inject/constants/constants.go +++ b/control-plane/connect-inject/constants/constants.go @@ -27,4 +27,10 @@ const ( // MetaKeyPodName is the meta key name for Kubernetes pod name used for the Consul services. MetaKeyPodName = "pod-name" + + // DefaultGracefulPort is the default port that consul-dataplane uses for graceful shutdown. + DefaultGracefulPort = 20600 + + // DefaultGracefulShutdownPath is the default path that consul-dataplane uses for graceful shutdown. + DefaultGracefulShutdownPath = "/graceful_shutdown" ) diff --git a/control-plane/connect-inject/lifecycle/lifecycle_configuration.go b/control-plane/connect-inject/lifecycle/lifecycle_configuration.go new file mode 100644 index 0000000000..651d4eecae --- /dev/null +++ b/control-plane/connect-inject/lifecycle/lifecycle_configuration.go @@ -0,0 +1,95 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package lifecycle + +import ( + "fmt" + "strconv" + + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/common" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" + corev1 "k8s.io/api/core/v1" +) + +// Config represents configuration common to connect-inject components related to proxy lifecycle management. +type Config struct { + DefaultEnableProxyLifecycle bool + DefaultEnableShutdownDrainListeners bool + DefaultShutdownGracePeriodSeconds int + DefaultGracefulPort string + DefaultGracefulShutdownPath string +} + +// EnableProxyLifecycle returns whether proxy lifecycle management is enabled either via the default value in the meshWebhook, or if it's been +// overridden via the annotation. +func (lc Config) EnableProxyLifecycle(pod corev1.Pod) (bool, error) { + enabled := lc.DefaultEnableProxyLifecycle + if raw, ok := pod.Annotations[constants.AnnotationEnableSidecarProxyLifecycle]; ok && raw != "" { + enableProxyLifecycle, err := strconv.ParseBool(raw) + if err != nil { + return false, fmt.Errorf("%s annotation value of %s was invalid: %s", constants.AnnotationEnableSidecarProxyLifecycle, raw, err) + } + enabled = enableProxyLifecycle + } + return enabled, nil +} + +// EnableShutdownDrainListeners returns whether proxy listener draining during shutdown is enabled either via the default value in the meshWebhook, or if it's been +// overridden via the annotation. +func (lc Config) EnableShutdownDrainListeners(pod corev1.Pod) (bool, error) { + enabled := lc.DefaultEnableShutdownDrainListeners + if raw, ok := pod.Annotations[constants.AnnotationEnableSidecarProxyLifecycleShutdownDrainListeners]; ok && raw != "" { + enableShutdownDrainListeners, err := strconv.ParseBool(raw) + if err != nil { + return false, fmt.Errorf("%s annotation value of %s was invalid: %s", constants.AnnotationEnableSidecarProxyLifecycleShutdownDrainListeners, raw, err) + } + enabled = enableShutdownDrainListeners + } + return enabled, nil +} + +// ShutdownGracePeriodSeconds returns how long the sidecar proxy should wait before shutdown, either via the default value in the meshWebhook, or if it's been +// overridden via the annotation. +func (lc Config) ShutdownGracePeriodSeconds(pod corev1.Pod) (int, error) { + shutdownGracePeriodSeconds := lc.DefaultShutdownGracePeriodSeconds + if shutdownGracePeriodSecondsAnnotation, ok := pod.Annotations[constants.AnnotationSidecarProxyLifecycleShutdownGracePeriodSeconds]; ok { + val, err := strconv.ParseUint(shutdownGracePeriodSecondsAnnotation, 10, 64) + if err != nil { + return 0, fmt.Errorf("unable to parse annotation %q: %w", constants.AnnotationSidecarProxyLifecycleShutdownGracePeriodSeconds, err) + } + shutdownGracePeriodSeconds = int(val) + } + return shutdownGracePeriodSeconds, nil +} + +// GracefulPort returns the port on which consul-dataplane should serve the proxy lifecycle management HTTP endpoints, either via the default value in the meshWebhook, or +// if it's been overridden via the annotation. It also validates the port is in the unprivileged port range. +func (lc Config) GracefulPort(pod corev1.Pod) (int, error) { + anno, err := common.DetermineAndValidatePort(pod, constants.AnnotationSidecarProxyLifecycleGracefulPort, lc.DefaultGracefulPort, false) + if err != nil { + return 0, err + } + + if anno == "" { + return constants.DefaultGracefulPort, nil + } + + port, _ := strconv.Atoi(anno) + + return port, nil +} + +// GracefulShutdownPath returns the path on which consul-dataplane should serve the graceful shutdown HTTP endpoint, either via the default value in the meshWebhook, or +// if it's been overridden via the annotation. +func (lc Config) GracefulShutdownPath(pod corev1.Pod) string { + if raw, ok := pod.Annotations[constants.AnnotationSidecarProxyLifecycleGracefulShutdownPath]; ok && raw != "" { + return raw + } + + if lc.DefaultGracefulShutdownPath == "" { + return constants.DefaultGracefulShutdownPath + } + + return lc.DefaultGracefulShutdownPath +} diff --git a/control-plane/connect-inject/lifecycle/lifecycle_configuration_test.go b/control-plane/connect-inject/lifecycle/lifecycle_configuration_test.go new file mode 100644 index 0000000000..64157a3d55 --- /dev/null +++ b/control-plane/connect-inject/lifecycle/lifecycle_configuration_test.go @@ -0,0 +1,351 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package lifecycle + +import ( + "testing" + + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" + "github.com/hashicorp/consul-k8s/control-plane/namespaces" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestLifecycleConfig_EnableSidecarProxyLifecycle(t *testing.T) { + cases := []struct { + Name string + Pod func(*corev1.Pod) *corev1.Pod + LifecycleConfig Config + Expected bool + Err string + }{ + { + Name: "Sidecar proxy lifecycle management enabled via meshWebhook", + Pod: func(pod *corev1.Pod) *corev1.Pod { + return pod + }, + LifecycleConfig: Config{ + DefaultEnableProxyLifecycle: true, + }, + Expected: true, + Err: "", + }, + { + Name: "Sidecar proxy lifecycle management enabled via annotation", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations[constants.AnnotationEnableSidecarProxyLifecycle] = "true" + return pod + }, + LifecycleConfig: Config{ + DefaultEnableProxyLifecycle: false, + }, + Expected: true, + Err: "", + }, + { + Name: "Sidecar proxy lifecycle management configured via invalid annotation", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations[constants.AnnotationEnableSidecarProxyLifecycle] = "not-a-bool" + return pod + }, + LifecycleConfig: Config{ + DefaultEnableProxyLifecycle: false, + }, + Expected: false, + Err: "consul.hashicorp.com/enable-sidecar-proxy-lifecycle annotation value of not-a-bool was invalid: strconv.ParseBool: parsing \"not-a-bool\": invalid syntax", + }, + } + + for _, tt := range cases { + t.Run(tt.Name, func(t *testing.T) { + require := require.New(t) + lc := tt.LifecycleConfig + + actual, err := lc.EnableProxyLifecycle(*tt.Pod(minimal())) + + if tt.Err == "" { + require.Equal(tt.Expected, actual) + require.NoError(err) + } else { + require.EqualError(err, tt.Err) + } + }) + } +} + +func TestLifecycleConfig_ShutdownDrainListeners(t *testing.T) { + cases := []struct { + Name string + Pod func(*corev1.Pod) *corev1.Pod + LifecycleConfig Config + Expected bool + Err string + }{ + { + Name: "Sidecar proxy shutdown listener draining enabled via meshWebhook", + Pod: func(pod *corev1.Pod) *corev1.Pod { + return pod + }, + LifecycleConfig: Config{ + DefaultEnableShutdownDrainListeners: true, + }, + Expected: true, + Err: "", + }, + { + Name: "Sidecar proxy shutdown listener draining enabled via annotation", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations[constants.AnnotationEnableSidecarProxyLifecycleShutdownDrainListeners] = "true" + return pod + }, + LifecycleConfig: Config{ + DefaultEnableShutdownDrainListeners: false, + }, + Expected: true, + Err: "", + }, + { + Name: "Sidecar proxy shutdown listener draining configured via invalid annotation", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations[constants.AnnotationEnableSidecarProxyLifecycleShutdownDrainListeners] = "not-a-bool" + return pod + }, + Expected: false, + Err: "consul.hashicorp.com/enable-sidecar-proxy-lifecycle-shutdown-drain-listeners annotation value of not-a-bool was invalid: strconv.ParseBool: parsing \"not-a-bool\": invalid syntax", + }, + } + + for _, tt := range cases { + t.Run(tt.Name, func(t *testing.T) { + require := require.New(t) + lc := tt.LifecycleConfig + + actual, err := lc.EnableShutdownDrainListeners(*tt.Pod(minimal())) + + if tt.Err == "" { + require.Equal(tt.Expected, actual) + require.NoError(err) + } else { + require.EqualError(err, tt.Err) + } + }) + } +} + +func TestLifecycleConfig_ShutdownGracePeriodSeconds(t *testing.T) { + cases := []struct { + Name string + Pod func(*corev1.Pod) *corev1.Pod + LifecycleConfig Config + Expected int + Err string + }{ + { + Name: "Sidecar proxy shutdown grace period set via meshWebhook", + Pod: func(pod *corev1.Pod) *corev1.Pod { + return pod + }, + LifecycleConfig: Config{ + DefaultShutdownGracePeriodSeconds: 10, + }, + Expected: 10, + Err: "", + }, + { + Name: "Sidecar proxy shutdown grace period set via annotation", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations[constants.AnnotationSidecarProxyLifecycleShutdownGracePeriodSeconds] = "20" + return pod + }, + LifecycleConfig: Config{ + DefaultShutdownGracePeriodSeconds: 10, + }, + Expected: 20, + Err: "", + }, + { + Name: "Sidecar proxy shutdown grace period configured via invalid annotation, negative number", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations[constants.AnnotationSidecarProxyLifecycleShutdownGracePeriodSeconds] = "-1" + return pod + }, + Err: "unable to parse annotation \"consul.hashicorp.com/sidecar-proxy-lifecycle-shutdown-grace-period-seconds\": strconv.ParseUint: parsing \"-1\": invalid syntax", + }, + { + Name: "Sidecar proxy shutdown grace period configured via invalid annotation, not-parseable string", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations[constants.AnnotationSidecarProxyLifecycleShutdownGracePeriodSeconds] = "not-int" + return pod + }, + Err: "unable to parse annotation \"consul.hashicorp.com/sidecar-proxy-lifecycle-shutdown-grace-period-seconds\": strconv.ParseUint: parsing \"not-int\": invalid syntax", + }, + } + + for _, tt := range cases { + t.Run(tt.Name, func(t *testing.T) { + require := require.New(t) + lc := tt.LifecycleConfig + + actual, err := lc.ShutdownGracePeriodSeconds(*tt.Pod(minimal())) + + if tt.Err == "" { + require.Equal(tt.Expected, actual) + require.NoError(err) + } else { + require.EqualError(err, tt.Err) + } + }) + } +} + +func TestLifecycleConfig_GracefulPort(t *testing.T) { + cases := []struct { + Name string + Pod func(*corev1.Pod) *corev1.Pod + LifecycleConfig Config + Expected int + Err string + }{ + { + Name: "Sidecar proxy lifecycle graceful port set to default", + Pod: func(pod *corev1.Pod) *corev1.Pod { + return pod + }, + Expected: constants.DefaultGracefulPort, + Err: "", + }, + { + Name: "Sidecar proxy lifecycle graceful port set via meshWebhook", + Pod: func(pod *corev1.Pod) *corev1.Pod { + return pod + }, + LifecycleConfig: Config{ + DefaultGracefulPort: "3000", + }, + Expected: 3000, + Err: "", + }, + { + Name: "Sidecar proxy lifecycle graceful port set via annotation", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations[constants.AnnotationSidecarProxyLifecycleGracefulPort] = "9000" + return pod + }, + LifecycleConfig: Config{ + DefaultGracefulPort: "3000", + }, + Expected: 9000, + Err: "", + }, + { + Name: "Sidecar proxy lifecycle graceful port configured via invalid annotation, negative number", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations[constants.AnnotationSidecarProxyLifecycleGracefulPort] = "-1" + return pod + }, + Err: "consul.hashicorp.com/sidecar-proxy-lifecycle-graceful-port annotation value of -1 is not in the unprivileged port range 1024-65535", + }, + { + Name: "Sidecar proxy lifecycle graceful port configured via invalid annotation, not-parseable string", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations[constants.AnnotationSidecarProxyLifecycleGracefulPort] = "not-int" + return pod + }, + Err: "consul.hashicorp.com/sidecar-proxy-lifecycle-graceful-port annotation value of not-int is not a valid integer", + }, + } + + for _, tt := range cases { + t.Run(tt.Name, func(t *testing.T) { + require := require.New(t) + lc := tt.LifecycleConfig + + actual, err := lc.GracefulPort(*tt.Pod(minimal())) + + if tt.Err == "" { + require.Equal(tt.Expected, actual) + require.NoError(err) + } else { + require.EqualError(err, tt.Err) + } + }) + } +} + +func TestLifecycleConfig_GracefulShutdownPath(t *testing.T) { + cases := []struct { + Name string + Pod func(*corev1.Pod) *corev1.Pod + LifecycleConfig Config + Expected string + Err string + }{ + { + Name: "Sidecar proxy lifecycle graceful shutdown path defaults to /graceful_shutdown", + Pod: func(pod *corev1.Pod) *corev1.Pod { + return pod + }, + Expected: "/graceful_shutdown", + Err: "", + }, + { + Name: "Sidecar proxy lifecycle graceful shutdown path set via meshWebhook", + Pod: func(pod *corev1.Pod) *corev1.Pod { + return pod + }, + LifecycleConfig: Config{ + DefaultGracefulShutdownPath: "/quit", + }, + Expected: "/quit", + Err: "", + }, + { + Name: "Sidecar proxy lifecycle graceful port set via annotation", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations[constants.AnnotationSidecarProxyLifecycleGracefulShutdownPath] = "/custom-shutdown-path" + return pod + }, + LifecycleConfig: Config{ + DefaultGracefulShutdownPath: "/quit", + }, + Expected: "/custom-shutdown-path", + Err: "", + }, + } + + for _, tt := range cases { + t.Run(tt.Name, func(t *testing.T) { + require := require.New(t) + lc := tt.LifecycleConfig + + actual := lc.GracefulShutdownPath(*tt.Pod(minimal())) + + require.Equal(tt.Expected, actual) + }) + } +} + +func minimal() *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespaces.DefaultNamespace, + Name: "minimal", + Annotations: map[string]string{ + constants.AnnotationService: "foo", + }, + }, + + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + }, + { + Name: "web-side", + }, + }, + }, + } +} diff --git a/control-plane/connect-inject/metrics/metrics_configuration.go b/control-plane/connect-inject/metrics/metrics_configuration.go index f5b819af3d..6f9c29c85b 100644 --- a/control-plane/connect-inject/metrics/metrics_configuration.go +++ b/control-plane/connect-inject/metrics/metrics_configuration.go @@ -98,13 +98,13 @@ func (mc Config) EnableMetricsMerging(pod corev1.Pod) (bool, error) { // MergedMetricsPort returns the port to run the merged metrics server on, either via the default value in the meshWebhook, // or if it's been overridden via the annotation. It also validates the port is in the unprivileged port range. func (mc Config) MergedMetricsPort(pod corev1.Pod) (string, error) { - return determineAndValidatePort(pod, constants.AnnotationMergedMetricsPort, mc.DefaultMergedMetricsPort, false) + return common.DetermineAndValidatePort(pod, constants.AnnotationMergedMetricsPort, mc.DefaultMergedMetricsPort, false) } // PrometheusScrapePort returns the port for Prometheus to scrape from, either via the default value in the meshWebhook, or // if it's been overridden via the annotation. It also validates the port is in the unprivileged port range. func (mc Config) PrometheusScrapePort(pod corev1.Pod) (string, error) { - return determineAndValidatePort(pod, constants.AnnotationPrometheusScrapePort, mc.DefaultPrometheusScrapePort, false) + return common.DetermineAndValidatePort(pod, constants.AnnotationPrometheusScrapePort, mc.DefaultPrometheusScrapePort, false) } // PrometheusScrapePath returns the path for Prometheus to scrape from, either via the default value in the meshWebhook, or @@ -133,14 +133,14 @@ func (mc Config) ServiceMetricsPort(pod corev1.Pod) (string, error) { // written their service in such a way that it expects to be able to use // privileged ports. So, the port metrics are exposed on the service can // be privileged. - return determineAndValidatePort(pod, constants.AnnotationServiceMetricsPort, raw, true) + return common.DetermineAndValidatePort(pod, constants.AnnotationServiceMetricsPort, raw, true) } // If the annotationPort is not set, the serviceMetrics port will be 0 // unless overridden by the service-metrics-port annotation. If the service // metrics port is 0, the consul sidecar will not run a merged metrics // server. - return determineAndValidatePort(pod, constants.AnnotationServiceMetricsPort, "0", true) + return common.DetermineAndValidatePort(pod, constants.AnnotationServiceMetricsPort, "0", true) } // ServiceMetricsPath returns a default of /metrics, or overrides @@ -180,37 +180,3 @@ func (mc Config) ShouldRunMergedMetricsServer(pod corev1.Pod) (bool, error) { } return false, nil } - -// determineAndValidatePort behaves as follows: -// If the annotation exists, validate the port and return it. -// If the annotation does not exist, return the default port. -// If the privileged flag is true, it will allow the port to be in the -// privileged port range of 1-1023. Otherwise, it will only allow ports in the -// unprivileged range of 1024-65535. -func determineAndValidatePort(pod corev1.Pod, annotation string, defaultPort string, privileged bool) (string, error) { - if raw, ok := pod.Annotations[annotation]; ok && raw != "" { - port, err := common.PortValue(pod, raw) - if err != nil { - return "", fmt.Errorf("%s annotation value of %s is not a valid integer", annotation, raw) - } - - if privileged && (port < 1 || port > 65535) { - return "", fmt.Errorf("%s annotation value of %d is not in the valid port range 1-65535", annotation, port) - } else if !privileged && (port < 1024 || port > 65535) { - return "", fmt.Errorf("%s annotation value of %d is not in the unprivileged port range 1024-65535", annotation, port) - } - - // If the annotation exists, return the validated port. - return fmt.Sprint(port), nil - } - - // If the annotation does not exist, return the default. - if defaultPort != "" { - port, err := common.PortValue(pod, defaultPort) - if err != nil { - return "", fmt.Errorf("%s is not a valid port on the pod %s", defaultPort, pod.Name) - } - return fmt.Sprint(port), nil - } - return "", nil -} diff --git a/control-plane/connect-inject/metrics/metrics_configuration_test.go b/control-plane/connect-inject/metrics/metrics_configuration_test.go index ec19d4f55a..12045e28d1 100644 --- a/control-plane/connect-inject/metrics/metrics_configuration_test.go +++ b/control-plane/connect-inject/metrics/metrics_configuration_test.go @@ -307,149 +307,6 @@ func TestMetricsConfigShouldRunMergedMetricsServer(t *testing.T) { } } -// Tests determineAndValidatePort, which in turn tests the -// PrometheusScrapePort() and MergedMetricsPort() functions because their logic -// is just to call out to determineAndValidatePort(). -func TestMetricsConfigDetermineAndValidatePort(t *testing.T) { - cases := []struct { - Name string - Pod func(*corev1.Pod) *corev1.Pod - Annotation string - Privileged bool - DefaultPort string - Expected string - Err string - }{ - { - Name: "Valid annotation", - Pod: func(pod *corev1.Pod) *corev1.Pod { - pod.Annotations["consul.hashicorp.com/test-annotation-port"] = "1234" - return pod - }, - Annotation: "consul.hashicorp.com/test-annotation-port", - Privileged: false, - Expected: "1234", - Err: "", - }, - { - Name: "Uses default when there's no annotation", - Pod: func(pod *corev1.Pod) *corev1.Pod { - return pod - }, - Annotation: "consul.hashicorp.com/test-annotation-port", - Privileged: false, - DefaultPort: "4321", - Expected: "4321", - Err: "", - }, - { - Name: "Gets the value of the named default port when there's no annotation", - Pod: func(pod *corev1.Pod) *corev1.Pod { - pod.Spec.Containers[0].Ports = []corev1.ContainerPort{ - { - Name: "web-port", - ContainerPort: 2222, - }, - } - return pod - }, - Annotation: "consul.hashicorp.com/test-annotation-port", - Privileged: false, - DefaultPort: "web-port", - Expected: "2222", - Err: "", - }, - { - Name: "Errors if the named default port doesn't exist on the pod", - Pod: func(pod *corev1.Pod) *corev1.Pod { - return pod - }, - Annotation: "consul.hashicorp.com/test-annotation-port", - Privileged: false, - DefaultPort: "web-port", - Expected: "", - Err: "web-port is not a valid port on the pod minimal", - }, - { - Name: "Gets the value of the named port", - Pod: func(pod *corev1.Pod) *corev1.Pod { - pod.Annotations["consul.hashicorp.com/test-annotation-port"] = "web-port" - pod.Spec.Containers[0].Ports = []corev1.ContainerPort{ - { - Name: "web-port", - ContainerPort: 2222, - }, - } - return pod - }, - Annotation: "consul.hashicorp.com/test-annotation-port", - Privileged: false, - DefaultPort: "4321", - Expected: "2222", - Err: "", - }, - { - Name: "Invalid annotation (not an integer)", - Pod: func(pod *corev1.Pod) *corev1.Pod { - pod.Annotations["consul.hashicorp.com/test-annotation-port"] = "not-an-int" - return pod - }, - Annotation: "consul.hashicorp.com/test-annotation-port", - Privileged: false, - Expected: "", - Err: "consul.hashicorp.com/test-annotation-port annotation value of not-an-int is not a valid integer", - }, - { - Name: "Invalid annotation (integer not in port range)", - Pod: func(pod *corev1.Pod) *corev1.Pod { - pod.Annotations["consul.hashicorp.com/test-annotation-port"] = "100000" - return pod - }, - Annotation: "consul.hashicorp.com/test-annotation-port", - Privileged: true, - Expected: "", - Err: "consul.hashicorp.com/test-annotation-port annotation value of 100000 is not in the valid port range 1-65535", - }, - { - Name: "Invalid annotation (integer not in unprivileged port range)", - Pod: func(pod *corev1.Pod) *corev1.Pod { - pod.Annotations["consul.hashicorp.com/test-annotation-port"] = "22" - return pod - }, - Annotation: "consul.hashicorp.com/test-annotation-port", - Privileged: false, - Expected: "", - Err: "consul.hashicorp.com/test-annotation-port annotation value of 22 is not in the unprivileged port range 1024-65535", - }, - { - Name: "Privileged ports allowed", - Pod: func(pod *corev1.Pod) *corev1.Pod { - pod.Annotations["consul.hashicorp.com/test-annotation-port"] = "22" - return pod - }, - Annotation: "consul.hashicorp.com/test-annotation-port", - Privileged: true, - Expected: "22", - Err: "", - }, - } - - for _, tt := range cases { - t.Run(tt.Name, func(t *testing.T) { - require := require.New(t) - - actual, err := determineAndValidatePort(*tt.Pod(minimal()), tt.Annotation, tt.DefaultPort, tt.Privileged) - - if tt.Err == "" { - require.NoError(err) - require.Equal(tt.Expected, actual) - } else { - require.EqualError(err, tt.Err) - } - }) - } -} - // Tests MergedMetricsServerConfiguration happy path and error case not covered by other Config tests. func TestMetricsConfigMergedMetricsServerConfiguration(t *testing.T) { cases := []struct { diff --git a/control-plane/connect-inject/webhook/consul_dataplane_sidecar.go b/control-plane/connect-inject/webhook/consul_dataplane_sidecar.go index fe37720b7d..68f57ed061 100644 --- a/control-plane/connect-inject/webhook/consul_dataplane_sidecar.go +++ b/control-plane/connect-inject/webhook/consul_dataplane_sidecar.go @@ -247,6 +247,45 @@ func (w *MeshWebhook) getContainerSidecarArgs(namespace corev1.Namespace, mpi mu args = append(args, fmt.Sprintf("-envoy-admin-bind-port=%d", 19000+mpi.serviceIndex)) } + // The consul-dataplane HTTP listener always starts for graceful shutdown. To avoid port conflicts, the + // graceful port always needs to be set + gracefulPort, err := w.LifecycleConfig.GracefulPort(pod) + if err != nil { + return nil, fmt.Errorf("unable to determine proxy lifecycle graceful port: %w", err) + } + + // To avoid conflicts + if mpi.serviceName != "" { + gracefulPort = gracefulPort + mpi.serviceIndex + } + args = append(args, fmt.Sprintf("-graceful-port=%d", gracefulPort)) + + enableProxyLifecycle, err := w.LifecycleConfig.EnableProxyLifecycle(pod) + if err != nil { + return nil, fmt.Errorf("unable to determine if proxy lifecycle management is enabled: %w", err) + } + if enableProxyLifecycle { + shutdownDrainListeners, err := w.LifecycleConfig.EnableShutdownDrainListeners(pod) + if err != nil { + return nil, fmt.Errorf("unable to determine if proxy lifecycle shutdown listener draining is enabled: %w", err) + } + if shutdownDrainListeners { + args = append(args, "-shutdown-drain-listeners") + } + + shutdownGracePeriodSeconds, err := w.LifecycleConfig.ShutdownGracePeriodSeconds(pod) + if err != nil { + return nil, fmt.Errorf("unable to determine proxy lifecycle shutdown grace period: %w", err) + } + args = append(args, fmt.Sprintf("-shutdown-grace-period-seconds=%d", shutdownGracePeriodSeconds)) + + gracefulShutdownPath := w.LifecycleConfig.GracefulShutdownPath(pod) + if err != nil { + return nil, fmt.Errorf("unable to determine proxy lifecycle graceful shutdown path: %w", err) + } + args = append(args, fmt.Sprintf("-graceful-shutdown-path=%s", gracefulShutdownPath)) + } + // Set a default scrape path that can be overwritten by the annotation. prometheusScrapePath := w.MetricsConfig.PrometheusScrapePath(pod) args = append(args, "-telemetry-prom-scrape-path="+prometheusScrapePath) diff --git a/control-plane/connect-inject/webhook/consul_dataplane_sidecar_test.go b/control-plane/connect-inject/webhook/consul_dataplane_sidecar_test.go index 0860293352..d83b094d99 100644 --- a/control-plane/connect-inject/webhook/consul_dataplane_sidecar_test.go +++ b/control-plane/connect-inject/webhook/consul_dataplane_sidecar_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/lifecycle" "github.com/hashicorp/consul-k8s/control-plane/consul" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" @@ -28,20 +29,20 @@ func TestHandlerConsulDataplaneSidecar(t *testing.T) { }{ "default": { webhookSetupFunc: nil, - additionalExpCmdArgs: " -tls-disabled -telemetry-prom-scrape-path=/metrics", + additionalExpCmdArgs: " -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", }, "with custom gRPC port": { webhookSetupFunc: func(w *MeshWebhook) { w.ConsulConfig.GRPCPort = 8602 }, - additionalExpCmdArgs: " -tls-disabled -telemetry-prom-scrape-path=/metrics", + additionalExpCmdArgs: " -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", }, "with ACLs": { webhookSetupFunc: func(w *MeshWebhook) { w.AuthMethod = "test-auth-method" }, additionalExpCmdArgs: " -credential-type=login -login-auth-method=test-auth-method -login-bearer-token-path=/var/run/secrets/kubernetes.io/serviceaccount/token " + - "-login-meta=pod=k8snamespace/test-pod -tls-disabled -telemetry-prom-scrape-path=/metrics", + "-login-meta=pod=k8snamespace/test-pod -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", }, "with ACLs and namespace mirroring": { webhookSetupFunc: func(w *MeshWebhook) { @@ -50,7 +51,7 @@ func TestHandlerConsulDataplaneSidecar(t *testing.T) { w.EnableK8SNSMirroring = true }, additionalExpCmdArgs: " -credential-type=login -login-auth-method=test-auth-method -login-bearer-token-path=/var/run/secrets/kubernetes.io/serviceaccount/token " + - "-login-meta=pod=k8snamespace/test-pod -login-namespace=default -service-namespace=k8snamespace -tls-disabled -telemetry-prom-scrape-path=/metrics", + "-login-meta=pod=k8snamespace/test-pod -login-namespace=default -service-namespace=k8snamespace -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", }, "with ACLs and single destination namespace": { webhookSetupFunc: func(w *MeshWebhook) { @@ -59,7 +60,7 @@ func TestHandlerConsulDataplaneSidecar(t *testing.T) { w.ConsulDestinationNamespace = "test-ns" }, additionalExpCmdArgs: " -credential-type=login -login-auth-method=test-auth-method -login-bearer-token-path=/var/run/secrets/kubernetes.io/serviceaccount/token " + - "-login-meta=pod=k8snamespace/test-pod -login-namespace=test-ns -service-namespace=test-ns -tls-disabled -telemetry-prom-scrape-path=/metrics", + "-login-meta=pod=k8snamespace/test-pod -login-namespace=test-ns -service-namespace=test-ns -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", }, "with ACLs and partitions": { webhookSetupFunc: func(w *MeshWebhook) { @@ -67,7 +68,7 @@ func TestHandlerConsulDataplaneSidecar(t *testing.T) { w.ConsulPartition = "test-part" }, additionalExpCmdArgs: " -credential-type=login -login-auth-method=test-auth-method -login-bearer-token-path=/var/run/secrets/kubernetes.io/serviceaccount/token " + - "-login-meta=pod=k8snamespace/test-pod -login-partition=test-part -service-partition=test-part -tls-disabled -telemetry-prom-scrape-path=/metrics", + "-login-meta=pod=k8snamespace/test-pod -login-partition=test-part -service-partition=test-part -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", }, "with TLS and CA cert provided": { webhookSetupFunc: func(w *MeshWebhook) { @@ -75,28 +76,28 @@ func TestHandlerConsulDataplaneSidecar(t *testing.T) { w.ConsulTLSServerName = "server.dc1.consul" w.ConsulCACert = "consul-ca-cert" }, - additionalExpCmdArgs: " -tls-server-name=server.dc1.consul -ca-certs=/consul/connect-inject/consul-ca.pem -telemetry-prom-scrape-path=/metrics", + additionalExpCmdArgs: " -tls-server-name=server.dc1.consul -ca-certs=/consul/connect-inject/consul-ca.pem -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", }, "with TLS and no CA cert provided": { webhookSetupFunc: func(w *MeshWebhook) { w.TLSEnabled = true w.ConsulTLSServerName = "server.dc1.consul" }, - additionalExpCmdArgs: " -tls-server-name=server.dc1.consul -telemetry-prom-scrape-path=/metrics", + additionalExpCmdArgs: " -tls-server-name=server.dc1.consul -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", }, "with single destination namespace": { webhookSetupFunc: func(w *MeshWebhook) { w.EnableNamespaces = true w.ConsulDestinationNamespace = "consul-namespace" }, - additionalExpCmdArgs: " -service-namespace=consul-namespace -tls-disabled -telemetry-prom-scrape-path=/metrics", + additionalExpCmdArgs: " -service-namespace=consul-namespace -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", }, "with namespace mirroring": { webhookSetupFunc: func(w *MeshWebhook) { w.EnableNamespaces = true w.EnableK8SNSMirroring = true }, - additionalExpCmdArgs: " -service-namespace=k8snamespace -tls-disabled -telemetry-prom-scrape-path=/metrics", + additionalExpCmdArgs: " -service-namespace=k8snamespace -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", }, "with namespace mirroring prefix": { webhookSetupFunc: func(w *MeshWebhook) { @@ -104,38 +105,38 @@ func TestHandlerConsulDataplaneSidecar(t *testing.T) { w.EnableK8SNSMirroring = true w.K8SNSMirroringPrefix = "foo-" }, - additionalExpCmdArgs: " -service-namespace=foo-k8snamespace -tls-disabled -telemetry-prom-scrape-path=/metrics", + additionalExpCmdArgs: " -service-namespace=foo-k8snamespace -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", }, "with partitions": { webhookSetupFunc: func(w *MeshWebhook) { w.ConsulPartition = "partition-1" }, - additionalExpCmdArgs: " -service-partition=partition-1 -tls-disabled -telemetry-prom-scrape-path=/metrics", + additionalExpCmdArgs: " -service-partition=partition-1 -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", }, "with different log level": { webhookSetupFunc: func(w *MeshWebhook) { w.LogLevel = "debug" }, - additionalExpCmdArgs: " -tls-disabled -telemetry-prom-scrape-path=/metrics", + additionalExpCmdArgs: " -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", }, "with different log level and log json": { webhookSetupFunc: func(w *MeshWebhook) { w.LogLevel = "debug" w.LogJSON = true }, - additionalExpCmdArgs: " -tls-disabled -telemetry-prom-scrape-path=/metrics", + additionalExpCmdArgs: " -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", }, "skip server watch enabled": { webhookSetupFunc: func(w *MeshWebhook) { w.SkipServerWatch = true }, - additionalExpCmdArgs: " -server-watch-disabled=true -tls-disabled -telemetry-prom-scrape-path=/metrics", + additionalExpCmdArgs: " -server-watch-disabled=true -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", }, "custom prometheus scrape path": { webhookSetupFunc: func(w *MeshWebhook) { w.MetricsConfig.DefaultPrometheusScrapePath = "/scrape-path" // Simulate what would be passed as a flag }, - additionalExpCmdArgs: " -tls-disabled -telemetry-prom-scrape-path=/scrape-path", + additionalExpCmdArgs: " -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/scrape-path", }, } @@ -622,18 +623,18 @@ func TestHandlerConsulDataplaneSidecar_Multiport(t *testing.T) { } expArgs := []string{ "-addresses 1.1.1.1 -grpc-port=8502 -proxy-service-id-path=/consul/connect-inject/proxyid-web " + - "-log-level=info -log-json=false -envoy-concurrency=0 -tls-disabled -envoy-admin-bind-port=19000 -telemetry-prom-scrape-path=/metrics -- --base-id 0", + "-log-level=info -log-json=false -envoy-concurrency=0 -tls-disabled -envoy-admin-bind-port=19000 -graceful-port=20600 -telemetry-prom-scrape-path=/metrics -- --base-id 0", "-addresses 1.1.1.1 -grpc-port=8502 -proxy-service-id-path=/consul/connect-inject/proxyid-web-admin " + - "-log-level=info -log-json=false -envoy-concurrency=0 -tls-disabled -envoy-admin-bind-port=19001 -telemetry-prom-scrape-path=/metrics -- --base-id 1", + "-log-level=info -log-json=false -envoy-concurrency=0 -tls-disabled -envoy-admin-bind-port=19001 -graceful-port=20601 -telemetry-prom-scrape-path=/metrics -- --base-id 1", } if aclsEnabled { expArgs = []string{ "-addresses 1.1.1.1 -grpc-port=8502 -proxy-service-id-path=/consul/connect-inject/proxyid-web " + "-log-level=info -log-json=false -envoy-concurrency=0 -credential-type=login -login-auth-method=test-auth-method " + - "-login-bearer-token-path=/var/run/secrets/kubernetes.io/serviceaccount/token -login-meta=pod=k8snamespace/test-pod -tls-disabled -envoy-admin-bind-port=19000 -telemetry-prom-scrape-path=/metrics -- --base-id 0", + "-login-bearer-token-path=/var/run/secrets/kubernetes.io/serviceaccount/token -login-meta=pod=k8snamespace/test-pod -tls-disabled -envoy-admin-bind-port=19000 -graceful-port=20600 -telemetry-prom-scrape-path=/metrics -- --base-id 0", "-addresses 1.1.1.1 -grpc-port=8502 -proxy-service-id-path=/consul/connect-inject/proxyid-web-admin " + "-log-level=info -log-json=false -envoy-concurrency=0 -credential-type=login -login-auth-method=test-auth-method " + - "-login-bearer-token-path=/consul/serviceaccount-web-admin/token -login-meta=pod=k8snamespace/test-pod -tls-disabled -envoy-admin-bind-port=19001 -telemetry-prom-scrape-path=/metrics -- --base-id 1", + "-login-bearer-token-path=/consul/serviceaccount-web-admin/token -login-meta=pod=k8snamespace/test-pod -tls-disabled -envoy-admin-bind-port=19001 -graceful-port=20601 -telemetry-prom-scrape-path=/metrics -- --base-id 1", } } expSAVolumeMounts := []corev1.VolumeMount{ @@ -1299,6 +1300,156 @@ func TestHandlerConsulDataplaneSidecar_Metrics(t *testing.T) { } } +func TestHandlerConsulDataplaneSidecar_Lifecycle(t *testing.T) { + gracefulShutdownSeconds := 10 + gracefulPort := "20307" + gracefulShutdownPath := "/exit" + + cases := []struct { + name string + webhook MeshWebhook + annotations map[string]string + expCmdArgs string + expErr string + }{ + { + name: "no defaults, no annotations", + webhook: MeshWebhook{}, + annotations: nil, + expCmdArgs: "", + }, + { + name: "all defaults, no annotations", + webhook: MeshWebhook{ + LifecycleConfig: lifecycle.Config{ + DefaultEnableProxyLifecycle: true, + DefaultEnableShutdownDrainListeners: true, + DefaultShutdownGracePeriodSeconds: gracefulShutdownSeconds, + DefaultGracefulPort: gracefulPort, + DefaultGracefulShutdownPath: gracefulShutdownPath, + }, + }, + annotations: nil, + expCmdArgs: "graceful-port=20307 -shutdown-drain-listeners -shutdown-grace-period-seconds=10 -graceful-shutdown-path=/exit", + }, + { + name: "no defaults, all annotations", + webhook: MeshWebhook{}, + annotations: map[string]string{ + constants.AnnotationEnableSidecarProxyLifecycle: "true", + constants.AnnotationEnableSidecarProxyLifecycleShutdownDrainListeners: "true", + constants.AnnotationSidecarProxyLifecycleShutdownGracePeriodSeconds: fmt.Sprint(gracefulShutdownSeconds), + constants.AnnotationSidecarProxyLifecycleGracefulPort: gracefulPort, + constants.AnnotationSidecarProxyLifecycleGracefulShutdownPath: gracefulShutdownPath, + }, + expCmdArgs: "-graceful-port=20307 -shutdown-drain-listeners -shutdown-grace-period-seconds=10 -graceful-shutdown-path=/exit", + }, + { + name: "annotations override defaults", + webhook: MeshWebhook{ + LifecycleConfig: lifecycle.Config{ + DefaultEnableProxyLifecycle: false, + DefaultEnableShutdownDrainListeners: true, + DefaultShutdownGracePeriodSeconds: gracefulShutdownSeconds, + DefaultGracefulPort: gracefulPort, + DefaultGracefulShutdownPath: gracefulShutdownPath, + }, + }, + annotations: map[string]string{ + constants.AnnotationEnableSidecarProxyLifecycle: "true", + constants.AnnotationEnableSidecarProxyLifecycleShutdownDrainListeners: "false", + constants.AnnotationSidecarProxyLifecycleShutdownGracePeriodSeconds: fmt.Sprint(gracefulShutdownSeconds + 5), + constants.AnnotationSidecarProxyLifecycleGracefulPort: "20317", + constants.AnnotationSidecarProxyLifecycleGracefulShutdownPath: "/foo", + }, + expCmdArgs: "-graceful-port=20317 -shutdown-grace-period-seconds=15 -graceful-shutdown-path=/foo", + }, + { + name: "lifecycle disabled, no annotations", + webhook: MeshWebhook{ + LifecycleConfig: lifecycle.Config{ + DefaultEnableProxyLifecycle: false, + DefaultEnableShutdownDrainListeners: true, + DefaultShutdownGracePeriodSeconds: gracefulShutdownSeconds, + DefaultGracefulPort: gracefulPort, + DefaultGracefulShutdownPath: gracefulShutdownPath, + }, + }, + annotations: nil, + expCmdArgs: "-graceful-port=20307", + }, + { + name: "lifecycle enabled, defaults omited, no annotations", + webhook: MeshWebhook{ + LifecycleConfig: lifecycle.Config{ + DefaultEnableProxyLifecycle: true, + }, + }, + annotations: nil, + expCmdArgs: "", + }, + { + name: "annotations disable lifecycle default", + webhook: MeshWebhook{ + LifecycleConfig: lifecycle.Config{ + DefaultEnableProxyLifecycle: true, + DefaultEnableShutdownDrainListeners: true, + DefaultShutdownGracePeriodSeconds: gracefulShutdownSeconds, + DefaultGracefulPort: gracefulPort, + DefaultGracefulShutdownPath: gracefulShutdownPath, + }, + }, + annotations: map[string]string{ + constants.AnnotationEnableSidecarProxyLifecycle: "false", + }, + expCmdArgs: "-graceful-port=20307", + }, + { + name: "annotations skip graceful shutdown", + webhook: MeshWebhook{ + LifecycleConfig: lifecycle.Config{ + DefaultEnableProxyLifecycle: false, + DefaultEnableShutdownDrainListeners: true, + DefaultShutdownGracePeriodSeconds: gracefulShutdownSeconds, + }, + }, + annotations: map[string]string{ + constants.AnnotationEnableSidecarProxyLifecycle: "false", + constants.AnnotationEnableSidecarProxyLifecycleShutdownDrainListeners: "false", + constants.AnnotationSidecarProxyLifecycleShutdownGracePeriodSeconds: "0", + }, + expCmdArgs: "", + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + c.webhook.ConsulConfig = &consul.Config{HTTPPort: 8500, GRPCPort: 8502} + require := require.New(t) + pod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: c.annotations, + }, + + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + }, + }, + }, + } + container, err := c.webhook.consulDataplaneSidecar(testNS, pod, multiPortInfo{}) + if c.expErr != "" { + require.NotNil(err) + require.Contains(err.Error(), c.expErr) + } else { + require.NoError(err) + require.Contains(strings.Join(container.Args, " "), c.expCmdArgs) + } + }) + } +} + // boolPtr returns pointer to b. func boolPtr(b bool) *bool { return &b diff --git a/control-plane/connect-inject/webhook/mesh_webhook.go b/control-plane/connect-inject/webhook/mesh_webhook.go index 96c73d93d4..d97bca6646 100644 --- a/control-plane/connect-inject/webhook/mesh_webhook.go +++ b/control-plane/connect-inject/webhook/mesh_webhook.go @@ -17,6 +17,7 @@ import ( "github.com/go-logr/logr" "github.com/hashicorp/consul-k8s/control-plane/connect-inject/common" "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/lifecycle" "github.com/hashicorp/consul-k8s/control-plane/connect-inject/metrics" "github.com/hashicorp/consul-k8s/control-plane/consul" "github.com/hashicorp/consul-k8s/control-plane/namespaces" @@ -151,6 +152,10 @@ type MeshWebhook struct { DefaultProxyMemoryRequest resource.Quantity DefaultProxyMemoryLimit resource.Quantity + // LifecycleConfig contains proxy lifecycle management configuration from the inject-connect command and has methods to determine whether + // configuration should come from the default flags or annotations. The meshWebhook uses this to configure container sidecar proxy args. + LifecycleConfig lifecycle.Config + // Default Envoy concurrency flag, this is the number of worker threads to be used by the proxy. DefaultEnvoyProxyConcurrency int @@ -306,6 +311,7 @@ func (w *MeshWebhook) Handle(ctx context.Context, req admission.Request) admissi w.Log.Error(err, "error configuring injection sidecar container", "request name", req.Name) return admission.Errored(http.StatusInternalServerError, fmt.Errorf("error configuring injection sidecar container: %s", err)) } + // TODO: invert to start the Envoy sidecar before the application container pod.Spec.Containers = append(pod.Spec.Containers, envoySidecar) } else { // For multi port pods, check for unsupported cases, mount all relevant service account tokens, and mount an init @@ -376,6 +382,8 @@ func (w *MeshWebhook) Handle(ctx context.Context, req admission.Request) admissi w.Log.Error(err, "error configuring injection sidecar container", "request name", req.Name) return admission.Errored(http.StatusInternalServerError, fmt.Errorf("error configuring injection sidecar container: %s", err)) } + // TODO: invert to start the Envoy sidecar container before the + // application container pod.Spec.Containers = append(pod.Spec.Containers, envoySidecar) } } diff --git a/control-plane/subcommand/inject-connect/command.go b/control-plane/subcommand/inject-connect/command.go index 8bada415db..6767e60130 100644 --- a/control-plane/subcommand/inject-connect/command.go +++ b/control-plane/subcommand/inject-connect/command.go @@ -19,8 +19,10 @@ import ( gatewaycontrollers "github.com/hashicorp/consul-k8s/control-plane/api-gateway/controllers" apicommon "github.com/hashicorp/consul-k8s/control-plane/api/common" "github.com/hashicorp/consul-k8s/control-plane/api/v1alpha1" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" "github.com/hashicorp/consul-k8s/control-plane/connect-inject/controllers/endpoints" "github.com/hashicorp/consul-k8s/control-plane/connect-inject/controllers/peering" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/lifecycle" "github.com/hashicorp/consul-k8s/control-plane/connect-inject/metrics" "github.com/hashicorp/consul-k8s/control-plane/connect-inject/webhook" "github.com/hashicorp/consul-k8s/control-plane/controllers" @@ -86,6 +88,13 @@ type Command struct { flagDefaultSidecarProxyMemoryRequest string flagDefaultEnvoyProxyConcurrency int + // Proxy lifecycle settings. + flagDefaultEnableSidecarProxyLifecycle bool + flagDefaultEnableSidecarProxyLifecycleShutdownDrainListeners bool + flagDefaultSidecarProxyLifecycleShutdownGracePeriodSeconds int + flagDefaultSidecarProxyLifecycleGracefulPort string + flagDefaultSidecarProxyLifecycleGracefulShutdownPath string + // Metrics settings. flagDefaultEnableMetrics bool flagEnableGatewayMetrics bool @@ -220,6 +229,13 @@ func (c *Command) init() { c.flagSet.StringVar(&c.flagDefaultSidecarProxyMemoryRequest, "default-sidecar-proxy-memory-request", "", "Default sidecar proxy memory request.") c.flagSet.StringVar(&c.flagDefaultSidecarProxyMemoryLimit, "default-sidecar-proxy-memory-limit", "", "Default sidecar proxy memory limit.") + // Proxy lifecycle setting flags. + c.flagSet.BoolVar(&c.flagDefaultEnableSidecarProxyLifecycle, "default-enable-sidecar-proxy-lifecycle", false, "Default for enabling sidecar proxy lifecycle management.") + c.flagSet.BoolVar(&c.flagDefaultEnableSidecarProxyLifecycleShutdownDrainListeners, "default-enable-sidecar-proxy-lifecycle-shutdown-drain-listeners", false, "Default for enabling sidecar proxy listener draining of inbound connections during shutdown.") + c.flagSet.IntVar(&c.flagDefaultSidecarProxyLifecycleShutdownGracePeriodSeconds, "default-sidecar-proxy-lifecycle-shutdown-grace-period-seconds", 0, "Default sidecar proxy shutdown grace period in seconds.") + c.flagSet.StringVar(&c.flagDefaultSidecarProxyLifecycleGracefulPort, "default-sidecar-proxy-lifecycle-graceful-port", strconv.Itoa(constants.DefaultGracefulPort), "Default port for sidecar proxy lifecycle management HTTP endpoints.") + c.flagSet.StringVar(&c.flagDefaultSidecarProxyLifecycleGracefulShutdownPath, "default-sidecar-proxy-lifecycle-graceful-shutdown-path", "/graceful_shutdown", "Default sidecar proxy lifecycle management graceful shutdown path.") + // Metrics setting flags. c.flagSet.BoolVar(&c.flagDefaultEnableMetrics, "default-enable-metrics", false, "Default for enabling connect service metrics.") c.flagSet.BoolVar(&c.flagEnableGatewayMetrics, "enable-gateway-metrics", false, "Allows enabling Consul gateway metrics.") @@ -422,6 +438,14 @@ func (c *Command) Run(args []string) int { return 1 } + lifecycleConfig := lifecycle.Config{ + DefaultEnableProxyLifecycle: c.flagDefaultEnableSidecarProxyLifecycle, + DefaultEnableShutdownDrainListeners: c.flagDefaultEnableSidecarProxyLifecycleShutdownDrainListeners, + DefaultShutdownGracePeriodSeconds: c.flagDefaultSidecarProxyLifecycleShutdownGracePeriodSeconds, + DefaultGracefulPort: c.flagDefaultSidecarProxyLifecycleGracefulPort, + DefaultGracefulShutdownPath: c.flagDefaultSidecarProxyLifecycleGracefulShutdownPath, + } + metricsConfig := metrics.Config{ DefaultEnableMetrics: c.flagDefaultEnableMetrics, EnableGatewayMetrics: c.flagEnableGatewayMetrics, @@ -725,6 +749,7 @@ func (c *Command) Run(args []string) int { DefaultProxyMemoryRequest: sidecarProxyMemoryRequest, DefaultProxyMemoryLimit: sidecarProxyMemoryLimit, DefaultEnvoyProxyConcurrency: c.flagDefaultEnvoyProxyConcurrency, + LifecycleConfig: lifecycleConfig, MetricsConfig: metricsConfig, InitContainerResources: initResources, ConsulPartition: c.consul.Partition,