-
Notifications
You must be signed in to change notification settings - Fork 323
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Backport of Proxy Lifecycle helm, connect-inject and acceptance tests…
… into release/1.2.x (#2233) (#2482) * Proxy Lifecycle helm, connect-inject and acceptance tests (#2233) Proxy Lifecycle helm, connect-inject and acceptance tests (#2233) * disable lifecycle test --------- Co-authored-by: Curt Bushko <[email protected]> Co-authored-by: Mike Morris <[email protected]> Co-authored-by: Nitya Dhanushkodi <[email protected]>
- Loading branch information
4 people
authored
Jun 29, 2023
1 parent
9efbcba
commit d4c4afb
Showing
21 changed files
with
1,364 additions
and
256 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| ```release-note:feature | ||
| Add support for configuring graceful shutdown proxy lifecycle management settings. | ||
| ``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
207 changes: 207 additions & 0 deletions
207
acceptance/tests/connect/connect_proxy_lifecycle_test.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,207 @@ | ||
| // Copyright (c) HashiCorp, Inc. | ||
| // SPDX-License-Identifier: MPL-2.0 | ||
|
|
||
| package connect | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "strconv" | ||
| "strings" | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/gruntwork-io/terratest/modules/k8s" | ||
| "github.com/hashicorp/consul-k8s/acceptance/framework/connhelper" | ||
| "github.com/hashicorp/consul-k8s/acceptance/framework/consul" | ||
| "github.com/hashicorp/consul-k8s/acceptance/framework/helpers" | ||
| "github.com/hashicorp/consul-k8s/acceptance/framework/logger" | ||
| "github.com/hashicorp/consul/sdk/testutil/retry" | ||
| "github.com/stretchr/testify/require" | ||
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| ) | ||
|
|
||
| type LifecycleShutdownConfig struct { | ||
| secure bool | ||
| helmValues map[string]string | ||
| } | ||
|
|
||
| const ( | ||
| helmDrainListenersKey = "connectInject.sidecarProxy.lifecycle.defaultEnableShutdownDrainListeners" | ||
| helmGracePeriodSecondsKey = "connectInject.sidecarProxy.lifecycle.defaultShutdownGracePeriodSeconds" | ||
| ) | ||
|
|
||
| // Test the endpoints controller cleans up force-killed pods. | ||
| func TestConnectInject_ProxyLifecycleShutdown(t *testing.T) { | ||
| t.Skipf("skiping this test, will be re-added in a future commit") | ||
| cfg := suite.Config() | ||
|
|
||
| for _, testCfg := range []LifecycleShutdownConfig{ | ||
| {secure: false, helmValues: map[string]string{ | ||
| helmDrainListenersKey: "true", | ||
| helmGracePeriodSecondsKey: "15", | ||
| }}, | ||
| {secure: true, helmValues: map[string]string{ | ||
| helmDrainListenersKey: "true", | ||
| helmGracePeriodSecondsKey: "15", | ||
| }}, | ||
| {secure: false, helmValues: map[string]string{ | ||
| helmDrainListenersKey: "false", | ||
| helmGracePeriodSecondsKey: "15", | ||
| }}, | ||
| {secure: true, helmValues: map[string]string{ | ||
| helmDrainListenersKey: "false", | ||
| helmGracePeriodSecondsKey: "15", | ||
| }}, | ||
| {secure: false, helmValues: map[string]string{ | ||
| helmDrainListenersKey: "false", | ||
| helmGracePeriodSecondsKey: "0", | ||
| }}, | ||
| {secure: true, helmValues: map[string]string{ | ||
| helmDrainListenersKey: "false", | ||
| helmGracePeriodSecondsKey: "0", | ||
| }}, | ||
| } { | ||
| // Determine if listeners should be expected to drain inbound connections | ||
| var drainListenersEnabled bool | ||
| var err error | ||
| val, ok := testCfg.helmValues[helmDrainListenersKey] | ||
| if ok { | ||
| drainListenersEnabled, err = strconv.ParseBool(val) | ||
| require.NoError(t, err) | ||
| } | ||
|
|
||
| // Determine expected shutdown grace period | ||
| var gracePeriodSeconds int64 | ||
| val, ok = testCfg.helmValues[helmGracePeriodSecondsKey] | ||
| if ok { | ||
| gracePeriodSeconds, err = strconv.ParseInt(val, 10, 64) | ||
| require.NoError(t, err) | ||
| } else { | ||
| // Half of the helm default to speed tests up | ||
| gracePeriodSeconds = 15 | ||
| } | ||
|
|
||
| name := fmt.Sprintf("secure: %t, drainListeners: %t, gracePeriodSeconds: %d", testCfg.secure, drainListenersEnabled, gracePeriodSeconds) | ||
| t.Run(name, func(t *testing.T) { | ||
| ctx := suite.Environment().DefaultContext(t) | ||
| releaseName := helpers.RandomName() | ||
|
|
||
| connHelper := connhelper.ConnectHelper{ | ||
| ClusterKind: consul.Helm, | ||
| Secure: testCfg.secure, | ||
| ReleaseName: releaseName, | ||
| Ctx: ctx, | ||
| Cfg: cfg, | ||
| HelmValues: testCfg.helmValues, | ||
| } | ||
|
|
||
| connHelper.Setup(t) | ||
| connHelper.Install(t) | ||
| connHelper.DeployClientAndServer(t) | ||
|
|
||
| // TODO: should this move into connhelper.DeployClientAndServer? | ||
| logger.Log(t, "waiting for static-client and static-server to be registered with Consul") | ||
| retry.Run(t, func(r *retry.R) { | ||
| for _, name := range []string{ | ||
| "static-client", | ||
| "static-client-sidecar-proxy", | ||
| "static-server", | ||
| "static-server-sidecar-proxy", | ||
| } { | ||
| logger.Logf(t, "checking for %s service in Consul catalog", name) | ||
| instances, _, err := connHelper.ConsulClient.Catalog().Service(name, "", nil) | ||
| r.Check(err) | ||
|
|
||
| if len(instances) != 1 { | ||
| r.Errorf("expected 1 instance of %s", name) | ||
| } | ||
| } | ||
| }) | ||
|
|
||
| if testCfg.secure { | ||
| connHelper.TestConnectionFailureWithoutIntention(t) | ||
| connHelper.CreateIntention(t) | ||
| } | ||
|
|
||
| connHelper.TestConnectionSuccess(t) | ||
|
|
||
| // Get static-client pod name | ||
| ns := ctx.KubectlOptions(t).Namespace | ||
| pods, err := ctx.KubernetesClient(t).CoreV1().Pods(ns).List( | ||
| context.Background(), | ||
| metav1.ListOptions{ | ||
| LabelSelector: "app=static-client", | ||
| }, | ||
| ) | ||
| require.NoError(t, err) | ||
| require.Len(t, pods.Items, 1) | ||
| clientPodName := pods.Items[0].Name | ||
|
|
||
| var terminationGracePeriod int64 = 60 | ||
| logger.Logf(t, "killing the %q pod with %dseconds termination grace period", clientPodName, terminationGracePeriod) | ||
| err = ctx.KubernetesClient(t).CoreV1().Pods(ns).Delete(context.Background(), clientPodName, metav1.DeleteOptions{GracePeriodSeconds: &terminationGracePeriod}) | ||
| require.NoError(t, err) | ||
|
|
||
| // Exec into terminating pod, not just any static-client pod | ||
| args := []string{"exec", clientPodName, "-c", connhelper.StaticClientName, "--", "curl", "-vvvsSf"} | ||
|
|
||
| if cfg.EnableTransparentProxy { | ||
| args = append(args, "http://static-server") | ||
| } else { | ||
| args = append(args, "http://localhost:1234") | ||
| } | ||
|
|
||
| if gracePeriodSeconds > 0 { | ||
| // Ensure outbound requests are still successful during grace | ||
| // period. | ||
| retry.RunWith(&retry.Timer{Timeout: time.Duration(gracePeriodSeconds) * time.Second, Wait: 2 * time.Second}, t, func(r *retry.R) { | ||
| output, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), args...) | ||
| require.NoError(r, err) | ||
| require.Condition(r, func() bool { | ||
| exists := false | ||
| if strings.Contains(output, "curl: (7) Failed to connect") { | ||
| exists = true | ||
| } | ||
| return !exists | ||
| }) | ||
| }) | ||
|
|
||
| // If listener draining is enabled, ensure inbound | ||
| // requests are rejected during grace period. | ||
| // connHelper.TestConnectionSuccess(t) | ||
| } else { | ||
| // Ensure outbound requests fail because proxy has terminated | ||
| retry.RunWith(&retry.Timer{Timeout: time.Duration(terminationGracePeriod) * time.Second, Wait: 2 * time.Second}, t, func(r *retry.R) { | ||
| output, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), args...) | ||
| require.Error(r, err) | ||
| require.Condition(r, func() bool { | ||
| exists := false | ||
| if strings.Contains(output, "curl: (7) Failed to connect") { | ||
| exists = true | ||
| } | ||
| return exists | ||
| }) | ||
| }) | ||
| } | ||
|
|
||
| logger.Log(t, "ensuring pod is deregistered after termination") | ||
| retry.Run(t, func(r *retry.R) { | ||
| for _, name := range []string{ | ||
| "static-client", | ||
| "static-client-sidecar-proxy", | ||
| } { | ||
| logger.Logf(t, "checking for %s service in Consul catalog", name) | ||
| instances, _, err := connHelper.ConsulClient.Catalog().Service(name, "", nil) | ||
| r.Check(err) | ||
|
|
||
| for _, instance := range instances { | ||
| if strings.Contains(instance.ServiceID, clientPodName) { | ||
| r.Errorf("%s is still registered", instance.ServiceID) | ||
| } | ||
| } | ||
| } | ||
| }) | ||
| }) | ||
| } | ||
| } |
Oops, something went wrong.