Acceptance tests: increase api-gateway retries (#2716)

* Increase the retries and add config entry retries

acceptance/tests/api-gateway/api_gateway_external_servers_test.go

@@ -96,7 +96,7 @@ func TestAPIGateway_ExternalServers(t *testing.T) {
 	// leader election so we may need to wait a long time for
 	// the reconcile loop to run (hence a ~1m timeout here).
 	var gatewayAddress string
-	retryCheck(t, 30, func(r *retry.R) {
+	retryCheck(t, 60, func(r *retry.R) {
 		var gateway gwv1beta1.Gateway
 		err := k8sClient.Get(context.Background(), types.NamespacedName{Name: "gateway", Namespace: "default"}, &gateway)
 		require.NoError(r, err)

acceptance/tests/api-gateway/api_gateway_lifecycle_test.go

@@ -196,7 +196,7 @@ func TestAPIGateway_Lifecycle(t *testing.T) {
 
 	// check that the route is unbound and all Consul objects and Kubernetes statuses are cleaned up
 	logger.Log(t, "checking that http route one is not bound to gateway two")
-	retryCheck(t, 30, func(r *retry.R) {
+	retryCheck(t, 60, func(r *retry.R) {
 		var route gwv1beta1.HTTPRoute
 		err := k8sClient.Get(context.Background(), types.NamespacedName{Name: routeOneName, Namespace: defaultNamespace}, &route)
 		require.NoError(r, err)
@@ -246,7 +246,7 @@ func TestAPIGateway_Lifecycle(t *testing.T) {
 
 	// check that the Kubernetes gateway is cleaned up
 	logger.Log(t, "checking that gateway one is cleaned up in Kubernetes")
-	retryCheck(t, 30, func(r *retry.R) {
+	retryCheck(t, 60, func(r *retry.R) {
 		var route gwv1beta1.Gateway
 		err := k8sClient.Get(context.Background(), types.NamespacedName{Name: controlledGatewayOneName, Namespace: defaultNamespace}, &route)
 		require.NoError(r, err)
@@ -299,7 +299,7 @@ func checkConsulNotExists(t *testing.T, client *api.Client, kind, name string, n
 		opts.Namespace = namespace[0]
 	}
 
-	retryCheck(t, 30, func(r *retry.R) {
+	retryCheck(t, 60, func(r *retry.R) {
 		_, _, err := client.ConfigEntries().Get(kind, name, opts)
 		require.Error(r, err)
 		require.EqualError(r, err, fmt.Sprintf("Unexpected response code: 404 (Config entry not found for %q / %q)", kind, name))
@@ -309,7 +309,7 @@ func checkConsulNotExists(t *testing.T, client *api.Client, kind, name string, n
 func checkConsulExists(t *testing.T, client *api.Client, kind, name string) {
 	t.Helper()
 
-	retryCheck(t, 30, func(r *retry.R) {
+	retryCheck(t, 60, func(r *retry.R) {
 		_, _, err := client.ConfigEntries().Get(kind, name, nil)
 		require.NoError(r, err)
 	})
@@ -318,7 +318,7 @@ func checkConsulExists(t *testing.T, client *api.Client, kind, name string) {
 func checkConsulRouteParent(t *testing.T, client *api.Client, name, parent string) {
 	t.Helper()
 
-	retryCheck(t, 30, func(r *retry.R) {
+	retryCheck(t, 60, func(r *retry.R) {
 		entry, _, err := client.ConfigEntries().Get(api.HTTPRoute, name, nil)
 		require.NoError(r, err)
 		route := entry.(*api.HTTPRouteConfigEntry)
@@ -331,7 +331,7 @@ func checkConsulRouteParent(t *testing.T, client *api.Client, name, parent strin
 func checkEmptyRoute(t *testing.T, client client.Client, name, namespace string) {
 	t.Helper()
 
-	retryCheck(t, 30, func(r *retry.R) {
+	retryCheck(t, 60, func(r *retry.R) {
 		var route gwv1beta1.HTTPRoute
 		err := client.Get(context.Background(), types.NamespacedName{Name: name, Namespace: namespace}, &route)
 		require.NoError(r, err)
@@ -344,7 +344,7 @@ func checkEmptyRoute(t *testing.T, client client.Client, name, namespace string)
 func checkRouteBound(t *testing.T, client client.Client, name, namespace, parent string) {
 	t.Helper()
 
-	retryCheck(t, 30, func(r *retry.R) {
+	retryCheck(t, 60, func(r *retry.R) {
 		var route gwv1beta1.HTTPRoute
 		err := client.Get(context.Background(), types.NamespacedName{Name: name, Namespace: namespace}, &route)
 		require.NoError(r, err)

acceptance/tests/api-gateway/api_gateway_tenancy_test.go

@@ -131,7 +131,7 @@ func TestAPIGateway_Tenancy(t *testing.T) {
 			k8sClient := ctx.ControllerRuntimeClient(t)
 			consulClient, _ := consulCluster.SetupConsulClient(t, c.secure)
 
-			retryCheck(t, 60, func(r *retry.R) {
+			retryCheck(t, 120, func(r *retry.R) {
 				var gateway gwv1beta1.Gateway
 				err := k8sClient.Get(context.Background(), types.NamespacedName{Name: "gateway", Namespace: gatewayNamespace}, &gateway)
 				require.NoError(r, err)
@@ -153,7 +153,7 @@ func TestAPIGateway_Tenancy(t *testing.T) {
 			checkConsulNotExists(t, consulClient, api.APIGateway, "gateway", namespaceForConsul(c.namespaceMirroring, gatewayNamespace))
 
 			// route failure
-			retryCheck(t, 30, func(r *retry.R) {
+			retryCheck(t, 60, func(r *retry.R) {
 				var httproute gwv1beta1.HTTPRoute
 				err := k8sClient.Get(context.Background(), types.NamespacedName{Name: "route", Namespace: routeNamespace}, &httproute)
 				require.NoError(r, err)
@@ -175,7 +175,7 @@ func TestAPIGateway_Tenancy(t *testing.T) {
 			createReferenceGrant(t, k8sClient, "route-service", routeNamespace, serviceNamespace)
 
 			// gateway updated with references allowed
-			retryCheck(t, 30, func(r *retry.R) {
+			retryCheck(t, 60, func(r *retry.R) {
 				var gateway gwv1beta1.Gateway
 				err := k8sClient.Get(context.Background(), types.NamespacedName{Name: "gateway", Namespace: gatewayNamespace}, &gateway)
 				require.NoError(r, err)

acceptance/tests/api-gateway/api_gateway_test.go

@@ -101,6 +101,11 @@ func TestAPIGateway_Basic(t *testing.T) {
 			logger.Log(t, "creating target http server")
 			k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-server-inject")
 
+			// We use the static-client pod so that we can make calls to the api gateway
+			// via kubectl exec without needing a route into the cluster from the test machine.
+			logger.Log(t, "creating static-client pod")
+			k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/bases/static-client")
+
 			logger.Log(t, "patching route to target http server")
 			k8s.RunKubectl(t, ctx.KubectlOptions(t), "patch", "httproute", "http-route", "-p", `{"spec":{"rules":[{"backendRefs":[{"name":"static-server","port":80}]}]}}`, "--type=merge")
 
@@ -115,11 +120,6 @@ func TestAPIGateway_Basic(t *testing.T) {
 				k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "delete", "-f", "../fixtures/cases/api-gateways/tcproute/route.yaml")
 			})
 
-			// We use the static-client pod so that we can make calls to the api gateway
-			// via kubectl exec without needing a route into the cluster from the test machine.
-			logger.Log(t, "creating static-client pod")
-			k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/bases/static-client")
-
 			// Grab a kubernetes client so that we can verify binding
 			// behavior prior to issuing requests through the gateway.
 			k8sClient := ctx.ControllerRuntimeClient(t)
@@ -128,7 +128,7 @@ func TestAPIGateway_Basic(t *testing.T) {
 			// leader election so we may need to wait a long time for
 			// the reconcile loop to run (hence the 1m timeout here).
 			var gatewayAddress string
-			counter := &retry.Counter{Count: 60, Wait: 2 * time.Second}
+			counter := &retry.Counter{Count: 120, Wait: 2 * time.Second}
 			retry.RunWith(counter, t, func(r *retry.R) {
 				var gateway gwv1beta1.Gateway
 				err := k8sClient.Get(context.Background(), types.NamespacedName{Name: "gateway", Namespace: "default"}, &gateway)
@@ -209,17 +209,22 @@ func TestAPIGateway_Basic(t *testing.T) {
 			checkStatusCondition(t, tcpRoute.Status.Parents[0].Conditions, trueCondition("ConsulAccepted", "Accepted"))
 
 			// check that the Consul entries were created
-			entry, _, err := consulClient.ConfigEntries().Get(api.APIGateway, "gateway", nil)
-			require.NoError(t, err)
-			gateway := entry.(*api.APIGatewayConfigEntry)
+			var gateway *api.APIGatewayConfigEntry
+			var httpRoute *api.HTTPRouteConfigEntry
+			var route *api.TCPRouteConfigEntry
+			retry.RunWith(counter, t, func(r *retry.R) {
+				entry, _, err := consulClient.ConfigEntries().Get(api.APIGateway, "gateway", nil)
+				require.NoError(r, err)
+				gateway = entry.(*api.APIGatewayConfigEntry)
 
-			entry, _, err = consulClient.ConfigEntries().Get(api.HTTPRoute, "http-route", nil)
-			require.NoError(t, err)
-			httpRoute := entry.(*api.HTTPRouteConfigEntry)
+				entry, _, err = consulClient.ConfigEntries().Get(api.HTTPRoute, "http-route", nil)
+				require.NoError(r, err)
+				httpRoute = entry.(*api.HTTPRouteConfigEntry)
 
-			entry, _, err = consulClient.ConfigEntries().Get(api.TCPRoute, "tcp-route", nil)
-			require.NoError(t, err)
-			route := entry.(*api.TCPRouteConfigEntry)
+				entry, _, err = consulClient.ConfigEntries().Get(api.TCPRoute, "tcp-route", nil)
+				require.NoError(r, err)
+				route = entry.(*api.TCPRouteConfigEntry)
+			})
 
 			// now check the gateway status conditions
 			checkConsulStatusCondition(t, gateway.Status.Conditions, trueConsulCondition("Accepted", "Accepted"))

acceptance/tests/partitions/partitions_gateway_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/hashicorp/consul/api"
 	"github.com/hashicorp/consul/sdk/testutil/retry"
 	"github.com/stretchr/testify/require"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 
 	gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"
@@ -255,6 +256,16 @@ func TestPartitions_Gateway(t *testing.T) {
 		logger.Log(t, "creating target server in secondary partition cluster")
 		k8s.DeployKustomize(t, secondaryPartitionClusterStaticServerOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-server-inject")
 
+		// Check that static-server injected 2 containers.
+		for _, labelSelector := range []string{"app=static-server"} {
+			podList, err := secondaryPartitionClusterContext.KubernetesClient(t).CoreV1().Pods(metav1.NamespaceAll).List(context.Background(), metav1.ListOptions{
+				LabelSelector: labelSelector,
+			})
+			require.NoError(t, err)
+			require.Len(t, podList.Items, 1)
+			require.Len(t, podList.Items[0].Spec.Containers, 2)
+		}
+
 		logger.Log(t, "patching route to target server")
 		k8s.RunKubectl(t, secondaryPartitionClusterStaticServerOpts, "patch", "httproute", "http-route", "-p", `{"spec":{"rules":[{"backendRefs":[{"group":"consul.hashicorp.com","kind":"MeshService","name":"mesh-service","port":80}]}]}}`, "--type=merge")
 
@@ -293,6 +304,16 @@ func TestPartitions_Gateway(t *testing.T) {
 		logger.Log(t, "creating target server in default partition cluster")
 		k8s.DeployKustomize(t, defaultPartitionClusterStaticServerOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-server-inject")
 
+		// Check that static-server injected 2 containers.
+		for _, labelSelector := range []string{"app=static-server"} {
+			podList, err := defaultPartitionClusterContext.KubernetesClient(t).CoreV1().Pods(metav1.NamespaceAll).List(context.Background(), metav1.ListOptions{
+				LabelSelector: labelSelector,
+			})
+			require.NoError(t, err)
+			require.Len(t, podList.Items, 1)
+			require.Len(t, podList.Items[0].Spec.Containers, 2)
+		}
+
 		logger.Log(t, "creating exported services")
 		k8s.KubectlApplyK(t, defaultPartitionClusterContext.KubectlOptions(t), "../fixtures/cases/crd-partitions/default-partition-ns1")
 		helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() {