test: Validate Service IP

Validate resolved service IP when waiting for service to become
available.

This will help avoid test flakes where kube-dns is returning a stale
IP of a service that was just removed, e.g., when running test with
`--force-deploy`.

Signed-off-by: Jarno Rajahalme <[email protected]>

connectivity/check/deployment.go

@@ -19,6 +19,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -436,6 +437,16 @@ func (ct *ConnectivityTest) validateDeployment(ctx context.Context) error {
 		}
 
 		for _, echoService := range echoServices.Items {
+			if ct.params.MultiCluster != "" {
+				if _, exists := ct.echoServices[echoService.Name]; exists {
+					// ct.clients.clients() lists the client cluster first.
+					// If we already have this service (for the client cluster), keep it
+					// so that we can rely on the service's ClusterIP being valid for the
+					// client pods.
+					continue
+				}
+			}
+
 			ct.echoServices[echoService.Name] = Service{
 				Service: echoService.DeepCopy(),
 			}
@@ -621,13 +632,25 @@ func (ct *ConnectivityTest) waitForService(ctx context.Context, service Service)
 
 		// Warning: ExecInPodWithStderr ignores ctx. Don't pass it here so we don't
 		// falsely expect the function to be able to be cancelled.
-		_, e, err := ct.client.ExecInPodWithStderr(context.TODO(),
+		stdout, e, err := ct.client.ExecInPodWithStderr(context.TODO(),
 			pod.Pod.Namespace, pod.Pod.Name, pod.Pod.Labels["name"],
 			[]string{"nslookup", service.Service.Name}) // BusyBox nslookup doesn't support any arguments.
 
 		// Lookup successful.
 		if err == nil {
-			return nil
+			svcIP := ""
+			switch service.Service.Spec.Type {
+			case corev1.ServiceTypeClusterIP, corev1.ServiceTypeNodePort:
+				svcIP = service.Service.Spec.ClusterIP
+			case corev1.ServiceTypeLoadBalancer:
+				if len(service.Service.Status.LoadBalancer.Ingress) > 0 {
+					svcIP = service.Service.Status.LoadBalancer.Ingress[0].IP
+				}
+			}
+			if strings.Contains(stdout.String(), svcIP) {
+				return nil
+			}
+			err = fmt.Errorf("Service IP %q not found in nslookup output %q", svcIP, stdout.String())
 		}
 
 		ct.Debugf("Error waiting for service %s: %s: %s", service.Name(), err, e.String())