refactor lb svc e2e

Signed-off-by: zhangzujian <[email protected]>

Makefile

@@ -784,7 +784,6 @@ kind-install-lb-svc:
 	$(call kind_load_image,kube-ovn,$(VPC_NAT_GW_IMG))
 	@$(MAKE) ENABLE_LB_SVC=true CNI_CONFIG_PRIORITY=10 kind-install
 	@$(MAKE) kind-install-multus
-	kubectl apply -f yamls/lb-svc-attachment.yaml
 
 .PHONY: kind-install-webhook
 kind-install-webhook: kind-install

test/e2e/framework/cni.go

@@ -0,0 +1,75 @@
+package framework
+
+import (
+	"encoding/json"
+
+	"github.com/containernetworking/cni/pkg/types"
+	nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	"github.com/onsi/ginkgo/v2"
+
+	"github.com/kubeovn/kube-ovn/pkg/netconf"
+	"github.com/kubeovn/kube-ovn/pkg/request"
+	"github.com/kubeovn/kube-ovn/pkg/util"
+)
+
+const CNIVersion = "0.3.1"
+
+// https://github.com/containernetworking/plugins/blob/main/plugins/main/macvlan/macvlan.go#L37
+type MacvlanNetConf struct {
+	netconf.NetConf
+	Master     string `json:"master"`
+	Mode       string `json:"mode"`
+	MTU        int    `json:"mtu"`
+	Mac        string `json:"mac,omitempty"`
+	LinkContNs bool   `json:"linkInContainer,omitempty"`
+
+	RuntimeConfig struct {
+		Mac string `json:"mac,omitempty"`
+	} `json:"runtimeConfig,omitempty"`
+}
+
+func MakeMacvlanNetworkAttachmentDefinition(name, namespace, master, mode, provider string, routes []request.Route) *nadv1.NetworkAttachmentDefinition {
+	ginkgo.GinkgoHelper()
+
+	config := &MacvlanNetConf{
+		NetConf: netconf.NetConf{
+			NetConf: types.NetConf{
+				CNIVersion: CNIVersion,
+				Type:       "macvlan",
+			},
+			IPAM: &netconf.IPAMConf{
+				Type:         util.CniTypeName,
+				ServerSocket: "/run/openvswitch/kube-ovn-daemon.sock",
+				Provider:     provider,
+				Routes:       routes,
+			},
+		},
+		Master:     master,
+		Mode:       mode,
+		LinkContNs: true,
+	}
+	buf, err := json.MarshalIndent(config, "", "  ")
+	framework.ExpectNoError(err)
+
+	return MakeNetworkAttachmentDefinition(name, namespace, string(buf))
+}
+
+func MakeOVNNetworkAttachmentDefinition(name, namespace, provider string, routes []request.Route) *nadv1.NetworkAttachmentDefinition {
+	ginkgo.GinkgoHelper()
+
+	config := &netconf.NetConf{
+		NetConf: types.NetConf{
+			CNIVersion: CNIVersion,
+			Type:       util.CniTypeName,
+		},
+		ServerSocket: "/run/openvswitch/kube-ovn-daemon.sock",
+		Provider:     provider,
+		Routes:       routes,
+	}
+	buf, err := json.MarshalIndent(config, "", "  ")
+	framework.ExpectNoError(err)
+
+	return MakeNetworkAttachmentDefinition(name, namespace, string(buf))
+}

test/e2e/lb-svc/e2e_test.go

@@ -7,10 +7,12 @@ import (
 	"math/big"
 	"math/rand/v2"
 	"net"
+	"strconv"
 	"testing"
 	"time"
 
 	dockernetwork "github.com/docker/docker/api/types/network"
+	nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
 	corev1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -21,6 +23,8 @@ import (
 	k8sframework "k8s.io/kubernetes/test/e2e/framework"
 	"k8s.io/kubernetes/test/e2e/framework/config"
 	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
+	e2epodoutput "k8s.io/kubernetes/test/e2e/framework/pod/output"
+	"k8s.io/utils/ptr"
 
 	"github.com/onsi/ginkgo/v2"
 
@@ -31,8 +35,6 @@ import (
 	"github.com/kubeovn/kube-ovn/test/e2e/framework/kind"
 )
 
-const subnetProvider = "lb-svc-attachment.kube-system"
-
 func init() {
 	klog.SetOutput(ginkgo.GinkgoWriter)
 
@@ -56,20 +58,27 @@ var _ = framework.SerialDescribe("[group:lb-svc]", func() {
 
 	var skip bool
 	var cs clientset.Interface
+	var podClient *framework.PodClient
 	var subnetClient *framework.SubnetClient
 	var serviceClient *framework.ServiceClient
 	var deploymentClient *framework.DeploymentClient
-	var clusterName, subnetName, namespaceName, serviceName, deploymentName string
+	var nadClient *framework.NetworkAttachmentDefinitionClient
+	var provider, nadName, clusterName, subnetName, namespaceName, serviceName, deploymentName, serverPodName, clientPodName string
 	var dockerNetwork *dockernetwork.Inspect
 	var cidr, gateway string
 	ginkgo.BeforeEach(func() {
 		cs = f.ClientSet
+		podClient = f.PodClient()
 		subnetClient = f.SubnetClient()
 		serviceClient = f.ServiceClient()
 		deploymentClient = f.DeploymentClient()
+		nadClient = f.NetworkAttachmentDefinitionClient()
 		namespaceName = f.Namespace.Name
+		nadName = "nad-" + framework.RandomSuffix()
 		subnetName = "subnet-" + framework.RandomSuffix()
 		serviceName = "service-" + framework.RandomSuffix()
+		serverPodName = "pod-" + framework.RandomSuffix()
+		clientPodName = "pod-" + framework.RandomSuffix()
 		deploymentName = lbSvcDeploymentName(serviceName)
 
 		if skip {
@@ -96,6 +105,13 @@ var _ = framework.SerialDescribe("[group:lb-svc]", func() {
 			dockerNetwork = network
 		}
 
+		provider = fmt.Sprintf("%s.%s", nadName, namespaceName)
+
+		ginkgo.By("Creating network attachment definition " + nadName)
+		nad := framework.MakeMacvlanNetworkAttachmentDefinition(nadName, namespaceName, "eth0", "bridge", provider, nil)
+		nad = nadClient.Create(nad)
+		framework.Logf("created network attachment definition config:\n%s", nad.Spec.Config)
+
 		ginkgo.By("Creating subnet " + subnetName)
 		for _, config := range dockerNetwork.IPAM.Config {
 			if util.CheckProtocol(config.Subnet) == apiv1.ProtocolIPv4 {
@@ -112,38 +128,54 @@ var _ = framework.SerialDescribe("[group:lb-svc]", func() {
 			}
 		}
 		subnet := framework.MakeSubnet(subnetName, "", cidr, gateway, "", "", excludeIPs, nil, []string{namespaceName})
-		subnet.Spec.Provider = subnetProvider
+		subnet.Spec.Provider = provider
 		_ = subnetClient.Create(subnet)
 	})
 	ginkgo.AfterEach(func() {
-		ginkgo.By("Deleting deployment " + deploymentName)
-		deploymentClient.DeleteSync(deploymentName)
+		ginkgo.By("Deleting pod " + clientPodName)
+		podClient.DeleteSync(clientPodName)
+
+		ginkgo.By("Deleting pod " + serverPodName)
+		podClient.DeleteSync(serverPodName)
 
 		ginkgo.By("Deleting service " + serviceName)
 		serviceClient.DeleteSync(serviceName)
 
+		ginkgo.By("Deleting deployment " + deploymentName)
+		deploymentClient.DeleteSync(deploymentName)
+
 		ginkgo.By("Deleting subnet " + subnetName)
 		subnetClient.DeleteSync(subnetName)
+
+		ginkgo.By("Deleting network attachment definition " + nadName)
+		nadClient.Delete(nadName)
 	})
 
 	framework.ConformanceIt("should allocate dynamic external IP for service", func() {
+		ginkgo.By("Creating server pod " + serverPodName)
+		labels := map[string]string{"app": serviceName}
+		port := 8000 + rand.Int32N(1000)
+		args := []string{"netexec", "--http-port", strconv.Itoa(int(port))}
+		serverPod := framework.MakePod(namespaceName, serverPodName, labels, nil, framework.AgnhostImage, nil, args)
+		_ = podClient.CreateSync(serverPod)
+
 		ginkgo.By("Creating service " + serviceName)
 		ports := []corev1.ServicePort{{
 			Name:       "tcp",
 			Protocol:   corev1.ProtocolTCP,
-			Port:       80,
-			TargetPort: intstr.FromInt32(80),
+			Port:       port,
+			TargetPort: intstr.FromInt32(port),
 		}}
 		annotations := map[string]string{
-			subnetProvider + ".kubernetes.io/logical_switch": subnetName,
+			util.AttachmentProvider: provider,
 		}
-		selector := map[string]string{"app": "lb-svc-dynamic"}
-		service := framework.MakeService(serviceName, corev1.ServiceTypeLoadBalancer, annotations, selector, ports, corev1.ServiceAffinityNone)
-		_ = serviceClient.CreateSync(service, func(s *corev1.Service) (bool, error) {
+		service := framework.MakeService(serviceName, corev1.ServiceTypeLoadBalancer, annotations, labels, ports, corev1.ServiceAffinityNone)
+		service.Spec.AllocateLoadBalancerNodePorts = ptr.To(false)
+		service = serviceClient.CreateSync(service, func(s *corev1.Service) (bool, error) {
 			return len(s.Spec.ClusterIPs) != 0, nil
 		}, "cluster ips are not empty")
 
-		ginkgo.By("Waiting for deployment " + deploymentName + " to be ready")
+		ginkgo.By("Waiting for LB deployment " + deploymentName + " to be ready")
 		framework.WaitUntil(2*time.Second, time.Minute, func(ctx context.Context) (bool, error) {
 			_, err := deploymentClient.DeploymentInterface.Get(ctx, deploymentName, metav1.GetOptions{})
 			if err == nil {
@@ -164,44 +196,91 @@ var _ = framework.SerialDescribe("[group:lb-svc]", func() {
 		framework.ExpectNoError(err)
 		framework.ExpectHaveLen(pods.Items, 1)
 
-		ginkgo.By("Checking pod annotations")
-		key := fmt.Sprintf(util.AllocatedAnnotationTemplate, subnetProvider)
-		framework.ExpectHaveKeyWithValue(pods.Items[0].Annotations, key, "true")
-		cidrKey := fmt.Sprintf(util.CidrAnnotationTemplate, subnetProvider)
-		ipKey := fmt.Sprintf(util.IPAddressAnnotationTemplate, subnetProvider)
-		framework.ExpectHaveKey(pods.Items[0].Annotations, cidrKey)
-		framework.ExpectHaveKey(pods.Items[0].Annotations, ipKey)
-		cidr := pods.Items[0].Annotations[cidrKey]
-		ip := pods.Items[0].Annotations[ipKey]
-		framework.ExpectTrue(util.CIDRContainIP(cidr, ip))
-
-		ginkgo.By("Checking service external IP")
+		ginkgo.By("Checking LB pod annotations")
+		pod := &pods.Items[0]
+		key := fmt.Sprintf(util.AllocatedAnnotationTemplate, provider)
+		framework.ExpectHaveKeyWithValue(pod.Annotations, key, "true")
+		cidrKey := fmt.Sprintf(util.CidrAnnotationTemplate, provider)
+		ipKey := fmt.Sprintf(util.IPAddressAnnotationTemplate, provider)
+		framework.ExpectHaveKey(pod.Annotations, cidrKey)
+		framework.ExpectHaveKey(pod.Annotations, ipKey)
+		lbIP := pod.Annotations[ipKey]
+		framework.ExpectIPInCIDR(lbIP, pod.Annotations[cidrKey])
+
+		ginkgo.By("Checking service status")
 		framework.WaitUntil(2*time.Second, time.Minute, func(_ context.Context) (bool, error) {
 			service = serviceClient.Get(serviceName)
 			return len(service.Status.LoadBalancer.Ingress) != 0, nil
 		}, ".status.loadBalancer.ingress is not empty")
-		framework.ExpectEqual(service.Status.LoadBalancer.Ingress[0].IP, ip)
+		framework.ExpectHaveLen(service.Status.LoadBalancer.Ingress, 1)
+		framework.ExpectEqual(service.Status.LoadBalancer.Ingress[0].IP, lbIP)
+
+		ginkgo.By("Creating client pod " + clientPodName)
+		annotations = map[string]string{nadv1.NetworkAttachmentAnnot: fmt.Sprintf("%s/%s", namespaceName, nadName)}
+		cmd := []string{"sh", "-c", "sleep infinity"}
+		clientPod := framework.MakePod(namespaceName, clientPodName, nil, annotations, f.KubeOVNImage, cmd, nil)
+		clientPod = podClient.CreateSync(clientPod)
+
+		ginkgo.By("Checking service connectivity from client pod " + clientPodName)
+		curlCmd := fmt.Sprintf("curl -q -s --connect-timeout 2 --max-time 2 %s/clientip", util.JoinHostPort(lbIP, port))
+		ginkgo.By(fmt.Sprintf(`Executing %q in pod %s/%s`, curlCmd, clientPod.Namespace, clientPod.Name))
+		_ = e2epodoutput.RunHostCmdOrDie(clientPod.Namespace, clientPod.Name, curlCmd)
+
+		ginkgo.By("Deleting lb svc pod " + pod.Name)
+		podClient.DeleteSync(pod.Name)
+
+		ginkgo.By("Waiting for LB deployment " + deploymentName + " to be ready")
+		err = deploymentClient.WaitToComplete(deployment)
+		framework.ExpectNoError(err, "deployment failed to complete")
+
+		ginkgo.By("Getting pods for deployment " + deploymentName)
+		pods, err = deploymentClient.GetPods(deployment)
+		framework.ExpectNoError(err)
+		framework.ExpectHaveLen(pods.Items, 1)
+		lbIP = pods.Items[0].Annotations[ipKey]
+
+		ginkgo.By("Checking service connectivity from client pod " + clientPodName)
+		curlCmd = fmt.Sprintf("curl -q -s --connect-timeout 2 --max-time 2 %s/clientip", util.JoinHostPort(lbIP, port))
+		framework.WaitUntil(2*time.Second, 30*time.Second, func(_ context.Context) (bool, error) {
+			ginkgo.By(fmt.Sprintf(`Executing %q in pod %s/%s`, curlCmd, clientPod.Namespace, clientPod.Name))
+			_, err = e2epodoutput.RunHostCmd(clientPod.Namespace, clientPod.Name, curlCmd)
+			return err == nil, nil
+		}, "")
+
+		ginkgo.By("Deleting service " + serviceName)
+		serviceClient.DeleteSync(serviceName)
+
+		ginkgo.By("Waiting for LB deployment " + deploymentName + " to be deleted automatically")
+		err = deploymentClient.WaitToDisappear(deploymentName, 2*time.Second, 2*time.Minute)
+		framework.ExpectNoError(err, "deployment failed to disappear")
 	})
 
 	framework.ConformanceIt("should allocate static external IP for service", func() {
+		ginkgo.By("Creating server pod " + serverPodName)
+		labels := map[string]string{"app": serviceName}
+		port := 8000 + rand.Int32N(1000)
+		args := []string{"netexec", "--http-port", strconv.Itoa(int(port))}
+		serverPod := framework.MakePod(namespaceName, serverPodName, labels, nil, framework.AgnhostImage, nil, args)
+		_ = podClient.CreateSync(serverPod)
+
 		ginkgo.By("Creating service " + serviceName)
-		base := util.IP2BigInt(gateway)
-		lbIP := util.BigInt2Ip(base.Add(base, big.NewInt(50+rand.Int64N(50))))
 		ports := []corev1.ServicePort{{
 			Name:       "tcp",
 			Protocol:   corev1.ProtocolTCP,
-			Port:       80,
-			TargetPort: intstr.FromInt32(80),
+			Port:       port,
+			TargetPort: intstr.FromInt32(port),
 		}}
 		annotations := map[string]string{
-			subnetProvider + ".kubernetes.io/logical_switch": subnetName,
+			util.AttachmentProvider: provider,
 		}
-		selector := map[string]string{"app": "lb-svc-static"}
-		service := framework.MakeService(serviceName, corev1.ServiceTypeLoadBalancer, annotations, selector, ports, corev1.ServiceAffinityNone)
+		base := util.IP2BigInt(gateway)
+		lbIP := util.BigInt2Ip(base.Add(base, big.NewInt(50+rand.Int64N(50))))
+		service := framework.MakeService(serviceName, corev1.ServiceTypeLoadBalancer, annotations, labels, ports, corev1.ServiceAffinityNone)
 		service.Spec.LoadBalancerIP = lbIP
+		service.Spec.AllocateLoadBalancerNodePorts = ptr.To(false)
 		_ = serviceClient.Create(service)
 
-		ginkgo.By("Waiting for deployment " + deploymentName + " to be ready")
+		ginkgo.By("Waiting for LB deployment " + deploymentName + " to be ready")
 		framework.WaitUntil(2*time.Second, time.Minute, func(ctx context.Context) (bool, error) {
 			_, err := deploymentClient.DeploymentInterface.Get(ctx, deploymentName, metav1.GetOptions{})
 			if err == nil {
@@ -221,19 +300,54 @@ var _ = framework.SerialDescribe("[group:lb-svc]", func() {
 		framework.ExpectNoError(err)
 		framework.ExpectHaveLen(pods.Items, 1)
 
-		ginkgo.By("Checking pod annotations")
-		key := fmt.Sprintf(util.AllocatedAnnotationTemplate, subnetProvider)
-		framework.ExpectHaveKeyWithValue(pods.Items[0].Annotations, key, "true")
-		ipKey := fmt.Sprintf(util.IPAddressAnnotationTemplate, subnetProvider)
-		framework.ExpectHaveKeyWithValue(pods.Items[0].Annotations, ipKey, lbIP)
-		cidr := pods.Items[0].Annotations[fmt.Sprintf(util.CidrAnnotationTemplate, subnetProvider)]
-		framework.ExpectTrue(util.CIDRContainIP(cidr, lbIP))
+		ginkgo.By("Checking LB pod annotations")
+		pod := &pods.Items[0]
+		key := fmt.Sprintf(util.AllocatedAnnotationTemplate, provider)
+		framework.ExpectHaveKeyWithValue(pod.Annotations, key, "true")
+		ipKey := fmt.Sprintf(util.IPAddressAnnotationTemplate, provider)
+		framework.ExpectHaveKeyWithValue(pod.Annotations, ipKey, lbIP)
+		cidrKey := fmt.Sprintf(util.CidrAnnotationTemplate, provider)
+		framework.ExpectHaveKey(pod.Annotations, cidrKey)
+		framework.ExpectIPInCIDR(lbIP, pod.Annotations[cidrKey])
 
-		ginkgo.By("Checking service external IP")
+		ginkgo.By("Checking service status")
 		framework.WaitUntil(2*time.Second, time.Minute, func(_ context.Context) (bool, error) {
 			service = serviceClient.Get(serviceName)
 			return len(service.Status.LoadBalancer.Ingress) != 0, nil
 		}, ".status.loadBalancer.ingress is not empty")
+		framework.ExpectHaveLen(service.Status.LoadBalancer.Ingress, 1)
 		framework.ExpectEqual(service.Status.LoadBalancer.Ingress[0].IP, lbIP)
+
+		ginkgo.By("Creating client pod " + clientPodName)
+		annotations = map[string]string{nadv1.NetworkAttachmentAnnot: fmt.Sprintf("%s/%s", namespaceName, nadName)}
+		cmd := []string{"sh", "-c", "sleep infinity"}
+		clientPod := framework.MakePod(namespaceName, clientPodName, nil, annotations, f.KubeOVNImage, cmd, nil)
+		clientPod = podClient.CreateSync(clientPod)
+
+		ginkgo.By("Checking service connectivity from client pod " + clientPodName)
+		curlCmd := fmt.Sprintf("curl -q -s --connect-timeout 2 --max-time 2 %s/clientip", util.JoinHostPort(lbIP, port))
+		ginkgo.By(fmt.Sprintf(`Executing %q in pod %s/%s`, curlCmd, clientPod.Namespace, clientPod.Name))
+		_ = e2epodoutput.RunHostCmdOrDie(clientPod.Namespace, clientPod.Name, curlCmd)
+
+		ginkgo.By("Deleting lb svc pod " + pod.Name)
+		podClient.DeleteSync(pod.Name)
+
+		ginkgo.By("Waiting for LB deployment " + deploymentName + " to be ready")
+		err = deploymentClient.WaitToComplete(deployment)
+		framework.ExpectNoError(err, "deployment failed to complete")
+
+		ginkgo.By("Checking service connectivity from client pod " + clientPodName)
+		framework.WaitUntil(2*time.Second, 30*time.Second, func(_ context.Context) (bool, error) {
+			ginkgo.By(fmt.Sprintf(`Executing %q in pod %s/%s`, curlCmd, clientPod.Namespace, clientPod.Name))
+			_, err = e2epodoutput.RunHostCmd(clientPod.Namespace, clientPod.Name, curlCmd)
+			return err == nil, nil
+		}, "")
+
+		ginkgo.By("Deleting service " + serviceName)
+		serviceClient.DeleteSync(serviceName)
+
+		ginkgo.By("Waiting for LB deployment " + deploymentName + " to be deleted automatically")
+		err = deploymentClient.WaitToDisappear(deploymentName, 2*time.Second, 2*time.Minute)
+		framework.ExpectNoError(err, "deployment failed to disappear")
 	})
 })