some fixes for e2e testing (#2207)

.github/workflows/build-x86-image.yaml

@@ -447,6 +447,7 @@ jobs:
       - name: Run E2E
         working-directory: ${{ env.E2E_DIR }}
         env:
+          E2E_BRANCH: ${{ github.base_ref || github.ref }}
           E2E_IP_FAMILY: ${{ matrix.ip-family }}
           E2E_NETWORK_MODE: ${{ matrix.mode }}
         run: make kube-ovn-conformance-e2e

Makefile.e2e

@@ -62,14 +62,23 @@ cyclonus-netpol-e2e:
 .PHONY: kube-ovn-conformance-e2e
 kube-ovn-conformance-e2e:
 	go test ./test/e2e/kube-ovn -c -o test/e2e/kube-ovn/e2e.test
+	E2E_BRANCH=$(E2E_BRANCH) \
+	E2E_IP_FAMILY=$(E2E_IP_FAMILY) \
+	E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \
 	./test/e2e/kube-ovn/e2e.test --ginkgo.focus=CNI:Kube-OVN
 
 .PHONY: kube-ovn-ic-conformance-e2e
 kube-ovn-ic-conformance-e2e:
 	go test ./test/e2e/ovn-ic -c -o test/e2e/ovn-ic/e2e.test
+	E2E_BRANCH=$(E2E_BRANCH) \
+	E2E_IP_FAMILY=$(E2E_IP_FAMILY) \
+	E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \
 	./test/e2e/ovn-ic/e2e.test --ginkgo.focus=CNI:Kube-OVN
 
 .PHONY: kube-ovn-lb-svc-conformance-e2e
 kube-ovn-lb-svc-conformance-e2e:
 	go test ./test/e2e/lb-svc -c -o test/e2e/lb-svc/e2e.test
+	E2E_BRANCH=$(E2E_BRANCH) \
+	E2E_IP_FAMILY=$(E2E_IP_FAMILY) \
+	E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \
 	./test/e2e/lb-svc/e2e.test --ginkgo.focus=CNI:Kube-OVN

test/e2e/framework/framework.go

@@ -1,7 +1,9 @@
 package framework
 
 import (
+	"fmt"
 	"os"
+	"strings"
 	"time"
 
 	"k8s.io/client-go/tools/clientcmd"
@@ -27,6 +29,9 @@ type Framework struct {
 
 	// master/release-1.10/...
 	ClusterVersion string
+	// 999.999 for master
+	ClusterVersionMajor uint
+	ClusterVersionMinor uint
 	// ipv4/ipv6/dual
 	ClusterIpFamily string
 	// overlay/underlay/underlay-hairpin
@@ -42,6 +47,14 @@ func NewDefaultFramework(baseName string) *Framework {
 	f.ClusterVersion = os.Getenv("E2E_BRANCH")
 	f.ClusterNetworkMode = os.Getenv("E2E_NETWORK_MODE")
 
+	if strings.HasPrefix(f.ClusterVersion, "release-") {
+		n, err := fmt.Sscanf(f.ClusterVersion, "release-%d.%d", &f.ClusterVersionMajor, &f.ClusterVersionMinor)
+		ExpectNoError(err)
+		ExpectEqual(n, 2)
+	} else {
+		f.ClusterVersionMajor, f.ClusterVersionMinor = 999, 999
+	}
+
 	ginkgo.BeforeEach(f.BeforeEach)
 
 	return f
@@ -112,6 +125,12 @@ func (f *Framework) BeforeEach() {
 	framework.TestContext.Host = ""
 }
 
+func (f *Framework) SkipVersionPriorTo(major, minor uint, message string) {
+	if f.ClusterVersionMajor <= major && f.ClusterVersionMinor < minor {
+		ginkgo.Skip(message)
+	}
+}
+
 func Describe(text string, body func()) bool {
 	return ginkgo.Describe("[CNI:Kube-OVN] "+text, body)
 }

test/e2e/framework/util.go

@@ -72,7 +72,7 @@ func RandomExcludeIPs(cidr string, count int) []string {
 	}
 
 	rangeCount := rand.Intn(count + 1)
-	ips := strings.Split(RandomIPPool(cidr, rangeCount*2+count-rangeCount), ";")
+	ips := strings.Split(RandomIPPool(cidr, ";", rangeCount*2+count-rangeCount), ";")
 	sortIPs(ips)
 
 	var idx int
@@ -92,7 +92,7 @@ func RandomExcludeIPs(cidr string, count int) []string {
 	return ret
 }
 
-func RandomIPPool(cidr string, count int) string {
+func RandomIPPool(cidr, sep string, count int) string {
 	fn := func(cidr string) []string {
 		if cidr == "" {
 			return nil
@@ -133,5 +133,5 @@ func RandomIPPool(cidr string, count int) string {
 		dual = append(dual, strings.Join(ips, ","))
 	}
 
-	return strings.Join(dual, ";")
+	return strings.Join(dual, sep)
 }

test/e2e/kube-ovn/ipam/ipam.go

@@ -48,7 +48,7 @@ var _ = framework.Describe("[group:ipam]", func() {
 	framework.ConformanceIt("should allocate static ipv4 and mac for pod", func() {
 		name := "pod-" + framework.RandomSuffix()
 		mac := util.GenerateMac()
-		ip := framework.RandomIPPool(cidr, 1)
+		ip := framework.RandomIPPool(cidr, ";", 1)
 
 		ginkgo.By("Creating pod " + name + " with ip " + ip + " and mac " + mac)
 		annotations := map[string]string{
@@ -76,10 +76,46 @@ var _ = framework.Describe("[group:ipam]", func() {
 		podClient.DeleteSync(pod.Name)
 	})
 
-	framework.ConformanceIt("should allocate static ipv4 for deployment with ippool", func() {
+	framework.ConformanceIt("should allocate static ip for pod with comma separated ippool", func() {
+		if f.ClusterIpFamily == "dual" {
+			ginkgo.Skip("Comma separated ippool is not supported for dual stack")
+		}
+
+		name := "pod-" + framework.RandomSuffix()
+		pool := framework.RandomIPPool(cidr, ",", 3)
+
+		ginkgo.By("Creating pod " + name + " with ippool " + pool)
+		annotations := map[string]string{util.IpPoolAnnotation: pool}
+		pod := framework.MakePod(namespaceName, name, nil, annotations, "", nil, nil)
+		pod = podClient.CreateSync(pod)
+
+		framework.ExpectHaveKeyWithValue(pod.Annotations, util.AllocatedAnnotation, "true")
+		framework.ExpectHaveKeyWithValue(pod.Annotations, util.CidrAnnotation, subnet.Spec.CIDRBlock)
+		framework.ExpectHaveKeyWithValue(pod.Annotations, util.GatewayAnnotation, subnet.Spec.Gateway)
+		framework.ExpectHaveKeyWithValue(pod.Annotations, util.IpPoolAnnotation, pool)
+		framework.ExpectEqual(pod.Annotations[util.IpAddressAnnotation], pod.Status.PodIP)
+		framework.ExpectHaveKeyWithValue(pod.Annotations, util.LogicalSwitchAnnotation, subnet.Name)
+		framework.ExpectMAC(pod.Annotations[util.MacAddressAnnotation])
+		framework.ExpectHaveKeyWithValue(pod.Annotations, util.RoutedAnnotation, "true")
+
+		framework.ExpectContainElement(strings.Split(pool, ","), pod.Status.PodIP)
+
+		ginkgo.By("Deleting pod " + name)
+		podClient.DeleteSync(pod.Name)
+	})
+
+	framework.ConformanceIt("should allocate static ip for deployment with ippool", func() {
+		ippoolSep := ";"
+		if f.ClusterVersionMajor <= 1 && f.ClusterVersionMinor < 11 {
+			if f.ClusterIpFamily == "dual" {
+				ginkgo.Skip("Support for dual stack ippool was introduced in v1.11")
+			}
+			ippoolSep = ","
+		}
+
 		replicas := 3
 		name := "deployment-" + framework.RandomSuffix()
-		ippool := framework.RandomIPPool(cidr, replicas)
+		ippool := framework.RandomIPPool(cidr, ippoolSep, replicas)
 		labels := map[string]string{"app": name}
 
 		ginkgo.By("Creating deployment " + name + " with ippool " + ippool)
@@ -95,7 +131,7 @@ var _ = framework.Describe("[group:ipam]", func() {
 		framework.ExpectNoError(err, "failed to get pods for deployment "+name)
 		framework.ExpectHaveLen(pods.Items, replicas)
 
-		ips := strings.Split(ippool, ";")
+		ips := strings.Split(ippool, ippoolSep)
 		for _, pod := range pods.Items {
 			framework.ExpectHaveKeyWithValue(pod.Annotations, util.AllocatedAnnotation, "true")
 			framework.ExpectHaveKeyWithValue(pod.Annotations, util.CidrAnnotation, subnet.Spec.CIDRBlock)
@@ -151,7 +187,7 @@ var _ = framework.Describe("[group:ipam]", func() {
 		framework.ExpectNoError(err, "failed to delete deployment "+name)
 	})
 
-	framework.ConformanceIt("should allocate static ipv4 for statefulset", func() {
+	framework.ConformanceIt("should allocate static ip for statefulset", func() {
 		replicas := 3
 		name := "statefulset-" + framework.RandomSuffix()
 		labels := map[string]string{"app": name}
@@ -212,10 +248,18 @@ var _ = framework.Describe("[group:ipam]", func() {
 		framework.ExpectNoError(err, "failed to delete statefulset "+name)
 	})
 
-	framework.ConformanceIt("should allocate static ipv4 for statefulset with ippool", func() {
+	framework.ConformanceIt("should allocate static ip for statefulset with ippool", func() {
+		ippoolSep := ";"
+		if f.ClusterVersionMajor <= 1 && f.ClusterVersionMinor < 11 {
+			if f.ClusterIpFamily == "dual" {
+				ginkgo.Skip("Support for dual stack ippool was introduced in v1.11")
+			}
+			ippoolSep = ","
+		}
+
 		replicas := 3
 		name := "statefulset-" + framework.RandomSuffix()
-		ippool := framework.RandomIPPool(cidr, replicas)
+		ippool := framework.RandomIPPool(cidr, ippoolSep, replicas)
 		labels := map[string]string{"app": name}
 
 		ginkgo.By("Creating statefulset " + name + " with ippool " + ippool)
@@ -248,7 +292,7 @@ var _ = framework.Describe("[group:ipam]", func() {
 			framework.ExpectConsistOf(podIPs, strings.Split(pod.Annotations[util.IpAddressAnnotation], ","))
 			ips = append(ips, pod.Annotations[util.IpAddressAnnotation])
 		}
-		framework.ExpectConsistOf(ips, strings.Split(ippool, ";"))
+		framework.ExpectConsistOf(ips, strings.Split(ippool, ippoolSep))
 
 		ginkgo.By("Deleting pods for statefulset " + name)
 		for _, pod := range pods.Items {

test/e2e/kube-ovn/qos/qos.go

@@ -87,7 +87,9 @@ var _ = framework.Describe("[group:qos]", func() {
 		}
 	})
 
-	framework.ConformanceIt(`should support netem QoS"`, func() {
+	framework.ConformanceIt("should support netem QoS", func() {
+		f.SkipVersionPriorTo(1, 9, "Support for netem QoS was introduced in v1.9")
+
 		name := "pod-" + framework.RandomSuffix()
 		ginkgo.By("Creating pod " + name)
 		latency, limit, loss := 600, 2000, 10
@@ -116,7 +118,9 @@ var _ = framework.Describe("[group:qos]", func() {
 		podClient.DeleteSync(pod.Name)
 	})
 
-	framework.ConformanceIt(`should be able to update netem QoS"`, func() {
+	framework.ConformanceIt("should be able to update netem QoS", func() {
+		f.SkipVersionPriorTo(1, 9, "Support for netem QoS was introduced in v1.9")
+
 		name := "pod-" + framework.RandomSuffix()
 		ginkgo.By("Creating pod " + name + " without QoS")
 		pod := framework.MakePod(namespaceName, name, nil, nil, "", nil, nil)
@@ -154,7 +158,7 @@ var _ = framework.Describe("[group:qos]", func() {
 		podClient.DeleteSync(pod.Name)
 	})
 
-	framework.ConformanceIt(`should support htb QoS"`, func() {
+	framework.ConformanceIt("should support htb QoS", func() {
 		name := "pod-" + framework.RandomSuffix()
 		ginkgo.By("Creating pod " + name)
 		priority, ingressRate := 50, 300
@@ -180,7 +184,7 @@ var _ = framework.Describe("[group:qos]", func() {
 		podClient.DeleteSync(pod.Name)
 	})
 
-	framework.ConformanceIt(`should be able to update htb QoS"`, func() {
+	framework.ConformanceIt("should be able to update htb QoS", func() {
 		subnetName = f.Namespace.Name
 		ginkgo.By("Creating subnet " + subnetName + " with htb QoS")
 		cidr := framework.RandomCIDR(f.ClusterIpFamily)

test/e2e/kube-ovn/subnet/subnet.go

@@ -105,7 +105,7 @@ var _ = framework.Describe("[group:subnet]", func() {
 				return ""
 			}
 			_, ipnet, _ := net.ParseCIDR(cidr)
-			ipnet.IP = net.ParseIP(framework.RandomIPPool(cidr, 1))
+			ipnet.IP = net.ParseIP(framework.RandomIPPool(cidr, ";", 1))
 			return ipnet.String()
 		}