diff --git a/plugins/router/README.md b/plugins/router/README.md index 960fa5d4..6b0e2dc1 100644 --- a/plugins/router/README.md +++ b/plugins/router/README.md @@ -7,7 +7,7 @@ ## How to start -1. 安装 Multus-underlay, maclan-type 选择为 macvlan-overlay. +1. 安装 Multus-underlay, macvlan-type 选择为 macvlan-overlay. 2. 在 pod 的annotations中注入: ```shell annotations: diff --git a/test/Makefile b/test/Makefile index a3424161..3ea5aae8 100644 --- a/test/Makefile +++ b/test/Makefile @@ -48,6 +48,7 @@ install: export CNI_PLUGINS_VERSION=${CNI_PLUGINS_VERSION} ; \ export RUN_ON_LOCAL=${RUN_ON_LOCAL} ; \ export META_PLUGINS_CI_REPO=${META_PLUGINS_CI_REPO} ; \ + export SPIDERPOOL_VERSION=${SPIDERPOOL_VERSION} ; \ for script in `ls scripts/install/` ; do \ echo "Running $${script}" ; \ chmod +x scripts/install/$${script} ; \ diff --git a/test/Makefile.defs b/test/Makefile.defs index 67af4f72..00e60dc1 100644 --- a/test/Makefile.defs +++ b/test/Makefile.defs @@ -24,3 +24,5 @@ CNI_PLUGINS_VERSION=v1.1.1 RUN_ON_LOCAL=false META_PLUGINS_CI_REPO=ghcr.m.daocloud.io/spidernet-io/cni-plugins/meta-plugins +# Spiderpool +SPIDERPOOL_VERSION=0.3.2 \ No newline at end of file diff --git a/test/e2e/common/common.go b/test/e2e/common/common.go index 98005c1d..92064c2e 100644 --- a/test/e2e/common/common.go +++ b/test/e2e/common/common.go @@ -1,6 +1,7 @@ package common import ( + "encoding/json" "fmt" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -8,8 +9,14 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/pointer" "net" + "strings" ) +type spiderpoolAnnotation struct { + IPV4 string `json:"ipv4"` + IPV6 string `json:"ipv6"` +} + func GenerateDeploymentYaml(dpmName, namespace string, labels, annotations map[string]string) *appsv1.Deployment { return &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ @@ -139,10 +146,47 @@ func GetIPsFromPods(pods *corev1.PodList) []string { return ips } +func GetAllIPsFromPods(pods *corev1.PodList) ([]string, error) { + var ips []string + var err error + for _, pod := range pods.Items { + calicoIPs := pod.Annotations["cni.projectcalico.org/podIPs"] + for _, v := range strings.Split(calicoIPs, ",") { + //if net.ParseIP(v).To4() == nil && !IPV6 { + if net.ParseIP(v).To4() == nil { + continue + } else if net.ParseIP(v).To4() != nil && !IPV4 { + continue + } + ip, _, _ := net.ParseCIDR(v) + ips = append(ips, ip.String()) + } + for _, v := range SpiderPoolIPAnnotationsKey { + spiderpoolIPs, ok := pod.Annotations[v] + if ok { + spiderpool := &spiderpoolAnnotation{} + err = json.Unmarshal([]byte(spiderpoolIPs), spiderpool) + if err != nil { + return ips, fmt.Errorf("unmarshal spiderpool annations err: %s", err) + } + if spiderpool.IPV4 != "" && IPV4 { + ip, _, _ := net.ParseCIDR(spiderpool.IPV4) + ips = append(ips, ip.String()) + } + if spiderpool.IPV6 != "" && IPV6 { + ip, _, _ := net.ParseCIDR(spiderpool.IPV6) + ips = append(ips, ip.String()) + } + } + } + } + return ips, nil +} + func GetCurlCommandByIPFamily(netIp string, port int32) string { args := fmt.Sprintf("%s:%d ", netIp, port) if net.ParseIP(netIp).To4() == nil { - args = fmt.Sprintf("-g [%s]:%d", netIp, port) + args = fmt.Sprintf("-g [%s]:%d ", netIp, port) } return "curl " + args } diff --git a/test/e2e/common/const.go b/test/e2e/common/const.go index eb7f6acd..d87a20d6 100644 --- a/test/e2e/common/const.go +++ b/test/e2e/common/const.go @@ -1,6 +1,9 @@ package common -import "time" +import ( + "os" + "time" +) // multus crd type const ( @@ -11,10 +14,10 @@ const ( // multus crd name var ( - MacvlanStandaloneVlan100Name = "macvlan-standalone-vlan0" - MacvlanStandaloneVlan200Name = "macvlan-standalone-vlan100" - MacvlanOverlayVlan100Name = "macvlan-overlay-vlan0" - MacvlanOverlayVlan200Name = "macvlan-overlay-vlan100" + MacvlanStandaloneVlan0Name = "macvlan-standalone-vlan0" + MacvlanStandaloneVlan100Name = "macvlan-standalone-vlan100" + MacvlanOverlayVlan0Name = "macvlan-overlay-vlan0" + MacvlanOverlayVlan100Name = "macvlan-overlay-vlan100" ) // annotations @@ -23,6 +26,11 @@ var ( MultusAddonAnnotation_Key = "k8s.v1.cni.cncf.io/networks" SpiderPoolIPPoolAnnotationKey = "ipam.spidernet.io/ippool" SpiderPoolIPPoolsAnnotationKey = "ipam.spidernet.io/ippools" + SpiderPoolIPAnnotationsKey = []string{ + "ipam.spidernet.io/assigned-net1", + "ipam.spidernet.io/assigned-net2", + "ipam.spidernet.io/assigned-eth0", + } ) var ( @@ -30,3 +38,13 @@ var ( CtxTimeout = 60 * time.Second ENV_VLAN_GATEWAY_CONTAINER = "VLAN_GATEWAY_CONTAINER" ) +var ( + IPV4 bool + IPV6 bool +) + +func init() { + IPV4 = os.Getenv("E2E_IPV4_ENABLED") == "true" + IPV6 = os.Getenv("E2E_IPV6_ENABLED") == "true" + +} diff --git a/test/e2e/common/kind.go b/test/e2e/common/kind.go index 1c19f484..b2bba102 100644 --- a/test/e2e/common/kind.go +++ b/test/e2e/common/kind.go @@ -26,6 +26,12 @@ func GetKindNodeIPs(ctx context.Context, f *e2e.Framework, nodeList []string) ([ if err != nil { return nil, err } + if nodeIP.To4() == nil && !IPV6 { + continue + } + if nodeIP.To4() != nil && !IPV4 { + continue + } nodeIPs = append(nodeIPs, nodeIP.String()) } diff --git a/test/e2e/macvlan-overlay-one/macvlan_overlay_one_suite_test.go b/test/e2e/macvlan-overlay-one/macvlan_overlay_one_suite_test.go index 8d5794cf..95fdc65d 100644 --- a/test/e2e/macvlan-overlay-one/macvlan_overlay_one_suite_test.go +++ b/test/e2e/macvlan-overlay-one/macvlan_overlay_one_suite_test.go @@ -1,13 +1,130 @@ package macvlan_overlay_one_test import ( - "testing" - + "context" + "fmt" + multus_v1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/spidernet-io/cni-plugins/pkg/schema" + "github.com/spidernet-io/cni-plugins/test/e2e/common" + e2e "github.com/spidernet-io/e2eframework/framework" + "github.com/spidernet-io/e2eframework/tools" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "net" + "testing" + "time" ) func TestMacvlanOverlayOne(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "MacvlanOverlayOne Suite") } + +var frame *e2e.Framework +var name, namespace, multuNs string + +var podList *corev1.PodList +var dp *appsv1.Deployment +var labels, annotations = make(map[string]string), make(map[string]string) + +var port int32 = 80 +var nodePorts []int32 +var podIPs, clusterIPs, nodeIPs []string + +var retryTimes = 5 + +var _ = BeforeSuite(func() { + defer GinkgoRecover() + var e error + frame, e = e2e.NewFramework(GinkgoT(), []func(*runtime.Scheme) error{multus_v1.AddToScheme, schema.SpiderPoolAddToScheme}) + Expect(e).NotTo(HaveOccurred()) + + // init namespace name and create + name = "one-macvlan-overlay" + namespace = "ns" + tools.RandomName() + multuNs = "kube-system" + labels["app"] = name + + err := frame.CreateNamespace(namespace) + Expect(err).NotTo(HaveOccurred(), "failed to create namespace %v", namespace) + GinkgoWriter.Printf("create namespace %v \n", namespace) + + // get macvlan-standalone multus crd instance by name + multusInstance, err := frame.GetMultusInstance(common.MacvlanOverlayVlan100Name, multuNs) + Expect(err).NotTo(HaveOccurred()) + Expect(multusInstance).NotTo(BeNil()) + annotations[common.MultusAddonAnnotation_Key] = fmt.Sprintf("%s/%s", multuNs, common.MacvlanOverlayVlan100Name) + annotations[common.SpiderPoolIPPoolAnnotationKey] = `{"interface": "net1", "ipv4": ["vlan100-v4"], "ipv6": ["vlan100-v6"] }` + + GinkgoWriter.Printf("create deploy: %v/%v \n", namespace, name) + dp = common.GenerateDeploymentYaml(name, namespace, labels, annotations) + Expect(frame.CreateDeployment(dp)).NotTo(HaveOccurred()) + + ctx, cancel := context.WithTimeout(context.Background(), 2*common.CtxTimeout) + defer cancel() + + err = frame.WaitPodListRunning(dp.Spec.Selector.MatchLabels, int(*dp.Spec.Replicas), ctx) + Expect(err).NotTo(HaveOccurred()) + + podList, err = frame.GetDeploymentPodList(dp) + Expect(err).NotTo(HaveOccurred()) + Expect(podList).NotTo(BeNil()) + + // create nodePort service + st := common.GenerateServiceYaml(name, namespace, port, dp.Spec.Selector.MatchLabels) + err = frame.CreateService(st) + Expect(err).NotTo(HaveOccurred()) + + GinkgoWriter.Printf("succeed to create nodePort service: %s/%s\n", namespace, name) + + // get clusterIPs & nodePorts + service, err := frame.GetService(name, namespace) + Expect(err).NotTo(HaveOccurred()) + Expect(service).NotTo(BeNil(), "failed to get service: %s/%s", namespace, name) + + for _, ip := range service.Spec.ClusterIPs { + if net.ParseIP(ip).To4() == nil && common.IPV6 { + clusterIPs = append(clusterIPs, ip) + } + + if net.ParseIP(ip).To4() != nil && common.IPV4 { + clusterIPs = append(clusterIPs, ip) + } + } + nodePorts = common.GetServiceNodePorts(service.Spec.Ports) + GinkgoWriter.Printf("clusterIPs: %v\n", clusterIPs) + + // check service ready by get endpoint + err = common.WaitEndpointReady(retryTimes, name, namespace, frame) + Expect(err).NotTo(HaveOccurred()) + + // get pod all ip + podIPs, err = common.GetAllIPsFromPods(podList) + Expect(err).NotTo(HaveOccurred(), err) + Expect(podIPs).NotTo(BeNil()) + GinkgoWriter.Printf("Get All PodIPs: %v\n", podIPs) + + nodeIPs, err = common.GetKindNodeIPs(context.TODO(), frame, frame.Info.KindNodeList) + Expect(err).NotTo(HaveOccurred(), "failed to get all node ips: %v", err) + Expect(nodeIPs).NotTo(BeNil()) + GinkgoWriter.Printf("Get All Node ips: %v\n", nodeIPs) + + GinkgoWriter.Printf("Node list : %v \n", frame.Info.KindNodeList) + time.Sleep(5 * time.Second) +}) + +var _ = AfterSuite(func() { + // delete deployment + err := frame.DeleteDeployment(name, namespace) + Expect(err).NotTo(HaveOccurred(), "failed to delete deployment %v/%v", namespace, name) + + // delete service + err = frame.DeleteService(name, namespace) + Expect(err).To(Succeed()) + + err = frame.DeleteNamespace(namespace) + Expect(err).NotTo(HaveOccurred(), "failed to delete namespace %v", namespace) +}) diff --git a/test/e2e/macvlan-overlay-one/macvlan_overlay_one_test.go b/test/e2e/macvlan-overlay-one/macvlan_overlay_one_test.go index 7faf5eaf..8f3f6976 100644 --- a/test/e2e/macvlan-overlay-one/macvlan_overlay_one_test.go +++ b/test/e2e/macvlan-overlay-one/macvlan_overlay_one_test.go @@ -1,9 +1,91 @@ package macvlan_overlay_one_test import ( + "context" + "fmt" . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/spidernet-io/cni-plugins/test/e2e/common" + "net" + "os" ) var _ = Describe("MacvlanOverlayOne", Label("overlay", "one-nic"), func() { + It("Host can be communicate with pod, including pod in same and different node", Label("ping"), func() { + for _, node := range frame.Info.KindNodeList { + for _, podIP := range podIPs { + command := common.GetPingCommandByIPFamily(podIP) + _, err := frame.DockerExecCommand(context.TODO(), node, command) + Expect(err).NotTo(HaveOccurred(), " host %s failed to ping %s: %v,", node, podIP, err) + } + } + }) + It("Pods in different node can be communicate with each other", Label("ping"), func() { + for _, pod := range podList.Items { + for _, podIP := range podIPs { + command := common.GetPingCommandByIPFamily(podIP) + _, err := frame.ExecCommandInPod(pod.Name, pod.Namespace, command, context.TODO()) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("pod %s/%s failed to %s", pod.Namespace, pod.Name, command)) + } + } + }) + + // ping calico pod ? + + It("remote's client should be access to pod", Label("ping"), func() { + // get remote's client container name + vlanGatewayName := os.Getenv(common.ENV_VLAN_GATEWAY_CONTAINER) + Expect(vlanGatewayName).NotTo(BeEmpty(), "failed to get env 'VLAN_GATEWAY_CONTAINER'") + + for _, podIP := range podIPs { + command := common.GetPingCommandByIPFamily(podIP) + _, err := frame.DockerExecCommand(context.TODO(), vlanGatewayName, command) + Expect(err).To(Succeed()) + } + }) + + It("Pod should be access to clusterIP, including ipv4 and ipv6", Label("curl"), func() { + for _, pod := range podList.Items { + for _, clusterIP := range clusterIPs { + // TODO https://github.com/projectcalico/calico/issues/6877 + if net.ParseIP(clusterIP).To4() == nil { + continue + } + command := common.GetCurlCommandByIPFamily(clusterIP, 80) + output, err := frame.ExecCommandInPod(pod.Name, pod.Namespace, command, context.TODO()) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("pod %s/%s failed to %s: %s \n", pod.Namespace, pod.Name, command, output)) + } + } + }) + + It("Host should be access to clusterIP, including ipv4 and ipv6", Label("curl"), func() { + for _, node := range frame.Info.KindNodeList { + for _, clusterIP := range clusterIPs { + // TODO https://github.com/projectcalico/calico/issues/6877 + if net.ParseIP(clusterIP).To4() == nil { + continue + } + command := common.GetCurlCommandByIPFamily(clusterIP, 80) + GinkgoWriter.Printf("docker exec -it %s %s \n", node, command) + output, err := frame.DockerExecCommand(context.TODO(), node, command) + Expect(err).NotTo(HaveOccurred(), " host %s failed to access to cluster service %s: %s,", node, clusterIP, output) + } + } + + }) + It("Host should be access to nodePort address, including ipv4 and ipv6", Label("curl"), func() { + for _, node := range frame.Info.KindNodeList { + for _, nodeIP := range nodeIPs { + // TODO https://github.com/projectcalico/calico/issues/6877 + if net.ParseIP(nodeIP).To4() == nil { + continue + } + command := common.GetCurlCommandByIPFamily(nodeIP, nodePorts[0]) + GinkgoWriter.Printf("docker exec -it %s %s \n", node, command) + output, err := frame.DockerExecCommand(context.TODO(), node, command) + Expect(err).NotTo(HaveOccurred(), " host %s failed to access to nodePort service %s/%d: %s \n", node, nodeIP, nodePorts[0], output) + } + } + }) }) diff --git a/test/e2e/macvlan-standalone-one/macvlan_standalone_one_suite_test.go b/test/e2e/macvlan-standalone-one/macvlan_standalone_one_suite_test.go index b0968199..19e17785 100644 --- a/test/e2e/macvlan-standalone-one/macvlan_standalone_one_suite_test.go +++ b/test/e2e/macvlan-standalone-one/macvlan_standalone_one_suite_test.go @@ -13,6 +13,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" + "net" "testing" "time" ) @@ -52,11 +53,11 @@ var _ = BeforeSuite(func() { GinkgoWriter.Printf("create namespace %v \n", namespace) // get macvlan-standalone multus crd instance by name - multusInstance, err := frame.GetMultusInstance(common.MacvlanStandaloneVlan100Name, multuNs) + multusInstance, err := frame.GetMultusInstance(common.MacvlanStandaloneVlan0Name, multuNs) Expect(err).NotTo(HaveOccurred()) Expect(multusInstance).NotTo(BeNil()) - annotations[common.MultusDefaultAnnotationKey] = fmt.Sprintf("%s/%s", multuNs, common.MacvlanStandaloneVlan100Name) + annotations[common.MultusDefaultAnnotationKey] = fmt.Sprintf("%s/%s", multuNs, common.MacvlanStandaloneVlan0Name) GinkgoWriter.Printf("create deploy: %v/%v \n", namespace, name) dp = common.GenerateDeploymentYaml(name, namespace, labels, annotations) @@ -83,30 +84,40 @@ var _ = BeforeSuite(func() { service, err := frame.GetService(name, namespace) Expect(err).NotTo(HaveOccurred()) Expect(service).NotTo(BeNil(), "failed to get service: %s/%s", namespace, name) - clusterIPs = service.Spec.ClusterIPs + for _, ip := range service.Spec.ClusterIPs { + if net.ParseIP(ip).To4() == nil && common.IPV6 { + clusterIPs = append(clusterIPs, ip) + } + + if net.ParseIP(ip).To4() != nil && common.IPV4 { + clusterIPs = append(clusterIPs, ip) + } + } nodePorts = common.GetServiceNodePorts(service.Spec.Ports) GinkgoWriter.Printf("clusterIPs: %v\n", clusterIPs) // check service ready by get endpoint err = common.WaitEndpointReady(retryTimes, name, namespace, frame) Expect(err).NotTo(HaveOccurred()) - // get pod all ip - podIPs = common.GetIPsFromPods(podList) + podIPs, err = common.GetAllIPsFromPods(podList) + Expect(err).NotTo(HaveOccurred(), err) Expect(podIPs).NotTo(BeNil()) GinkgoWriter.Printf("Get All PodIPs: %v\n", podIPs) + GinkgoWriter.Printf("Node list : %v \n", frame.Info.KindNodeList) time.Sleep(5 * time.Second) }) var _ = AfterSuite(func() { - err := frame.DeleteNamespace(namespace) - Expect(err).NotTo(HaveOccurred(), "failed to delete namespace %v", namespace) - - err = frame.DeleteDeployment(name, namespace) + // delete deployment + err := frame.DeleteDeployment(name, namespace) Expect(err).NotTo(HaveOccurred(), "failed to delete deployment %v/%v", namespace, name) // delete service err = frame.DeleteService(name, namespace) Expect(err).To(Succeed()) + + err = frame.DeleteNamespace(namespace) + Expect(err).NotTo(HaveOccurred(), "failed to delete namespace %v", namespace) }) diff --git a/test/scripts/install/02-multus-underlay.sh b/test/scripts/install/02-multus-underlay.sh index 5f6b43f2..188d17ab 100755 --- a/test/scripts/install/02-multus-underlay.sh +++ b/test/scripts/install/02-multus-underlay.sh @@ -15,7 +15,7 @@ PROJECT_ROOT_PATH=$( cd ${CURRENT_DIR_PATH}/../.. && pwd ) #[ -n ${INSTALLED} ] && echo "Warning!! multus-underlay has been deployed, skip install multus-underlay" && exit 0 # Multus config -MULTUS_UNDERLAY_VERSION=${MULTUS_UNDERLAY_VERSION:-0.1.3} +MULTUS_UNDERLAY_VERSION=${MULTUS_UNDERLAY_VERSION:-0.1.4} MACVLAN_MASTER=${MACVLAN_MASTER:-eth0} MACVLAN_TYPE=${MACVLAN_TYPE:-macvlan-overlay} diff --git a/test/scripts/install/03-spiderpool.sh b/test/scripts/install/03-spiderpool.sh index ca11558e..faf10200 100755 --- a/test/scripts/install/03-spiderpool.sh +++ b/test/scripts/install/03-spiderpool.sh @@ -88,6 +88,41 @@ helm install spiderpool daocloud/spiderpool --wait -n kube-system --kubeconfig $ kubectl wait --for=condition=ready -l app.kubernetes.io/name=spiderpool --timeout=${INSTALL_TIME_OUT} pod -n kube-system \ --kubeconfig ${E2E_KUBECONFIG} + + +case ${IP_FAMILY} in + ipv4) +cat <