From 4750d2d2d0eabe9deaf97573e6da56a531e8d145 Mon Sep 17 00:00:00 2001 From: Cyclinder Date: Thu, 17 Aug 2023 14:15:33 +0800 Subject: [PATCH] coordinator: auto mode is supported (#2178) --- api/v1/agent/models/coordinator_config.go | 3 + api/v1/agent/openapi.yaml | 4 ++ api/v1/agent/server/embedded_spec.go | 12 ++++ charts/spiderpool/README.md | 20 +++--- ...rpool.spidernet.io_spidercoordinators.yaml | 1 + ...pool.spidernet.io_spidermultusconfigs.yaml | 1 + charts/spiderpool/values.yaml | 4 +- cmd/coordinator/cmd/cni_types.go | 1 + cmd/coordinator/cmd/command_add.go | 6 +- cmd/coordinator/cmd/command_del.go | 15 ++-- cmd/coordinator/cmd/utils.go | 44 ++++++++++-- cmd/spiderpool-agent/cmd/coordinator.go | 22 +++++- cmd/spiderpool-init/cmd/config.go | 11 ++- docs/reference/crd-spidercoordinator.md | 2 +- docs/usage/coordinator-zh_CN.md | 9 ++- .../v2beta1/spidercoordinator_types.go | 2 +- pkg/networking/networking/route.go | 34 --------- test/Makefile | 4 +- test/Makefile.defs | 8 +-- test/e2e/annotation/annotation_test.go | 2 +- test/e2e/common/constant.go | 13 ++-- .../macvlan_overlay_one_test.go | 2 +- test/e2e/subnet/subnet_test.go | 4 +- test/scripts/debugEnv.sh | 6 ++ test/scripts/install-multus.sh | 71 ++++++++++++------- 25 files changed, 184 insertions(+), 117 deletions(-) diff --git a/api/v1/agent/models/coordinator_config.go b/api/v1/agent/models/coordinator_config.go index 4cb7dc336a..bdc5899ea6 100644 --- a/api/v1/agent/models/coordinator_config.go +++ b/api/v1/agent/models/coordinator_config.go @@ -51,6 +51,9 @@ type CoordinatorConfig struct { // pod m a c prefix PodMACPrefix string `json:"podMACPrefix,omitempty"` + // pod n i cs + PodNICs []string `json:"podNICs"` + // service c ID r // Required: true ServiceCIDR []string `json:"serviceCIDR"` diff --git a/api/v1/agent/openapi.yaml b/api/v1/agent/openapi.yaml index 14aada2b07..c205d1c0a1 100644 --- a/api/v1/agent/openapi.yaml +++ b/api/v1/agent/openapi.yaml @@ -328,6 +328,10 @@ definitions: type: boolean detectGateway: type: boolean + podNICs: + type: array + items: + type: string required: - mode - overlayPodCIDR diff --git a/api/v1/agent/server/embedded_spec.go b/api/v1/agent/server/embedded_spec.go index 7dd998a6fc..7a8b931ac4 100644 --- a/api/v1/agent/server/embedded_spec.go +++ b/api/v1/agent/server/embedded_spec.go @@ -302,6 +302,12 @@ func init() { "podMACPrefix": { "type": "string" }, + "podNICs": { + "type": "array", + "items": { + "type": "string" + } + }, "serviceCIDR": { "type": "array", "items": { @@ -799,6 +805,12 @@ func init() { "podMACPrefix": { "type": "string" }, + "podNICs": { + "type": "array", + "items": { + "type": "string" + } + }, "serviceCIDR": { "type": "array", "items": { diff --git a/charts/spiderpool/README.md b/charts/spiderpool/README.md index 792eab3026..986848e627 100644 --- a/charts/spiderpool/README.md +++ b/charts/spiderpool/README.md @@ -132,16 +132,16 @@ helm install spiderpool spiderpool/spiderpool --wait --namespace kube-system \ ### coordinator parameters -| Name | Description | Value | -| ------------------------------ | ------------------------------------------------------------------------- | ---------- | -| `coordinator.enabled` | enable SpiderCoordinator | `true` | -| `coordinator.name` | the name of the default SpiderCoordinator CR | `default` | -| `coordinator.mode` | optional network mode, ["underlay", "overlay", "disabled"] | `underlay` | -| `coordinator.podCIDRType` | Pod CIDR type that should be collected, [ "cluster", "calico", "cilium" ] | `cluster` | -| `coordinator.detectGateway` | detect the reachability of the gateway | `false` | -| `coordinator.detectIPConflict` | detect IP address conflicts | `false` | -| `coordinator.tunePodRoutes` | tune Pod routes | `true` | -| `coordinator.hijackCIDR` | Additional subnets that need to be hijacked to the host forward | `[]` | +| Name | Description | Value | +| ------------------------------ | ------------------------------------------------------------------------- | --------- | +| `coordinator.enabled` | enable SpiderCoordinator | `true` | +| `coordinator.name` | the name of the default SpiderCoordinator CR | `default` | +| `coordinator.mode` | optional network mode, ["auto","underlay", "overlay", "disabled"] | `auto` | +| `coordinator.podCIDRType` | Pod CIDR type that should be collected, [ "cluster", "calico", "cilium" ] | `cluster` | +| `coordinator.detectGateway` | detect the reachability of the gateway | `false` | +| `coordinator.detectIPConflict` | detect IP address conflicts | `false` | +| `coordinator.tunePodRoutes` | tune Pod routes | `true` | +| `coordinator.hijackCIDR` | Additional subnets that need to be hijacked to the host forward | `[]` | ### multus parameters diff --git a/charts/spiderpool/crds/spiderpool.spidernet.io_spidercoordinators.yaml b/charts/spiderpool/crds/spiderpool.spidernet.io_spidercoordinators.yaml index 9ff57d0bf4..bf60af9b3d 100644 --- a/charts/spiderpool/crds/spiderpool.spidernet.io_spidercoordinators.yaml +++ b/charts/spiderpool/crds/spiderpool.spidernet.io_spidercoordinators.yaml @@ -53,6 +53,7 @@ spec: type: integer mode: enum: + - auto - underlay - overlay - disabled diff --git a/charts/spiderpool/crds/spiderpool.spidernet.io_spidermultusconfigs.yaml b/charts/spiderpool/crds/spiderpool.spidernet.io_spidermultusconfigs.yaml index 5c38eea61c..7b80f6f4bf 100644 --- a/charts/spiderpool/crds/spiderpool.spidernet.io_spidermultusconfigs.yaml +++ b/charts/spiderpool/crds/spiderpool.spidernet.io_spidermultusconfigs.yaml @@ -62,6 +62,7 @@ spec: type: integer mode: enum: + - auto - underlay - overlay - disabled diff --git a/charts/spiderpool/values.yaml b/charts/spiderpool/values.yaml index f9816a0632..f4f855ef6e 100644 --- a/charts/spiderpool/values.yaml +++ b/charts/spiderpool/values.yaml @@ -90,8 +90,8 @@ coordinator: ## @param coordinator.name the name of the default SpiderCoordinator CR name: "default" - ## @param coordinator.mode optional network mode, ["underlay", "overlay", "disabled"] - mode: "underlay" + ## @param coordinator.mode optional network mode, ["auto","underlay", "overlay", "disabled"] + mode: "auto" ## @param coordinator.podCIDRType Pod CIDR type that should be collected, [ "cluster", "calico", "cilium" ] podCIDRType: "cluster" diff --git a/cmd/coordinator/cmd/cni_types.go b/cmd/coordinator/cmd/cni_types.go index 7e858268de..700e14432e 100644 --- a/cmd/coordinator/cmd/cni_types.go +++ b/cmd/coordinator/cmd/cni_types.go @@ -36,6 +36,7 @@ var ( type Mode string const ( + ModeAuto Mode = "auto" ModeUnderlay Mode = "underlay" ModeOverlay Mode = "overlay" ModeDisable Mode = "disable" diff --git a/cmd/coordinator/cmd/command_add.go b/cmd/coordinator/cmd/command_add.go index aac381fc2c..1057f05481 100644 --- a/cmd/coordinator/cmd/command_add.go +++ b/cmd/coordinator/cmd/command_add.go @@ -55,6 +55,7 @@ func CmdAdd(args *skel.CmdArgs) (err error) { if err != nil { return err } + if conf.Mode == ModeDisable { return types.PrintResult(conf.PrevResult, conf.CNIVersion) } @@ -98,6 +99,7 @@ func CmdAdd(args *skel.CmdArgs) (err error) { currentInterface: args.IfName, tuneMode: conf.Mode, interfacePrefix: conf.MultusNicPrefix, + podNics: coordinatorConfig.PodNICs, } c.HijackCIDR = append(c.HijackCIDR, conf.ServiceCIDR...) c.HijackCIDR = append(c.HijackCIDR, conf.HijackCIDR...) @@ -110,14 +112,14 @@ func CmdAdd(args *skel.CmdArgs) (err error) { defer c.netns.Close() // check if it's first time invoke - err = c.coordinatorFirstInvoke(conf.PodDefaultCniNic) + err = c.coordinatorModeAndFirstInvoke(logger, conf.PodDefaultCniNic) if err != nil { logger.Error(err.Error()) return err } // get basic info - switch conf.Mode { + switch c.tuneMode { case ModeUnderlay: c.podVethName = defaultUnderlayVethName c.hostVethName = getHostVethName(args.ContainerID) diff --git a/cmd/coordinator/cmd/command_del.go b/cmd/coordinator/cmd/command_del.go index 4426b104c6..fc6c7cb51f 100644 --- a/cmd/coordinator/cmd/command_del.go +++ b/cmd/coordinator/cmd/command_del.go @@ -84,17 +84,16 @@ func CmdDel(args *skel.CmdArgs) (err error) { } defer c.netns.Close() - if conf.Mode == ModeUnderlay { - hostVeth := getHostVethName(args.ContainerID) - vethLink, err := netlink.LinkByName(hostVeth) - if err != nil { - if _, ok := err.(netlink.LinkNotFoundError); ok { - logger.Sugar().Debug("Host veth has gone, nothing to do", zap.String("HostVeth", hostVeth)) - return nil - } + hostVeth := getHostVethName(args.ContainerID) + vethLink, err := netlink.LinkByName(hostVeth) + if err != nil { + if _, ok := err.(netlink.LinkNotFoundError); ok { + logger.Sugar().Debug("Host veth has gone, nothing to do", zap.String("HostVeth", hostVeth)) + } else { logger.Sugar().Warn(fmt.Sprintf("failed to get host veth device %s: %v", hostVeth, err)) return fmt.Errorf("failed to get host veth device %s: %v", hostVeth, err) } + } else { if err = netlink.LinkDel(vethLink); err != nil { logger.Sugar().Warn("failed to del hostVeth", zap.Error(err)) return fmt.Errorf("failed to del hostVeth %s: %w", hostVeth, err) diff --git a/cmd/coordinator/cmd/utils.go b/cmd/coordinator/cmd/utils.go index 37a9053f04..42b09c8f0e 100644 --- a/cmd/coordinator/cmd/utils.go +++ b/cmd/coordinator/cmd/utils.go @@ -26,20 +26,56 @@ type coordinator struct { ipFamily, currentRuleTable, hostRuleTable int tuneMode Mode hostVethName, podVethName, currentInterface, interfacePrefix string - HijackCIDR []string + HijackCIDR, podNics []string netns ns.NetNS hostVethHwAddress, podVethHwAddress net.HardwareAddr currentAddress []netlink.Addr hostIPRouteForPod []net.IP } +func (c *coordinator) autoModeToSpecificMode(mode Mode, podFirstInterface string) error { + if mode != ModeAuto { + return nil + } + + if c.currentInterface == podFirstInterface { + c.firstInvoke = true + c.tuneMode = ModeUnderlay + return nil + } + + // veth0 must be present in underlay mode + vethExist, err := networking.CheckInterfaceExist(c.netns, defaultUnderlayVethName) + if err != nil { + return fmt.Errorf("failed to check interface: %v exist: %v", defaultUnderlayVethName, err) + } + + if vethExist { + c.tuneMode = ModeUnderlay + } else { + c.tuneMode = ModeOverlay + // If spinderpool only assigns a NIC to the pod, Indicates that it is the first invoke + if len(c.podNics) == 1 { + c.firstInvoke = true + } + } + + return nil +} + // firstInvoke check if coordinator is first called and do some checks: // underlay mode only works with underlay mode, which can't work with overlay // mode, and which can't be called in first cni invoked by using multus's // annotations: v1.multus-cni.io/default-network -func (c *coordinator) coordinatorFirstInvoke(podFirstInterface string) error { +func (c *coordinator) coordinatorModeAndFirstInvoke(logger *zap.Logger, podFirstInterface string) error { var err error switch c.tuneMode { + case ModeAuto: + if err = c.autoModeToSpecificMode(ModeAuto, podFirstInterface); err != nil { + return err + } + logger.Sugar().Infof("Successfully auto detect mode, change mode from auto to %v", c.tuneMode) + return nil case ModeUnderlay: c.firstInvoke = c.currentInterface == podFirstInterface // underlay mode can't work with calico/cilium(overlay) @@ -70,8 +106,8 @@ func (c *coordinator) coordinatorFirstInvoke(podFirstInterface string) error { return fmt.Errorf("when creating interface %s in overlay mode, it detects that the auxiliary interface %s of underlay mode exists. It seems that the previous interface work in underlay mode. ", c.currentInterface, defaultUnderlayVethName) } - c.firstInvoke, err = networking.IsFirstModeOverlayInvoke(c.netns, c.interfacePrefix) - return err + c.firstInvoke = len(c.podNics) == 1 + return nil case ModeDisable: return nil } diff --git a/cmd/spiderpool-agent/cmd/coordinator.go b/cmd/spiderpool-agent/cmd/coordinator.go index 10753bb8cb..03923a9599 100644 --- a/cmd/spiderpool-agent/cmd/coordinator.go +++ b/cmd/spiderpool-agent/cmd/coordinator.go @@ -13,6 +13,7 @@ import ( "github.com/spidernet-io/spiderpool/pkg/coordinatormanager" spiderpoolv2beta1 "github.com/spidernet-io/spiderpool/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" ) var unixGetCoordinatorConfig = &_unixGetCoordinatorConfig{} @@ -24,6 +25,7 @@ func (g *_unixGetCoordinatorConfig) Handle(params daemonset.GetCoordinatorConfig ctx := params.HTTPRequest.Context() crdClient := agentContext.CRDManager.GetClient() podClient := agentContext.PodManager + epClient := agentContext.EndpointManager var coordList spiderpoolv2beta1.SpiderCoordinatorList if err := crdClient.List(ctx, &coordList); err != nil { @@ -39,11 +41,25 @@ func (g *_unixGetCoordinatorConfig) Handle(params daemonset.GetCoordinatorConfig return daemonset.NewGetCoordinatorConfigFailure().WithPayload(models.Error(fmt.Sprintf("spidercoordinator: %s no ready", coord.Name))) } - var pod *corev1.Pod var err error + var spNics []string + var se *spiderpoolv2beta1.SpiderEndpoint + // get spiderendpoint + se, err = epClient.GetEndpointByName(ctx, params.GetCoordinatorConfig.PodNamespace, params.GetCoordinatorConfig.PodName, constant.UseCache) + if err != nil && !apierrors.IsNotFound(err) { + return daemonset.NewGetCoordinatorConfigFailure().WithPayload(models.Error(fmt.Sprintf("failed to get spiderendpoint %s/%s", params.GetCoordinatorConfig.PodNamespace, params.GetCoordinatorConfig.PodName))) + } + + if se != nil { + for _, spip := range se.Status.Current.IPs { + spNics = append(spNics, spip.NIC) + } + } + + var pod *corev1.Pod pod, err = podClient.GetPodByName(ctx, params.GetCoordinatorConfig.PodNamespace, params.GetCoordinatorConfig.PodName, constant.UseCache) if err != nil { - return daemonset.NewGetCoordinatorConfigFailure().WithPayload(models.Error(fmt.Sprintf("failed to get coordinator config: pod %s/%s not found", params.GetCoordinatorConfig.PodNamespace, params.GetCoordinatorConfig.PodName))) + return daemonset.NewGetCoordinatorConfigFailure().WithPayload(models.Error(fmt.Sprintf("failed to get pod %s/%s", params.GetCoordinatorConfig.PodNamespace, params.GetCoordinatorConfig.PodName))) } var prefix string @@ -73,7 +89,7 @@ func (g *_unixGetCoordinatorConfig) Handle(params daemonset.GetCoordinatorConfig HostRPFilter: int64(*coord.Spec.HostRPFilter), DetectGateway: *coord.Spec.DetectGateway, DetectIPConflict: *coord.Spec.DetectIPConflict, + PodNICs: spNics, } - return daemonset.NewGetCoordinatorConfigOK().WithPayload(config) } diff --git a/cmd/spiderpool-init/cmd/config.go b/cmd/spiderpool-init/cmd/config.go index 482b0f2f56..8f965d8ba9 100644 --- a/cmd/spiderpool-init/cmd/config.go +++ b/cmd/spiderpool-init/cmd/config.go @@ -12,6 +12,7 @@ import ( "strconv" "strings" + coordinatorcmd "github.com/spidernet-io/spiderpool/cmd/coordinator/cmd" "github.com/spidernet-io/spiderpool/pkg/constant" spiderpoolip "github.com/spidernet-io/spiderpool/pkg/ip" spiderpoolv2beta1 "github.com/spidernet-io/spiderpool/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1" @@ -108,6 +109,9 @@ func parseENVAsDefault() InitDefaultConfig { config.CoordinatorName = strings.ReplaceAll(os.Getenv(ENVDefaultCoordinatorName), "\"", "") if len(config.CoordinatorName) != 0 { config.CoordinatorMode = strings.ReplaceAll(os.Getenv(ENVDefaultCoordinatorTuneMode), "\"", "") + if config.CoordinatorMode == "" { + config.CoordinatorMode = string(coordinatorcmd.ModeAuto) + } config.CoordinatorPodCIDRType = strings.ReplaceAll(os.Getenv(ENVDefaultCoordinatorPodCIDRType), "\"", "") edg := strings.ReplaceAll(os.Getenv(ENVDefaultCoordinatorDetectGateway), "\"", "") @@ -130,12 +134,7 @@ func parseENVAsDefault() InitDefaultConfig { logger.Sugar().Fatalf("ENV %s %s: %v", ENVDefaultCoordinatorTunePodRoutes, etpr, err) } config.CoordinatorTunePodRoutes = tpr - switch config.CoordinatorMode { - case "underlay": - config.CoordinatorPodDefaultRouteNic = "eth0" - case "overlay": - config.CoordinatorPodDefaultRouteNic = "net1" - } + config.CoordinatorPodDefaultRouteNic = "" config.CoordinatorPodMACPrefix = "" v := os.Getenv(ENVDefaultCoordiantorHijackCIDR) if len(v) > 0 { diff --git a/docs/reference/crd-spidercoordinator.md b/docs/reference/crd-spidercoordinator.md index d8358af398..b4edfd4fe0 100644 --- a/docs/reference/crd-spidercoordinator.md +++ b/docs/reference/crd-spidercoordinator.md @@ -45,7 +45,7 @@ This is the Spidercoordinators spec for users to configure. | Field | Description | Schema | Validation | Values | Default | |--------------------|--------------------------------------------------------------|----------------------|------------|------------------------------|------------------------------| -| mode | The mode in which the coordinator. underlay: coordinator creates veth devices to solve the problem that CNIs such as macvlan cannot communicate with clusterIP. overlay: fix the problem that CNIs such as Macvlan cannot access ClusterIP through the Calico network card attached to the pod,coordinate policy route between interfaces to ensure consistence data path of request and reply packets | string | require | underlay,overlay | underlay | +| mode | The mode in which the coordinator. auto: automatically determine if it's overlay or underlay. underlay: coordinator creates veth devices to solve the problem that CNIs such as macvlan cannot communicate with clusterIP. overlay: fix the problem that CNIs such as Macvlan cannot access ClusterIP through the Calico network card attached to the pod,coordinate policy route between interfaces to ensure consistence data path of request and reply packets | string | require | auto,underlay,overlay | auto | | podCIDRType | The ways to fetch the CIDR of the cluster | string | require | cluster,calico,cilium,none | cluster | | tunePodRoutes | tune pod's route while the pod is attached to multiple NICs | bool | optional | true,false | true | | podDefaultRouteNIC | The NIC where the pod's default route resides | string | optional | "",eth0,net1... | underlay: eth0,overlay: net1 | diff --git a/docs/usage/coordinator-zh_CN.md b/docs/usage/coordinator-zh_CN.md index 55e92c451d..2e9984da68 100644 --- a/docs/usage/coordinator-zh_CN.md +++ b/docs/usage/coordinator-zh_CN.md @@ -23,6 +23,9 @@ ClusterIP 的路由，导致无法访问。 ### 配置 coordinator 运行在 underlay 模式 +> 在默认情况下 mode 的值为auto(spidercoordinator CR 中 spec.mode 为 auto), coordinator 将通过对比当前 CNI 网卡是否是 `eth0`, 如果是，则自动判断为 Underlay 模式。 +> 如果当前网卡不是 `eth0`，那么 coordinator 将检测 Pod 中是否存在 `veth0` 网卡，如果是，则判断为 Underlay 模式。 + 当您的业务部署在"传统网络"或者 IAAS 环境上时，业务 Pod 的 IP 地址可能直接从宿主机的 IP 子网分配。应用 Pod 可直接使用自己的 IP 地址进行东西向和南北向通。 该模式的优点有: @@ -69,7 +72,7 @@ spec: } ``` -- mode: 指定 coordinator 运行在 underlay 模式 +- mode: 指定 coordinator 运行在 underlay 模式。或默认为 auto 模式，您只需要在 Pod 注入注解: `v1.multus-cni.io/default-network: kube-system/macvlan-underlay`, coordinator 将会自动判断 mode 为 underlay。 当以 macvlan-underlay 创建 Pod，我们进入到 Pod 内部，看看路由等信息: @@ -100,6 +103,8 @@ default via 10.6.0.1 dev eth0 与 Underlay 模式相对应，我们有时候并不关心集群部署环境的底层网络是什么，我们希望集群能够运行在大多数的底层网络。常常会用到如[Calico](https://github.com/projectcalico/calico) 和 [Cilium](https://github.com/cilium/cilium) 等CNI, 这些插件多数使用了 vxlan 等隧道技术，搭建起一个 Overlay 网络平面，再借用 NAT 技术实现南北向的通信。 +> 在默认情况下 mode 的值为auto(spidercoordinator CR 中 spec.mode 为 auto), coordinator 将通过对比当前 CNI 调用网卡是否不是 `eth0`。如果不是，确认 Pod 中不存在 `veth0` 网卡，则自动判断为 overlay 模式。 + 此模式的优点有: - IP 地址充沛，几乎不存在地址短缺的问题 @@ -146,7 +151,7 @@ spec: } ``` -- mode: 指定 coordinator 运行在 overlay 模式 +- mode: 指定 coordinator 运行在 overlay 模式。或默认为 auto 模式，您只需要在 Pod 注入注解: `k8s.v1.cni.cncf.io/networks: kube-system/macvlan-overlay`，coordinator 将会自动判断 mode 为 overlay。 当以 macvlan-overlay 创建 Pod，我们进入到 Pod 内部，看看路由等信息: diff --git a/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spidercoordinator_types.go b/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spidercoordinator_types.go index 20a34116b7..a358161e1f 100644 --- a/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spidercoordinator_types.go +++ b/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spidercoordinator_types.go @@ -9,7 +9,7 @@ import ( // CoordinationSpec defines the desired state of SpiderCoordinator. type CoordinatorSpec struct { - // +kubebuilder:validation:Enum=underlay;overlay;disabled + // +kubebuilder:validation:Enum=auto;underlay;overlay;disabled // +kubebuilder:validation:Optional Mode *string `json:"mode,omitempty"` diff --git a/pkg/networking/networking/route.go b/pkg/networking/networking/route.go index 632612e160..89f3102de2 100644 --- a/pkg/networking/networking/route.go +++ b/pkg/networking/networking/route.go @@ -7,7 +7,6 @@ import ( "fmt" "net" "os" - "strings" "github.com/containernetworking/plugins/pkg/ns" "github.com/vishvananda/netlink" @@ -286,39 +285,6 @@ func getDefaultRouteIface(linkIndex int, ignore string) (string, error) { return link.Attrs().Name, nil } -// IsFirstModeOverlayInvoke return true if the number of NICs in the pod prefixed with interfacePrefix is equal to 1 -func IsFirstModeOverlayInvoke(netns ns.NetNS, interfacePrefix string) (bool, error) { - var interfaces []net.Interface - var err error - - err = netns.Do(func(netNS ns.NetNS) error { - interfaces, err = net.Interfaces() - if err != nil { - return err - } - return nil - }) - - if err != nil { - return false, err - } - - count := 0 - for _, iface := range interfaces { - if strings.HasPrefix(iface.Name, interfacePrefix) { - count += 1 - } - } - - if count > 1 { - return false, nil - } else if count == 1 { - return true, nil - } else { - return false, fmt.Errorf("overlay mode can't work with multus pod's annotation: v1.multus-cni.io/default-network") - } -} - func ConvertMaxMaskIPNet(nip net.IP) *net.IPNet { mIPNet := &net.IPNet{ IP: nip, diff --git a/test/Makefile b/test/Makefile index c307f7e5f3..c833529ef0 100644 --- a/test/Makefile +++ b/test/Makefile @@ -182,7 +182,9 @@ setup_spiderpool: else \ HELM_OPTION+=" --set coordinator.enabled=false " ; \ fi ; \ - HELM_OPTION+=" --set coordinator.detectIPConflict=false --set coordinator.detectGateway=false " ; \ + if [ "$(INSTALL_OVERLAY_CNI)" == "true" ] && [ "$(INSTALL_CILIUM)" == "true" ] ; then \ + HELM_OPTION+=" --set coordinator.podCIDRType=cilium " ; \ + fi ; \ if [ "$(INSTALL_MULTUS)" == "true" ] ; then \ HELM_OPTION+=" --set multus.multusCNI.install=true " ; \ HELM_OPTION+=" --set multus.multusCNI.image.registry= " ; \ diff --git a/test/Makefile.defs b/test/Makefile.defs index afbd4910c4..1cf9ca2cfb 100644 --- a/test/Makefile.defs +++ b/test/Makefile.defs @@ -108,11 +108,9 @@ VLAN_GATEWAY_CONTAINER := vlan-gateway #================= multus MULTUS_DEFAULT_CNI_CALICO := k8s-pod-network MULTUS_DEFAULT_CNI_CILIUM := cilium -MULTUS_DEFAULT_CNI_VLAN0 := macvlan-vlan0-underlay -MULTUS_DEFAULT_CNI_VLAN100 := macvlan-vlan100-underlay -MULTUS_DEFAULT_CNI_VLAN200 := macvlan-vlan200-underlay -MULTUS_ADDITIONAL_CNI_VLAN100 := macvlan-vlan100-overlay -MULTUS_ADDITIONAL_CNI_VLAN200 := macvlan-vlan200-overlay +MULTUS_DEFAULT_CNI_VLAN0 := macvlan-vlan0 +MULTUS_DEFAULT_CNI_VLAN100 := macvlan-vlan100 +MULTUS_DEFAULT_CNI_VLAN200 := macvlan-vlan200 ifeq ($(E2E_CHINA_IMAGE_REGISTRY),true) E2E_MULTUS_IMAGE_REGISTER ?= ghcr.m.daocloud.io diff --git a/test/e2e/annotation/annotation_test.go b/test/e2e/annotation/annotation_test.go index f61125bc3e..100fc33cbb 100644 --- a/test/e2e/annotation/annotation_test.go +++ b/test/e2e/annotation/annotation_test.go @@ -727,7 +727,7 @@ var _ = Describe("test annotation", Label("annotation"), func() { podYaml := common.GenerateExamplePodYaml(podName, nsName) podYaml.Annotations = map[string]string{ pkgconstant.AnnoPodIPPools: annoPodIPPoolsStr, - common.MultusNetworks: fmt.Sprintf("%s/%s", common.MultusNs, common.MacvlanUnderlayVlan100), + common.MultusNetworks: fmt.Sprintf("%s/%s", common.MultusNs, common.MacvlanVlan100), } Expect(podYaml).NotTo(BeNil()) GinkgoWriter.Printf("succeeded to generate pod yaml: %+v. \n", podYaml) diff --git a/test/e2e/common/constant.go b/test/e2e/common/constant.go index 67519b567e..fac950504e 100644 --- a/test/e2e/common/constant.go +++ b/test/e2e/common/constant.go @@ -48,14 +48,11 @@ var ( MultusDefaultNetwork = "v1.multus-cni.io/default-network" MultusNetworks = "k8s.v1.cni.cncf.io/networks" - CalicoCNIName string = "k8s-pod-network" - CiliumCNIName string = "cilium" - MacvlanUnderlayVlan0 string = "macvlan-vlan0-underlay" - MacvlanUnderlayVlan100 string = "macvlan-vlan100-underlay" - MacvlanUnderlayVlan200 string = "macvlan-vlan200-underlay" - MacvlanOverlayVlan100 string = "macvlan-vlan100-overlay" - MacvlanOverlayVlan200 string = "macvlan-vlan200-overlay" - + CalicoCNIName string = "k8s-pod-network" + CiliumCNIName string = "cilium" + MacvlanUnderlayVlan0 string = "macvlan-vlan0" + MacvlanVlan100 string = "macvlan-vlan100" + MacvlanVlan200 string = "macvlan-vlan200" SpiderPoolIPv4SubnetVlan100 string = "vlan100-v4" SpiderPoolIPv6SubnetVlan100 string = "vlan100-v6" SpiderPoolIPv4SubnetVlan200 string = "vlan200-v4" diff --git a/test/e2e/coordinator/macvlan-overlay-one/macvlan_overlay_one_test.go b/test/e2e/coordinator/macvlan-overlay-one/macvlan_overlay_one_test.go index 0331c69d60..f868ba2ab9 100644 --- a/test/e2e/coordinator/macvlan-overlay-one/macvlan_overlay_one_test.go +++ b/test/e2e/coordinator/macvlan-overlay-one/macvlan_overlay_one_test.go @@ -45,7 +45,7 @@ var _ = Describe("MacvlanOverlayOne", Label("overlay", "one-nic", "coordinator") condition = new(spiderdoctorV1.NetSuccessCondition) name = "one-macvlan-overlay-" + tools.RandomName() - annotations[common.MultusNetworks] = fmt.Sprintf("%s/%s", common.MultusNs, common.MacvlanOverlayVlan100) + annotations[common.MultusNetworks] = fmt.Sprintf("%s/%s", common.MultusNs, common.MacvlanVlan100) if frame.Info.IpV4Enabled && frame.Info.IpV6Enabled { annotations[constant.AnnoPodIPPool] = `{"interface": "net1", "ipv4": ["vlan100-v4"], "ipv6": ["vlan100-v6"]}` } else if frame.Info.IpV4Enabled && !frame.Info.IpV6Enabled { diff --git a/test/e2e/subnet/subnet_test.go b/test/e2e/subnet/subnet_test.go index 7cbbe9d48f..4ff6a11002 100644 --- a/test/e2e/subnet/subnet_test.go +++ b/test/e2e/subnet/subnet_test.go @@ -1217,7 +1217,7 @@ var _ = Describe("test subnet", Label("subnet"), func() { deployName = "deploy-" + tools.RandomName() deployObj := common.GenerateExampleDeploymentYaml(deployName, namespace, 1) deployObj.Spec.Template.Annotations = map[string]string{ - common.MultusNetworks: fmt.Sprintf("%s/%s", common.MultusNs, common.MacvlanUnderlayVlan100), + common.MultusNetworks: fmt.Sprintf("%s/%s", common.MultusNs, common.MacvlanVlan100), // second Interface constant.AnnoSpiderSubnets: string(subnetsAnnoMarshal), } @@ -1685,7 +1685,7 @@ var _ = Describe("test subnet", Label("subnet"), func() { GinkgoWriter.Printf("Generate multi-NIC annotations for same name app %v/%v \n", namespace, longAppName) annotationMap[constant.AnnoSpiderSubnets] = string(subnetsAnnoMarshal) - annotationMap[common.MultusNetworks] = fmt.Sprintf("%s/%s", common.MultusNs, common.MacvlanUnderlayVlan100) + annotationMap[common.MultusNetworks] = fmt.Sprintf("%s/%s", common.MultusNs, common.MacvlanVlan100) // Delete Single Card Annotations delete(annotationMap, constant.AnnoSpiderSubnet) diff --git a/test/scripts/debugEnv.sh b/test/scripts/debugEnv.sh index 9f18735042..4853e4b780 100755 --- a/test/scripts/debugEnv.sh +++ b/test/scripts/debugEnv.sh @@ -175,6 +175,12 @@ elif [ "$TYPE"x == "detail"x ] ; then docker exec $NODE cat /var/log/spidernet/spiderpool.log echo "--------- coordinator logs from node ${NODE}" docker exec $NODE cat /var/log/spidernet/coordinator.log + echo "--------- ip rule from ${NODE}" + ip rule + echo "--------- ip route show table 500 from ${NODE}" + ip route show table 500 + echo "--------- ip link show from ${NODE}" + ip link show done diff --git a/test/scripts/install-multus.sh b/test/scripts/install-multus.sh index 705df57580..beae2780a5 100755 --- a/test/scripts/install-multus.sh +++ b/test/scripts/install-multus.sh @@ -35,12 +35,6 @@ echo "$CURRENT_FILENAME : MULTUS_DEFAULT_CNI_VLAN100 $MULTUS_DEFAULT_CNI_VLAN100 [ -z "$MULTUS_DEFAULT_CNI_VLAN200" ] && echo "error, miss MULTUS_DEFAULT_CNI_VLAN200" && exit 1 echo "$CURRENT_FILENAME : MULTUS_DEFAULT_CNI_VLAN200 $MULTUS_DEFAULT_CNI_VLAN200 " -[ -z "$MULTUS_ADDITIONAL_CNI_VLAN100" ] && echo "error, miss MULTUS_ADDITIONAL_CNI_VLAN100" && exit 1 -echo "$CURRENT_FILENAME : MULTUS_DEFAULT_CNI_VLAN100 $MULTUS_ADDITIONAL_CNI_VLAN100 " - -[ -z "$MULTUS_ADDITIONAL_CNI_VLAN200" ] && echo "error, miss MULTUS_ADDITIONAL_CNI_VLAN200" && exit 1 -echo "$CURRENT_FILENAME : MULTUS_DEFAULT_CNI_VLAN200 $MULTUS_ADDITIONAL_CNI_VLAN200 " - #============== OS=$(uname | tr 'A-Z' 'a-z') SED_COMMAND=sed @@ -59,50 +53,75 @@ spec: macvlan: master: ["<>"] vlanID: <> + ippools: + ipv4: [<>] + ipv6: [<>] coordinator: mode: "<>" ' + case ${E2E_IP_FAMILY} in + ipv4) + DEFAULT_IPV4_IPPOOLS=\"default-v4-ippool\" + DEFAULT_IPV6_IPPOOLS="" + VLAN100_IPV4_IPPOOLS=vlan100-v4 + VLAN100_IPV6_IPPOOLS="" + VLAN200_IPV4_IPPOOLS=vlan200-v4 + VLAN200_IPV6_IPPOOLS="" + ;; + + ipv6) + DEFAULT_IPV4_IPPOOLS='' + DEFAULT_IPV6_IPPOOLS=\"default-v6-ippool\" + VLAN100_IPV4_IPPOOLS='' + VLAN100_IPV6_IPPOOLS=vlan100-v6 + VLAN200_IPV4_IPPOOLS='' + VLAN200_IPV6_IPPOOLS=vlan200-v6 + ;; + + dual) + DEFAULT_IPV4_IPPOOLS=\"default-v4-ippool\" + DEFAULT_IPV6_IPPOOLS=\"default-v6-ippool\" + VLAN100_IPV4_IPPOOLS=vlan100-v4 + VLAN100_IPV6_IPPOOLS=vlan100-v6 + VLAN200_IPV4_IPPOOLS=vlan200-v4 + VLAN200_IPV6_IPPOOLS=vlan200-v6 + ;; + + *) + echo "the value of IP_FAMILY: ipv4 or ipv6 or dual" + exit 1 + esac + echo "${MACVLAN_CR_TEMPLATE}" \ | sed 's?<>?'""${MULTUS_DEFAULT_CNI_NAME}""'?g' \ | sed 's?<>?'"${RELEASE_NAMESPACE}"'?g' \ - | sed 's?<>?underlay?g' \ + | sed 's?<>?auto?g' \ | sed 's?<>?eth0?g' \ | sed 's?<>?0?g' \ + | sed 's?<>?'""${DEFAULT_IPV4_IPPOOLS}""'?g' \ + | sed 's?<>?'""${DEFAULT_IPV6_IPPOOLS}""'?g' \ | kubectl apply --kubeconfig ${E2E_KUBECONFIG} -f - echo "${MACVLAN_CR_TEMPLATE}" \ | sed 's?<>?'""${MULTUS_DEFAULT_CNI_VLAN100}""'?g' \ | sed 's?<>?'"${RELEASE_NAMESPACE}"'?g' \ - | sed 's?<>?underlay?g' \ - | sed 's?<>?eth0?g' \ - | sed 's?<>?100?g' \ - | kubectl apply --kubeconfig ${E2E_KUBECONFIG} -f - - - echo "${MACVLAN_CR_TEMPLATE}" \ - | sed 's?<>?'""${MULTUS_ADDITIONAL_CNI_VLAN100}""'?g' \ - | sed 's?<>?'"${RELEASE_NAMESPACE}"'?g' \ - | sed 's?<>?overlay?g' \ + | sed 's?<>?auto?g' \ | sed 's?<>?eth0?g' \ | sed 's?<>?100?g' \ - | kubectl apply --kubeconfig ${E2E_KUBECONFIG} -f - - - echo "${MACVLAN_CR_TEMPLATE}" \ - | sed 's?<>?'""${MULTUS_ADDITIONAL_CNI_VLAN200}""'?g' \ - | sed 's?<>?'"${RELEASE_NAMESPACE}"'?g' \ - | sed 's?<>?overlay?g' \ - | sed 's?<>?eth0?g' \ - | sed 's?<>?200?g' \ + | sed 's?<>?vlan100-v4?g' \ + | sed 's?<>?vlan100-v6?g' \ | kubectl apply --kubeconfig ${E2E_KUBECONFIG} -f - echo "${MACVLAN_CR_TEMPLATE}" \ | sed 's?<>?'""${MULTUS_DEFAULT_CNI_VLAN200}""'?g' \ | sed 's?<>?'"${RELEASE_NAMESPACE}"'?g' \ - | sed 's?<>?underlay?g' \ + | sed 's?<>?auto?g' \ | sed 's?<>?eth0?g' \ | sed 's?<>?200?g' \ + | sed 's?<>?vlan200-v4?g' \ + | sed 's?<>?vlan200-v6?g' \ | kubectl apply --kubeconfig ${E2E_KUBECONFIG} -f - - }