From 10a4cca10e2b5bc1cc0aa9439bedc6caa757274a Mon Sep 17 00:00:00 2001 From: cyclinder Date: Fri, 25 Oct 2024 15:48:11 +0800 Subject: [PATCH] Add a flag to configure an link-local address to veth0 for istio Signed-off-by: cyclinder --- api/v1/agent/models/coordinator_config.go | 3 + api/v1/agent/openapi.yaml | 2 + api/v1/agent/server/embedded_spec.go | 4 + charts/spiderpool/README.md | 1 + ...rpool.spidernet.io_spidercoordinators.yaml | 5 + ...pool.spidernet.io_spidermultusconfigs.yaml | 5 + charts/spiderpool/templates/pod.yaml | 2 + charts/spiderpool/values.yaml | 3 + cmd/coordinator/cmd/cni_types.go | 4 + cmd/coordinator/cmd/command_add.go | 1 + cmd/coordinator/cmd/utils.go | 28 +++--- cmd/spiderpool-agent/cmd/coordinator.go | 6 ++ cmd/spiderpool-init/cmd/config.go | 18 ++-- cmd/spiderpool-init/cmd/root.go | 1 + docs/concepts/coordinator-zh_CN.md | 21 ++++ docs/concepts/coordinator.md | 28 +++++- docs/mkdocs.yml | 1 + docs/reference/crd-spidercoordinator.md | 8 +- docs/usage/istio-zh_CN.md | 98 +++++++++++++++++++ docs/usage/istio.md | 98 +++++++++++++++++++ pkg/coordinatormanager/coordinator_mutate.go | 4 + .../coordinator_validate.go | 20 ++-- .../v2beta1/spidercoordinator_types.go | 5 + .../v2beta1/zz_generated.deepcopy.go | 5 + pkg/multuscniconfig/multusconfig_informer.go | 3 + pkg/multuscniconfig/multusconfig_mutate.go | 1 + pkg/multuscniconfig/utils.go | 1 + test/doc/spidermultus.md | 1 + test/e2e/spidermultus/spidermultus_test.go | 79 ++++++++++++++- test/scripts/install-multus.sh | 4 + 30 files changed, 426 insertions(+), 34 deletions(-) create mode 100644 docs/usage/istio-zh_CN.md create mode 100644 docs/usage/istio.md diff --git a/api/v1/agent/models/coordinator_config.go b/api/v1/agent/models/coordinator_config.go index c91a2db688..aebec903a9 100644 --- a/api/v1/agent/models/coordinator_config.go +++ b/api/v1/agent/models/coordinator_config.go @@ -58,6 +58,9 @@ type CoordinatorConfig struct { // tune pod routes // Required: true TunePodRoutes *bool `json:"tunePodRoutes"` + + // veth link address + VethLinkAddress string `json:"vethLinkAddress,omitempty"` } // Validate validates this coordinator config diff --git a/api/v1/agent/openapi.yaml b/api/v1/agent/openapi.yaml index 73259e2976..d9ed46405d 100644 --- a/api/v1/agent/openapi.yaml +++ b/api/v1/agent/openapi.yaml @@ -328,6 +328,8 @@ definitions: type: boolean detectGateway: type: boolean + vethLinkAddress: + type: string required: - overlayPodCIDR - serviceCIDR diff --git a/api/v1/agent/server/embedded_spec.go b/api/v1/agent/server/embedded_spec.go index 0b98f6cb73..7ae04adaf5 100644 --- a/api/v1/agent/server/embedded_spec.go +++ b/api/v1/agent/server/embedded_spec.go @@ -310,6 +310,8 @@ func init() { }, "tunePodRoutes": { "type": "boolean" + "vethLinkAddress": { + "type": "string" } } }, @@ -807,6 +809,8 @@ func init() { }, "tunePodRoutes": { "type": "boolean" + "vethLinkAddress": { + "type": "string" } } }, diff --git a/charts/spiderpool/README.md b/charts/spiderpool/README.md index 67b388d15c..174fb0ab51 100644 --- a/charts/spiderpool/README.md +++ b/charts/spiderpool/README.md @@ -161,6 +161,7 @@ helm install spiderpool spiderpool/spiderpool --wait --namespace kube-system \ | `coordinator.detectIPConflict` | detect IP address conflicts | `false` | | `coordinator.tunePodRoutes` | tune Pod routes | `true` | | `coordinator.hijackCIDR` | Additional subnets that need to be hijacked to the host forward, the default link-local range "169.254.0.0/16" is used for NodeLocal DNS | `["169.254.0.0/16"]` | +| `coordinator.vethLinkAddress` | configure an link-local address for veth0 device. empty means disable. default is empty. Format is like 169.254.100.1 | `""` | ### rdma parameters diff --git a/charts/spiderpool/crds/spiderpool.spidernet.io_spidercoordinators.yaml b/charts/spiderpool/crds/spiderpool.spidernet.io_spidercoordinators.yaml index ff1908eedf..f28da302f5 100644 --- a/charts/spiderpool/crds/spiderpool.spidernet.io_spidercoordinators.yaml +++ b/charts/spiderpool/crds/spiderpool.spidernet.io_spidercoordinators.yaml @@ -109,6 +109,11 @@ spec: description: TunePodRoutes specifies whether to tune pod routes of multiple NICs on pods. type: boolean + vethLinkAddress: + description: VethLinkAddress configure a ipv4 link-local address for + veth0 device. empty means disable. default is empty. Format is like + 169.254.100.1 + type: string type: object status: description: CoordinationStatus defines the observed state of SpiderCoordinator. diff --git a/charts/spiderpool/crds/spiderpool.spidernet.io_spidermultusconfigs.yaml b/charts/spiderpool/crds/spiderpool.spidernet.io_spidermultusconfigs.yaml index 0a94e11bf0..951d583355 100644 --- a/charts/spiderpool/crds/spiderpool.spidernet.io_spidermultusconfigs.yaml +++ b/charts/spiderpool/crds/spiderpool.spidernet.io_spidermultusconfigs.yaml @@ -121,6 +121,11 @@ spec: description: TunePodRoutes specifies whether to tune pod routes of multiple NICs on pods. type: boolean + vethLinkAddress: + description: VethLinkAddress configure a ipv4 link-local address + for veth0 device. empty means disable. default is empty. Format + is like 169.254.100.1 + type: string type: object customCNI: description: OtherCniTypeConfig only used for CniType custom, valid diff --git a/charts/spiderpool/templates/pod.yaml b/charts/spiderpool/templates/pod.yaml index b39034cabc..048b1c6ef6 100644 --- a/charts/spiderpool/templates/pod.yaml +++ b/charts/spiderpool/templates/pod.yaml @@ -47,6 +47,8 @@ spec: value: {{ .Values.coordinator.tunePodRoutes | quote }} - name: SPIDERPOOL_INIT_DEFAULT_COORDINATOR_HIJACK_CIDR value: {{ toJson .Values.coordinator.hijackCIDR | quote }} + - name: SPIDERPOOL_INIT_DEFAULT_COORDINATOR_VETH_LINK_ADDRESS + value: {{ .Values.coordinator.vethLinkAddress | quote }} {{- end }} {{- if and .Values.clusterDefaultPool.installIPv4IPPool .Values.ipam.enableIPv4 }} - name: SPIDERPOOL_INIT_DEFAULT_IPV4_IPPOOL_NAME diff --git a/charts/spiderpool/values.yaml b/charts/spiderpool/values.yaml index 38c097781c..51c7c502b9 100644 --- a/charts/spiderpool/values.yaml +++ b/charts/spiderpool/values.yaml @@ -116,6 +116,9 @@ coordinator: ## @param coordinator.hijackCIDR Additional subnets that need to be hijacked to the host forward, the default link-local range "169.254.0.0/16" is used for NodeLocal DNS hijackCIDR: ["169.254.0.0/16"] + ## @param coordinator.vethLinkAddress configure an link-local address for veth0 device. empty means disable. default is empty. Format is like 169.254.100.1 + vethLinkAddress: "" + ## @section rdma parameters ## rdma: diff --git a/cmd/coordinator/cmd/cni_types.go b/cmd/coordinator/cmd/cni_types.go index 31925611d3..6ebcba01ee 100644 --- a/cmd/coordinator/cmd/cni_types.go +++ b/cmd/coordinator/cmd/cni_types.go @@ -45,6 +45,7 @@ const ( type Config struct { types.NetConf DetectGateway *bool `json:"detectGateway,omitempty"` + VethLinkAddress string `json:"vethLinkAddress,omitempty"` MacPrefix string `json:"podMACPrefix,omitempty"` MultusNicPrefix string `json:"multusNicPrefix,omitempty"` PodDefaultCniNic string `json:"podDefaultCniNic,omitempty"` @@ -173,6 +174,9 @@ func ParseConfig(stdin []byte, coordinatorConfig *models.CoordinatorConfig) (*Co conf.PodDefaultRouteNIC = coordinatorConfig.PodDefaultRouteNIC } + if conf.VethLinkAddress == "" { + conf.VethLinkAddress = coordinatorConfig.VethLinkAddress + } return &conf, nil } diff --git a/cmd/coordinator/cmd/command_add.go b/cmd/coordinator/cmd/command_add.go index 48e5cd21af..ebda672014 100644 --- a/cmd/coordinator/cmd/command_add.go +++ b/cmd/coordinator/cmd/command_add.go @@ -99,6 +99,7 @@ func CmdAdd(args *skel.CmdArgs) (err error) { ipFamily: ipFamily, currentInterface: args.IfName, tuneMode: conf.Mode, + vethLinkAddress: conf.VethLinkAddress, } c.HijackCIDR = append(c.HijackCIDR, conf.ServiceCIDR...) c.HijackCIDR = append(c.HijackCIDR, conf.HijackCIDR...) diff --git a/cmd/coordinator/cmd/utils.go b/cmd/coordinator/cmd/utils.go index a7209cc161..c7696ca15c 100644 --- a/cmd/coordinator/cmd/utils.go +++ b/cmd/coordinator/cmd/utils.go @@ -21,17 +21,17 @@ import ( ) type coordinator struct { - firstInvoke bool - ipFamily, currentRuleTable, hostRuleTable int - tuneMode Mode - hostVethName, podVethName, currentInterface string - v4HijackRouteGw, v6HijackRouteGw net.IP - HijackCIDR []string - netns, hostNs ns.NetNS - hostVethHwAddress, podVethHwAddress net.HardwareAddr - currentAddress []netlink.Addr - v4PodOverlayNicAddr, v6PodOverlayNicAddr *net.IPNet - hostIPRouteForPod []net.IP + firstInvoke bool + ipFamily, currentRuleTable, hostRuleTable int + tuneMode Mode + hostVethName, podVethName, vethLinkAddress, currentInterface string + v4HijackRouteGw, v6HijackRouteGw net.IP + HijackCIDR []string + netns, hostNs ns.NetNS + hostVethHwAddress, podVethHwAddress net.HardwareAddr + currentAddress []netlink.Addr + v4PodOverlayNicAddr, v6PodOverlayNicAddr *net.IPNet + hostIPRouteForPod []net.IP } func (c *coordinator) autoModeToSpecificMode(mode Mode, podFirstInterface string, vethExist bool) error { @@ -189,9 +189,13 @@ func (c *coordinator) setupVeth(logger *zap.Logger, containerID string) error { return nil } + if c.vethLinkAddress == "" { + return nil + } + if err = netlink.AddrAdd(link, &netlink.Addr{ IPNet: &net.IPNet{ - IP: net.ParseIP("169.254.200.1"), + IP: net.ParseIP(c.vethLinkAddress), Mask: net.CIDRMask(32, 32), }, }); err != nil { diff --git a/cmd/spiderpool-agent/cmd/coordinator.go b/cmd/spiderpool-agent/cmd/coordinator.go index 67469fc16d..61b4cb7d20 100644 --- a/cmd/spiderpool-agent/cmd/coordinator.go +++ b/cmd/spiderpool-agent/cmd/coordinator.go @@ -94,6 +94,11 @@ func (g *_unixGetCoordinatorConfig) Handle(params daemonset.GetCoordinatorConfig nic = *coord.Spec.PodDefaultRouteNIC } + var vethLinkAddress string + if coord.Spec.VethLinkAddress != nil { + vethLinkAddress = *coord.Spec.VethLinkAddress + } + defaultRouteNic, ok := pod.Annotations[constant.AnnoDefaultRouteInterface] if ok { nic = defaultRouteNic @@ -107,6 +112,7 @@ func (g *_unixGetCoordinatorConfig) Handle(params daemonset.GetCoordinatorConfig PodMACPrefix: prefix, TunePodRoutes: coord.Spec.TunePodRoutes, PodDefaultRouteNIC: nic, + VethLinkAddress: vethLinkAddress, HostRuleTable: int64(*coord.Spec.HostRuleTable), PodRPFilter: int64(*coord.Spec.PodRPFilter), DetectGateway: *coord.Spec.DetectGateway, diff --git a/cmd/spiderpool-init/cmd/config.go b/cmd/spiderpool-init/cmd/config.go index 955fbd5e51..1907b2da23 100644 --- a/cmd/spiderpool-init/cmd/config.go +++ b/cmd/spiderpool-init/cmd/config.go @@ -45,13 +45,14 @@ const ( ENVDefaultIPv6IPRanges = "SPIDERPOOL_INIT_DEFAULT_IPV6_IPPOOL_IPRANGES" ENVDefaultIPv6Gateway = "SPIDERPOOL_INIT_DEFAULT_IPV6_IPPOOL_GATEWAY" - ENVEnableMultusConfig = "SPIDERPOOL_INIT_ENABLE_MULTUS_CONFIG" - ENVInstallMultusCNI = "SPIDERPOOL_INIT_INSTALL_MULTUS" - ENVDefaultCNIDir = "SPIDERPOOL_INIT_DEFAULT_CNI_DIR" - ENVDefaultCNIName = "SPIDERPOOL_INIT_DEFAULT_CNI_NAME" - ENVDefaultCNINamespace = "SPIDERPOOL_INIT_DEFAULT_CNI_NAMESPACE" - ENVDefaultMultusConfigMap = "SPIDERPOOL_INIT_MULTUS_CONFIGMAP" - ENVDefaultReadinessFile = "SPIDERPOOL_INIT_READINESS_FILE" + ENVEnableMultusConfig = "SPIDERPOOL_INIT_ENABLE_MULTUS_CONFIG" + ENVInstallMultusCNI = "SPIDERPOOL_INIT_INSTALL_MULTUS" + ENVDefaultCNIDir = "SPIDERPOOL_INIT_DEFAULT_CNI_DIR" + ENVDefaultCNIName = "SPIDERPOOL_INIT_DEFAULT_CNI_NAME" + ENVDefaultCNINamespace = "SPIDERPOOL_INIT_DEFAULT_CNI_NAMESPACE" + ENVDefaultMultusConfigMap = "SPIDERPOOL_INIT_MULTUS_CONFIGMAP" + ENVDefaultReadinessFile = "SPIDERPOOL_INIT_READINESS_FILE" + ENVDefaultCoordinatorVethLinkAddress = "SPIDERPOOL_INIT_DEFAULT_COORDINATOR_VETH_LINK_ADDRESS" ) var ( @@ -70,6 +71,7 @@ type InitDefaultConfig struct { CoordinatorPodCIDRType string CoordinatorPodDefaultRouteNic string CoordinatorPodMACPrefix string + CoordinatorVethLinkAddress string CoordinatorDetectGateway bool CoordinatorDetectIPConflict bool CoordinatorTunePodRoutes bool @@ -169,6 +171,8 @@ func parseENVAsDefault() InitDefaultConfig { } else { config.CoordinatorHijackCIDR = []string{} } + + config.CoordinatorVethLinkAddress = strings.ReplaceAll(os.Getenv(ENVDefaultCoordinatorVethLinkAddress), "\"", "") } else { logger.Info("Ignore creating default Coordinator") } diff --git a/cmd/spiderpool-init/cmd/root.go b/cmd/spiderpool-init/cmd/root.go index 8fce482af3..26d8478f3d 100644 --- a/cmd/spiderpool-init/cmd/root.go +++ b/cmd/spiderpool-init/cmd/root.go @@ -47,6 +47,7 @@ func Execute() { DetectGateway: &config.CoordinatorDetectGateway, PodDefaultRouteNIC: &config.CoordinatorPodDefaultRouteNic, PodMACPrefix: &config.CoordinatorPodMACPrefix, + VethLinkAddress: &config.CoordinatorVethLinkAddress, HijackCIDR: config.CoordinatorHijackCIDR, }, } diff --git a/docs/concepts/coordinator-zh_CN.md b/docs/concepts/coordinator-zh_CN.md index ca7bebca52..720a141fc0 100644 --- a/docs/concepts/coordinator-zh_CN.md +++ b/docs/concepts/coordinator-zh_CN.md @@ -129,6 +129,27 @@ spec: 当 Pod 创建完成，我们可以检测 Pod 的 Mac 地址的前缀是否是 "0a:1b" +## 为 Pod 的 veth0 网卡配置本地链路地址，支持服务网格场景 + +默认情况下，Coordinator 不会为 veth0 网卡配置本地链路地址。但有些场景下(比如服务网格)，经过 veth0 网卡流入的网格流量会随 istio 设置的 iptables 规则重定向，如果 veth0 没有 IP 地址，这会导致这部分流量被丢弃(见[#Issue3568](https://github.com/spidernet-io/spiderpool/issues/3568))。所以在这个场景下，我们需要为 veth0 配置一个本地链路地址。 + +```yaml +apiVersion: spiderpool.spidernet.io/v2beta1 +kind: SpiderMultusConfig +metadata: + name: istio-demo + namespace: default +spec: + cniType: macvlan + macvlan: + master: ["eth0"] + enableCoordinator: true + coordinator: + vethLinkAddress: "169.254.200.1" +``` + +> `vethLinkAddress` 默认为空，表示不配置。不为空则必须是一个合法的本地链路地址。 + ## 已知问题 - underlay 模式下，underlay Pod 与 Overlay Pod(calico or cilium) 进行 TCP 通信失败 diff --git a/docs/concepts/coordinator.md b/docs/concepts/coordinator.md index 20f604cfc9..4ce15103f0 100644 --- a/docs/concepts/coordinator.md +++ b/docs/concepts/coordinator.md @@ -10,7 +10,7 @@ Spiderpool incorporates a CNI meta-plugin called `coordinator` that works after - Check the reachability of Pod gateways - Support fixed Mac address prefixes for Pods -Note: If your OS(such as Fedora, CentOS, etc.) uses NetworkManager, highly recommend configuring following configuration file at `/etc/NetworkManager/conf.d/spidernet.conf` to +Note: If your OS(such as Fedora, CentOS, etc.) uses NetworkManager, highly recommend configuring following configuration file at `/etc/NetworkManager/conf.d/spidernet.conf` to prevent interference from NetworkManager with veth interfaces created through `coordinator`: ```shell @@ -97,17 +97,16 @@ spec: > Note: There are some switches that are not allowed to be probed by arp, otherwise an alarm will be issued, in this case, we need to set detectGateway to false - ## Fix MAC address prefix for Pods(alpha) -Some traditional applications may require a fixed MAC address or IP address to couple the behavior of the application. For example, the License Server may need to apply a fixed Mac address -or IP address to issue a license for the app. If the MAC address of a pod changes, the issued license may be invalid. Therefore, you need to fix the MAC address of the pod. Spiderpool can fix +Some traditional applications may require a fixed MAC address or IP address to couple the behavior of the application. For example, the License Server may need to apply a fixed Mac address +or IP address to issue a license for the app. If the MAC address of a pod changes, the issued license may be invalid. Therefore, you need to fix the MAC address of the pod. Spiderpool can fix the MAC address of the application through `coordinator`, and the fixed rule is to configure the MAC address prefix (2 bytes) + convert the IP of the pod (4 bytes). Note: > currently supports updating Macvlan and SR-IOV as pods for CNI. In IPVlan L2 mode, the MAC addresses of the primary interface and the sub-interface are the same and cannot be modified. -> +> > The fixed rule is to configure the MAC address prefix (2 bytes) + the IP of the converted pod (4 bytes). An IPv4 address is 4 bytes long and can be fully converted to 2 hexadecimal numbers. For IPv6 addresses, only the last 4 bytes are taken. We can configure it via Spidermultusconfig: @@ -129,6 +128,25 @@ spec: You can check if the MAC address prefix of the Pod starts with "0a:1b" after a Pod is created. +By default, Coordinator does not configure a link-local address for the veth0 interface. However, in some scenarios (such as service mesh), mesh traffic flowing through the veth0 interface will be redirected according to iptables rules set by Istio. If veth0 does not have an IP address, this can cause that traffic to be dropped (see #Issue3568). Therefore, in this scenario, we need to configure a link-local address for veth0. + +```yaml +apiVersion: spiderpool.spidernet.io/v2beta1 +kind: SpiderMultusConfig +metadata: + name: istio-demo + namespace: default +spec: + cniType: macvlan + macvlan: + master: ["eth0"] + enableCoordinator: true + coordinator: + vethLinkAddress: "169.254.100.1" +``` + +> `vethLinkAddress` default to "", It means that we don't configure an address for veth0. It must an valid link-local address if it isn't empty. + ## Known issues - Underlay mode: TCP communication between underlay Pods and overlay Pods (Calico or Cilium) fails diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 86a7d579df..2292a693a5 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -92,6 +92,7 @@ nav: - Access Service for Underlay CNI: usage/underlay_cni_service.md - Coexistence of multi CNIs: usage/multi_cni_coexist.md - Kubevirt: usage/kubevirt.md + - Istio: usage/istio.md - FAQ: usage/faq.md - Reference: - Annotations: reference/annotation.md diff --git a/docs/reference/crd-spidercoordinator.md b/docs/reference/crd-spidercoordinator.md index 554ef8148c..7e19c6e6a5 100644 --- a/docs/reference/crd-spidercoordinator.md +++ b/docs/reference/crd-spidercoordinator.md @@ -14,6 +14,7 @@ metadata: spec: detectGateway: false detectIPConflict: false + enableVethLinkLocalAddress: false hostRPFilter: 0 podRPFilter: 0 hostRuleTable: 500 @@ -50,9 +51,10 @@ This is the Spidercoordinators spec for users to configure. | podCIDRType | The ways to fetch the CIDR of the cluster. auto(default), This means that it will automatically switch podCIDRType to cluster or calico or cilium. based on cluster CNI. calico: auto fetch the subnet of the pod from the ip pools of calico, This only works if the cluster CNI is calico; cilium: Auto fetch the pod's subnet from cilium's configMap or ip pools. Supported IPAM modes: ["cluster-pool","kubernetes","multi-pool"]; cluster: auto fetch the subnet of the pod from the kubeadm-config configmap, This is useful if there is only a globally unique default pod's subnet; none: don't get the subnet of the pod, which is useful for some special cases. In this case,you can manually configure the hijackCIDR field | string | require | auto,cluster,calico,cilium,none | auto | | tunePodRoutes | tune pod's route while the pod is attached to multiple NICs | bool | optional | true,false | true | | podDefaultRouteNIC | The NIC where the pod's default route resides | string | optional | "",eth0,net1... | underlay: eth0,overlay: net1 | -| detectGateway | enable detect gateway while launching pod, If the gateway is unreachable, pod will be failed to created; Note: We use ARP probes to detect if the gateway is reachable, and some gateway routers may warn about this | boolean | optional | true,false | false | -| detectIPConflict | enable the pod's ip if is conflicting while launching pod. If an IP conflict of the pod is detected, pod will be failed to created | boolean | optional | true,false | false | -| podMACPrefix | fix the pod's mac address with this prefix + 4 bytes IP | string | optional | a invalid mac address prefix | "" | +| detectGateway | enable detect gateway while launching pod, If the gateway is unreachable, pod will be failed to created; Note: We use ARP probes to detect if the gateway is reachable, and some gateway routers may warn about this | boolean | optional | true,false | false | +| detectIPConflict | enable the pod's ip if is conflicting while launching pod. If an IP conflict of the pod is detected, pod will be failed to created | boolean | optional | true,false | false | +| vethLinkAddress | configure an link-local address for veth0 device, fix the istio case | boolean | optional | true,false | false | +| podMACPrefix | fix the pod's mac address with this prefix + 4 bytes IP | string | optional | a invalid mac address prefix | "" | | podRPFilter | set rp_filter sysctl for the pod | int | required | 0,1,2;suggest to be 0 | 0 | | hostRPFilter | (deprecated) set rp_filter sysctl for the node | int | required | 0,1,2;suggest to be 0 | 0 | | hostRuleTable | The directly routing table of the host accessing the pod's underlay IP will be placed in this policy routing table | int | required | int | 500 | diff --git a/docs/usage/istio-zh_CN.md b/docs/usage/istio-zh_CN.md new file mode 100644 index 0000000000..f70dd52d6d --- /dev/null +++ b/docs/usage/istio-zh_CN.md @@ -0,0 +1,98 @@ +# Istio + +**简体中文** | [**English**](./istio.md) + +## 介绍 + +在 Istio 场景下，使用 Spiderpool 配置服务网格应用使用 Underlay 网络时，可能会出现流量无法被 istio 劫持的问题。这是因为： + +1. 访问服务网格 Pod 的流量通过其 veth0 网卡(由 Spiderpool 创建)转发。流量随后会通过 istio 设置的 iptables redirect 规则，被劫持到 sidecar 容器中。但由于 iptables redirect 规则必须要求接收流量的网卡必须配置 IP 地址，否则该数据包会被内核沉默的丢弃。 + +2. 在默认情况下，Spiderpool 不会为使用 Underlay 网络的 Pod 的 veth0 网卡配置 IP 地址, 所以这会导致访问服务网格的流量被丢弃。 + +参考 [#Issue 3568](https://github.com/spidernet-io/spiderpool/issues/3568)。为了解决这个问题， Spiderpool 提供一个配置: `vethLinkAddress`，用于为 veth0 网卡配置一个 link-local 地址。 + +## 如何配置 + +1. 使用 Helm 安装 Spiderpool 时，可通过以下命令开启这个功能： + + ```shell + helm repo add spiderpool https://spidernet-io.github.io/spiderpool + helm repo update spiderpool + kubectl create namespace spiderpool + helm install spiderpool spiderpool/spiderpool -n spiderpool --set coordinator.vethLinkAddress=169.254.100.1 + ``` + + > - `vethLinkAddress` 必须是一个合法的 IP 地址。 + > - 如果您是中国用户，可以指定参数 `--set global.imageRegistryOverride=ghcr.m.daocloud.io` 来使用国内的镜像源。 + +2. 安装完成后，查看 Spidercoordinator 的配置，确保 `vethLinkAddress` 已配置正确： + + ```shell + ~# kubectl get spidercoordinators.spiderpool.spidernet.io default -o yaml + apiVersion: spiderpool.spidernet.io/v2beta1 + kind: SpiderCoordinator + metadata: + creationTimestamp: "2024-10-30T08:31:09Z" + finalizers: + - spiderpool.spidernet.io + generation: 7 + name: default + resourceVersion: "195405" + uid: 8bdceced-15db-497b-be07-81cbcba7caac + spec: + detectGateway: false + detectIPConflict: false + hijackCIDR: + - 169.254.0.0/16 + podRPFilter: 0 + hostRPFilter: 0 + hostRuleTable: 500 + mode: auto + podCIDRType: calico + podDefaultRouteNIC: "" + vethLinkAddress: 169.254.100.1 + podMACPrefix: "" + tunePodRoutes: true + status: + overlayPodCIDR: + - 10.222.64.0/18 + - 10.223.64.0/18 + phase: Synced + serviceCIDR: + - 10.233.0.0/18 + ``` + +3. 如果您已经安装 Spiderpool, 您可以直接修改 Spidercoordinator 中关于 vethLinkAddress 的配置: + + ```shell + kubectl patch spidercoordinators default --type='merge' -p '{"spec": {"vethLinkAddress": "169.254.100.1"}}' + ``` + +4. 步骤 3 中是集群默认设置，如果您不希望整个集群默认都配置 vethLinkAddress，您可以为单个网卡配置： + + ```shell + MACVLAN_MASTER_INTERFACE="eth0" + cat < -n -- ip addr show veth0 +``` diff --git a/docs/usage/istio.md b/docs/usage/istio.md new file mode 100644 index 0000000000..e8507c5843 --- /dev/null +++ b/docs/usage/istio.md @@ -0,0 +1,98 @@ +# Istio + +**English** | [**简体中文**](./istio-zh_CN.md) + +## Introduction + +In the context of Istio, when using Spiderpool to configure the network for service mesh applications with an Underlay network, there may be issues where traffic cannot be intercepted by Istio. This is because: + +1. Traffic accessing the service mesh Pod is forwarded through its veth0 network interface (created by Spiderpool). The traffic is then intercepted to the sidecar container through the iptables redirect rules set by Istio. However, since iptables redirect rules require the receiving network interface to be configured with an IP address, otherwise the packet will be silently dropped by the kernel. + +2. By default, Spiderpool does not configure an IP address for the veth0 network interface of Pods using the Underlay network, which leads to the traffic accessing the service mesh being dropped. + +Refer to [#Issue 3568](https://github.com/spidernet-io/spiderpool/issues/3568). To solve this problem, Spiderpool provides a configuration: `vethLinkAddress`, which is used to configure a link-local address for the veth0 network interface. + +## How to Configure + +1. When installing Spiderpool using Helm, you can enable this feature with the following command: + + ```shell + helm repo add spiderpool https://spidernet-io.github.io/spiderpool + helm repo update spiderpool + kubectl create namespace spiderpool + helm install spiderpool spiderpool/spiderpool -n spiderpool --set coordinator.vethLinkAddress=169.254.100.1 + ``` + + > - `vethLinkAddress` must be a valid IP address. + > - If you are a user in China, you can specify the parameter `--set global.imageRegistryOverride=ghcr.m.daocloud.io` to use a domestic image source. + +2. After installation, check the configuration of the Spidercoordinator to ensure that `vethLinkAddress` is configured correctly: + + ```shell + ~# kubectl get spidercoordinators.spiderpool.spidernet.io default -o yaml + apiVersion: spiderpool.spidernet.io/v2beta1 + kind: SpiderCoordinator + metadata: + creationTimestamp: "2024-10-30T08:31:09Z" + finalizers: + - spiderpool.spidernet.io + generation: 7 + name: default + resourceVersion: "195405" + uid: 8bdceced-15db-497b-be07-81cbcba7caac + spec: + detectGateway: false + detectIPConflict: false + hijackCIDR: + - 169.254.0.0/16 + podRPFilter: 0 + hostRPFilter: 0 + hostRuleTable: 500 + mode: auto + podCIDRType: calico + podDefaultRouteNIC: "" + vethLinkAddress: 169.254.100.1 + podMACPrefix: "" + tunePodRoutes: true + status: + overlayPodCIDR: + - 10.222.64.0/18 + - 10.223.64.0/18 + phase: Synced + serviceCIDR: + - 10.233.0.0/18 + ``` + +3. If you have already installed Spiderpool, you can directly modify the configuration of `vethLinkAddress` in the Spidercoordinator: + + ```shell + kubectl patch spidercoordinators default --type='merge' -p '{"spec": {"vethLinkAddress": "169.254.100.1"}}' + ``` + +4. Step 3 is the default setting for the cluster. If you do not want the entire cluster to default to configuring `vethLinkAddress`, you can configure it for a single network interface: + + ```shell + MACVLAN_MASTER_INTERFACE="eth0" + cat < -n -- ip addr show veth0 +``` diff --git a/pkg/coordinatormanager/coordinator_mutate.go b/pkg/coordinatormanager/coordinator_mutate.go index cb54f4f58d..c89b2af914 100644 --- a/pkg/coordinatormanager/coordinator_mutate.go +++ b/pkg/coordinatormanager/coordinator_mutate.go @@ -45,6 +45,10 @@ func mutateCoordinator(ctx context.Context, coord *spiderpoolv2beta1.SpiderCoord coord.Spec.DetectGateway = ptr.To(false) } + if coord.Spec.VethLinkAddress == nil { + coord.Spec.VethLinkAddress = ptr.To("") + } + if coord.DeletionTimestamp != nil { logger.Info("Terminating Coordinator, noting to mutate") return nil diff --git a/pkg/coordinatormanager/coordinator_validate.go b/pkg/coordinatormanager/coordinator_validate.go index 6f69859a36..a3f1900eff 100644 --- a/pkg/coordinatormanager/coordinator_validate.go +++ b/pkg/coordinatormanager/coordinator_validate.go @@ -5,6 +5,7 @@ package coordinatormanager import ( "fmt" + "net/netip" "strconv" "strings" @@ -16,11 +17,12 @@ import ( ) var ( - podCIDRTypeField *field.Path = field.NewPath("spec").Child("podCIDRType") - extraCIDRField *field.Path = field.NewPath("spec").Child("extraCIDR") - podMACPrefixField *field.Path = field.NewPath("spec").Child("podMACPrefix") - hostRPFilterField *field.Path = field.NewPath("spec").Child("hostRPFilter") - podRPFilterField *field.Path = field.NewPath("spec").Child("podRPFilter") + podCIDRTypeField *field.Path = field.NewPath("spec").Child("podCIDRType") + extraCIDRField *field.Path = field.NewPath("spec").Child("extraCIDR") + podMACPrefixField *field.Path = field.NewPath("spec").Child("podMACPrefix") + hostRPFilterField *field.Path = field.NewPath("spec").Child("hostRPFilter") + podRPFilterField *field.Path = field.NewPath("spec").Child("podRPFilter") + vethLinkAddressField *field.Path = field.NewPath("spec").Child("vethLinkAddress") ) func validateCreateCoordinator(coord *spiderpoolv2beta1.SpiderCoordinator) field.ErrorList { @@ -50,7 +52,6 @@ func validateUpdateCoordinator(oldCoord, newCoord *spiderpoolv2beta1.SpiderCoord } func ValidateCoordinatorSpec(spec *spiderpoolv2beta1.CoordinatorSpec, requireOptionalType bool) *field.Error { - if requireOptionalType && spec.PodCIDRType == nil { return field.NotSupported( podCIDRTypeField, @@ -97,6 +98,13 @@ func ValidateCoordinatorSpec(spec *spiderpoolv2beta1.CoordinatorSpec, requireOpt } } + if spec.VethLinkAddress != nil && *spec.VethLinkAddress != "" { + _, err := netip.ParseAddr(*spec.VethLinkAddress) + if err != nil { + return field.Invalid(vethLinkAddressField, *spec.VethLinkAddress, "vethLinkAddress is an invalid IP address") + } + } + return nil } diff --git a/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spidercoordinator_types.go b/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spidercoordinator_types.go index b87ac69ec1..2dc2ed3ddc 100644 --- a/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spidercoordinator_types.go +++ b/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spidercoordinator_types.go @@ -78,6 +78,11 @@ type CoordinatorSpec struct { // +kubebuilder:validation:Optional DetectGateway *bool `json:"detectGateway,omitempty"` + + // VethLinkAddress configure a ipv4 link-local address + // for veth0 device. empty means disable. default is empty. + // Format is like 169.254.100.1 + VethLinkAddress *string `json:"vethLinkAddress,omitempty"` } // CoordinationStatus defines the observed state of SpiderCoordinator. diff --git a/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/zz_generated.deepcopy.go b/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/zz_generated.deepcopy.go index c03456fd1f..14fef01377 100644 --- a/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/zz_generated.deepcopy.go +++ b/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/zz_generated.deepcopy.go @@ -91,6 +91,11 @@ func (in *CoordinatorSpec) DeepCopyInto(out *CoordinatorSpec) { *out = new(bool) **out = **in } + if in.VethLinkAddress != nil { + in, out := &in.VethLinkAddress, &out.VethLinkAddress + *out = new(string) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CoordinatorSpec. diff --git a/pkg/multuscniconfig/multusconfig_informer.go b/pkg/multuscniconfig/multusconfig_informer.go index e4b2a5b974..582936762a 100644 --- a/pkg/multuscniconfig/multusconfig_informer.go +++ b/pkg/multuscniconfig/multusconfig_informer.go @@ -644,6 +644,9 @@ func generateCoordinatorCNIConf(coordinatorSpec *spiderpoolv2beta1.CoordinatorSp if coordinatorSpec.DetectGateway != nil { coordinatorNetConf.DetectGateway = coordinatorSpec.DetectGateway } + if coordinatorSpec.VethLinkAddress != nil { + coordinatorNetConf.VethLinkAddress = *coordinatorSpec.VethLinkAddress + } if coordinatorSpec.TunePodRoutes != nil { coordinatorNetConf.TunePodRoutes = coordinatorSpec.TunePodRoutes } diff --git a/pkg/multuscniconfig/multusconfig_mutate.go b/pkg/multuscniconfig/multusconfig_mutate.go index fbd9370e44..ccf0156169 100644 --- a/pkg/multuscniconfig/multusconfig_mutate.go +++ b/pkg/multuscniconfig/multusconfig_mutate.go @@ -133,6 +133,7 @@ func setCoordinatorDefaultConfig(coordinator *spiderpoolv2beta1.CoordinatorSpec) HijackCIDR: []string{}, DetectGateway: ptr.To(false), DetectIPConflict: ptr.To(false), + VethLinkAddress: ptr.To(""), PodMACPrefix: ptr.To(""), PodDefaultRouteNIC: ptr.To(""), HostRPFilter: ptr.To(0), diff --git a/pkg/multuscniconfig/utils.go b/pkg/multuscniconfig/utils.go index 384e6d11cf..fa004f6fc0 100644 --- a/pkg/multuscniconfig/utils.go +++ b/pkg/multuscniconfig/utils.go @@ -88,6 +88,7 @@ type IfacerNetConf struct { type CoordinatorConfig struct { IPConflict *bool `json:"detectIPConflict,omitempty"` DetectGateway *bool `json:"detectGateway,omitempty"` + VethLinkAddress string `json:"vethLinkAddress,omitempty"` MacPrefix string `json:"podMACPrefix,omitempty"` Mode coordinatorcmd.Mode `json:"mode,omitempty"` Type string `json:"type"` diff --git a/test/doc/spidermultus.md b/test/doc/spidermultus.md index b1e54a5e76..a74717d9d8 100644 --- a/test/doc/spidermultus.md +++ b/test/doc/spidermultus.md @@ -27,3 +27,4 @@ | M00023 | set hostRPFilter and podRPFilter to a invalid value | p3 | | done | | | M00024 | verify the podMACPrefix filed | p3 | | done | | | M00025 | The custom net-attach-conf name from the annotation multus.spidernet.io/cr-name doesn't follow Kubernetes naming rules and can't be created. | p3 | | done | | +| M00026 | check the coordinatorConfig: enableVethLinkLocakAddress works | p3 | | done | | diff --git a/test/e2e/spidermultus/spidermultus_test.go b/test/e2e/spidermultus/spidermultus_test.go index e78945e3dd..1732c1ece9 100644 --- a/test/e2e/spidermultus/spidermultus_test.go +++ b/test/e2e/spidermultus/spidermultus_test.go @@ -5,6 +5,7 @@ package spidermultus_test import ( "encoding/json" "fmt" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -552,7 +553,83 @@ var _ = Describe("test spidermultus", Label("SpiderMultusConfig"), func() { Expect(err).To(HaveOccurred(), "create spiderMultus instance failed: %v\n", err) }) - It("verify the podMACPrefix filed", Label("M00028"), func() { + It("set disableIPAM to true and see if multus's nad has ipam config", Label("M00017"), func() { + var smcName string = "disable-ipam-multus-" + common.GenerateString(10, true) + + // Define Spidermultus cr with DisableIPAM=true + smc := &spiderpoolv2beta1.SpiderMultusConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: smcName, + Namespace: namespace, + }, + Spec: spiderpoolv2beta1.MultusCNIConfigSpec{ + CniType: ptr.To(constant.MacvlanCNI), + MacvlanConfig: &spiderpoolv2beta1.SpiderMacvlanCniConfig{ + Master: []string{common.NIC1}, + }, + DisableIPAM: ptr.To(true), + }, + } + GinkgoWriter.Printf("spidermultus cr: %+v \n", smc) + err := frame.CreateSpiderMultusInstance(smc) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func() error { + netAttachDef, err := frame.GetMultusInstance(smcName, namespace) + if nil != err { + return err + } + if netAttachDef.Spec.Config == "" { + return fmt.Errorf("SpiderMultusConfig %s/%s corresponding net-attach-def resource doesn't have CNI configuration", smcName, namespace) + } + + configByte, err := netutils.GetCNIConfigFromSpec(netAttachDef.Spec.Config, netAttachDef.Name) + if nil != err { + return fmt.Errorf("GetCNIConfig: err in getCNIConfigFromSpec: %v", err) + } + + networkConfigList, err := libcni.ConfListFromBytes(configByte) + if nil != err { + return err + } + + if len(networkConfigList.Plugins) != 2 { + return fmt.Errorf("unexpected CNI configuration: %s", netAttachDef.Spec.Config) + } + + Expect(netAttachDef.Spec.Config).NotTo(ContainSubstring(`"type":"spiderpool"`)) + GinkgoWriter.Printf("SpiderMultusConfig with disableIPAM: %s\n", netAttachDef.Spec.Config) + + return nil + }).WithTimeout(time.Minute * 3).WithPolling(time.Second * 5).Should(BeNil()) + }) + + It("check the enableVethLinkLocakAddress works", Label("M00026"), func() { + // create a pod + name := "veth-address-test" + var annotations = make(map[string]string) + annotations[common.MultusDefaultNetwork] = fmt.Sprintf("%s/%s", common.MultusNs, common.MacvlanUnderlayVlan0) + deployObject := common.GenerateExampleDeploymentYaml("veth-address-test", namespace, int32(1)) + deployObject.Spec.Template.Annotations = annotations + Expect(frame.CreateDeployment(deployObject)).NotTo(HaveOccurred()) + + ctx, cancel := context.WithTimeout(context.Background(), common.PodStartTimeout) + defer cancel() + + depObject, err := frame.WaitDeploymentReady(name, namespace, ctx) + Expect(err).NotTo(HaveOccurred(), "waiting for deploy ready failed: %v ", err) + podList, err := frame.GetPodListByLabel(depObject.Spec.Template.Labels) + Expect(err).NotTo(HaveOccurred(), "failed to get podList: %v ", err) + + commandString := "ip a show veth0 | grep 169.254.200.1 &> /dev/null" + ctx, cancel = context.WithTimeout(context.Background(), common.ExecCommandTimeout) + defer cancel() + + _, err = frame.ExecCommandInPod(podList.Items[0].Name, podList.Items[0].Namespace, commandString, ctx) + Expect(err).NotTo(HaveOccurred(), "failed to execute command, err: %v ", err) + }) + + It("verify the podMACPrefix filed", Label("M00024"), func() { smcName := "test-multus-" + common.GenerateString(10, true) smc := &spiderpoolv2beta1.SpiderMultusConfig{ ObjectMeta: metav1.ObjectMeta{ diff --git a/test/scripts/install-multus.sh b/test/scripts/install-multus.sh index 6eebc0e9e2..4c61e21bfb 100755 --- a/test/scripts/install-multus.sh +++ b/test/scripts/install-multus.sh @@ -68,6 +68,7 @@ spec: ipv6: [<>] coordinator: mode: "<>" + vethLinkAddress: <> ' OVS_CR_TEMPLATE=' @@ -150,6 +151,7 @@ spec: | sed 's?<>?0?g' \ | sed 's?<>?'""${DEFAULT_IPV4_IPPOOLS}""'?g' \ | sed 's?<>?'""${DEFAULT_IPV6_IPPOOLS}""'?g' \ + | sed 's?<>?169.254.100.1?g' \ | kubectl apply --kubeconfig ${E2E_KUBECONFIG} -f - echo "${MACVLAN_CR_TEMPLATE}" \ @@ -161,6 +163,7 @@ spec: | sed 's?<>?100?g' \ | sed 's?<>?'""${VLAN100_IPV4_IPPOOLS}""'?g' \ | sed 's?<>?'""${VLAN100_IPV6_IPPOOLS}""'?g' \ + | sed 's?<>?""?g' \ | kubectl apply --kubeconfig ${E2E_KUBECONFIG} -f - echo "${MACVLAN_CR_TEMPLATE}" \ @@ -172,6 +175,7 @@ spec: | sed 's?<>?200?g' \ | sed 's?<>?'""${VLAN200_IPV4_IPPOOLS}""'?g' \ | sed 's?<>?'""${VLAN200_IPV6_IPPOOLS}""'?g' \ + | sed 's?<>?""?g' \ | kubectl apply --kubeconfig ${E2E_KUBECONFIG} -f - if [ "${INSTALL_OVS}" == "true" ] ; then