Add a flag to configure an link-local address to veth0 for istio

Signed-off-by: cyclinder <[email protected]>

api/v1/agent/openapi.yaml

@@ -328,6 +328,8 @@ definitions:
         type: boolean
       detectGateway:
         type: boolean
+      vethLinkAddress:
+        type: string
     required:
       - overlayPodCIDR
       - serviceCIDR

charts/spiderpool/README.md

@@ -161,6 +161,7 @@ helm install spiderpool spiderpool/spiderpool --wait --namespace kube-system \
 | `coordinator.detectIPConflict` | detect IP address conflicts                                                                                                              | `false`              |
 | `coordinator.tunePodRoutes`    | tune Pod routes                                                                                                                          | `true`               |
 | `coordinator.hijackCIDR`       | Additional subnets that need to be hijacked to the host forward, the default link-local range "169.254.0.0/16" is used for NodeLocal DNS | `["169.254.0.0/16"]` |
+| `coordinator.vethLinkAddress`  | configure an link-local address for veth0 device. empty means disable. default is empty. Format is like 169.254.100.1                    | `""`                 |
 
 ### rdma parameters
 

charts/spiderpool/crds/spiderpool.spidernet.io_spidercoordinators.yaml

@@ -109,6 +109,11 @@ spec:
                 description: TunePodRoutes specifies whether to tune pod routes of
                   multiple NICs on pods.
                 type: boolean
+              vethLinkAddress:
+                description: VethLinkAddress configure a ipv4 link-local address for
+                  veth0 device. empty means disable. default is empty. Format is like
+                  169.254.100.1
+                type: string
             type: object
           status:
             description: CoordinationStatus defines the observed state of SpiderCoordinator.

charts/spiderpool/crds/spiderpool.spidernet.io_spidermultusconfigs.yaml

@@ -121,6 +121,11 @@ spec:
                     description: TunePodRoutes specifies whether to tune pod routes
                       of multiple NICs on pods.
                     type: boolean
+                  vethLinkAddress:
+                    description: VethLinkAddress configure a ipv4 link-local address
+                      for veth0 device. empty means disable. default is empty. Format
+                      is like 169.254.100.1
+                    type: string
                 type: object
               customCNI:
                 description: OtherCniTypeConfig only used for CniType custom, valid

charts/spiderpool/templates/pod.yaml

@@ -47,6 +47,8 @@ spec:
           value: {{ .Values.coordinator.tunePodRoutes | quote }}
         - name: SPIDERPOOL_INIT_DEFAULT_COORDINATOR_HIJACK_CIDR
           value: {{ toJson .Values.coordinator.hijackCIDR | quote }}
+        - name: SPIDERPOOL_INIT_DEFAULT_COORDINATOR_VETH_LINK_ADDRESS
+          value: {{ .Values.coordinator.vethLinkAddress | quote }}
         {{- end }}
         {{- if and .Values.clusterDefaultPool.installIPv4IPPool .Values.ipam.enableIPv4 }}
         - name: SPIDERPOOL_INIT_DEFAULT_IPV4_IPPOOL_NAME

charts/spiderpool/values.yaml

@@ -116,6 +116,9 @@ coordinator:
   ## @param coordinator.hijackCIDR Additional subnets that need to be hijacked to the host forward, the default link-local range "169.254.0.0/16" is used for NodeLocal DNS
   hijackCIDR: ["169.254.0.0/16"]
 
+  ## @param coordinator.vethLinkAddress configure an link-local address for veth0 device. empty means disable. default is empty. Format is like 169.254.100.1
+  vethLinkAddress: ""
+
 ## @section rdma parameters
 ##
 rdma:

cmd/coordinator/cmd/cni_types.go

@@ -45,6 +45,7 @@ const (
 type Config struct {
 	types.NetConf
 	DetectGateway      *bool          `json:"detectGateway,omitempty"`
+	VethLinkAddress    string         `json:"vethLinkAddress,omitempty"`
 	MacPrefix          string         `json:"podMACPrefix,omitempty"`
 	MultusNicPrefix    string         `json:"multusNicPrefix,omitempty"`
 	PodDefaultCniNic   string         `json:"podDefaultCniNic,omitempty"`
@@ -173,6 +174,9 @@ func ParseConfig(stdin []byte, coordinatorConfig *models.CoordinatorConfig) (*Co
 		conf.PodDefaultRouteNIC = coordinatorConfig.PodDefaultRouteNIC
 	}
 
+	if conf.VethLinkAddress == "" {
+		conf.VethLinkAddress = coordinatorConfig.VethLinkAddress
+	}
 	return &conf, nil
 }
 

cmd/coordinator/cmd/command_add.go

@@ -99,6 +99,7 @@ func CmdAdd(args *skel.CmdArgs) (err error) {
 		ipFamily:         ipFamily,
 		currentInterface: args.IfName,
 		tuneMode:         conf.Mode,
+		vethLinkAddress:  conf.VethLinkAddress,
 	}
 	c.HijackCIDR = append(c.HijackCIDR, conf.ServiceCIDR...)
 	c.HijackCIDR = append(c.HijackCIDR, conf.HijackCIDR...)

cmd/coordinator/cmd/utils.go

@@ -21,17 +21,17 @@ import (
 )
 
 type coordinator struct {
-	firstInvoke                                 bool
-	ipFamily, currentRuleTable, hostRuleTable   int
-	tuneMode                                    Mode
-	hostVethName, podVethName, currentInterface string
-	v4HijackRouteGw, v6HijackRouteGw            net.IP
-	HijackCIDR                                  []string
-	netns, hostNs                               ns.NetNS
-	hostVethHwAddress, podVethHwAddress         net.HardwareAddr
-	currentAddress                              []netlink.Addr
-	v4PodOverlayNicAddr, v6PodOverlayNicAddr    *net.IPNet
-	hostIPRouteForPod                           []net.IP
+	firstInvoke                                                  bool
+	ipFamily, currentRuleTable, hostRuleTable                    int
+	tuneMode                                                     Mode
+	hostVethName, podVethName, vethLinkAddress, currentInterface string
+	v4HijackRouteGw, v6HijackRouteGw                             net.IP
+	HijackCIDR                                                   []string
+	netns, hostNs                                                ns.NetNS
+	hostVethHwAddress, podVethHwAddress                          net.HardwareAddr
+	currentAddress                                               []netlink.Addr
+	v4PodOverlayNicAddr, v6PodOverlayNicAddr                     *net.IPNet
+	hostIPRouteForPod                                            []net.IP
 }
 
 func (c *coordinator) autoModeToSpecificMode(mode Mode, podFirstInterface string, vethExist bool) error {
@@ -189,9 +189,13 @@ func (c *coordinator) setupVeth(logger *zap.Logger, containerID string) error {
 			return nil
 		}
 
+		if c.vethLinkAddress == "" {
+			return nil
+		}
+
 		if err = netlink.AddrAdd(link, &netlink.Addr{
 			IPNet: &net.IPNet{
-				IP:   net.ParseIP("169.254.200.1"),
+				IP:   net.ParseIP(c.vethLinkAddress),
 				Mask: net.CIDRMask(32, 32),
 			},
 		}); err != nil {

cmd/spiderpool-agent/cmd/coordinator.go

@@ -94,6 +94,11 @@ func (g *_unixGetCoordinatorConfig) Handle(params daemonset.GetCoordinatorConfig
 		nic = *coord.Spec.PodDefaultRouteNIC
 	}
 
+	var vethLinkAddress string
+	if coord.Spec.VethLinkAddress != nil {
+		vethLinkAddress = *coord.Spec.VethLinkAddress
+	}
+
 	defaultRouteNic, ok := pod.Annotations[constant.AnnoDefaultRouteInterface]
 	if ok {
 		nic = defaultRouteNic
@@ -107,6 +112,7 @@ func (g *_unixGetCoordinatorConfig) Handle(params daemonset.GetCoordinatorConfig
 		PodMACPrefix:       prefix,
 		TunePodRoutes:      coord.Spec.TunePodRoutes,
 		PodDefaultRouteNIC: nic,
+		VethLinkAddress:    vethLinkAddress,
 		HostRuleTable:      int64(*coord.Spec.HostRuleTable),
 		PodRPFilter:        int64(*coord.Spec.PodRPFilter),
 		DetectGateway:      *coord.Spec.DetectGateway,

cmd/spiderpool-init/cmd/config.go

@@ -45,13 +45,14 @@ const (
 	ENVDefaultIPv6IPRanges   = "SPIDERPOOL_INIT_DEFAULT_IPV6_IPPOOL_IPRANGES"
 	ENVDefaultIPv6Gateway    = "SPIDERPOOL_INIT_DEFAULT_IPV6_IPPOOL_GATEWAY"
 
-	ENVEnableMultusConfig     = "SPIDERPOOL_INIT_ENABLE_MULTUS_CONFIG"
-	ENVInstallMultusCNI       = "SPIDERPOOL_INIT_INSTALL_MULTUS"
-	ENVDefaultCNIDir          = "SPIDERPOOL_INIT_DEFAULT_CNI_DIR"
-	ENVDefaultCNIName         = "SPIDERPOOL_INIT_DEFAULT_CNI_NAME"
-	ENVDefaultCNINamespace    = "SPIDERPOOL_INIT_DEFAULT_CNI_NAMESPACE"
-	ENVDefaultMultusConfigMap = "SPIDERPOOL_INIT_MULTUS_CONFIGMAP"
-	ENVDefaultReadinessFile   = "SPIDERPOOL_INIT_READINESS_FILE"
+	ENVEnableMultusConfig                = "SPIDERPOOL_INIT_ENABLE_MULTUS_CONFIG"
+	ENVInstallMultusCNI                  = "SPIDERPOOL_INIT_INSTALL_MULTUS"
+	ENVDefaultCNIDir                     = "SPIDERPOOL_INIT_DEFAULT_CNI_DIR"
+	ENVDefaultCNIName                    = "SPIDERPOOL_INIT_DEFAULT_CNI_NAME"
+	ENVDefaultCNINamespace               = "SPIDERPOOL_INIT_DEFAULT_CNI_NAMESPACE"
+	ENVDefaultMultusConfigMap            = "SPIDERPOOL_INIT_MULTUS_CONFIGMAP"
+	ENVDefaultReadinessFile              = "SPIDERPOOL_INIT_READINESS_FILE"
+	ENVDefaultCoordinatorVethLinkAddress = "SPIDERPOOL_INIT_DEFAULT_COORDINATOR_VETH_LINK_ADDRESS"
 )
 
 var (
@@ -70,6 +71,7 @@ type InitDefaultConfig struct {
 	CoordinatorPodCIDRType        string
 	CoordinatorPodDefaultRouteNic string
 	CoordinatorPodMACPrefix       string
+	CoordinatorVethLinkAddress    string
 	CoordinatorDetectGateway      bool
 	CoordinatorDetectIPConflict   bool
 	CoordinatorTunePodRoutes      bool
@@ -169,6 +171,8 @@ func parseENVAsDefault() InitDefaultConfig {
 		} else {
 			config.CoordinatorHijackCIDR = []string{}
 		}
+
+		config.CoordinatorVethLinkAddress = strings.ReplaceAll(os.Getenv(ENVDefaultCoordinatorVethLinkAddress), "\"", "")
 	} else {
 		logger.Info("Ignore creating default Coordinator")
 	}

cmd/spiderpool-init/cmd/root.go

@@ -47,6 +47,7 @@ func Execute() {
 				DetectGateway:      &config.CoordinatorDetectGateway,
 				PodDefaultRouteNIC: &config.CoordinatorPodDefaultRouteNic,
 				PodMACPrefix:       &config.CoordinatorPodMACPrefix,
+				VethLinkAddress:    &config.CoordinatorVethLinkAddress,
 				HijackCIDR:         config.CoordinatorHijackCIDR,
 			},
 		}

docs/concepts/coordinator-zh_CN.md

@@ -129,6 +129,27 @@ spec:
 
 当 Pod 创建完成，我们可以检测 Pod 的 Mac 地址的前缀是否是 "0a:1b"
 
+## 为 Pod 的 veth0 网卡配置本地链路地址，支持服务网格场景
+
+默认情况下，Coordinator 不会为 veth0 网卡配置本地链路地址。但有些场景下(比如服务网格)，经过 veth0 网卡流入的网格流量会随 istio 设置的 iptables 规则重定向，如果 veth0 没有 IP 地址，这会导致这部分流量被丢弃(见[#Issue3568](https://github.com/spidernet-io/spiderpool/issues/3568))。所以在这个场景下，我们需要为 veth0 配置一个本地链路地址。
+
+```yaml
+apiVersion: spiderpool.spidernet.io/v2beta1
+kind: SpiderMultusConfig
+metadata:
+  name: istio-demo 
+  namespace: default
+spec:
+  cniType: macvlan
+  macvlan:
+    master: ["eth0"]
+  enableCoordinator: true
+  coordinator:
+    vethLinkAddress: "169.254.200.1"
+```
+
+> `vethLinkAddress` 默认为空，表示不配置。不为空则必须是一个合法的本地链路地址。
+
 ## 已知问题
 
 - underlay 模式下，underlay Pod 与 Overlay Pod(calico or cilium) 进行 TCP 通信失败

docs/concepts/coordinator.md

@@ -10,7 +10,7 @@ Spiderpool incorporates a CNI meta-plugin called `coordinator` that works after
 - Check the reachability of Pod gateways
 - Support fixed Mac address prefixes for Pods
 
-Note: If your OS(such as Fedora, CentOS, etc.) uses NetworkManager, highly recommend configuring following configuration file at `/etc/NetworkManager/conf.d/spidernet.conf` to 
+Note: If your OS(such as Fedora, CentOS, etc.) uses NetworkManager, highly recommend configuring following configuration file at `/etc/NetworkManager/conf.d/spidernet.conf` to
 prevent interference from NetworkManager with veth interfaces created through `coordinator`:
 
 ```shell
@@ -97,17 +97,16 @@ spec:
 
 > Note: There are some switches that are not allowed to be probed by arp, otherwise an alarm will be issued, in this case, we need to set detectGateway to false
 
-
 ## Fix MAC address prefix for Pods(alpha)
 
-Some traditional applications may require a fixed MAC address or IP address to couple the behavior of the application. For example, the License Server may need to apply a fixed Mac address 
-or IP address to issue a license for the app. If the MAC address of a pod changes, the issued license may be invalid. Therefore, you need to fix the MAC address of the pod. Spiderpool can fix 
+Some traditional applications may require a fixed MAC address or IP address to couple the behavior of the application. For example, the License Server may need to apply a fixed Mac address
+or IP address to issue a license for the app. If the MAC address of a pod changes, the issued license may be invalid. Therefore, you need to fix the MAC address of the pod. Spiderpool can fix
 the MAC address of the application through `coordinator`, and the fixed rule is to configure the MAC address prefix (2 bytes) + convert the IP of the pod (4 bytes).
 
 Note:
 
 > currently supports updating  Macvlan and SR-IOV as pods for CNI. In IPVlan L2 mode, the MAC addresses of the primary interface and the sub-interface are the same and cannot be modified.
-> 
+>
 > The fixed rule is to configure the MAC address prefix (2 bytes) + the IP of the converted pod (4 bytes). An IPv4 address is 4 bytes long and can be fully converted to 2 hexadecimal numbers. For IPv6 addresses, only the last 4 bytes are taken.
 
 We can configure it via Spidermultusconfig:
@@ -129,6 +128,25 @@ spec:
 
 You can check if the MAC address prefix of the Pod starts with "0a:1b" after a Pod is created.
 
+By default, Coordinator does not configure a link-local address for the veth0 interface. However, in some scenarios (such as service mesh), mesh traffic flowing through the veth0 interface will be redirected according to iptables rules set by Istio. If veth0 does not have an IP address, this can cause that traffic to be dropped (see #Issue3568). Therefore, in this scenario, we need to configure a link-local address for veth0.
+
+```yaml
+apiVersion: spiderpool.spidernet.io/v2beta1
+kind: SpiderMultusConfig
+metadata:
+  name: istio-demo 
+  namespace: default
+spec:
+  cniType: macvlan
+  macvlan:
+    master: ["eth0"]
+  enableCoordinator: true
+  coordinator:
+    vethLinkAddress: "169.254.100.1"
+```
+
+> `vethLinkAddress` default to "", It means that we don't configure an address for veth0. It must an valid link-local address if it isn't empty.
+
 ## Known issues
 
 - Underlay mode: TCP communication between underlay Pods and overlay Pods (Calico or Cilium) fails

docs/mkdocs.yml

@@ -92,6 +92,7 @@ nav:
       - Access Service for Underlay CNI: usage/underlay_cni_service.md
       - Coexistence of multi CNIs: usage/multi_cni_coexist.md
       - Kubevirt: usage/kubevirt.md
+      - Istio: usage/istio.md
       - FAQ: usage/faq.md
   - Reference:
       - Annotations: reference/annotation.md