fix reconcile routes

pkg/daemon/controller_linux.go

@@ -16,6 +16,7 @@ import (
 	"github.com/kubeovn/felix/ipsets"
 	"github.com/kubeovn/go-iptables/iptables"
 	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
 	v1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/labels"
@@ -270,13 +271,18 @@ func (c *Controller) reconcileRouters(event *subnetEvent) error {
 		return err
 	}
 	nodeIPv4, nodeIPv6 := util.GetNodeInternalIP(*node)
+	var joinIPv4, joinIPv6 string
+	if len(node.Annotations) != 0 {
+		joinIPv4, joinIPv6 = util.SplitStringIP(node.Annotations[util.IPAddressAnnotation])
+	}
 
 	joinCIDR := make([]string, 0, 2)
 	cidrs := make([]string, 0, len(subnets)*2)
 	for _, subnet := range subnets {
 		// The route for overlay subnet cidr via ovn0 should not be deleted even though subnet.Status has changed to not ready
 		if subnet.Spec.Vpc != c.config.ClusterRouter ||
-			(subnet.Spec.Vlan != "" && !subnet.Spec.LogicalGateway && (!subnet.Spec.U2OInterconnection || (subnet.Spec.EnableLb != nil && *subnet.Spec.EnableLb))) {
+			(subnet.Spec.Vlan != "" && !subnet.Spec.LogicalGateway && (!subnet.Spec.U2OInterconnection || (subnet.Spec.EnableLb != nil && *subnet.Spec.EnableLb))) ||
+			(subnet.Name != c.config.NodeSwitch && !subnet.Status.IsReady()) {
 			continue
 		}
 
@@ -319,7 +325,7 @@ func (c *Controller) reconcileRouters(event *subnetEvent) error {
 		klog.Error(err)
 		return err
 	}
-	toAdd, toDel := routeDiff(nodeNicRoutes, allRoutes, cidrs, joinCIDR, gateway, net.ParseIP(nodeIPv4), net.ParseIP(nodeIPv6))
+	toAdd, toDel := routeDiff(nodeNicRoutes, allRoutes, cidrs, joinCIDR, joinIPv4, joinIPv6, gateway, net.ParseIP(nodeIPv4), net.ParseIP(nodeIPv6))
 	for _, r := range toDel {
 		if err = netlink.RouteDel(&netlink.Route{Dst: r.Dst}); err != nil {
 			klog.Errorf("failed to del route %v", err)
@@ -353,7 +359,10 @@ func getNicExistRoutes(nic netlink.Link, gateway string) ([]netlink.Route, error
 	return existRoutes, nil
 }
 
-func routeDiff(nodeNicRoutes, allRoutes []netlink.Route, cidrs, joinCIDR []string, gateway string, srcIPv4, srcIPv6 net.IP) (toAdd, toDel []netlink.Route) {
+func routeDiff(nodeNicRoutes, allRoutes []netlink.Route, cidrs, joinCIDR []string, joinIPv4, joinIPv6, gateway string, srcIPv4, srcIPv6 net.IP) (toAdd, toDel []netlink.Route) {
+	// joinIPv6 is not used for now
+	_ = joinIPv6
+
 	for _, route := range nodeNicRoutes {
 		if route.Scope == netlink.SCOPE_LINK || route.Dst == nil || route.Dst.IP.IsLinkLocalUnicast() {
 			continue
@@ -382,16 +391,12 @@ func routeDiff(nodeNicRoutes, allRoutes []netlink.Route, cidrs, joinCIDR []strin
 		}
 	}
 	if len(toDel) > 0 {
-		klog.Infof("route to del %v", toDel)
+		klog.Infof("routes to delete: %v", toDel)
 	}
 
 	ipv4, ipv6 := util.SplitStringIP(gateway)
 	gwV4, gwV6 := net.ParseIP(ipv4), net.ParseIP(ipv6)
 	for _, c := range cidrs {
-		if slices.Contains(joinCIDR, c) {
-			continue
-		}
-
 		var src, gw net.IP
 		switch util.CheckProtocol(c) {
 		case kubeovnv1.ProtocolIPv4:
@@ -426,17 +431,31 @@ func routeDiff(nodeNicRoutes, allRoutes []netlink.Route, cidrs, joinCIDR []strin
 			}
 		}
 		if !found {
+			var priority int
+			scope := netlink.SCOPE_UNIVERSE
+			proto := netlink.RouteProtocol(syscall.RTPROT_STATIC)
+			if slices.Contains(joinCIDR, c) {
+				if util.CheckProtocol(c) == kubeovnv1.ProtocolIPv4 {
+					src = net.ParseIP(joinIPv4)
+				} else {
+					src, priority = nil, 256
+				}
+				gw, scope = nil, netlink.SCOPE_LINK
+				proto = netlink.RouteProtocol(unix.RTPROT_KERNEL)
+			}
 			_, cidr, _ := net.ParseCIDR(c)
 			toAdd = append(toAdd, netlink.Route{
-				Dst:   cidr,
-				Src:   src,
-				Gw:    gw,
-				Scope: netlink.SCOPE_UNIVERSE,
+				Dst:      cidr,
+				Src:      src,
+				Gw:       gw,
+				Protocol: proto,
+				Scope:    scope,
+				Priority: priority,
 			})
 		}
 	}
 	if len(toAdd) > 0 {
-		klog.Infof("route to add %v", toAdd)
+		klog.Infof("routes to add: %v", toAdd)
 	}
 	return
 }
@@ -572,9 +591,8 @@ func (c *Controller) getPolicyRouting(subnet *kubeovnv1.Subnet) ([]netlink.Rule,
 	// routes
 	var routes []netlink.Route
 	for i := range protocols {
-		family, _ := util.ProtocolToFamily(protocols[i])
 		routes = append(routes, netlink.Route{
-			Protocol: netlink.RouteProtocol(family),
+			Protocol: netlink.RouteProtocol(syscall.RTPROT_STATIC),
 			Table:    int(subnet.Spec.PolicyRoutingTableID),
 			Gw:       net.ParseIP(egw[i]),
 		})

pkg/daemon/controller_windows.go

@@ -55,7 +55,8 @@ func (c *Controller) reconcileRouters(_ *subnetEvent) error {
 	for _, subnet := range subnets {
 		// The route for overlay subnet cidr via ovn0 should not be deleted even though subnet.Status has changed to not ready
 		if (subnet.Spec.Vlan != "" && !subnet.Spec.LogicalGateway) ||
-			subnet.Spec.Vpc != c.config.ClusterRouter {
+			subnet.Spec.Vpc != c.config.ClusterRouter ||
+			(subnet.Name != c.config.NodeSwitch && !subnet.Status.IsReady()) {
 			continue
 		}
 

pkg/daemon/gateway_linux.go

@@ -324,7 +324,7 @@ func (c *Controller) deletePodPolicyRouting(podProtocol, externalEgressGateway s
 
 func (c *Controller) addPolicyRouting(family int, gateway string, priority, tableID uint32, ips ...string) error {
 	route := &netlink.Route{
-		Protocol: netlink.RouteProtocol(family),
+		Protocol: netlink.RouteProtocol(syscall.RTPROT_STATIC),
 		Gw:       net.ParseIP(gateway),
 		Table:    int(tableID),
 	}

pkg/daemon/ovs_linux.go

@@ -606,19 +606,36 @@ func configureNodeNic(portName, ip, gw, joinCIDR string, macAddr net.HardwareAdd
 			}
 		}
 		if !found {
+			protocol := util.CheckProtocol(c)
+			var src net.IP
+			var priority int
+			if protocol == kubeovnv1.ProtocolIPv4 {
+				for _, ip := range strings.Split(ipStr, ",") {
+					if util.CheckProtocol(ip) == protocol {
+						src = net.ParseIP(ip)
+						break
+					}
+				}
+			} else {
+				priority = 256
+			}
 			_, cidr, _ := net.ParseCIDR(c)
 			toAdd = append(toAdd, netlink.Route{
-				Dst:   cidr,
-				Scope: netlink.SCOPE_UNIVERSE,
+				Dst:      cidr,
+				Src:      src,
+				Protocol: netlink.RouteProtocol(unix.RTPROT_KERNEL),
+				Scope:    netlink.SCOPE_LINK,
+				Priority: priority,
 			})
 		}
 	}
 	if len(toAdd) > 0 {
-		klog.Infof("route to add for nic %s, %v", util.NodeNic, toAdd)
+		klog.Infof("routes to be added on nic %s: %v", util.NodeNic, toAdd)
 	}
 
 	for _, r := range toAdd {
 		r.LinkIndex = hostLink.Attrs().Index
+		klog.Infof("adding route %q on %s", r.String(), hostLink.Attrs().Name)
 		if err = netlink.RouteReplace(&r); err != nil && !errors.Is(err, syscall.EEXIST) {
 			klog.Errorf("failed to replace route %v: %v", r, err)
 		}

test/e2e/framework/iproute/iproute.go

@@ -101,8 +101,10 @@ func (e *execer) exec(cmd string, result interface{}) error {
 		return fmt.Errorf("failed to exec cmd %q: %v\nstdout:\n%s\nstderr:\n%s", cmd, err, stdout, stderr)
 	}
 
-	if err = json.Unmarshal(stdout, result); err != nil {
-		return fmt.Errorf("failed to decode json %q: %v", string(stdout), err)
+	if result != nil {
+		if err = json.Unmarshal(stdout, result); err != nil {
+			return fmt.Errorf("failed to decode json %q: %v", string(stdout), err)
+		}
 	}
 
 	return nil
@@ -150,6 +152,16 @@ func RouteShow(table, device string, execFunc ExecFunc) ([]Route, error) {
 	return append(routes, routes6...), nil
 }
 
+func RouteDel(table, dst string, execFunc ExecFunc) error {
+	e := execer{fn: execFunc}
+	args := dst
+	if table != "" {
+		args = " table " + table
+	}
+
+	return e.exec("ip route del "+args, nil)
+}
+
 func RuleShow(device string, execFunc ExecFunc) ([]Rule, error) {
 	e := execer{fn: execFunc}