diff --git a/pkg/daemon/controller_linux.go b/pkg/daemon/controller_linux.go index f58e29897de..a7158d13c9d 100644 --- a/pkg/daemon/controller_linux.go +++ b/pkg/daemon/controller_linux.go @@ -16,6 +16,7 @@ import ( "github.com/kubeovn/felix/ipsets" "github.com/kubeovn/go-iptables/iptables" "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" v1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" @@ -270,13 +271,18 @@ func (c *Controller) reconcileRouters(event *subnetEvent) error { return err } nodeIPv4, nodeIPv6 := util.GetNodeInternalIP(*node) + var joinIPv4, joinIPv6 string + if len(node.Annotations) != 0 { + joinIPv4, joinIPv6 = util.SplitStringIP(node.Annotations[util.IPAddressAnnotation]) + } joinCIDR := make([]string, 0, 2) cidrs := make([]string, 0, len(subnets)*2) for _, subnet := range subnets { // The route for overlay subnet cidr via ovn0 should not be deleted even though subnet.Status has changed to not ready if subnet.Spec.Vpc != c.config.ClusterRouter || - (subnet.Spec.Vlan != "" && !subnet.Spec.LogicalGateway && (!subnet.Spec.U2OInterconnection || (subnet.Spec.EnableLb != nil && *subnet.Spec.EnableLb))) { + (subnet.Spec.Vlan != "" && !subnet.Spec.LogicalGateway && (!subnet.Spec.U2OInterconnection || (subnet.Spec.EnableLb != nil && *subnet.Spec.EnableLb))) || + (subnet.Name != c.config.NodeSwitch && !subnet.Status.IsReady()) { continue } @@ -319,7 +325,7 @@ func (c *Controller) reconcileRouters(event *subnetEvent) error { klog.Error(err) return err } - toAdd, toDel := routeDiff(nodeNicRoutes, allRoutes, cidrs, joinCIDR, gateway, net.ParseIP(nodeIPv4), net.ParseIP(nodeIPv6)) + toAdd, toDel := routeDiff(nodeNicRoutes, allRoutes, cidrs, joinCIDR, joinIPv4, joinIPv6, gateway, net.ParseIP(nodeIPv4), net.ParseIP(nodeIPv6)) for _, r := range toDel { if err = netlink.RouteDel(&netlink.Route{Dst: r.Dst}); err != nil { klog.Errorf("failed to del route %v", err) @@ -353,7 +359,10 @@ func getNicExistRoutes(nic netlink.Link, gateway string) ([]netlink.Route, error return existRoutes, nil } -func routeDiff(nodeNicRoutes, allRoutes []netlink.Route, cidrs, joinCIDR []string, gateway string, srcIPv4, srcIPv6 net.IP) (toAdd, toDel []netlink.Route) { +func routeDiff(nodeNicRoutes, allRoutes []netlink.Route, cidrs, joinCIDR []string, joinIPv4, joinIPv6, gateway string, srcIPv4, srcIPv6 net.IP) (toAdd, toDel []netlink.Route) { + // joinIPv6 is not used for now + _ = joinIPv6 + for _, route := range nodeNicRoutes { if route.Scope == netlink.SCOPE_LINK || route.Dst == nil || route.Dst.IP.IsLinkLocalUnicast() { continue @@ -382,16 +391,12 @@ func routeDiff(nodeNicRoutes, allRoutes []netlink.Route, cidrs, joinCIDR []strin } } if len(toDel) > 0 { - klog.Infof("route to del %v", toDel) + klog.Infof("routes to delete: %v", toDel) } ipv4, ipv6 := util.SplitStringIP(gateway) gwV4, gwV6 := net.ParseIP(ipv4), net.ParseIP(ipv6) for _, c := range cidrs { - if slices.Contains(joinCIDR, c) { - continue - } - var src, gw net.IP switch util.CheckProtocol(c) { case kubeovnv1.ProtocolIPv4: @@ -426,17 +431,31 @@ func routeDiff(nodeNicRoutes, allRoutes []netlink.Route, cidrs, joinCIDR []strin } } if !found { + var priority int + scope := netlink.SCOPE_UNIVERSE + proto := netlink.RouteProtocol(syscall.RTPROT_STATIC) + if slices.Contains(joinCIDR, c) { + if util.CheckProtocol(c) == kubeovnv1.ProtocolIPv4 { + src = net.ParseIP(joinIPv4) + } else { + src, priority = nil, 256 + } + gw, scope = nil, netlink.SCOPE_LINK + proto = netlink.RouteProtocol(unix.RTPROT_KERNEL) + } _, cidr, _ := net.ParseCIDR(c) toAdd = append(toAdd, netlink.Route{ - Dst: cidr, - Src: src, - Gw: gw, - Scope: netlink.SCOPE_UNIVERSE, + Dst: cidr, + Src: src, + Gw: gw, + Protocol: proto, + Scope: scope, + Priority: priority, }) } } if len(toAdd) > 0 { - klog.Infof("route to add %v", toAdd) + klog.Infof("routes to add: %v", toAdd) } return } @@ -572,9 +591,8 @@ func (c *Controller) getPolicyRouting(subnet *kubeovnv1.Subnet) ([]netlink.Rule, // routes var routes []netlink.Route for i := range protocols { - family, _ := util.ProtocolToFamily(protocols[i]) routes = append(routes, netlink.Route{ - Protocol: netlink.RouteProtocol(family), + Protocol: netlink.RouteProtocol(syscall.RTPROT_STATIC), Table: int(subnet.Spec.PolicyRoutingTableID), Gw: net.ParseIP(egw[i]), }) diff --git a/pkg/daemon/controller_windows.go b/pkg/daemon/controller_windows.go index 0406949e78f..7e94d2b745b 100644 --- a/pkg/daemon/controller_windows.go +++ b/pkg/daemon/controller_windows.go @@ -55,7 +55,8 @@ func (c *Controller) reconcileRouters(_ *subnetEvent) error { for _, subnet := range subnets { // The route for overlay subnet cidr via ovn0 should not be deleted even though subnet.Status has changed to not ready if (subnet.Spec.Vlan != "" && !subnet.Spec.LogicalGateway) || - subnet.Spec.Vpc != c.config.ClusterRouter { + subnet.Spec.Vpc != c.config.ClusterRouter || + (subnet.Name != c.config.NodeSwitch && !subnet.Status.IsReady()) { continue } diff --git a/pkg/daemon/gateway_linux.go b/pkg/daemon/gateway_linux.go index 2ed8c556f83..57c876e8723 100644 --- a/pkg/daemon/gateway_linux.go +++ b/pkg/daemon/gateway_linux.go @@ -324,7 +324,7 @@ func (c *Controller) deletePodPolicyRouting(podProtocol, externalEgressGateway s func (c *Controller) addPolicyRouting(family int, gateway string, priority, tableID uint32, ips ...string) error { route := &netlink.Route{ - Protocol: netlink.RouteProtocol(family), + Protocol: netlink.RouteProtocol(syscall.RTPROT_STATIC), Gw: net.ParseIP(gateway), Table: int(tableID), } diff --git a/pkg/daemon/ovs_linux.go b/pkg/daemon/ovs_linux.go index 8d709cd6eb6..4aa97c6f372 100644 --- a/pkg/daemon/ovs_linux.go +++ b/pkg/daemon/ovs_linux.go @@ -606,19 +606,36 @@ func configureNodeNic(portName, ip, gw, joinCIDR string, macAddr net.HardwareAdd } } if !found { + protocol := util.CheckProtocol(c) + var src net.IP + var priority int + if protocol == kubeovnv1.ProtocolIPv4 { + for _, ip := range strings.Split(ipStr, ",") { + if util.CheckProtocol(ip) == protocol { + src = net.ParseIP(ip) + break + } + } + } else { + priority = 256 + } _, cidr, _ := net.ParseCIDR(c) toAdd = append(toAdd, netlink.Route{ - Dst: cidr, - Scope: netlink.SCOPE_UNIVERSE, + Dst: cidr, + Src: src, + Protocol: netlink.RouteProtocol(unix.RTPROT_KERNEL), + Scope: netlink.SCOPE_LINK, + Priority: priority, }) } } if len(toAdd) > 0 { - klog.Infof("route to add for nic %s, %v", util.NodeNic, toAdd) + klog.Infof("routes to be added on nic %s: %v", util.NodeNic, toAdd) } for _, r := range toAdd { r.LinkIndex = hostLink.Attrs().Index + klog.Infof("adding route %q on %s", r.String(), hostLink.Attrs().Name) if err = netlink.RouteReplace(&r); err != nil && !errors.Is(err, syscall.EEXIST) { klog.Errorf("failed to replace route %v: %v", r, err) } diff --git a/test/e2e/framework/iproute/iproute.go b/test/e2e/framework/iproute/iproute.go index 34ef7722b95..7ac28c5d564 100644 --- a/test/e2e/framework/iproute/iproute.go +++ b/test/e2e/framework/iproute/iproute.go @@ -101,8 +101,10 @@ func (e *execer) exec(cmd string, result interface{}) error { return fmt.Errorf("failed to exec cmd %q: %v\nstdout:\n%s\nstderr:\n%s", cmd, err, stdout, stderr) } - if err = json.Unmarshal(stdout, result); err != nil { - return fmt.Errorf("failed to decode json %q: %v", string(stdout), err) + if result != nil { + if err = json.Unmarshal(stdout, result); err != nil { + return fmt.Errorf("failed to decode json %q: %v", string(stdout), err) + } } return nil @@ -150,6 +152,16 @@ func RouteShow(table, device string, execFunc ExecFunc) ([]Route, error) { return append(routes, routes6...), nil } +func RouteDel(table, dst string, execFunc ExecFunc) error { + e := execer{fn: execFunc} + args := dst + if table != "" { + args = " table " + table + } + + return e.exec("ip route del "+args, nil) +} + func RuleShow(device string, execFunc ExecFunc) ([]Rule, error) { e := execer{fn: execFunc} diff --git a/test/e2e/kube-ovn/node/node.go b/test/e2e/kube-ovn/node/node.go index 35c9a129379..76b368975fb 100644 --- a/test/e2e/kube-ovn/node/node.go +++ b/test/e2e/kube-ovn/node/node.go @@ -7,9 +7,9 @@ import ( "net" "strconv" "strings" + "time" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" clientset "k8s.io/client-go/kubernetes" e2enode "k8s.io/kubernetes/test/e2e/framework/node" @@ -90,8 +90,8 @@ var _ = framework.OrderedDescribe("[group:node]", func() { pod.Spec.HostNetwork = true pod = podClient.CreateSync(pod) - ginkgo.By("Checking ip addresses on ovn0") - links, err := iproute.AddressShow("ovn0", func(cmd ...string) ([]byte, []byte, error) { + ginkgo.By("Checking ip addresses on " + util.NodeNic) + links, err := iproute.AddressShow(util.NodeNic, func(cmd ...string) ([]byte, []byte, error) { return framework.KubectlExec(pod.Namespace, pod.Name, cmd...) }) framework.ExpectNoError(err) @@ -107,91 +107,6 @@ var _ = framework.OrderedDescribe("[group:node]", func() { } }) - framework.ConformanceIt("should add default route on node for join subnet", func() { - f.SkipVersionPriorTo(1, 13, "This feature was introduced in v1.13") - ginkgo.By("Getting join subnet") - join := subnetClient.Get("join") - - ginkgo.By("Getting nodes") - nodeList, err := e2enode.GetReadySchedulableNodes(context.Background(), cs) - framework.ExpectNoError(err) - - ginkgo.By("Validating node annotations") - node := nodeList.Items[0] - framework.ExpectHaveKeyWithValue(node.Annotations, util.AllocatedAnnotation, "true") - framework.ExpectHaveKeyWithValue(node.Annotations, util.CidrAnnotation, join.Spec.CIDRBlock) - framework.ExpectHaveKeyWithValue(node.Annotations, util.GatewayAnnotation, join.Spec.Gateway) - framework.ExpectIPInCIDR(node.Annotations[util.IPAddressAnnotation], join.Spec.CIDRBlock) - framework.ExpectHaveKeyWithValue(node.Annotations, util.LogicalSwitchAnnotation, join.Name) - - podName = "pod-" + framework.RandomSuffix() - ginkgo.By("Creating pod " + podName + " with host network") - cmd := []string{"sh", "-c", "sleep infinity"} - pod := framework.MakePod(namespaceName, podName, nil, nil, image, cmd, nil) - pod.Spec.NodeName = node.Name - pod.Spec.HostNetwork = true - pod = podClient.CreateSync(pod) - - ginkgo.By("Getting node routes") - nodeRoutes, err := iproute.RouteShow("", "ovn0", func(cmd ...string) ([]byte, []byte, error) { - return framework.KubectlExec(pod.Namespace, pod.Name, cmd...) - }) - framework.ExpectNoError(err) - for _, joinCidr := range strings.Split(join.Spec.CIDRBlock, ",") { - found := false - for _, nodeRoute := range nodeRoutes { - if nodeRoute.Dst == joinCidr { - ginkgo.By("Getting ovn0 default route for cidr " + nodeRoute.Dst) - found = true - break - } - if found { - break - } - } - framework.ExpectTrue(found) - } - - ginkgo.By("Getting and recreate kube-ovn-cni pod") - kubePodClient := f.PodClientNS(framework.KubeOvnNamespace) - pods, err := kubePodClient.List(context.Background(), metav1.ListOptions{LabelSelector: metav1.FormatLabelSelector(&metav1.LabelSelector{MatchLabels: map[string]string{"app": "kube-ovn-cni"}}), FieldSelector: fmt.Sprintf("spec.nodeName=%s", node.Name)}) - framework.ExpectNoError(err) - framework.ExpectNotNil(pods) - framework.Logf("Delete kube-ovn-cni pod: %s", pods.Items[0].Name) - err = kubePodClient.Delete(pods.Items[0].Name) - framework.ExpectNoError(err) - kubePodClient.WaitForNotFound(pods.Items[0].Name) - - ginkgo.By("Getting new created kube-ovn-cni pod") - pods, err = kubePodClient.List(context.Background(), metav1.ListOptions{LabelSelector: metav1.FormatLabelSelector(&metav1.LabelSelector{MatchLabels: map[string]string{"app": "kube-ovn-cni"}}), FieldSelector: fmt.Sprintf("spec.nodeName=%s", node.Name)}) - framework.ExpectNoError(err) - framework.ExpectNotNil(pods) - kubePodClient.WaitForRunning(pods.Items[0].Name) - framework.Logf("Get new kube-ovn-cni pod: %s", pods.Items[0].Name) - - nodeRoutes, err = iproute.RouteShow("", "ovn0", func(cmd ...string) ([]byte, []byte, error) { - return framework.KubectlExec(pod.Namespace, pod.Name, cmd...) - }) - framework.ExpectNoError(err) - for _, joinCidr := range strings.Split(join.Spec.CIDRBlock, ",") { - found := false - for _, nodeRoute := range nodeRoutes { - if nodeRoute.Dst == joinCidr { - ginkgo.By("Getting ovn0 default route for cidr " + nodeRoute.Dst) - found = true - break - } - if found { - break - } - } - framework.ExpectTrue(found) - } - - err = podClient.Delete(podName) - framework.ExpectNoError(err) - }) - framework.ConformanceIt("should access overlay pods using node ip", func() { f.SkipVersionPriorTo(1, 12, "This feature was introduced in v1.12") @@ -286,3 +201,107 @@ var _ = framework.OrderedDescribe("[group:node]", func() { } }) }) + +var _ = framework.SerialDescribe("[group:node]", func() { + f := framework.NewDefaultFramework("node") + + var cs clientset.Interface + var podClient *framework.PodClient + var subnetClient *framework.SubnetClient + var podName, namespaceName, image string + ginkgo.BeforeEach(func() { + cs = f.ClientSet + podClient = f.PodClient() + subnetClient = f.SubnetClient() + namespaceName = f.Namespace.Name + podName = "pod-" + framework.RandomSuffix() + if image == "" { + image = framework.GetKubeOvnImage(cs) + } + }) + ginkgo.AfterEach(func() { + ginkgo.By("Deleting pod " + podName) + podClient.DeleteSync(podName) + }) + + framework.ConformanceIt("should add missing routes on node for the join subnet", func() { + f.SkipVersionPriorTo(1, 9, "This feature was introduced in v1.9") + ginkgo.By("Getting join subnet") + join := subnetClient.Get("join") + + ginkgo.By("Getting nodes") + nodeList, err := e2enode.GetReadySchedulableNodes(context.Background(), cs) + framework.ExpectNoError(err) + + ginkgo.By("Validating node annotations") + node := nodeList.Items[0] + framework.ExpectHaveKeyWithValue(node.Annotations, util.AllocatedAnnotation, "true") + framework.ExpectHaveKeyWithValue(node.Annotations, util.CidrAnnotation, join.Spec.CIDRBlock) + framework.ExpectHaveKeyWithValue(node.Annotations, util.GatewayAnnotation, join.Spec.Gateway) + framework.ExpectIPInCIDR(node.Annotations[util.IPAddressAnnotation], join.Spec.CIDRBlock) + framework.ExpectHaveKeyWithValue(node.Annotations, util.LogicalSwitchAnnotation, join.Name) + + podName = "pod-" + framework.RandomSuffix() + ginkgo.By("Creating pod " + podName + " with host network") + cmd := []string{"sh", "-c", "sleep infinity"} + pod := framework.MakePrivilegedPod(namespaceName, podName, nil, nil, image, cmd, nil) + pod.Spec.NodeName = node.Name + pod.Spec.HostNetwork = true + pod = podClient.CreateSync(pod) + + ginkgo.By("Getting node routes on " + util.NodeNic) + cidrs := strings.Split(join.Spec.CIDRBlock, ",") + execFunc := func(cmd ...string) ([]byte, []byte, error) { + return framework.KubectlExec(pod.Namespace, pod.Name, cmd...) + } + routes, err := iproute.RouteShow("", util.NodeNic, execFunc) + framework.ExpectNoError(err) + found := make([]bool, len(cidrs)) + for i, cidr := range cidrs { + for _, route := range routes { + if route.Dst == cidr { + framework.Logf("Found route for cidr " + cidr + " on " + util.NodeNic) + found[i] = true + break + } + } + } + for i, cidr := range cidrs { + framework.ExpectTrue(found[i], "Route for cidr "+cidr+" not found on "+util.NodeNic) + } + + for _, cidr := range strings.Split(join.Spec.CIDRBlock, ",") { + ginkgo.By("Deleting route for " + cidr + " on node " + node.Name) + err = iproute.RouteDel("", cidr, execFunc) + framework.ExpectNoError(err) + } + + ginkgo.By("Waiting for routes for subnet " + join.Name + " to be created") + framework.WaitUntil(2*time.Second, 10*time.Second, func(_ context.Context) (bool, error) { + if routes, err = iproute.RouteShow("", util.NodeNic, execFunc); err != nil { + return false, err + } + + found = make([]bool, len(cidrs)) + for i, cidr := range cidrs { + for _, route := range routes { + if route.Dst == cidr { + framework.Logf("Found route for cidr " + cidr + " on " + util.NodeNic) + found[i] = true + break + } + } + } + for i, cidr := range cidrs { + if !found[i] { + framework.Logf("Route for cidr " + cidr + " not found on " + util.NodeNic) + return false, nil + } + } + return true, nil + }, "") + + err = podClient.Delete(podName) + framework.ExpectNoError(err) + }) +})