From 4846496fb9951927e0c2785c4dc2a0fc87fcf7b1 Mon Sep 17 00:00:00 2001 From: zhangbingbing Date: Fri, 14 Oct 2022 01:05:59 +0000 Subject: [PATCH] feature: support default vpc use nat gw pod as cust vpc --- pkg/controller/controller.go | 4 +- pkg/controller/gc.go | 16 ++++ pkg/controller/node.go | 13 +++- pkg/controller/subnet.go | 8 +- pkg/controller/vpc.go | 143 ++++++++++++++++------------------- 5 files changed, 99 insertions(+), 85 deletions(-) diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index a7bbb8c2a60..d0861f6dc3b 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -578,9 +578,7 @@ func (c *Controller) Run(stopCh <-chan struct{}) { } } - // The static route for node gw can be deleted when gc static route, so add it after gc process - dstIp := "0.0.0.0/0,::/0" - if err := c.ovnLegacyClient.AddStaticRoute("", dstIp, c.config.NodeSwitchGateway, c.config.ClusterRouter, util.NormalRouteType); err != nil { + if err := c.addNodeGwStaticRoute(); err != nil { klog.Errorf("failed to add static route for node gw: %v", err) } diff --git a/pkg/controller/gc.go b/pkg/controller/gc.go index f50af248ef1..9992fb9e316 100644 --- a/pkg/controller/gc.go +++ b/pkg/controller/gc.go @@ -644,7 +644,23 @@ func (c *Controller) gcStaticRoute() error { klog.Errorf("failed to list static route %v", err) return err } + defaultVpc, err := c.vpcsLister.Get(util.DefaultVpc) + if err != nil { + klog.Errorf("failed to get default vpc, %v", err) + return err + } + var keepStaticRoute bool for _, route := range routes { + keepStaticRoute = false + for _, item := range defaultVpc.Spec.StaticRoutes { + if route.CIDR == item.CIDR && route.NextHop == item.NextHopIP { + keepStaticRoute = true + break + } + } + if keepStaticRoute { + continue + } if route.CIDR != "0.0.0.0/0" && route.CIDR != "::/0" && c.ipam.ContainAddress(route.CIDR) { exist, err := c.ovnLegacyClient.NatRuleExists(route.CIDR) if exist || err != nil { diff --git a/pkg/controller/node.go b/pkg/controller/node.go index df6e3ca252a..800549192cb 100644 --- a/pkg/controller/node.go +++ b/pkg/controller/node.go @@ -1032,13 +1032,23 @@ func (c *Controller) validateChassis(node *v1.Node) error { } func (c *Controller) addNodeGwStaticRoute() error { + // If user not manage static route for default vpc, just add route about ovn-default to join + if vpc, err := c.vpcsLister.Get(util.DefaultVpc); err != nil || vpc.Spec.StaticRoutes != nil { + existRoute, err := c.ovnLegacyClient.GetStaticRouteList(c.config.ClusterRouter) + if err != nil { + klog.Errorf("failed to get vpc %s static route list, %v", c.config.ClusterRouter, err) + } + if len(existRoute) != 0 { + klog.Infof("skip add static route for node gw") + return nil + } + } dstCidr := "0.0.0.0/0,::/0" for _, cidrBlock := range strings.Split(dstCidr, ",") { for _, nextHop := range strings.Split(c.config.NodeSwitchGateway, ",") { if util.CheckProtocol(cidrBlock) != util.CheckProtocol(nextHop) { continue } - exist, err := c.checkRouteExist(nextHop, cidrBlock, ovs.PolicyDstIP) if err != nil { klog.Errorf("get static route for node gw error %v", err) @@ -1046,6 +1056,7 @@ func (c *Controller) addNodeGwStaticRoute() error { } if !exist { + klog.Infof("add static route for node gw") if err := c.ovnLegacyClient.AddStaticRoute("", cidrBlock, nextHop, c.config.ClusterRouter, util.NormalRouteType); err != nil { klog.Errorf("failed to add static route for node gw: %v", err) return err diff --git a/pkg/controller/subnet.go b/pkg/controller/subnet.go index 13239145343..c40d1866639 100644 --- a/pkg/controller/subnet.go +++ b/pkg/controller/subnet.go @@ -734,7 +734,7 @@ func (c *Controller) handleDeleteRoute(subnet *kubeovnv1.Subnet) error { } return err } - return c.deleteStaticRoute(subnet.Spec.CIDRBlock, vpc.Status.Router, subnet) + return c.deleteStaticRoute(subnet.Spec.CIDRBlock, vpc.Status.Router) } func (c *Controller) handleDeleteLogicalSwitch(key string) (err error) { @@ -1006,13 +1006,13 @@ func (c *Controller) reconcileOvnRoute(subnet *kubeovnv1.Subnet) error { if subnet.Spec.Vlan != "" && !subnet.Spec.LogicalGateway { for _, pod := range pods { if pod.Annotations[util.LogicalSwitchAnnotation] == subnet.Name && pod.Annotations[util.IpAddressAnnotation] != "" { - if err := c.deleteStaticRoute(pod.Annotations[util.IpAddressAnnotation], c.config.ClusterRouter, subnet); err != nil { + if err := c.deleteStaticRoute(pod.Annotations[util.IpAddressAnnotation], c.config.ClusterRouter); err != nil { return err } } } - if err := c.deleteStaticRoute(subnet.Spec.CIDRBlock, c.config.ClusterRouter, subnet); err != nil { + if err := c.deleteStaticRoute(subnet.Spec.CIDRBlock, c.config.ClusterRouter); err != nil { return err } @@ -1284,7 +1284,7 @@ func (c *Controller) reconcileOvnRoute(subnet *kubeovnv1.Subnet) error { return nil } -func (c *Controller) deleteStaticRoute(ip, router string, subnet *kubeovnv1.Subnet) error { +func (c *Controller) deleteStaticRoute(ip, router string) error { for _, ipStr := range strings.Split(ip, ",") { if err := c.ovnLegacyClient.DeleteStaticRoute(ipStr, router); err != nil { klog.Errorf("failed to delete static route %s, %v", ipStr, err) diff --git a/pkg/controller/vpc.go b/pkg/controller/vpc.go index 24028cf2bdb..fca4e8486d2 100644 --- a/pkg/controller/vpc.go +++ b/pkg/controller/vpc.go @@ -364,56 +364,54 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error { } } - if vpc.Name != util.DefaultVpc { - // handle static route - existRoute, err := c.ovnLegacyClient.GetStaticRouteList(vpc.Name) - if err != nil { - klog.Errorf("failed to get vpc %s static route list, %v", vpc.Name, err) - return err - } + // handle static route + existRoute, err := c.ovnLegacyClient.GetStaticRouteList(vpc.Name) + if err != nil { + klog.Errorf("failed to get vpc %s static route list, %v", vpc.Name, err) + return err + } - routeNeedDel, routeNeedAdd, err := diffStaticRoute(existRoute, vpc.Spec.StaticRoutes) - if err != nil { - klog.Errorf("failed to diff vpc %s static route, %v", vpc.Name, err) + routeNeedDel, routeNeedAdd, err := diffStaticRoute(existRoute, vpc.Spec.StaticRoutes) + if err != nil { + klog.Errorf("failed to diff vpc %s static route, %v", vpc.Name, err) + return err + } + for _, item := range routeNeedDel { + if err = c.ovnLegacyClient.DeleteStaticRoute(item.CIDR, vpc.Name); err != nil { + klog.Errorf("del vpc %s static route failed, %v", vpc.Name, err) return err } - for _, item := range routeNeedDel { - if err = c.ovnLegacyClient.DeleteStaticRoute(item.CIDR, vpc.Name); err != nil { - klog.Errorf("del vpc %s static route failed, %v", vpc.Name, err) - return err - } - } + } - for _, item := range routeNeedAdd { - if err = c.ovnLegacyClient.AddStaticRoute(convertPolicy(item.Policy), item.CIDR, item.NextHopIP, vpc.Name, util.NormalRouteType); err != nil { - klog.Errorf("add static route to vpc %s failed, %v", vpc.Name, err) - return err - } - } - // handle policy route - existPolicyRoute, err := c.ovnLegacyClient.GetPolicyRouteList(vpc.Name) - if err != nil { - klog.Errorf("failed to get vpc %s policy route list, %v", vpc.Name, err) + for _, item := range routeNeedAdd { + if err = c.ovnLegacyClient.AddStaticRoute(convertPolicy(item.Policy), item.CIDR, item.NextHopIP, vpc.Name, util.NormalRouteType); err != nil { + klog.Errorf("add static route to vpc %s failed, %v", vpc.Name, err) return err } + } + // handle policy route + existPolicyRoute, err := c.ovnLegacyClient.GetPolicyRouteList(vpc.Name) + if err != nil { + klog.Errorf("failed to get vpc %s policy route list, %v", vpc.Name, err) + return err + } - policyRouteNeedDel, policyRouteNeedAdd, err := diffPolicyRoute(existPolicyRoute, vpc.Spec.PolicyRoutes) - if err != nil { - klog.Errorf("failed to diff vpc %s policy route, %v", vpc.Name, err) + policyRouteNeedDel, policyRouteNeedAdd, err := diffPolicyRoute(existPolicyRoute, vpc.Spec.PolicyRoutes) + if err != nil { + klog.Errorf("failed to diff vpc %s policy route, %v", vpc.Name, err) + return err + } + for _, item := range policyRouteNeedDel { + if err = c.ovnLegacyClient.DeletePolicyRoute(vpc.Name, item.Priority, item.Match); err != nil { + klog.Errorf("del vpc %s policy route failed, %v", vpc.Name, err) return err } - for _, item := range policyRouteNeedDel { - if err = c.ovnLegacyClient.DeletePolicyRoute(vpc.Name, item.Priority, item.Match); err != nil { - klog.Errorf("del vpc %s policy route failed, %v", vpc.Name, err) - return err - } - } - for _, item := range policyRouteNeedAdd { - externalIDs := map[string]string{"vendor": util.CniTypeName} - if err = c.ovnLegacyClient.AddPolicyRoute(vpc.Name, item.Priority, item.Match, string(item.Action), item.NextHopIP, externalIDs); err != nil { - klog.Errorf("add policy route to vpc %s failed, %v", vpc.Name, err) - return err - } + } + for _, item := range policyRouteNeedAdd { + externalIDs := map[string]string{"vendor": util.CniTypeName} + if err = c.ovnLegacyClient.AddPolicyRoute(vpc.Name, item.Priority, item.Match, string(item.Action), item.NextHopIP, externalIDs); err != nil { + klog.Errorf("add policy route to vpc %s failed, %v", vpc.Name, err) + return err } } @@ -538,47 +536,38 @@ func getStaticRouteItemKey(item *kubeovnv1.StaticRoute) (key string) { func formatVpc(vpc *kubeovnv1.Vpc, c *Controller) error { var changed bool - - // default vpc does not support custom route - if vpc.Status.Default { - if len(vpc.Spec.StaticRoutes) > 0 { + for _, item := range vpc.Spec.StaticRoutes { + // check policy + if item.Policy == "" { + item.Policy = kubeovnv1.PolicyDst changed = true - vpc.Spec.StaticRoutes = nil } - } else { - for _, item := range vpc.Spec.StaticRoutes { - // check policy - if item.Policy == "" { - item.Policy = kubeovnv1.PolicyDst - changed = true - } - if item.Policy != kubeovnv1.PolicyDst && item.Policy != kubeovnv1.PolicySrc { - return fmt.Errorf("unknown policy type: %s", item.Policy) - } - // check cidr - if strings.Contains(item.CIDR, "/") { - if _, _, err := net.ParseCIDR(item.CIDR); err != nil { - return fmt.Errorf("invalid cidr %s: %w", item.CIDR, err) - } - } else if ip := net.ParseIP(item.CIDR); ip == nil { - return fmt.Errorf("invalid IP %s", item.CIDR) - } - // check next hop ip - if ip := net.ParseIP(item.NextHopIP); ip == nil { - return fmt.Errorf("invalid next hop IP %s", item.NextHopIP) + if item.Policy != kubeovnv1.PolicyDst && item.Policy != kubeovnv1.PolicySrc { + return fmt.Errorf("unknown policy type: %s", item.Policy) + } + // check cidr + if strings.Contains(item.CIDR, "/") { + if _, _, err := net.ParseCIDR(item.CIDR); err != nil { + return fmt.Errorf("invalid cidr %s: %w", item.CIDR, err) } + } else if ip := net.ParseIP(item.CIDR); ip == nil { + return fmt.Errorf("invalid IP %s", item.CIDR) + } + // check next hop ip + if ip := net.ParseIP(item.NextHopIP); ip == nil { + return fmt.Errorf("invalid next hop IP %s", item.NextHopIP) } + } - for _, route := range vpc.Spec.PolicyRoutes { - if route.Action != kubeovnv1.PolicyRouteActionReroute { - if route.NextHopIP != "" { - route.NextHopIP = "" - changed = true - } - } else { - if ip := net.ParseIP(route.NextHopIP); ip == nil { - return fmt.Errorf("bad next hop ip: %s", route.NextHopIP) - } + for _, route := range vpc.Spec.PolicyRoutes { + if route.Action != kubeovnv1.PolicyRouteActionReroute { + if route.NextHopIP != "" { + route.NextHopIP = "" + changed = true + } + } else { + if ip := net.ParseIP(route.NextHopIP); ip == nil { + return fmt.Errorf("bad next hop ip: %s", route.NextHopIP) } } }