Skip to content

Commit

Permalink
feature: support default vpc use nat gw pod as cust vpc (#1979)
Browse files Browse the repository at this point in the history 
Co-authored-by: zhangbingbing <[email protected]>

(cherry picked from commit eea7888)
  • Loading branch information
@bobz965 @oilbeater
bobz965 authored and oilbeater committed Oct 21, 2022
1 parent e7f3fb5 commit 8c9232c
Show file tree
Hide file tree
Showing 5 changed files with 100 additions and 85 deletions.
5 changes: 2 additions & 3 deletions pkg/controller/controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -527,9 +527,8 @@ func (c *Controller) Run(stopCh <-chan struct{}) {
if err := c.initSyncCrdVlans(); err != nil {
klog.Errorf("failed to sync crd vlans: %v", err)
}
// The static route for node gw can be deleted when gc static route, so add it after gc process
dstIp := "0.0.0.0/0,::/0"
if err := c.ovnLegacyClient.AddStaticRoute("", dstIp, c.config.NodeSwitchGateway, c.config.ClusterRouter, util.NormalRouteType); err != nil {

if err := c.addNodeGwStaticRoute(); err != nil {
klog.Errorf("failed to add static route for node gw: %v", err)
}

Expand Down
16 changes: 16 additions & 0 deletions pkg/controller/gc.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -605,7 +605,23 @@ func (c *Controller) gcStaticRoute() error {
klog.Errorf("failed to list static route %v", err)
return err
}
defaultVpc, err := c.vpcsLister.Get(util.DefaultVpc)
if err != nil {
klog.Errorf("failed to get default vpc, %v", err)
return err
}
var keepStaticRoute bool
for _, route := range routes {
keepStaticRoute = false
for _, item := range defaultVpc.Spec.StaticRoutes {
if route.CIDR == item.CIDR && route.NextHop == item.NextHopIP {
keepStaticRoute = true
break
}
}
if keepStaticRoute {
continue
}
if route.CIDR != "0.0.0.0/0" && route.CIDR != "::/0" && c.ipam.ContainAddress(route.CIDR) {
klog.Infof("gc static route %s %s %s", route.Policy, route.CIDR, route.NextHop)
exist, err := c.ovnLegacyClient.NatRuleExists(route.CIDR)
Expand Down
13 changes: 12 additions & 1 deletion pkg/controller/node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1031,20 +1031,31 @@ func (c *Controller) EnsureChassisOnlyForClusterNodes(node *v1.Node) error {
}

func (c *Controller) addNodeGwStaticRoute() error {
// If user not manage static route for default vpc, just add route about ovn-default to join
if vpc, err := c.vpcsLister.Get(util.DefaultVpc); err != nil || vpc.Spec.StaticRoutes != nil {
existRoute, err := c.ovnLegacyClient.GetStaticRouteList(c.config.ClusterRouter)
if err != nil {
klog.Errorf("failed to get vpc %s static route list, %v", c.config.ClusterRouter, err)
}
if len(existRoute) != 0 {
klog.Infof("skip add static route for node gw")
return nil
}
}
dstCidr := "0.0.0.0/0,::/0"
for _, cidrBlock := range strings.Split(dstCidr, ",") {
for _, nextHop := range strings.Split(c.config.NodeSwitchGateway, ",") {
if util.CheckProtocol(cidrBlock) != util.CheckProtocol(nextHop) {
continue
}

exist, err := c.checkRouteExist(nextHop, cidrBlock, ovs.PolicyDstIP)
if err != nil {
klog.Errorf("get static route for node gw error %v", err)
return err
}

if !exist {
klog.Infof("add static route for node gw")
if err := c.ovnLegacyClient.AddStaticRoute("", cidrBlock, nextHop, c.config.ClusterRouter, util.NormalRouteType); err != nil {
klog.Errorf("failed to add static route for node gw: %v", err)
return err
Expand Down
8 changes: 4 additions & 4 deletions pkg/controller/subnet.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -737,7 +737,7 @@ func (c *Controller) handleDeleteRoute(subnet *kubeovnv1.Subnet) error {
}
return err
}
return c.deleteStaticRoute(subnet.Spec.CIDRBlock, vpc.Status.Router, subnet)
return c.deleteStaticRoute(subnet.Spec.CIDRBlock, vpc.Status.Router)
}

func (c *Controller) handleDeleteLogicalSwitch(key string) (err error) {
Expand Down Expand Up @@ -1010,13 +1010,13 @@ func (c *Controller) reconcileOvnRoute(subnet *kubeovnv1.Subnet) error {
if subnet.Spec.Vlan != "" && !subnet.Spec.LogicalGateway {
for _, pod := range pods {
if pod.Annotations[util.LogicalSwitchAnnotation] == subnet.Name && pod.Annotations[util.IpAddressAnnotation] != "" {
if err := c.deleteStaticRoute(pod.Annotations[util.IpAddressAnnotation], c.config.ClusterRouter, subnet); err != nil {
if err := c.deleteStaticRoute(pod.Annotations[util.IpAddressAnnotation], c.config.ClusterRouter); err != nil {
return err
}
}
}

if err := c.deleteStaticRoute(subnet.Spec.CIDRBlock, c.config.ClusterRouter, subnet); err != nil {
if err := c.deleteStaticRoute(subnet.Spec.CIDRBlock, c.config.ClusterRouter); err != nil {
return err
}

Expand Down Expand Up @@ -1293,7 +1293,7 @@ func (c *Controller) reconcileOvnRoute(subnet *kubeovnv1.Subnet) error {
return nil
}

func (c *Controller) deleteStaticRoute(ip, router string, subnet *kubeovnv1.Subnet) error {
func (c *Controller) deleteStaticRoute(ip, router string) error {
for _, ipStr := range strings.Split(ip, ",") {
if err := c.ovnLegacyClient.DeleteStaticRoute(ipStr, router); err != nil {
klog.Errorf("failed to delete static route %s, %v", ipStr, err)
Expand Down
143 changes: 66 additions & 77 deletions pkg/controller/vpc.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -364,56 +364,54 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error {
}
}

if vpc.Name != util.DefaultVpc {
// handle static route
existRoute, err := c.ovnLegacyClient.GetStaticRouteList(vpc.Name)
if err != nil {
klog.Errorf("failed to get vpc %s static route list, %v", vpc.Name, err)
return err
}
// handle static route
existRoute, err := c.ovnLegacyClient.GetStaticRouteList(vpc.Name)
if err != nil {
klog.Errorf("failed to get vpc %s static route list, %v", vpc.Name, err)
return err
}

routeNeedDel, routeNeedAdd, err := diffStaticRoute(existRoute, vpc.Spec.StaticRoutes)
if err != nil {
klog.Errorf("failed to diff vpc %s static route, %v", vpc.Name, err)
routeNeedDel, routeNeedAdd, err := diffStaticRoute(existRoute, vpc.Spec.StaticRoutes)
if err != nil {
klog.Errorf("failed to diff vpc %s static route, %v", vpc.Name, err)
return err
}
for _, item := range routeNeedDel {
if err = c.ovnLegacyClient.DeleteStaticRoute(item.CIDR, vpc.Name); err != nil {
klog.Errorf("del vpc %s static route failed, %v", vpc.Name, err)
return err
}
for _, item := range routeNeedDel {
if err = c.ovnLegacyClient.DeleteStaticRoute(item.CIDR, vpc.Name); err != nil {
klog.Errorf("del vpc %s static route failed, %v", vpc.Name, err)
return err
}
}
}

for _, item := range routeNeedAdd {
if err = c.ovnLegacyClient.AddStaticRoute(convertPolicy(item.Policy), item.CIDR, item.NextHopIP, vpc.Name, util.NormalRouteType); err != nil {
klog.Errorf("add static route to vpc %s failed, %v", vpc.Name, err)
return err
}
}
// handle policy route
existPolicyRoute, err := c.ovnLegacyClient.GetPolicyRouteList(vpc.Name)
if err != nil {
klog.Errorf("failed to get vpc %s policy route list, %v", vpc.Name, err)
for _, item := range routeNeedAdd {
if err = c.ovnLegacyClient.AddStaticRoute(convertPolicy(item.Policy), item.CIDR, item.NextHopIP, vpc.Name, util.NormalRouteType); err != nil {
klog.Errorf("add static route to vpc %s failed, %v", vpc.Name, err)
return err
}
}
// handle policy route
existPolicyRoute, err := c.ovnLegacyClient.GetPolicyRouteList(vpc.Name)
if err != nil {
klog.Errorf("failed to get vpc %s policy route list, %v", vpc.Name, err)
return err
}

policyRouteNeedDel, policyRouteNeedAdd, err := diffPolicyRoute(existPolicyRoute, vpc.Spec.PolicyRoutes)
if err != nil {
klog.Errorf("failed to diff vpc %s policy route, %v", vpc.Name, err)
policyRouteNeedDel, policyRouteNeedAdd, err := diffPolicyRoute(existPolicyRoute, vpc.Spec.PolicyRoutes)
if err != nil {
klog.Errorf("failed to diff vpc %s policy route, %v", vpc.Name, err)
return err
}
for _, item := range policyRouteNeedDel {
if err = c.ovnLegacyClient.DeletePolicyRoute(vpc.Name, item.Priority, item.Match); err != nil {
klog.Errorf("del vpc %s policy route failed, %v", vpc.Name, err)
return err
}
for _, item := range policyRouteNeedDel {
if err = c.ovnLegacyClient.DeletePolicyRoute(vpc.Name, item.Priority, item.Match); err != nil {
klog.Errorf("del vpc %s policy route failed, %v", vpc.Name, err)
return err
}
}
for _, item := range policyRouteNeedAdd {
externalIDs := map[string]string{"vendor": util.CniTypeName}
if err = c.ovnLegacyClient.AddPolicyRoute(vpc.Name, item.Priority, item.Match, string(item.Action), item.NextHopIP, externalIDs); err != nil {
klog.Errorf("add policy route to vpc %s failed, %v", vpc.Name, err)
return err
}
}
for _, item := range policyRouteNeedAdd {
externalIDs := map[string]string{"vendor": util.CniTypeName}
if err = c.ovnLegacyClient.AddPolicyRoute(vpc.Name, item.Priority, item.Match, string(item.Action), item.NextHopIP, externalIDs); err != nil {
klog.Errorf("add policy route to vpc %s failed, %v", vpc.Name, err)
return err
}
}

Expand Down Expand Up @@ -538,47 +536,38 @@ func getStaticRouteItemKey(item *kubeovnv1.StaticRoute) (key string) {

func formatVpc(vpc *kubeovnv1.Vpc, c *Controller) error {
var changed bool

// default vpc does not support custom route
if vpc.Status.Default {
if len(vpc.Spec.StaticRoutes) > 0 {
for _, item := range vpc.Spec.StaticRoutes {
// check policy
if item.Policy == "" {
item.Policy = kubeovnv1.PolicyDst
changed = true
vpc.Spec.StaticRoutes = nil
}
} else {
for _, item := range vpc.Spec.StaticRoutes {
// check policy
if item.Policy == "" {
item.Policy = kubeovnv1.PolicyDst
changed = true
}
if item.Policy != kubeovnv1.PolicyDst && item.Policy != kubeovnv1.PolicySrc {
return fmt.Errorf("unknown policy type: %s", item.Policy)
}
// check cidr
if strings.Contains(item.CIDR, "/") {
if _, _, err := net.ParseCIDR(item.CIDR); err != nil {
return fmt.Errorf("invalid cidr %s: %w", item.CIDR, err)
}
} else if ip := net.ParseIP(item.CIDR); ip == nil {
return fmt.Errorf("invalid IP %s", item.CIDR)
}
// check next hop ip
if ip := net.ParseIP(item.NextHopIP); ip == nil {
return fmt.Errorf("invalid next hop IP %s", item.NextHopIP)
if item.Policy != kubeovnv1.PolicyDst && item.Policy != kubeovnv1.PolicySrc {
return fmt.Errorf("unknown policy type: %s", item.Policy)
}
// check cidr
if strings.Contains(item.CIDR, "/") {
if _, _, err := net.ParseCIDR(item.CIDR); err != nil {
return fmt.Errorf("invalid cidr %s: %w", item.CIDR, err)
}
} else if ip := net.ParseIP(item.CIDR); ip == nil {
return fmt.Errorf("invalid IP %s", item.CIDR)
}
// check next hop ip
if ip := net.ParseIP(item.NextHopIP); ip == nil {
return fmt.Errorf("invalid next hop IP %s", item.NextHopIP)
}
}

for _, route := range vpc.Spec.PolicyRoutes {
if route.Action != kubeovnv1.PolicyRouteActionReroute {
if route.NextHopIP != "" {
route.NextHopIP = ""
changed = true
}
} else {
if ip := net.ParseIP(route.NextHopIP); ip == nil {
return fmt.Errorf("bad next hop ip: %s", route.NextHopIP)
}
for _, route := range vpc.Spec.PolicyRoutes {
if route.Action != kubeovnv1.PolicyRouteActionReroute {
if route.NextHopIP != "" {
route.NextHopIP = ""
changed = true
}
} else {
if ip := net.ParseIP(route.NextHopIP); ip == nil {
return fmt.Errorf("bad next hop ip: %s", route.NextHopIP)
}
}
}
Expand Down

0 comments on commit 8c9232c

Please sign in to comment.