use route policy to reimplement northGateway

Signed-off-by: oilbeater <[email protected]>
kubeovn · Jul 17, 2024 · 2ded787 · 2ded787
1 parent 69befe3
commit 2ded787
Show file tree

Hide file tree

Showing 4 changed files with 86 additions and 42 deletions.
diff --git a/pkg/controller/gc.go b/pkg/controller/gc.go
@@ -3,6 +3,7 @@ package controller
 import (
 	"context"
 	"fmt"
+	"slices"
 	"strings"
 	"unicode"
 
@@ -32,6 +33,7 @@ func (c *Controller) gc() error {
 		c.gcLogicalSwitchPort,
 		c.gcLoadBalancer,
 		c.gcPortGroup,
+		c.gcRoutePolicy,
 		c.gcStaticRoute,
 		c.gcVpcNatGateway,
 		c.gcLogicalRouterPort,
@@ -671,6 +673,45 @@ func (c *Controller) gcPortGroup() error {
 	return nil
 }
 
+func (c *Controller) gcRoutePolicy() error {
+	klog.Infof("start to gc route policy")
+
+	policies, err := c.OVNNbClient.ListLogicalRouterPolicies(c.config.ClusterRouter, util.NorthGatewayRoutePolicyPriority, nil, true)
+	if err != nil {
+		klog.Errorf("failed to list route policy, %v", err)
+		return err
+	}
+
+	podIPs := []string{}
+	pods, err := c.podsLister.List(labels.Everything())
+	if err != nil {
+		klog.Errorf("failed to list pods, %v", err)
+		return err
+	}
+	for _, pod := range pods {
+		if pod.Annotations != nil && pod.Annotations[util.NorthGatewayAnnotation] != "" {
+			podIPs = append(podIPs, strings.Split(pod.Annotations[util.IPAddressAnnotation], ",")...)
+		}
+	}
+
+	for _, policy := range policies {
+		parts := strings.Split(policy.Match, "==")
+		if len(parts) != 2 {
+			continue
+		}
+		srcIP := strings.TrimSpace(parts[1])
+		if !slices.Contains(podIPs, srcIP) {
+			klog.Infof("gc route policy %s", policy.Match)
+			if err := c.OVNNbClient.DeleteLogicalRouterPolicy(c.config.ClusterRouter, policy.Priority, policy.Match); err != nil {
+				klog.Errorf("failed to delete route policy %s: %v", policy.Match, err)
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
 func (c *Controller) gcStaticRoute() error {
 	klog.Infof("start to gc static routes")
 	routes, err := c.OVNNbClient.ListLogicalRouterStaticRoutes(c.config.ClusterRouter, nil, nil, "", nil)

diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go
@@ -821,18 +821,32 @@ func (c *Controller) reconcileRouteSubnets(cachedPod, pod *v1.Pod, needRoutePodN
 					}
 				}
 
-				if pod.Annotations[util.NorthGatewayAnnotation] != "" {
-					if err := c.addStaticRouteToVpc(
-						subnet.Spec.Vpc,
-						&kubeovnv1.StaticRoute{
-							Policy:     kubeovnv1.PolicySrc,
-							CIDR:       podIP,
-							NextHopIP:  pod.Annotations[util.NorthGatewayAnnotation],
-							RouteTable: subnet.Spec.RouteTable,
-						},
-					); err != nil {
-						klog.Errorf("failed to add static route, %v", err)
-						return err
+				if pod.Annotations[util.NorthGatewayAnnotation] != "" && pod.Annotations[util.IPAddressAnnotation] != "" {
+					for _, podAddr := range strings.Split(pod.Annotations[util.IPAddressAnnotation], ",") {
+						if util.CheckProtocol(podAddr) != util.CheckProtocol(pod.Annotations[util.NorthGatewayAnnotation]) {
+							continue
+						}
+						ipSuffix := "ip4"
+						if util.CheckProtocol(podAddr) == kubeovnv1.ProtocolIPv6 {
+							ipSuffix = "ip6"
+						}
+
+						if err := c.addPolicyRouteToVpc(
+							subnet.Spec.Vpc,
+							&kubeovnv1.PolicyRoute{
+								Priority:  util.NorthGatewayRoutePolicyPriority,
+								Match:     fmt.Sprintf("%s.src == %s", ipSuffix, podAddr),
+								Action:    kubeovnv1.PolicyRouteActionReroute,
+								NextHopIP: pod.Annotations[util.NorthGatewayAnnotation],
+							},
+							map[string]string{
+								"vendor": util.CniTypeName,
+								"subnet": subnet.Name,
+							},
+						); err != nil {
+							klog.Errorf("failed to add policy route, %v", err)
+							return err
+						}
 					}
 				} else if c.config.EnableEipSnat {
 					if err = c.deleteStaticRouteFromVpc(
@@ -971,19 +985,22 @@ func (c *Controller) handleDeletePod(key string) error {
 					klog.Error(err)
 					return err
 				}
-				// If pod has snat or eip, also need delete staticRoute when delete pod
+
 				if vpc.Name == c.config.ClusterRouter {
-					if err = c.deleteStaticRouteFromVpc(
+					ipSuffix := "ip4"
+					if util.CheckProtocol(address.IP) == kubeovnv1.ProtocolIPv6 {
+						ipSuffix = "ip6"
+					}
+					if err = c.deletePolicyRouteFromVpc(
 						vpc.Name,
-						subnet.Spec.RouteTable,
-						address.IP,
-						"",
-						kubeovnv1.PolicyDst,
+						util.NorthGatewayRoutePolicyPriority,
+						fmt.Sprintf("%s.src == %s", ipSuffix, address.IP),
 					); err != nil {
 						klog.Errorf("failed to delete static route, %v", err)
 						return err
 					}
 				}
+
 				if c.config.EnableEipSnat {
 					if pod.Annotations[util.EipAnnotation] != "" {
 						if err = c.OVNNbClient.DeleteNat(c.config.ClusterRouter, ovnnb.NATTypeDNATAndSNAT, pod.Annotations[util.EipAnnotation], address.IP); err != nil {

diff --git a/pkg/controller/subnet.go b/pkg/controller/subnet.go
@@ -1546,24 +1546,9 @@ func (c *Controller) reconcileDistributedSubnetRouteInDefaultVpc(subnet *kubeovn
 				continue
 			}
 
-			if pod.Annotations[util.NorthGatewayAnnotation] != "" {
-				if err := c.addStaticRouteToVpc(
-					subnet.Spec.Vpc,
-					&kubeovnv1.StaticRoute{
-						Policy:     kubeovnv1.PolicySrc,
-						CIDR:       pod.Annotations[fmt.Sprintf(util.IPAddressAnnotationTemplate, podNet.ProviderName)],
-						NextHopIP:  pod.Annotations[util.NorthGatewayAnnotation],
-						RouteTable: util.MainRouteTable,
-					},
-				); err != nil {
-					klog.Errorf("add static route failed, %v", err)
-					return err
-				}
-			} else {
-				podName := c.getNameByPod(pod)
-				portName := ovs.PodNameToPortName(podName, pod.Namespace, podNet.ProviderName)
-				podPorts = append(podPorts, portName)
-			}
+			podName := c.getNameByPod(pod)
+			portName := ovs.PodNameToPortName(podName, pod.Namespace, podNet.ProviderName)
+			podPorts = append(podPorts, portName)
 		}
 
 		if pod.Annotations[util.NorthGatewayAnnotation] != "" {

diff --git a/pkg/util/const.go b/pkg/util/const.go
@@ -190,12 +190,13 @@ const (
 	OvnFip      = "ovn"
 	IptablesFip = "iptables"
 
-	U2OSubnetPolicyPriority     = 29400
-	GatewayRouterPolicyPriority = 29000
-	OvnICPolicyPriority         = 29500
-	NodeRouterPolicyPriority    = 30000
-	NodeLocalDNSPolicyPriority  = 30100
-	SubnetRouterPolicyPriority  = 31000
+	U2OSubnetPolicyPriority         = 29400
+	GatewayRouterPolicyPriority     = 29000
+	NorthGatewayRoutePolicyPriority = 29250
+	OvnICPolicyPriority             = 29500
+	NodeRouterPolicyPriority        = 30000
+	NodeLocalDNSPolicyPriority      = 30100
+	SubnetRouterPolicyPriority      = 31000
 
 	OffloadType  = "offload-port"
 	InternalType = "internal-port"