fix EIP/SNAT on dynamic Pod annotation (#1918)

docs/snat-and-eip.md

@@ -5,7 +5,7 @@ By using snat, a group of pods can share one same ip address to communicate with
 By using eip, external services can visit a pod with a stable ip and pod will visit external services using the same ip.
 
 ## Prerequisite
-* To take use of OVN L3 Gateway, a dedicated nic *MUST* be bridged into ovs to act as the gateway between overlay and underlay, ops should use other nics to manage the host server.
+* To take use of OVN L3 Gateway, a dedicated nic *MUST* be bridged into ovs to act as the gateway between overlay and underlay, ops should use other NICs to manage the host server.
 * As the nic will emit packets with nat ip directly into underlay network, administrators *MUST* make sure that these packets will not be denied by security rules.
 * SNAT and EIP functions *CANNOT* work together with Cluster interconnection network
 
@@ -25,7 +25,7 @@ data:
   external-gw-nodes: "kube-ovn-worker"  # NodeName in kubernetes which will act the overlay to underlay gateway functions
   external-gw-nic: "eth1"               # The nic that will be bridged into ovs and act as overlay to underlay gateway
   external-gw-addr: "172.56.0.1/16"     # The ip and mask of the underlay physical gateway
-  nic-ip: "172.56.0.254/16"             # The ip and mask of the underlay physical network for logical route externel gw port
+  nic-ip: "172.56.0.100/16"             # The ip and mask of the underlay physical network for logical route external gw port
   nic-mac: "16:52:f3:13:6a:25"          # The mac of nic-ip
 ```
 

pkg/controller/pod.go

@@ -805,6 +805,13 @@ func (c *Controller) handleUpdatePod(key string) error {
 		subnet = podNet.Subnet
 
 		if podIP != "" && subnet.Spec.Vlan == "" && subnet.Spec.Vpc == util.DefaultVpc {
+			node, err := c.nodesLister.Get(pod.Spec.NodeName)
+			if err != nil {
+				klog.Errorf("failed to get node %s: %v", pod.Spec.NodeName, err)
+				return err
+			}
+
+			pgName := getOverlaySubnetsPortGroupName(subnet.Name, node.Name)
 			if c.config.EnableEipSnat && (pod.Annotations[util.EipAnnotation] != "" || pod.Annotations[util.SnatAnnotation] != "") {
 				cm, err := c.configMapsLister.ConfigMaps(c.config.ExternalGatewayConfigNS).Get(util.ExternalGatewayConfig)
 				if err != nil {
@@ -824,20 +831,24 @@ func (c *Controller) handleUpdatePod(key string) error {
 					klog.Errorf("failed to add static route, %v", err)
 					return err
 				}
+
+				// remove lsp from port group to make EIP/SNAT work
+				portName := ovs.PodNameToPortName(podName, pod.Namespace, podNet.ProviderName)
+				c.ovnPgKeyMutex.Lock(pgName)
+				if err = c.ovnClient.PortGroupRemovePort(pgName, portName); err != nil {
+					c.ovnPgKeyMutex.Unlock(pgName)
+					return err
+				}
+				c.ovnPgKeyMutex.Unlock(pgName)
+
 			} else {
 				if subnet.Spec.GatewayType == kubeovnv1.GWDistributedType && pod.Annotations[util.NorthGatewayAnnotation] == "" {
-					node, err := c.nodesLister.Get(pod.Spec.NodeName)
-					if err != nil {
-						klog.Errorf("get node %s failed %v", pod.Spec.NodeName, err)
-						return err
-					}
-
 					nodeTunlIPAddr, err := getNodeTunlIP(node)
 					if err != nil {
 						return err
 					}
 
-					pgName := getOverlaySubnetsPortGroupName(subnet.Name, node.Name)
+					var added bool
 					for _, nodeAddr := range nodeTunlIPAddr {
 						for _, podAddr := range strings.Split(podIP, ",") {
 							if util.CheckProtocol(nodeAddr.String()) != util.CheckProtocol(podAddr) {
@@ -851,6 +862,12 @@ func (c *Controller) handleUpdatePod(key string) error {
 								return err
 							}
 							c.ovnPgKeyMutex.Unlock(pgName)
+
+							added = true
+							break
+						}
+						if added {
+							break
 						}
 					}
 				}

yamls/ovn-external-gw-config.yaml

@@ -5,7 +5,9 @@ metadata:
   namespace: kube-system
 data:
   enable-external-gw: "true"
+  type: "centralized"
   external-gw-nodes: "kube-ovn-worker"  # NodeName in kubernetes which will act the overlay to underlay gateway functions
   external-gw-nic: "eth1"               # The nic that will be bridged into ovs and act as overlay to underlay gateway
-  nic-ip: "172.56.0.1/16"               # The ip and mask of the underlay physical gateway
+  external-gw-addr: "172.56.0.1/16"     # The ip and mask of the underlay physical gateway
+  nic-ip: "172.56.0.100/16"             # The ip and mask of the underlay physical network for logical route external gw port
   nic-mac: "16:52:f3:13:6a:25"          # The mac of the underlay physical gateway