From 98f7bc08abb4f083a1653c231189ef830e6e9e9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E7=A5=96=E5=BB=BA?= Date: Tue, 20 Sep 2022 10:16:25 +0800 Subject: [PATCH] fix EIP/SNAT on dynamic Pod annotation (#1918) --- docs/snat-and-eip.md | 4 ++-- pkg/controller/pod.go | 31 ++++++++++++++++++++++++------- yamls/ovn-external-gw-config.yaml | 4 +++- 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/docs/snat-and-eip.md b/docs/snat-and-eip.md index 66d1988b49b..8b0f9404df6 100644 --- a/docs/snat-and-eip.md +++ b/docs/snat-and-eip.md @@ -5,7 +5,7 @@ By using snat, a group of pods can share one same ip address to communicate with By using eip, external services can visit a pod with a stable ip and pod will visit external services using the same ip. ## Prerequisite -* To take use of OVN L3 Gateway, a dedicated nic *MUST* be bridged into ovs to act as the gateway between overlay and underlay, ops should use other nics to manage the host server. +* To take use of OVN L3 Gateway, a dedicated nic *MUST* be bridged into ovs to act as the gateway between overlay and underlay, ops should use other NICs to manage the host server. * As the nic will emit packets with nat ip directly into underlay network, administrators *MUST* make sure that these packets will not be denied by security rules. * SNAT and EIP functions *CANNOT* work together with Cluster interconnection network @@ -25,7 +25,7 @@ data: external-gw-nodes: "kube-ovn-worker" # NodeName in kubernetes which will act the overlay to underlay gateway functions external-gw-nic: "eth1" # The nic that will be bridged into ovs and act as overlay to underlay gateway external-gw-addr: "172.56.0.1/16" # The ip and mask of the underlay physical gateway - nic-ip: "172.56.0.254/16" # The ip and mask of the underlay physical network for logical route externel gw port + nic-ip: "172.56.0.100/16" # The ip and mask of the underlay physical network for logical route external gw port nic-mac: "16:52:f3:13:6a:25" # The mac of nic-ip ``` diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go index ee4c33dddd3..453615945c7 100644 --- a/pkg/controller/pod.go +++ b/pkg/controller/pod.go @@ -805,6 +805,13 @@ func (c *Controller) handleUpdatePod(key string) error { subnet = podNet.Subnet if podIP != "" && subnet.Spec.Vlan == "" && subnet.Spec.Vpc == util.DefaultVpc { + node, err := c.nodesLister.Get(pod.Spec.NodeName) + if err != nil { + klog.Errorf("failed to get node %s: %v", pod.Spec.NodeName, err) + return err + } + + pgName := getOverlaySubnetsPortGroupName(subnet.Name, node.Name) if c.config.EnableEipSnat && (pod.Annotations[util.EipAnnotation] != "" || pod.Annotations[util.SnatAnnotation] != "") { cm, err := c.configMapsLister.ConfigMaps(c.config.ExternalGatewayConfigNS).Get(util.ExternalGatewayConfig) if err != nil { @@ -824,20 +831,24 @@ func (c *Controller) handleUpdatePod(key string) error { klog.Errorf("failed to add static route, %v", err) return err } + + // remove lsp from port group to make EIP/SNAT work + portName := ovs.PodNameToPortName(podName, pod.Namespace, podNet.ProviderName) + c.ovnPgKeyMutex.Lock(pgName) + if err = c.ovnClient.PortGroupRemovePort(pgName, portName); err != nil { + c.ovnPgKeyMutex.Unlock(pgName) + return err + } + c.ovnPgKeyMutex.Unlock(pgName) + } else { if subnet.Spec.GatewayType == kubeovnv1.GWDistributedType && pod.Annotations[util.NorthGatewayAnnotation] == "" { - node, err := c.nodesLister.Get(pod.Spec.NodeName) - if err != nil { - klog.Errorf("get node %s failed %v", pod.Spec.NodeName, err) - return err - } - nodeTunlIPAddr, err := getNodeTunlIP(node) if err != nil { return err } - pgName := getOverlaySubnetsPortGroupName(subnet.Name, node.Name) + var added bool for _, nodeAddr := range nodeTunlIPAddr { for _, podAddr := range strings.Split(podIP, ",") { if util.CheckProtocol(nodeAddr.String()) != util.CheckProtocol(podAddr) { @@ -851,6 +862,12 @@ func (c *Controller) handleUpdatePod(key string) error { return err } c.ovnPgKeyMutex.Unlock(pgName) + + added = true + break + } + if added { + break } } } diff --git a/yamls/ovn-external-gw-config.yaml b/yamls/ovn-external-gw-config.yaml index 7468a6e3d14..678a99abc40 100644 --- a/yamls/ovn-external-gw-config.yaml +++ b/yamls/ovn-external-gw-config.yaml @@ -5,7 +5,9 @@ metadata: namespace: kube-system data: enable-external-gw: "true" + type: "centralized" external-gw-nodes: "kube-ovn-worker" # NodeName in kubernetes which will act the overlay to underlay gateway functions external-gw-nic: "eth1" # The nic that will be bridged into ovs and act as overlay to underlay gateway - nic-ip: "172.56.0.1/16" # The ip and mask of the underlay physical gateway + external-gw-addr: "172.56.0.1/16" # The ip and mask of the underlay physical gateway + nic-ip: "172.56.0.100/16" # The ip and mask of the underlay physical network for logical route external gw port nic-mac: "16:52:f3:13:6a:25" # The mac of the underlay physical gateway