feat: host-local keep pod IP

Signed-off-by: Armin Schlegel <[email protected]>

keep pod IP:
add relationship between podNs/podName and pod IP and relationship is stored in a disk file that named by podIP_podns_podName. If podName file exists, we will use the already reserved IP for the pod. If podName file do not exists, go through the original process. 
fix IP of deleted pod can be reused when IP address walks around the IP range.

Co-authored-by: openyurt <openyurt.io>

plugins/ipam/host-local/backend/allocator/allocator.go

@@ -40,11 +40,51 @@ func NewIPAllocator(s *RangeSet, store backend.Store, id int) *IPAllocator {
 	}
 }
 
-// Get allocates an IP
-func (a *IPAllocator) Get(id string, ifname string, requestedIP net.IP) (*current.IPConfig, error) {
+// GetByPodNsAndName allocates an IP or used reserved IP for specified pod
+func (a *IPAllocator) GetByPodNsAndName(id string, ifname string, requestedIP net.IP, podNs, podName string) (*current.IPConfig, error) {
 	a.store.Lock()
 	defer a.store.Unlock()
+	if len(podName) != 0 {
+		podIPIsExist, knownIP := a.store.HasReservedIP(podNs, podName)
+
+		if podIPIsExist {
+			// podName file is exist, update ip file with new container id.
+			_, err := a.store.ReservePodInfo(id, knownIP, podNs, podName, podIPIsExist)
+			if err != nil {
+				return nil, err
+			}
+
+			reservedIP, gw := a.GetGWofKnowIP(knownIP)
+			if reservedIP == nil {
+				return nil, fmt.Errorf("no IP addresses available in range set: %s", a.rangeset.String())
+			}
+
+			return &current.IPConfig{
+				Address: *reservedIP,
+				Gateway: gw,
+			}, nil
+		} else {
+			// reserve ip for new pod
+			ipCfg, err := a.Get(id, ifname, requestedIP)
+			if err != nil {
+				return ipCfg, err
+			}
 
+			if ipCfg != nil {
+				_, err := a.store.ReservePodInfo(id, ipCfg.Address.IP, podNs, podName, podIPIsExist)
+				if err != nil {
+					return ipCfg, err
+				}
+			}
+			return ipCfg, nil
+		}
+	}
+
+	return a.Get(id, ifname, requestedIP)
+}
+
+// Get allocates an IP
+func (a *IPAllocator) Get(id string, ifname string, requestedIP net.IP) (*current.IPConfig, error) {
 	var reservedIP *net.IPNet
 	var gw net.IP
 
@@ -123,6 +163,18 @@ func (a *IPAllocator) Release(id string, ifname string) error {
 	return a.store.ReleaseByID(id, ifname)
 }
 
+// GetGWofKnowIP returns the known IP, its mask, and its gateway
+func (a *IPAllocator) GetGWofKnowIP(ip net.IP) (*net.IPNet, net.IP) {
+	rg := Range{}
+	for i, r := range *a.rangeset {
+		if r.Contains(ip) {
+			rg = (*a.rangeset)[i]
+			break
+		}
+	}
+	return &net.IPNet{IP: ip, Mask: rg.Subnet.Mask}, rg.Gateway
+}
+
 type RangeIter struct {
 	rangeset *RangeSet
 

plugins/ipam/host-local/backend/disk/backend.go

@@ -15,6 +15,7 @@
 package disk
 
 import (
+	"fmt"
 	"net"
 	"os"
 	"path/filepath"
@@ -200,3 +201,115 @@ func GetEscapedPath(dataDir string, fname string) string {
 	}
 	return filepath.Join(dataDir, fname)
 }
+
+// HasReservedIP verify the pod already had reserved ip or not.
+// and return the reserved ip on the other hand.
+func (s *Store) HasReservedIP(podNs, podName string) (bool, net.IP) {
+	ip := net.IP{}
+	if len(podName) == 0 {
+		return false, ip
+	}
+
+	// Pod, ip mapping info are recorded with file name: PodIP_PodNs_PodName
+	podIPNsNameFileName, err := s.findPodFileName("", podNs, podName)
+	if err != nil {
+		return false, ip
+	}
+
+	if len(podIPNsNameFileName) != 0 {
+		ipStr, ns, name := resolvePodFileName(podIPNsNameFileName)
+		if ns == podNs && name == podName {
+			ip = net.ParseIP(ipStr)
+			if ip != nil {
+				return true, ip
+			}
+		}
+	}
+
+	return false, ip
+}
+
+// ReservePodInfo create podName file for storing ip or update ip file with container id
+// in terms of podIPIsExist
+func (s *Store) ReservePodInfo(id string, ip net.IP, podNs, podName string, podIPIsExist bool) (bool, error) {
+	if podIPIsExist {
+		// pod Ns/Name file is exist, update ip file with new container id.
+		fname := GetEscapedPath(s.dataDir, ip.String())
+		err := os.WriteFile(fname, []byte(strings.TrimSpace(id)), 0644)
+		if err != nil {
+			return false, err
+		}
+	} else {
+		// for new pod, create a new file named "PodIP_PodNs_PodName",
+		// if there is already file named with prefix "ip_", rename the old file with new PodNs and PodName.
+		if len(podName) != 0 {
+			podIPNsNameFile := GetEscapedPath(s.dataDir, podFileName(ip.String(), podNs, podName))
+			podIPNsNameFileName, err := s.findPodFileName(ip.String(), "", "")
+			if err != nil {
+				return false, err
+			}
+
+			if len(podIPNsNameFileName) != 0 {
+				oldPodIPNsNameFile := GetEscapedPath(s.dataDir, podIPNsNameFileName)
+				err = os.Rename(oldPodIPNsNameFile, podIPNsNameFile)
+				if err != nil {
+					return false, err
+				} else {
+					return true, nil
+				}
+			}
+
+			err = os.WriteFile(podIPNsNameFile, []byte{}, 0644)
+			if err != nil {
+				return false, err
+			}
+		}
+	}
+
+	return true, nil
+}
+
+func podFileName(ip, ns, name string) string {
+	if len(ip) != 0 && len(ns) != 0 {
+		return fmt.Sprintf("%s_%s_%s", ip, ns, name)
+	}
+
+	return name
+}
+
+func resolvePodFileName(fName string) (ip, ns, name string) {
+	parts := strings.Split(fName, "_")
+	if len(parts) == 3 {
+		ip = parts[0]
+		ns = parts[1]
+		name = parts[2]
+	}
+
+	return
+}
+
+func (s *Store) findPodFileName(ip, ns, name string) (string, error) {
+	var pattern string
+	if len(ip) != 0 {
+		pattern = fmt.Sprintf("%s_*", ip)
+	} else if len(ns) != 0 && len(name) != 0 {
+		pattern = fmt.Sprintf("*_%s_%s", ns, name)
+	} else {
+		return "", nil
+	}
+	pattern = GetEscapedPath(s.dataDir, pattern)
+
+	podFiles, err := filepath.Glob(pattern)
+	if err != nil {
+		return "", err
+	}
+
+	if len(podFiles) == 1 {
+		_, fName := filepath.Split(podFiles[0])
+		if strings.Count(fName, "_") == 2 {
+			return fName, nil
+		}
+	}
+
+	return "", nil
+}

plugins/ipam/host-local/backend/store.go

@@ -24,4 +24,6 @@ type Store interface {
 	LastReservedIP(rangeID string) (net.IP, error)
 	ReleaseByID(id string, ifname string) error
 	GetByID(id string, ifname string) []net.IP
+	HasReservedIP(podNs, podName string) (bool, net.IP)
+	ReservePodInfo(id string, ip net.IP, podNs, podName string, podIPIsExist bool) (bool, error)
 }

plugins/ipam/host-local/backend/testing/fake_store.go

@@ -89,3 +89,19 @@ func (s *FakeStore) GetByID(id string, _ string) []net.IP {
 func (s *FakeStore) SetIPMap(m map[string]string) {
 	s.ipMap = m
 }
+
+func (s *FakeStore) ReleaseByPodName(podName string) error {
+	return nil
+}
+
+func (s *FakeStore) HasReservedIP(podNs, podName string) (bool, net.IP) {
+	ip := net.IP{}
+	if podName == "" {
+		return false, ip
+	}
+	return false, ip
+}
+
+func (s *FakeStore) ReservePodInfo(id string, ip net.IP, podName string, podIsExist bool) (bool, error) {
+	return true, nil
+}

plugins/ipam/host-local/main.go

@@ -54,6 +54,38 @@ func cmdCheck(args *skel.CmdArgs) error {
 	return nil
 }
 
+// Args: [][2]string{
+// {"IgnoreUnknown", "1"},
+// {"K8S_POD_NAMESPACE", podNs},
+// {"K8S_POD_NAME", podName},
+// {"K8S_POD_INFRA_CONTAINER_ID", podSandboxID.ID},
+// },
+func resolvePodNsAndNameFromEnvArgs(envArgs string) (string, string, error) {
+	var ns, name string
+	if envArgs == "" {
+		return ns, name, nil
+	}
+
+	pairs := strings.Split(envArgs, ";")
+	for _, pair := range pairs {
+		kv := strings.Split(pair, "=")
+		if len(kv) != 2 {
+			return ns, name, fmt.Errorf("ARGS: invalid pair %q", pair)
+		}
+
+		if kv[0] == "K8S_POD_NAMESPACE" {
+			ns = kv[1]
+		} else if kv[0] == "K8S_POD_NAME" {
+			name = kv[1]
+		}
+	}
+
+	if len(ns)+len(name) > 230 {
+		return "", "", fmt.Errorf("ARGS: length of pod ns and name exceed the length limit")
+	}
+	return ns, name, nil
+}
+
 func cmdAdd(args *skel.CmdArgs) error {
 	ipamConf, confVersion, err := allocator.LoadIPAMConfig(args.StdinData, args.Args)
 	if err != nil {
@@ -101,7 +133,14 @@ func cmdAdd(args *skel.CmdArgs) error {
 			}
 		}
 
-		ipConf, err := allocator.Get(args.ContainerID, args.IfName, requestedIP)
+		// get pod namespace and pod name
+		podNs, podName, err := resolvePodNsAndNameFromEnvArgs(args.Args)
+		if err != nil {
+			return fmt.Errorf("failed to get pod ns/name from env args: %s", err)
+		}
+
+		ipConf, err := allocator.GetByPodNsAndName(args.ContainerID, args.IfName, requestedIP, podNs, podName)
+
 		if err != nil {
 			// Deallocate all already allocated IPs
 			for _, alloc := range allocs {