diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 0e4a435a912..8d6311a6db2 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -21,6 +21,8 @@ updates: - dependency-name: "antrea.io/ofnet" - dependency-name: "antrea.io/libOpenflow" - dependency-name: "github.com/ClickHouse/clickhouse-go/v2" # auto-upgrade involves dependency conflicts + - dependency-name: "github.com/alexflint/go-filemutex" + update-types: ["version-update:semver-major"] # ignore major updates only - package-ecosystem: "github-actions" # Workflow files stored in the default location of `.github/workflows` directory: "/" diff --git a/go.mod b/go.mod index f852c268146..254d7e0b2ac 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/Microsoft/go-winio v0.6.1 github.com/Microsoft/hcsshim v0.9.10 github.com/TomCodeLV/OVSDB-golang-lib v0.0.0-20200116135253-9bbdfadcd881 + github.com/alexflint/go-filemutex v1.2.0 github.com/aws/aws-sdk-go-v2 v1.16.10 github.com/aws/aws-sdk-go-v2/config v1.16.0 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.23 diff --git a/go.sum b/go.sum index 57e466cb1eb..49434f216c8 100644 --- a/go.sum +++ b/go.sum @@ -123,6 +123,8 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alessio/shellescape v1.2.2/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= +github.com/alexflint/go-filemutex v1.2.0 h1:1v0TJPDtlhgpW4nJ+GvxCLSlUDC3+gW0CQQvlmfDR/s= +github.com/alexflint/go-filemutex v1.2.0/go.mod h1:mYyQSWvw9Tx2/H2n9qXPb52tTYfE0pZAWcBq5mK025c= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= diff --git a/pkg/agent/cniserver/ipam/hostlocal/gc.go b/pkg/agent/cniserver/ipam/hostlocal/gc.go new file mode 100644 index 00000000000..3555488611e --- /dev/null +++ b/pkg/agent/cniserver/ipam/hostlocal/gc.go @@ -0,0 +1,117 @@ +// Copyright 2023 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hostlocal + +import ( + "fmt" + "net" + "os" + "path/filepath" + "runtime" + "strings" + + "github.com/spf13/afero" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2" +) + +const dataDir = "/var/lib/cni/networks" + +func networkDir(network string) string { + return filepath.Join(dataDir, network) +} + +// This is a hacky approach as we access the internals of the host-local plugin, +// instead of using the CNI interface. However, crafting a CNI DEL request from +// scratch would also be hacky. +func GarbageCollectContainerIPs(network string, desiredIPs sets.Set[string]) error { + dir := networkDir(network) + + info, err := os.Stat(dir) + if os.IsNotExist(err) { + klog.V(2).InfoS("Host-local IPAM data directory does not exist, nothing to do", "dir", dir) + return nil + } + if !info.IsDir() { + return fmt.Errorf("path '%s' is not a directory: %w", dir, err) + } + + lk, err := NewFileLock(dataDir) + if err != nil { + return err + } + defer lk.Close() + lk.Lock() + defer lk.Unlock() + + fs := afero.NewOsFs() + return gcContainerIPs(fs, dir, desiredIPs) +} + +// Internal version of GarbageCollectContainerIPs which does not acquire the +// file lock and can work with an arbitrary afero filesystem. +func gcContainerIPs(fs afero.Fs, dir string, desiredIPs sets.Set[string]) error { + paths := make([]string, 0) + + if err := afero.Walk(fs, dir, func(path string, info os.FileInfo, err error) error { + if err != nil || info.IsDir() { + return nil + } + paths = append(paths, path) + return nil + }); err != nil { + return fmt.Errorf("error when gathering IP filenames in the host-local data directory: %w", err) + } + + allocatedIPs := sets.New[string]() + for _, p := range paths { + ip := getIPFromPath(p) + if net.ParseIP(ip) == nil { + // not a valid IP, nothing to do + continue + } + allocatedIPs.Insert(ip) + if desiredIPs.Has(ip) { + // IP is in-use + continue + } + if err := fs.Remove(p); err != nil { + klog.ErrorS(err, "Failed to release unused IP from host-local IPAM plugin", "IP", ip) + continue + } + allocatedIPs.Delete(ip) + klog.InfoS("Unused IP was successfully released from host-local IPAM plugin", "IP", ip) + } + + if allocatedIPs.Difference(desiredIPs).Len() > 0 { + return fmt.Errorf("not all unused IPs could be released from host-local IPAM plugin, some IPs may be leaked") + } + + // Note that it is perfectly possible for some IPs to be in desiredIPs but not in + // allocatedIPs. This can be the case when another IPAM plugin (e.g., AntreaIPAM) is also + // used. + + return nil +} + +func getIPFromPath(path string) string { + fname := filepath.Base(path) + // need to unespace IPv6 addresses on Windows + // see https://github.com/containernetworking/plugins/blob/38f18d26ecfef550b8bac02656cc11103fd7cff1/plugins/ipam/host-local/backend/disk/backend.go#L197 + if runtime.GOOS == "windows" { + fname = strings.ReplaceAll(fname, "_", ":") + } + return fname +} diff --git a/pkg/agent/cniserver/ipam/hostlocal/gc_test.go b/pkg/agent/cniserver/ipam/hostlocal/gc_test.go new file mode 100644 index 00000000000..6ac29abd7a8 --- /dev/null +++ b/pkg/agent/cniserver/ipam/hostlocal/gc_test.go @@ -0,0 +1,178 @@ +// Copyright 2023 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hostlocal + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/util/sets" +) + +type testFs struct { + afero.Fs + removeError error + removedFiles []string +} + +func (fs *testFs) Remove(name string) error { + if fs.removeError != nil { + err := &os.PathError{Op: "remove", Path: name, Err: fs.removeError} + // reset error + fs.removeError = nil + return err + } + if err := fs.Fs.Remove(name); err != nil { + return err + } + fs.removedFiles = append(fs.removedFiles, name) + return nil +} + +// forceRemoveError forces the next Remove call to fail. Error will be cleared after the first call. +func (fs *testFs) forceRemoveError() { + fs.removeError = fmt.Errorf("permission denied") +} + +func (fs *testFs) removedIPs() sets.Set[string] { + s := sets.New[string]() + for _, p := range fs.removedFiles { + s.Insert(getIPFromPath(p)) + } + return s +} + +// from https://github.com/containernetworking/plugins/blob/38f18d26ecfef550b8bac02656cc11103fd7cff1/plugins/ipam/host-local/backend/disk/backend.go#L197 +func getEscapedPath(dir string, fname string) string { + if runtime.GOOS == "windows" { + fname = strings.ReplaceAll(fname, ":", "_") + } + return filepath.Join(dir, fname) +} + +func allocateIPs(t *testing.T, fs afero.Fs, dir string, ips ...string) { + for _, ip := range ips { + path := getEscapedPath(dir, ip) + // The real host-local IPAM plugin writes the container ID + interface name to the + // file, but it is irrelevant in our case. + require.NoError(t, afero.WriteFile(fs, path, []byte("foo"), 0o600)) + } +} + +func TestGcContainerIPs(t *testing.T) { + dir := networkDir("antrea") + + newTestFs := func() *testFs { + return &testFs{ + Fs: afero.NewMemMapFs(), + } + } + + t.Run("missing directory", func(t *testing.T) { + fs := newTestFs() + // create the plugin data directory, but not the "network" sub-directory + require.NoError(t, fs.MkdirAll(dataDir, 0o755)) + assert.NoError(t, gcContainerIPs(fs, dir, sets.New[string]())) + removedIPs := fs.removedIPs() + assert.Empty(t, removedIPs) + }) + + t.Run("remove error", func(t *testing.T) { + ips := []string{"10.0.0.1", "10.0.0.2"} + fs := newTestFs() + require.NoError(t, fs.MkdirAll(dir, 0o755)) + allocateIPs(t, fs, dir, ips...) + fs.forceRemoveError() + require.Error(t, gcContainerIPs(fs, dir, sets.New[string]())) + // one of the IPs will fail to be released, the other one will succeed + removedIPs := fs.removedIPs() + assert.Len(t, removedIPs, 1) + }) + + resolveIP := func(id int, ipv6 bool) string { + if ipv6 { + return fmt.Sprintf("2001:db8:a::%d", id) + } else { + return fmt.Sprintf("10.0.0.%d", id) + } + } + + // some success test cases, will be run for both IPv4 and IPv6 + testCases := []struct { + name string + desiredIPs []int + allocatedIPs []int + expectedRemovedIPs []int + }{ + { + name: "same sets", + desiredIPs: []int{1, 2}, + allocatedIPs: []int{1, 2}, + expectedRemovedIPs: []int{}, + }, + { + name: "multiple removals", + desiredIPs: []int{1, 3}, + allocatedIPs: []int{1, 2, 3, 4}, + expectedRemovedIPs: []int{2, 4}, + }, + { + name: "extra ip", + desiredIPs: []int{1, 2, 3}, + allocatedIPs: []int{1, 2}, + expectedRemovedIPs: []int{}, + }, + } + + runTests := func(t *testing.T, ipv6 bool) { + name := "ipv4" + if ipv6 { + name = "ipv6" + } + + toIPSet := func(ids []int) sets.Set[string] { + ips := sets.New[string]() + for _, id := range ids { + ip := resolveIP(id, ipv6) + require.NotEmpty(t, ip) + ips.Insert(ip) + } + return ips + } + + t.Run(name, func(t *testing.T) { + for _, tc := range testCases { + fs := newTestFs() + require.NoError(t, fs.MkdirAll(dir, 0o755)) + desiredIPs := toIPSet(tc.desiredIPs) + allocatedIPs := toIPSet(tc.allocatedIPs) + expectedRemovedIPs := toIPSet(tc.expectedRemovedIPs) + allocateIPs(t, fs, dir, allocatedIPs.UnsortedList()...) + require.NoError(t, gcContainerIPs(fs, dir, desiredIPs)) + assert.Equal(t, expectedRemovedIPs, fs.removedIPs()) + } + }) + } + + runTests(t, false) + runTests(t, true) +} diff --git a/pkg/agent/cniserver/ipam/hostlocal/lock.go b/pkg/agent/cniserver/ipam/hostlocal/lock.go new file mode 100644 index 00000000000..e2eada4ff62 --- /dev/null +++ b/pkg/agent/cniserver/ipam/hostlocal/lock.go @@ -0,0 +1,62 @@ +// Copyright 2023 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hostlocal + +import ( + "os" + "path" + + "github.com/alexflint/go-filemutex" +) + +// This code was copied from https://github.com/containernetworking/plugins/blob/v1.3.0/plugins/ipam/host-local/backend/disk/lock.go + +// FileLock wraps os.File to be used as a lock using flock +type FileLock struct { + f *filemutex.FileMutex +} + +// NewFileLock opens file/dir at path and returns unlocked FileLock object +func NewFileLock(lockPath string) (*FileLock, error) { + fi, err := os.Stat(lockPath) + if err != nil { + return nil, err + } + + if fi.IsDir() { + lockPath = path.Join(lockPath, "lock") + } + + f, err := filemutex.New(lockPath) + if err != nil { + return nil, err + } + + return &FileLock{f}, nil +} + +func (l *FileLock) Close() error { + return l.f.Close() +} + +// Lock acquires an exclusive lock +func (l *FileLock) Lock() error { + return l.f.Lock() +} + +// Unlock releases the lock +func (l *FileLock) Unlock() error { + return l.f.Unlock() +} diff --git a/pkg/agent/cniserver/ipam/ipam_delegator.go b/pkg/agent/cniserver/ipam/ipam_delegator.go index d94d1370167..eb9d15c38d9 100644 --- a/pkg/agent/cniserver/ipam/ipam_delegator.go +++ b/pkg/agent/cniserver/ipam/ipam_delegator.go @@ -22,8 +22,10 @@ import ( "github.com/containernetworking/cni/pkg/invoke" "github.com/containernetworking/cni/pkg/types" current "github.com/containernetworking/cni/pkg/types/100" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" + "antrea.io/antrea/pkg/agent/cniserver/ipam/hostlocal" argtypes "antrea.io/antrea/pkg/agent/cniserver/types" ) @@ -86,6 +88,18 @@ func (d *IPAMDelegator) Check(args *invoke.Args, k8sArgs *argtypes.K8sArgs, netw return true, nil } +// GarbageCollectContainerIPs will IPs allocated by the delegated IPAM plugin +// that are no longer in-use (if there is any). It should be called on an agent +// restart to provide garbage collection for IPs, and to avoid IP leakage in +// case of missed CNI DEL events. Normally, it is not Antrea's responsibility to +// implement this, as the above layers should ensure that there is always one +// successful CNI DEL for every corresponding CNI ADD. However, we include this +// support to increase robustness in case of a container runtime bug. +// Only the host-local plugin is supported. +func GarbageCollectContainerIPs(network string, desiredIPs sets.Set[string]) error { + return hostlocal.GarbageCollectContainerIPs(network, desiredIPs) +} + var defaultExec invoke.Exec = &invoke.DefaultExec{ RawExec: &invoke.RawExec{Stderr: os.Stderr}, } diff --git a/pkg/agent/cniserver/pod_configuration.go b/pkg/agent/cniserver/pod_configuration.go index ac4e183c91a..a67efe2cc99 100644 --- a/pkg/agent/cniserver/pod_configuration.go +++ b/pkg/agent/cniserver/pod_configuration.go @@ -421,6 +421,8 @@ func (pc *podConfigurator) reconcile(pods []corev1.Pod, containerAccess *contain // desiredPods is the set of Pods that should be present, based on the // current list of Pods got from the Kubernetes API. desiredPods := sets.New[string]() + // desiredPodIPs is the set of IPs allocated to desiredPods. + desiredPodIPs := sets.New[string]() // knownInterfaces is the list of interfaces currently in the local cache. knownInterfaces := pc.ifaceStore.GetInterfacesByType(interfacestore.ContainerInterface) @@ -430,11 +432,16 @@ func (pc *podConfigurator) reconcile(pods []corev1.Pod, containerAccess *contain continue } desiredPods.Insert(k8s.NamespacedName(pod.Namespace, pod.Name)) + for _, podIP := range pod.Status.PodIPs { + desiredPodIPs.Insert(podIP.IP) + } } missingIfConfigs := make([]*interfacestore.InterfaceConfig, 0) for _, containerConfig := range knownInterfaces { - namespacedName := k8s.NamespacedName(containerConfig.PodNamespace, containerConfig.PodName) + namespace := containerConfig.PodNamespace + name := containerConfig.PodName + namespacedName := k8s.NamespacedName(namespace, name) if desiredPods.Has(namespacedName) { // Find the OVS ports which are not connected to host interfaces. This is useful on Windows if the runtime is // containerd, because the host interface is created async from the OVS port. @@ -446,7 +453,7 @@ func (pc *podConfigurator) reconcile(pods []corev1.Pod, containerAccess *contain // We rely on the interface cache / store - which is initialized from the persistent // OVSDB - to map the Pod to its interface configuration. The interface // configuration includes the parameters we need to replay the flows. - klog.V(4).Infof("Syncing interface %s for Pod %s", containerConfig.InterfaceName, namespacedName) + klog.V(4).InfoS("Syncing Pod interface", "Pod", klog.KRef(namespace, name), "iface", containerConfig.InterfaceName) if err := pc.ofClient.InstallPodFlows( containerConfig.InterfaceName, containerConfig.IPs, @@ -455,13 +462,13 @@ func (pc *podConfigurator) reconcile(pods []corev1.Pod, containerAccess *contain containerConfig.VLANID, nil, ); err != nil { - klog.Errorf("Error when re-installing flows for Pod %s", namespacedName) + klog.ErrorS(err, "Error when re-installing flows for Pod", "Pod", klog.KRef(namespace, name)) } } else { // clean-up and delete interface - klog.V(4).Infof("Deleting interface %s", containerConfig.InterfaceName) + klog.V(4).InfoS("Deleting interface", "Pod", klog.KRef(namespace, name), "iface", containerConfig.InterfaceName) if err := pc.removeInterfaces(containerConfig.ContainerID); err != nil { - klog.Errorf("Failed to delete interface %s: %v", containerConfig.InterfaceName, err) + klog.ErrorS(err, "Failed to delete interface", "Pod", klog.KRef(namespace, name), "iface", containerConfig.InterfaceName) } // interface should no longer be in store after the call to removeInterfaces } @@ -469,6 +476,13 @@ func (pc *podConfigurator) reconcile(pods []corev1.Pod, containerAccess *contain if len(missingIfConfigs) > 0 { pc.reconcileMissingPods(missingIfConfigs, containerAccess) } + + // clean-up IPs that may still be allocated + klog.V(4).InfoS("Running IPAM garbage collection for unused Pod IPs") + if err := ipam.GarbageCollectContainerIPs(antreaCNIType, desiredPodIPs); err != nil { + klog.ErrorS(err, "Error when garbage collecting previously-allocated IPs") + } + return nil } diff --git a/test/integration/agent/cniserver_test.go b/test/integration/agent/cniserver_test.go index c2f7cb6fe6d..846980eec0b 100644 --- a/test/integration/agent/cniserver_test.go +++ b/test/integration/agent/cniserver_test.go @@ -34,12 +34,15 @@ import ( "github.com/containernetworking/plugins/pkg/ns" "github.com/containernetworking/plugins/pkg/testutils" "github.com/containernetworking/plugins/plugins/ipam/host-local/backend/allocator" + "github.com/containernetworking/plugins/plugins/ipam/host-local/backend/disk" "github.com/google/uuid" "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/vishvananda/netlink" mock "go.uber.org/mock/gomock" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8sFake "k8s.io/client-go/kubernetes/fake" "k8s.io/component-base/metrics/legacyregistry" @@ -151,7 +154,6 @@ var ( ipamMock *ipamtest.MockIPAMDriver ovsServiceMock *ovsconfigtest.MockOVSBridgeClient ofServiceMock *openflowtest.MockClient - testNodeConfig *config.NodeConfig routeMock *routetest.MockInterface ) @@ -567,7 +569,7 @@ func newTester() *cmdAddDelTester { tester.networkReadyCh = make(chan struct{}) tester.server = cniserver.New(testSock, "", - testNodeConfig, + getTestNodeConfig(false), k8sFake.NewSimpleClientset(), routeMock, false, false, false, false, &config.NetworkConfig{InterfaceMTU: 1450}, @@ -721,9 +723,8 @@ func TestAntreaServerFunc(t *testing.T) { } func setupChainTest( - controller *mock.Controller, inServer *cniserver.CNIServer, netNS ns.NetNS, newServer bool) ( - server *cniserver.CNIServer, hostVeth, containerVeth net.Interface, err error) { - + testNodeConfig *config.NodeConfig, controller *mock.Controller, inServer *cniserver.CNIServer, netNS ns.NetNS, newServer bool, +) (server *cniserver.CNIServer, hostVeth, containerVeth net.Interface, err error) { if newServer { routeMock = routetest.NewMockInterface(controller) networkReadyCh := make(chan struct{}) @@ -763,6 +764,7 @@ func TestCNIServerChaining(t *testing.T) { t.Skipf("Skip test runs only in container") } + testNodeConfig := getTestNodeConfig(false) testRequire := require.New(t) controller := mock.NewController(t) var server *cniserver.CNIServer @@ -785,7 +787,7 @@ func TestCNIServerChaining(t *testing.T) { testRequire.Nil(err) var hostVeth net.Interface - server, hostVeth, _, err = setupChainTest(controller, server, netNS, newServer) + server, hostVeth, _, err = setupChainTest(testNodeConfig, controller, server, netNS, newServer) testRequire.Nil(err) if newServer { ovsServiceMock = ovsconfigtest.NewMockOVSBridgeClient(controller) @@ -850,12 +852,108 @@ func TestCNIServerChaining(t *testing.T) { } } -func init() { - nodeName := "node1" - gwIP := net.ParseIP("192.168.1.1") - gwMAC, _ = net.ParseMAC("11:11:11:11:11:11") - nodeGateway := &config.GatewayConfig{IPv4: gwIP, MAC: gwMAC, Name: ""} - _, nodePodCIDR, _ := net.ParseCIDR("192.168.1.0/24") +func TestCNIServerGCForHostLocalIPAM(t *testing.T) { + // Running in a container is required as we modify /var/lib/cni/networks. + if _, inContainer := os.LookupEnv("INCONTAINER"); !inContainer { + t.Skipf("Skip test runs only in container") + } + + testNodeConfig := getTestNodeConfig(true) + + usedIPv4 := net.ParseIP("10.0.0.1") + usedIPv6 := net.ParseIP("2001:db8:a::1") + unusedIPv4 := net.ParseIP("10.0.0.2") + unusedIPv6 := net.ParseIP("2001:db8:a::2") + + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + }, + Spec: corev1.PodSpec{ + NodeName: testNodeConfig.Name, + }, + Status: corev1.PodStatus{ + PodIP: usedIPv4.String(), + PodIPs: []corev1.PodIP{ + {IP: usedIPv4.String()}, + {IP: usedIPv6.String()}, + }, + }, + } - testNodeConfig = &config.NodeConfig{Name: nodeName, PodIPv4CIDR: nodePodCIDR, GatewayConfig: nodeGateway} + // we need to use the default data dir, as it is hardcoded in the Antrea logic + const ipamDataDir = "/var/lib/cni/networks" + // make sure we start from a clean state, this will not interfere with any other test + require.NoError(t, os.RemoveAll(ipamDataDir)) + // clean up directory at the end of the test + defer os.RemoveAll(ipamDataDir) + + // reserve all 4 IPs using host-local IPAM + const rangeID = "0" + reserveIPs := func(cID string, ips ...net.IP) { + ipamStore, err := disk.New("antrea", ipamDataDir) + require.NoError(t, err) + defer ipamStore.Close() + ipamStore.Lock() + defer ipamStore.Unlock() + for _, ip := range ips { + reserved, err := ipamStore.Reserve(cID, IFName, ip, rangeID) + require.NoError(t, err) + require.True(t, reserved) // should not have been previously reserved + } + } + reserveIPs("c0", usedIPv4, usedIPv6) + reserveIPs("c1", unusedIPv4, unusedIPv6) + + // create mocks and test CNIServer + controller := mock.NewController(t) + ovsServiceMock := ovsconfigtest.NewMockOVSBridgeClient(controller) + ovsServiceMock.EXPECT().GetPortList().Return([]ovsconfig.OVSPortData{}, nil).AnyTimes() + ovsServiceMock.EXPECT().IsHardwareOffloadEnabled().Return(false).AnyTimes() + ovsServiceMock.EXPECT().GetOVSDatapathType().Return(ovsconfig.OVSDatapathSystem).AnyTimes() + ofServiceMock := openflowtest.NewMockClient(controller) + routeMock := routetest.NewMockInterface(controller) + ifaceStore := interfacestore.NewInterfaceStore() + networkReadyCh := make(chan struct{}) + k8sClient := k8sFake.NewSimpleClientset(pod) + server := cniserver.New( + testSock, + "", + testNodeConfig, + k8sClient, + routeMock, + false, false, false, false, &config.NetworkConfig{InterfaceMTU: 1450}, + networkReadyCh, + ) + + // call Initialize, which will run reconciliation and perform host-local IPAM garbage collection + server.Initialize(ovsServiceMock, ofServiceMock, ifaceStore, channel.NewSubscribableChannel("PodUpdate", 100), nil) + + getIPs := func(cID string) []net.IP { + ipamStore, err := disk.New("antrea", "") + require.NoError(t, err) + defer ipamStore.Close() + ipamStore.Lock() + defer ipamStore.Unlock() + return ipamStore.GetByID(cID, IFName) + } + assert.ElementsMatch(t, getIPs("c0"), []net.IP{usedIPv4, usedIPv6}) + assert.Empty(t, getIPs("c1")) +} + +func getTestNodeConfig(dualStack bool) *config.NodeConfig { + nodeName := "node1" + gwIPv4 := net.ParseIP("192.168.1.1") + _, nodePodCIDRv4, _ := net.ParseCIDR("192.168.1.0/24") + gwMAC, _ := net.ParseMAC("11:11:11:11:11:11") + gateway := &config.GatewayConfig{Name: "", IPv4: gwIPv4, MAC: gwMAC} + nodeConfig := &config.NodeConfig{Name: nodeName, PodIPv4CIDR: nodePodCIDRv4, GatewayConfig: gateway} + if dualStack { + gwIPv6 := net.ParseIP("fd74:ca9b:172:18::1") + _, nodePodCIDRv6, _ := net.ParseCIDR("fd74:ca9b:172:18::/64") + gateway.IPv6 = gwIPv6 + nodeConfig.PodIPv6CIDR = nodePodCIDRv6 + } + return nodeConfig }