diff --git a/test/framework/resources/k8s/utils/container.go b/test/framework/resources/k8s/utils/container.go index 6d447d5aaa..1a5209a52f 100644 --- a/test/framework/resources/k8s/utils/container.go +++ b/test/framework/resources/k8s/utils/container.go @@ -60,6 +60,7 @@ func AddOrUpdateEnvironmentVariable(containers []v1.Container, containerName str // RemoveEnvironmentVariables removes the environment variable from the specified container func RemoveEnvironmentVariables(containers []v1.Container, containerName string, envVars map[string]struct{}) error { + var updatedEnvVar []v1.EnvVar containerIndex := -1 for i, container := range containers { if container.Name != containerName { @@ -67,9 +68,8 @@ func RemoveEnvironmentVariables(containers []v1.Container, containerName string, } containerIndex = i for j := 0; j < len(container.Env); j++ { - if _, ok := envVars[container.Env[j].Name]; ok { - container.Env = append(container.Env[:j], container.Env[j+1:]...) - j-- + if _, ok := envVars[container.Env[j].Name]; !ok { + updatedEnvVar = append(updatedEnvVar, container.Env[j]) } } } @@ -79,5 +79,7 @@ func RemoveEnvironmentVariables(containers []v1.Container, containerName string, containerName) } + containers[containerIndex].Env = updatedEnvVar + return nil } diff --git a/test/framework/utils/const.go b/test/framework/utils/const.go index 4217de8440..f829759659 100644 --- a/test/framework/utils/const.go +++ b/test/framework/utils/const.go @@ -17,6 +17,8 @@ import "time" const ( DefaultTestNamespace = "cni-automation" + AwsNodeNamespace = "kube-system" + AwsNodeName = "aws-node" ) const ( diff --git a/test/integration-new/README.md b/test/integration-new/README.md index 0f1a06c260..da6b386f81 100644 --- a/test/integration-new/README.md +++ b/test/integration-new/README.md @@ -7,6 +7,7 @@ The integration test requires - At least 2 nodes in a node group. - Nodes in the nodegroup shouldn't have existing pods. - Ginkgo installed on your environment. To install `go get github.com/onsi/ginkgo/ginkgo` +- Supports instance types having at least 3 ENIs and 16+ Secondary IPv4 Addresses across all ENIs. ####Testing Set the environment variables that will be passed to Ginkgo script. If you want to directly pass the arguments you can skip to next step. diff --git a/test/integration-new/ipamd/ipamd_suite_test.go b/test/integration-new/ipamd/ipamd_suite_test.go new file mode 100644 index 0000000000..1ac39b4934 --- /dev/null +++ b/test/integration-new/ipamd/ipamd_suite_test.go @@ -0,0 +1,63 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package ipamd + +import ( + "testing" + + "github.com/aws/amazon-vpc-cni-k8s/test/framework" + k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" + + "github.com/aws/aws-sdk-go/service/ec2" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" +) + +var err error +var f *framework.Framework +var primaryNode v1.Node +var primaryInstance *ec2.Instance + +func TestIPAMD(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "VPC IPAMD Test Suite") +} + +var _ = BeforeSuite(func() { + f = framework.New(framework.GlobalOptions) + + By("creating test namespace") + f.K8sResourceManagers.NamespaceManager(). + CreateNamespace(utils.DefaultTestNamespace) + + nodeList, err := f.K8sResourceManagers.NodeManager().GetNodes(f.Options.NgNameLabelKey, + f.Options.NgNameLabelVal) + Expect(err).ToNot(HaveOccurred()) + Expect(len(nodeList.Items)).Should(BeNumerically(">", 1)) + + // Nominate the first node as the primary node + primaryNode = nodeList.Items[0] + + instanceID := k8sUtils.GetInstanceIDFromNode(primaryNode) + primaryInstance, err = f.CloudServices.EC2().DescribeInstance(instanceID) + Expect(err).ToNot(HaveOccurred()) +}) + +var _ = AfterSuite(func() { + By("deleting test namespace") + f.K8sResourceManagers.NamespaceManager(). + DeleteAndWaitTillNamespaceDeleted(utils.DefaultTestNamespace) +}) diff --git a/test/integration-new/ipamd/warm_target_test.go b/test/integration-new/ipamd/warm_target_test.go new file mode 100644 index 0000000000..8dc9bff51b --- /dev/null +++ b/test/integration-new/ipamd/warm_target_test.go @@ -0,0 +1,190 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package ipamd + +import ( + "strconv" + "time" + + k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +// IMPORTANT: THE NODEGROUP TO RUN THE TEST MUST NOT HAVE ANY POD +// Ideally we should drain the node, but drain from go client is non trivial +// IMPORTANT: Only support nodes that can have 16+ Secondary IPV4s across at least 3 ENI +var _ = Describe("test warm target variables", func() { + + Context("when warm ENI target is used", func() { + var warmENITarget int + var maxENI int + + JustBeforeEach(func() { + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, + utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, + map[string]string{ + "WARM_ENI_TARGET": strconv.Itoa(warmENITarget), + "MAX_ENI": strconv.Itoa(maxENI), + }) + + // Allow for IPAMD to reconcile it's state + time.Sleep(utils.PollIntervalLong * 5) + + primaryInstance, err = f.CloudServices. + EC2().DescribeInstance(*primaryInstance.InstanceId) + Expect(err).ToNot(HaveOccurred()) + + Expect(len(primaryInstance.NetworkInterfaces)). + Should(Equal(MinIgnoreZero(warmENITarget, maxENI))) + }) + + JustAfterEach(func() { + k8sUtils.RemoveVarFromDaemonSetAndWaitTillUpdated(f, + utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, + map[string]struct{}{"WARM_ENI_TARGET": {}, "MAX_ENI": {}}) + }) + + Context("when WARM_ENI_TARGET = 2 and MAX_ENI = 1", func() { + BeforeEach(func() { + warmENITarget = 2 + maxENI = 1 + }) + + It("instance should have only 1 ENI", func() {}) + }) + + Context("when WARM_ENI_TARGET = 3", func() { + BeforeEach(func() { + warmENITarget = 3 + maxENI = 0 + }) + + It("instance should have only 3 ENIs", func() {}) + }) + + Context("when WARM_ENI_TARGET = 1", func() { + BeforeEach(func() { + warmENITarget = 1 + maxENI = 0 + }) + + It("instance should have only 1 ENI", func() {}) + }) + + }) + + Context("when warm IP target is set", func() { + var warmIPTarget int + var minIPTarget int + + JustBeforeEach(func() { + var availIPs int + + // Set the WARM IP TARGET + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, + utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, + map[string]string{ + "WARM_IP_TARGET": strconv.Itoa(warmIPTarget), + "MINIMUM_IP_TARGET": strconv.Itoa(minIPTarget), + }) + + // Allow for IPAMD to reconcile it's state + time.Sleep(utils.PollIntervalLong) + + // Query the EC2 Instance to get the list of available IPs on the instance + primaryInstance, err = f.CloudServices. + EC2().DescribeInstance(*primaryInstance.InstanceId) + Expect(err).ToNot(HaveOccurred()) + + // Sum all the IPs on all network interfaces minus the primary IPv4 address per ENI + for _, networkInterface := range primaryInstance.NetworkInterfaces { + availIPs += len(networkInterface.PrivateIpAddresses) - 1 + } + + // Validated avail IP equals the warm IP Size + Expect(availIPs).Should(Equal(Max(warmIPTarget, minIPTarget))) + }) + + JustAfterEach(func() { + k8sUtils.RemoveVarFromDaemonSetAndWaitTillUpdated(f, + utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, + map[string]struct{}{"WARM_IP_TARGET": {}, "MINIMUM_IP_TARGET": {}}) + }) + + Context("when WARM_IP_TARGET = 2", func() { + BeforeEach(func() { + warmIPTarget = 2 + minIPTarget = 0 + }) + + It("should have 2 secondary IPv4 addresses", func() {}) + }) + + Context("when WARM_IP_TARGET = 16", func() { + BeforeEach(func() { + warmIPTarget = 16 + minIPTarget = 0 + }) + + It("should have 16 secondary IPv4 addresses", func() {}) + }) + + Context("when MINIMUM_IP_TARGET = 2", func() { + BeforeEach(func() { + warmIPTarget = 0 + minIPTarget = 2 + }) + + It("should have 2 secondary IPv4 addresses", func() {}) + }) + + Context("when MINIMUM_IP_TARGET = 16", func() { + BeforeEach(func() { + warmIPTarget = 0 + minIPTarget = 16 + }) + + It("should have 16 secondary IPv4 addresses", func() {}) + }) + + Context("when MINIMUM_IP_TARGET = 6 and WARM_IP_TARGET = 10", func() { + BeforeEach(func() { + warmIPTarget = 6 + minIPTarget = 10 + }) + + It("should have 10 secondary IPv4 addresses", func() {}) + }) + }) +}) + +func Max(x, y int) int { + if x < y { + return y + } + return x +} + +// MinIgnoreZero returns smaller of two number, if any number is zero returns the other number +func MinIgnoreZero(x, y int) int { + if x == 0 {return y} + if y == 0 {return x} + if x < y { + return x + } + return y +}