From bf4667f43d306068a908cafad563803c20ed9e2c Mon Sep 17 00:00:00 2001 From: oraz Date: Tue, 12 Dec 2023 08:59:16 +0200 Subject: [PATCH 1/4] Print remediations time in e2e tests, and their average Checking remediation time in e2e could help us identify slow remediations and identify if the pr changes impact the remediations time. In addition there is no need to test pod creation time --- test/e2e/far_e2e_test.go | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/test/e2e/far_e2e_test.go b/test/e2e/far_e2e_test.go index c46432f0..ef88f742 100644 --- a/test/e2e/far_e2e_test.go +++ b/test/e2e/far_e2e_test.go @@ -79,11 +79,12 @@ var _ = Describe("FAR E2e", func() { Context("stress cluster", func() { var ( - nodes, filteredNodes *corev1.NodeList - nodeName string - pod *corev1.Pod - creationTimePod, nodeBootTimeBefore time.Time - err error + nodes, filteredNodes *corev1.NodeList + nodeName string + pod *corev1.Pod + startTime, nodeBootTimeBefore time.Time + err error + remediationTimes []time.Duration ) BeforeEach(func() { nodes = &corev1.NodeList{} @@ -116,30 +117,41 @@ var _ = Describe("FAR E2e", func() { nodeBootTimeBefore, err = e2eUtils.GetBootTime(clientSet, nodeName, testNsName, log) Expect(err).ToNot(HaveOccurred(), "failed to get boot time of the node") - // create tested pod, and save its creation time - // it will be deleted by FAR CR + // create tested pod which will be deleted by the far CR pod = e2eUtils.GetPod(nodeName, testContainerName) pod.Name = testPodName pod.Namespace = testNsName Expect(k8sClient.Create(context.Background(), pod)).To(Succeed()) - log.Info("Tested pod has been created", "pod", testPodName) - creationTimePod = metav1.Now().Time DeferCleanup(cleanupTestedResources, pod) // set the node as "unhealthy" by disabling kubelet makeNodeUnready(selectedNode) + startTime = time.Now() far := createFAR(nodeName, fenceAgent, testShareParam, testNodeParam) DeferCleanup(deleteFAR, far) }) When("running FAR to reboot two nodes", func() { It("should successfully remediate the first node", func() { - checkRemediation(nodeName, nodeBootTimeBefore, creationTimePod, pod) + checkRemediation(nodeName, nodeBootTimeBefore, pod) + remediationTimes = append(remediationTimes, time.Since(startTime)) }) It("should successfully remediate the second node", func() { - checkRemediation(nodeName, nodeBootTimeBefore, creationTimePod, pod) + checkRemediation(nodeName, nodeBootTimeBefore, pod) + remediationTimes = append(remediationTimes, time.Since(startTime)) }) }) + AfterEach(func() { + if len(remediationTimes) == 2 { + averageTimeDuration := 0.0 + for index, remTime := range remediationTimes { + averageTimeDuration += remTime.Seconds() + fmt.Printf("\nRemediation time #%d: %s\n", index+1, remTime) + } + averageTime := int(averageTimeDuration)/ len(remediationTimes) + fmt.Printf("\nAverage remediation time: %d minutes and %d seconds\n", averageTime/60, averageTime%60) + } + }) }) }) @@ -398,7 +410,7 @@ func verifyStatusCondition(nodeName, conditionType string, conditionStatus *meta } // checkRemediation verify whether the node was remediated -func checkRemediation(nodeName string, nodeBootTimeBefore time.Time, oldPodCreationTime time.Time, pod *corev1.Pod) { +func checkRemediation(nodeName string, nodeBootTimeBefore time.Time, pod *corev1.Pod) { By("Check if FAR NoExecute taint was added") wasFarTaintAdded(nodeName) From e14fcc692aa055f6f4fd7739aa86d5a9986f743a Mon Sep 17 00:00:00 2001 From: oraz Date: Wed, 13 Dec 2023 14:09:55 +0200 Subject: [PATCH 2/4] Use log instead of fmt.Printf --- test/e2e/far_e2e_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/e2e/far_e2e_test.go b/test/e2e/far_e2e_test.go index ef88f742..5fc1cca9 100644 --- a/test/e2e/far_e2e_test.go +++ b/test/e2e/far_e2e_test.go @@ -56,7 +56,7 @@ var _ = Describe("FAR E2e", func() { // create FAR CR spec based on OCP platformn clusterPlatform, err := e2eUtils.GetClusterInfo(configClient) Expect(err).ToNot(HaveOccurred(), "can't identify the cluster platform") - fmt.Printf("\ncluster name: %s and PlatformType: %s \n", string(clusterPlatform.Name), string(clusterPlatform.Status.PlatformStatus.Type)) + log.Info("Begin e2e test", "Cluster name", string(clusterPlatform.Name), "PlatformType", string(clusterPlatform.Status.PlatformStatus.Type)) switch clusterPlatform.Status.PlatformStatus.Type { case configv1.AWSPlatformType: @@ -148,7 +148,7 @@ var _ = Describe("FAR E2e", func() { averageTimeDuration += remTime.Seconds() fmt.Printf("\nRemediation time #%d: %s\n", index+1, remTime) } - averageTime := int(averageTimeDuration)/ len(remediationTimes) + averageTime := int(averageTimeDuration) / len(remediationTimes) fmt.Printf("\nAverage remediation time: %d minutes and %d seconds\n", averageTime/60, averageTime%60) } }) @@ -178,7 +178,7 @@ func buildSharedParameters(clusterPlatform *configv1.Infrastructure, action stri if clusterPlatformType == configv1.AWSPlatformType { accessKey, secretKey, err := e2eUtils.GetSecretData(clientSet, secretAWSName, secretAWSNamespace, secretKeyAWS, secretValAWS) if err != nil { - fmt.Printf("can't get AWS credentials\n") + log.Info("Can't get AWS credentials") return nil, err } @@ -199,7 +199,7 @@ func buildSharedParameters(clusterPlatform *configv1.Infrastructure, action stri // then parse ip username, password, err := e2eUtils.GetSecretData(clientSet, secretBMHName, secretBMHNamespace, secretKeyBM, secretValBM) if err != nil { - fmt.Printf("can't get BMH credentials\n") + log.Info("Can't get BMH credentials") return nil, err } testShareParam = map[v1alpha1.ParameterName]string{ @@ -225,7 +225,7 @@ func buildNodeParameters(clusterPlatformType configv1.PlatformType) (map[v1alpha if clusterPlatformType == configv1.AWSPlatformType { nodeListParam, err = e2eUtils.GetAWSNodeInfoList(machineClient) if err != nil { - fmt.Printf("can't get nodes' information - AWS instance ID\n") + log.Info("Can't get nodes' information - AWS instance ID is missing") return nil, err } nodeIdentifier = v1alpha1.ParameterName(nodeIdentifierPrefixAWS) @@ -233,7 +233,7 @@ func buildNodeParameters(clusterPlatformType configv1.PlatformType) (map[v1alpha } else if clusterPlatformType == configv1.BareMetalPlatformType { nodeListParam, err = e2eUtils.GetBMHNodeInfoList(machineClient) if err != nil { - fmt.Printf("can't get nodes' information - ports\n") + log.Info("Can't get nodes' information - ports are missing") return nil, err } nodeIdentifier = v1alpha1.ParameterName(nodeIdentifierPrefixIPMI) From 2d428da67d952cbf91674fab78dc901540f63f4c Mon Sep 17 00:00:00 2001 From: oraz Date: Wed, 13 Dec 2023 14:27:29 +0200 Subject: [PATCH 3/4] Print remediations time in e2e tests using AfterSuite AfterSuite is run after the whole suite is finished --- test/e2e/far_e2e_test.go | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/test/e2e/far_e2e_test.go b/test/e2e/far_e2e_test.go index 5fc1cca9..69ba2bce 100644 --- a/test/e2e/far_e2e_test.go +++ b/test/e2e/far_e2e_test.go @@ -46,6 +46,8 @@ const ( pollInterval = 10 * time.Second ) +var remediationsTime []time.Duration + var _ = Describe("FAR E2e", func() { var ( fenceAgent, nodeIdentifierPrefix string @@ -84,7 +86,6 @@ var _ = Describe("FAR E2e", func() { pod *corev1.Pod startTime, nodeBootTimeBefore time.Time err error - remediationTimes []time.Duration ) BeforeEach(func() { nodes = &corev1.NodeList{} @@ -134,27 +135,28 @@ var _ = Describe("FAR E2e", func() { When("running FAR to reboot two nodes", func() { It("should successfully remediate the first node", func() { checkRemediation(nodeName, nodeBootTimeBefore, pod) - remediationTimes = append(remediationTimes, time.Since(startTime)) + remediationsTime = append(remediationsTime, time.Since(startTime)) }) It("should successfully remediate the second node", func() { checkRemediation(nodeName, nodeBootTimeBefore, pod) - remediationTimes = append(remediationTimes, time.Since(startTime)) + remediationsTime = append(remediationsTime, time.Since(startTime)) }) }) - AfterEach(func() { - if len(remediationTimes) == 2 { - averageTimeDuration := 0.0 - for index, remTime := range remediationTimes { - averageTimeDuration += remTime.Seconds() - fmt.Printf("\nRemediation time #%d: %s\n", index+1, remTime) - } - averageTime := int(averageTimeDuration) / len(remediationTimes) - fmt.Printf("\nAverage remediation time: %d minutes and %d seconds\n", averageTime/60, averageTime%60) - } - }) }) }) +var _ = AfterSuite(func() { + if len(remediationsTime) > 0 { + averageTimeDuration := 0.0 + for _, remTime := range remediationsTime { + averageTimeDuration += remTime.Seconds() + log.Info("Remediation was finished", "remediation time", remTime) + } + averageTime := int(averageTimeDuration) / len(remediationsTime) + log.Info("Average remediation time", "minutes", averageTime/60, "seconds", averageTime%60) + } +}) + // buildSharedParameters returns a map key-value of shared parameters based on cluster platform type if it finds the credentials, otherwise an error func buildSharedParameters(clusterPlatform *configv1.Infrastructure, action string) (map[v1alpha1.ParameterName]string, error) { const ( From 492326e58ad5ad3b93890c46043e2b1738809901 Mon Sep 17 00:00:00 2001 From: oraz Date: Thu, 14 Dec 2023 08:02:18 +0200 Subject: [PATCH 4/4] Typo fix for var name s/remediationsTime/remediationTimes --- test/e2e/far_e2e_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/e2e/far_e2e_test.go b/test/e2e/far_e2e_test.go index 69ba2bce..1befa432 100644 --- a/test/e2e/far_e2e_test.go +++ b/test/e2e/far_e2e_test.go @@ -46,7 +46,7 @@ const ( pollInterval = 10 * time.Second ) -var remediationsTime []time.Duration +var remediationTimes []time.Duration var _ = Describe("FAR E2e", func() { var ( @@ -135,24 +135,24 @@ var _ = Describe("FAR E2e", func() { When("running FAR to reboot two nodes", func() { It("should successfully remediate the first node", func() { checkRemediation(nodeName, nodeBootTimeBefore, pod) - remediationsTime = append(remediationsTime, time.Since(startTime)) + remediationTimes = append(remediationTimes, time.Since(startTime)) }) It("should successfully remediate the second node", func() { checkRemediation(nodeName, nodeBootTimeBefore, pod) - remediationsTime = append(remediationsTime, time.Since(startTime)) + remediationTimes = append(remediationTimes, time.Since(startTime)) }) }) }) }) var _ = AfterSuite(func() { - if len(remediationsTime) > 0 { + if len(remediationTimes) > 0 { averageTimeDuration := 0.0 - for _, remTime := range remediationsTime { + for _, remTime := range remediationTimes { averageTimeDuration += remTime.Seconds() log.Info("Remediation was finished", "remediation time", remTime) } - averageTime := int(averageTimeDuration) / len(remediationsTime) + averageTime := int(averageTimeDuration) / len(remediationTimes) log.Info("Average remediation time", "minutes", averageTime/60, "seconds", averageTime%60) } })