From 79723a2636c9cfd9f9dfe1160941ed875dc4d311 Mon Sep 17 00:00:00 2001 From: oraz Date: Thu, 28 Dec 2023 11:15:33 +0200 Subject: [PATCH 1/2] Use human readable time and minimize timeouts and poll intervals Use shorter version of time as 6m0s instead of 6 * time.Minute. Minimize timeouts and poll intervals for shorter tests. --- test/e2e/far_e2e_test.go | 16 ++++++++-------- test/e2e/utils/command.go | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/test/e2e/far_e2e_test.go b/test/e2e/far_e2e_test.go index 1b08c24e..88048494 100644 --- a/test/e2e/far_e2e_test.go +++ b/test/e2e/far_e2e_test.go @@ -37,11 +37,11 @@ const ( //TODO: try to minimize timeout // eventually parameters - timeoutLogs = 3 * time.Minute - timeoutReboot = 6 * time.Minute // fencing with fence_aws should be completed within 6 minutes - timeoutDeletion = 10 * time.Second // this timeout is used after all the other steps have been succesfult - pollDeletion = 250 * time.Millisecond - pollInterval = 10 * time.Second + timeoutLogs = "3m0s" + timeoutReboot = "6m0s" // fencing with fence_aws should be completed within 6 minutes + timeoutShort = "5s" // this timeout is used after all the other steps have been succesfult + pollShort = "250ms" + pollInterval = "1s" ) var remediationTimes []time.Duration @@ -299,7 +299,7 @@ func wasFarTaintAdded(nodeName string) { node, err = utils.GetNodeWithName(k8sClient, nodeName) g.Expect(err).ToNot(HaveOccurred()) return utils.TaintExists(node.Spec.Taints, &farTaint) - }, 1*time.Second, "200ms").Should(BeTrue()) + }, timeoutShort, pollShort).Should(BeTrue()) log.Info("FAR taint was added", "node name", node.Name, "taint key", farTaint.Key, "taint effect", farTaint.Effect) } @@ -356,7 +356,7 @@ func checkPodDeleted(pod *corev1.Pod) { newPod := &corev1.Pod{} err := k8sClient.Get(context.Background(), client.ObjectKeyFromObject(pod), newPod) return apiErrors.IsNotFound(err) - }, timeoutDeletion, pollDeletion).Should(BeTrue()) + }, timeoutShort, pollShort).Should(BeTrue()) log.Info("Pod has already been deleted", "pod name", pod.Name) } @@ -373,7 +373,7 @@ func verifyStatusCondition(nodeName, conditionType string, conditionStatus *meta g.Expect(condition).ToNot(BeNil(), "expected condition %v to be set", conditionType) g.Expect(condition.Status).To(Equal(*conditionStatus), "expected condition %v to have status %v", conditionType, *conditionStatus) } - }, timeoutDeletion, pollInterval).Should(Succeed()) + }, timeoutShort, pollShort).Should(Succeed()) } // checkRemediation verify whether the node was remediated diff --git a/test/e2e/utils/command.go b/test/e2e/utils/command.go index ec45cebb..c1992e32 100644 --- a/test/e2e/utils/command.go +++ b/test/e2e/utils/command.go @@ -82,7 +82,7 @@ func runCommandInCluster(c *kubernetes.Clientset, nodeName string, ns string, co func waitForPodOutput(c *kubernetes.Clientset, pod *corev1.Pod, command []string) ([]byte, error) { var out []byte - if err := wait.PollImmediate(15*time.Second, time.Minute, func() (done bool, err error) { + if err := wait.PollImmediate(1*time.Second, time.Minute, func() (done bool, err error) { out, err = execCommandOnPod(c, pod, command) if err != nil { return false, err From 09224320a5c1a7c571833767f8afaac0b9c8f381 Mon Sep 17 00:00:00 2001 From: oraz Date: Thu, 28 Dec 2023 11:16:46 +0200 Subject: [PATCH 2/2] Use meaningful timeout and poll vars Replace timeoutShort with timeoutTaint, and timeoutAfterReboot. Replace pollShort with pollTaint, and pollAfterReboot --- test/e2e/far_e2e_test.go | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/test/e2e/far_e2e_test.go b/test/e2e/far_e2e_test.go index 88048494..e03c5015 100644 --- a/test/e2e/far_e2e_test.go +++ b/test/e2e/far_e2e_test.go @@ -37,11 +37,13 @@ const ( //TODO: try to minimize timeout // eventually parameters - timeoutLogs = "3m0s" - timeoutReboot = "6m0s" // fencing with fence_aws should be completed within 6 minutes - timeoutShort = "5s" // this timeout is used after all the other steps have been succesfult - pollShort = "250ms" - pollInterval = "1s" + timeoutLogs = "3m0s" + timeoutTaint = "2s" // Timeout for checking the FAR taint + timeoutReboot = "6m0s" // fencing with fence_aws should be completed within 6 minutes + timeoutAfterReboot = "5s" // Timeout for verifying steps after the node has been rebooted + pollTaint = "100ms" + pollReboot = "1s" + pollAfterReboot = "250ms" ) var remediationTimes []time.Duration @@ -299,7 +301,7 @@ func wasFarTaintAdded(nodeName string) { node, err = utils.GetNodeWithName(k8sClient, nodeName) g.Expect(err).ToNot(HaveOccurred()) return utils.TaintExists(node.Spec.Taints, &farTaint) - }, timeoutShort, pollShort).Should(BeTrue()) + }, timeoutTaint, pollTaint).Should(BeTrue()) log.Info("FAR taint was added", "node name", node.Name, "taint key", farTaint.Key, "taint effect", farTaint.Effect) } @@ -313,7 +315,7 @@ func waitForNodeHealthyCondition(node *corev1.Node, condStatus corev1.ConditionS } } return corev1.ConditionStatus("failure") - }, timeoutReboot, pollInterval).Should(Equal(condStatus)) + }, timeoutReboot, pollReboot).Should(Equal(condStatus)) } // makeNodeUnready stops kubelet and wait for the node condition to be not ready unless the node was already unready @@ -344,7 +346,7 @@ func wasNodeRebooted(nodeName string, nodeBootTimeBefore time.Time) { log.Error(errBootAfter, "Can't get boot time of the node") } return nodeBootTimeAfter, errBootAfter - }, timeoutReboot, pollInterval).Should( + }, timeoutReboot, pollReboot).Should( BeTemporally(">", nodeBootTimeBefore), "Timeout for node reboot has passed, even though FAR CR has been created") log.Info("successful reboot", "node", nodeName, "offset between last boot", nodeBootTimeAfter.Sub(nodeBootTimeBefore), "new boot time", nodeBootTimeAfter) @@ -356,7 +358,7 @@ func checkPodDeleted(pod *corev1.Pod) { newPod := &corev1.Pod{} err := k8sClient.Get(context.Background(), client.ObjectKeyFromObject(pod), newPod) return apiErrors.IsNotFound(err) - }, timeoutShort, pollShort).Should(BeTrue()) + }, timeoutAfterReboot, pollAfterReboot).Should(BeTrue()) log.Info("Pod has already been deleted", "pod name", pod.Name) } @@ -373,7 +375,7 @@ func verifyStatusCondition(nodeName, conditionType string, conditionStatus *meta g.Expect(condition).ToNot(BeNil(), "expected condition %v to be set", conditionType) g.Expect(condition.Status).To(Equal(*conditionStatus), "expected condition %v to have status %v", conditionType, *conditionStatus) } - }, timeoutShort, pollShort).Should(Succeed()) + }, timeoutAfterReboot, pollAfterReboot).Should(Succeed()) } // checkRemediation verify whether the node was remediated