From 2157b47b60edcbb728d2e0921d42972f4969f290 Mon Sep 17 00:00:00 2001 From: Parker Mossman Date: Thu, 24 Feb 2022 14:29:09 -0800 Subject: [PATCH] Log pod state if init pod wait condition times out (for debugging transient test issue) (#10639) * log pod state if init pod search times out * increase test timeout from 5 to 6 minutes to give kube pod process timeout time to trigger * format --- .../airbyte/workers/process/KubePodProcess.java | 15 +++++++++++++-- .../process/KubePodProcessIntegrationTest.java | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/process/KubePodProcess.java b/airbyte-workers/src/main/java/io/airbyte/workers/process/KubePodProcess.java index 28c0f1f3825b..b56126881c2c 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/process/KubePodProcess.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/process/KubePodProcess.java @@ -32,6 +32,7 @@ import io.fabric8.kubernetes.api.model.VolumeMount; import io.fabric8.kubernetes.api.model.VolumeMountBuilder; import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.dsl.PodResource; import io.fabric8.kubernetes.client.internal.readiness.Readiness; import java.io.IOException; import java.io.InputStream; @@ -300,8 +301,18 @@ public static void copyFilesToKubeConfigVolume(final KubernetesClient client, */ private static void waitForInitPodToRun(final KubernetesClient client, final Pod podDefinition) throws InterruptedException { LOGGER.info("Waiting for init container to be ready before copying files..."); - client.pods().inNamespace(podDefinition.getMetadata().getNamespace()).withName(podDefinition.getMetadata().getName()) - .waitUntilCondition(p -> p.getStatus().getInitContainerStatuses().size() != 0, 5, TimeUnit.MINUTES); + final PodResource pod = + client.pods().inNamespace(podDefinition.getMetadata().getNamespace()).withName(podDefinition.getMetadata().getName()); + try { + pod.waitUntilCondition(p -> p.getStatus().getInitContainerStatuses().size() != 0, 5, TimeUnit.MINUTES); + } catch (InterruptedException e) { + LOGGER.error("Init pod not found after 5 minutes"); + LOGGER.error("Pod search executed in namespace {} for pod name {} resulted in: {}", + podDefinition.getMetadata().getNamespace(), + podDefinition.getMetadata().getName(), + pod.get().toString()); + throw e; + } LOGGER.info("Init container present.."); client.pods().inNamespace(podDefinition.getMetadata().getNamespace()).withName(podDefinition.getMetadata().getName()) .waitUntilCondition(p -> p.getStatus().getInitContainerStatuses().get(0).getState().getRunning() != null, 5, TimeUnit.MINUTES); diff --git a/airbyte-workers/src/test-integration/java/io/airbyte/workers/process/KubePodProcessIntegrationTest.java b/airbyte-workers/src/test-integration/java/io/airbyte/workers/process/KubePodProcessIntegrationTest.java index ca29c6ccba9c..b8ade4102074 100644 --- a/airbyte-workers/src/test-integration/java/io/airbyte/workers/process/KubePodProcessIntegrationTest.java +++ b/airbyte-workers/src/test-integration/java/io/airbyte/workers/process/KubePodProcessIntegrationTest.java @@ -43,7 +43,7 @@ // requires kube running locally to run. If using Minikube it requires MINIKUBE=true // Must have a timeout on this class because it tests child processes that may misbehave; otherwise // this can hang forever during failures. -@Timeout(value = 5, +@Timeout(value = 6, unit = TimeUnit.MINUTES) public class KubePodProcessIntegrationTest {