Log pod state if init pod wait condition times out (for debugging tra…

…nsient test issue) (#10639) * log pod state if init pod search times out * increase test timeout from 5 to 6 minutes to give kube pod process timeout time to trigger * format
airbytehq · Feb 24, 2022 · 2157b47 · 2157b47
1 parent def938a
commit 2157b47
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 3 deletions.
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/process/KubePodProcess.java b/airbyte-workers/src/main/java/io/airbyte/workers/process/KubePodProcess.java
@@ -32,6 +32,7 @@
 import io.fabric8.kubernetes.api.model.VolumeMount;
 import io.fabric8.kubernetes.api.model.VolumeMountBuilder;
 import io.fabric8.kubernetes.client.KubernetesClient;
+import io.fabric8.kubernetes.client.dsl.PodResource;
 import io.fabric8.kubernetes.client.internal.readiness.Readiness;
 import java.io.IOException;
 import java.io.InputStream;
@@ -300,8 +301,18 @@ public static void copyFilesToKubeConfigVolume(final KubernetesClient client,
    */
   private static void waitForInitPodToRun(final KubernetesClient client, final Pod podDefinition) throws InterruptedException {
     LOGGER.info("Waiting for init container to be ready before copying files...");
-    client.pods().inNamespace(podDefinition.getMetadata().getNamespace()).withName(podDefinition.getMetadata().getName())
-        .waitUntilCondition(p -> p.getStatus().getInitContainerStatuses().size() != 0, 5, TimeUnit.MINUTES);
+    final PodResource<Pod> pod =
+        client.pods().inNamespace(podDefinition.getMetadata().getNamespace()).withName(podDefinition.getMetadata().getName());
+    try {
+      pod.waitUntilCondition(p -> p.getStatus().getInitContainerStatuses().size() != 0, 5, TimeUnit.MINUTES);
+    } catch (InterruptedException e) {
+      LOGGER.error("Init pod not found after 5 minutes");
+      LOGGER.error("Pod search executed in namespace {} for pod name {} resulted in: {}",
+          podDefinition.getMetadata().getNamespace(),
+          podDefinition.getMetadata().getName(),
+          pod.get().toString());
+      throw e;
+    }
     LOGGER.info("Init container present..");
     client.pods().inNamespace(podDefinition.getMetadata().getNamespace()).withName(podDefinition.getMetadata().getName())
         .waitUntilCondition(p -> p.getStatus().getInitContainerStatuses().get(0).getState().getRunning() != null, 5, TimeUnit.MINUTES);

diff --git a/...s/src/test-integration/java/io/airbyte/workers/process/KubePodProcessIntegrationTest.java b/...s/src/test-integration/java/io/airbyte/workers/process/KubePodProcessIntegrationTest.java
@@ -43,7 +43,7 @@
 // requires kube running locally to run. If using Minikube it requires MINIKUBE=true
 // Must have a timeout on this class because it tests child processes that may misbehave; otherwise
 // this can hang forever during failures.
-@Timeout(value = 5,
+@Timeout(value = 6,
          unit = TimeUnit.MINUTES)
 public class KubePodProcessIntegrationTest {