From 2a60bc880bcf6e639c461f86dc32ab6a3546248d Mon Sep 17 00:00:00 2001
From: attilapiros <piros.attila.zsolt@gmail.com>
Date: Fri, 2 Jul 2021 09:33:20 +0200
Subject: [PATCH] Retry in case IOException (exponential backoff)

---
 CHANGELOG.md                                  |  1 +
 .../client/dsl/base/OperationSupport.java     | 32 ++++++++++++-------
 .../client/dsl/base/BaseOperationTest.java    | 14 ++++++--
 3 files changed, 32 insertions(+), 15 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f8a6433b103..0e5a4a3be23 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@
 #### Dependency Upgrade
 
 #### New Features
+* Fix #3291: Retrying the HTTP operation in case of IOException too
 
 ### 5.5.0 (2021-06-30)
 
diff --git a/kubernetes-client/src/main/java/io/fabric8/kubernetes/client/dsl/base/OperationSupport.java b/kubernetes-client/src/main/java/io/fabric8/kubernetes/client/dsl/base/OperationSupport.java
index 0af6cadd633..624a63e2623 100644
--- a/kubernetes-client/src/main/java/io/fabric8/kubernetes/client/dsl/base/OperationSupport.java
+++ b/kubernetes-client/src/main/java/io/fabric8/kubernetes/client/dsl/base/OperationSupport.java
@@ -578,20 +578,28 @@ protected <T> T handleResponse(OkHttpClient client, Request.Builder requestBuild
   }
 
   protected Response retryWithExponentialBackoff(OkHttpClient client, Request request) throws InterruptedException, IOException {
-    Response response;
-    boolean doRetry;
     int numRetries = 0;
-    do {
-      response = client.newCall(request).execute();
-      doRetry = numRetries < requestRetryBackoffLimit && response.code() >= 500;
-      if (doRetry) {
-        long retryInterval= retryIntervalCalculator.getInterval(numRetries);
-        LOG.debug("HTTP operation on url: {} should be retried as the response code was {}, retrying after {} millis", request.url(), response.code(), retryInterval);
-        Thread.sleep(retryInterval);
-        numRetries++;
+    long retryInterval;
+    while (true) {
+      try {
+        Response response = client.newCall(request).execute();
+        if (numRetries < requestRetryBackoffLimit && response.code() >= 500) {
+          retryInterval = retryIntervalCalculator.getInterval(numRetries);
+          LOG.debug("HTTP operation on url: {} should be retried as the response code was {}, retrying after {} millis", request.url(), response.code(), retryInterval);
+        } else {
+          return response;
+        }
+      } catch (IOException ie) {
+        if (numRetries < requestRetryBackoffLimit) {
+          retryInterval = retryIntervalCalculator.getInterval(numRetries);
+          LOG.debug(String.format("HTTP operation on url: %s should be retried after %d millis because of IOException", request.url(), retryInterval), ie);
+        } else {
+          throw ie;
+        }
       }
-    } while(doRetry);
-    return response;
+      Thread.sleep(retryInterval);
+      numRetries++;
+    }
   }
 
   /**
diff --git a/kubernetes-client/src/test/java/io/fabric8/kubernetes/client/dsl/base/BaseOperationTest.java b/kubernetes-client/src/test/java/io/fabric8/kubernetes/client/dsl/base/BaseOperationTest.java
index b10e5d52e52..283c5fd5c82 100644
--- a/kubernetes-client/src/test/java/io/fabric8/kubernetes/client/dsl/base/BaseOperationTest.java
+++ b/kubernetes-client/src/test/java/io/fabric8/kubernetes/client/dsl/base/BaseOperationTest.java
@@ -239,7 +239,13 @@ private OkHttpClient newHttpClientWithSomeFailures(final AtomicInteger httpExecu
         when(mockCall.execute()).thenAnswer(i -> {
           int count = httpExecutionCounter.getAndIncrement();
           if (count < numFailures) {
-            return new Response.Builder().request(req).message("Internal Server Error").protocol(HTTP_1_1).code(500).build();
+            // Altering the type of the error for each call:
+            // even numbered calls (including the first call) fail with an IOException and odd numbered calls fail with HTTP response 500
+            if (count % 2 == 0) {
+              throw new IOException("For example java.net.ConnectException");
+            } else {
+              return new Response.Builder().request(req).message("Internal Server Error").protocol(HTTP_1_1).code(500).build();
+            }
           } else {
             Pod podNoLabels = new PodBuilder().withNewMetadata().withName("pod1").withNamespace("test").and().build();
             ResponseBody body = ResponseBody.create(MediaType.get("application/json"), Serialization.asJson(podNoLabels));
@@ -269,7 +275,8 @@ void testNoHttpRetryWithDefaultConfig() throws MalformedURLException, IOExceptio
     });
 
     // Then
-    assertTrue(exception.getMessage().contains("Internal Server Error"));
+    assertTrue("As the first failure is an IOException the message of the causedBy expected to contain the given text: 'For example java.net.ConnectException'!",
+      exception.getCause().getMessage().contains("For example java.net.ConnectException"));
     assertEquals(1, httpExecutionCounter.get());
   }
 
@@ -290,7 +297,8 @@ void testHttpRetryWithMoreFailuresThanRetries() throws MalformedURLException, IO
     });
 
     // Then
-    assertTrue(exception.getMessage().contains("Internal Server Error"));
+    assertTrue("As the last failure, the 3rd one, is not an IOException the message expected to contain: 'Internal Server Error'!",
+        exception.getMessage().contains("Internal Server Error"));
     assertEquals("Expected 4 calls: one normal try and 3 backoff retries!", 4, httpExecutionCounter.get());
   }