Skip to content

Commit

Permalink
Retry HTTP for status >= 500 (exponential backoff)
Browse files Browse the repository at this point in the history
  • Loading branch information
@attilapiros
attilapiros committed May 24, 2021
1 parent b81f54e commit 11bdb2b
Show file tree
Hide file tree
Showing 6 changed files with 205 additions and 9 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

#### New Features
* Fix #3133: Add DSL Support for `authorization.openshift.io/v1` resources in OpenShiftClient
* Fix #3087: Support HTTP operation retry with exponential backoff (for status code >= 500)

### 5.4.0 (2021-05-19)

Expand Down
3 changes: 3 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,9 @@ System properties are preferred over environment variables. The following system
| `kubernetes.watch.reconnectLimit` / `KUBERNETES_WATCH_RECONNECTLIMIT` | Number of reconnect attempts (-1 for infinite) | `-1` |
| `kubernetes.connection.timeout` / `KUBERNETES_CONNECTION_TIMEOUT` | Connection timeout in ms (0 for no timeout) | `10000` |
| `kubernetes.request.timeout` / `KUBERNETES_REQUEST_TIMEOUT` | Read timeout in ms | `10000` |
| `kubernetes.request.retry.backoff.count` / `KUBERNETES_REQUEST_RETRY_BACKOFF_COUNT_SYSTEM_PROPERTY` | Retry count | `0` |
| `kubernetes.request.retry.backoff.initial` / `KUBERNETES_REQUEST_RETRY_BACKOFF_INITIAL_SYSTEM_PROPERTY` | Retry initial backoff in ms | `1000` |
| `kubernetes.request.retry.backoff.multipier` / `KUBERNETES_REQUEST_RETRY_BACKOFF_MULTIPIER_SYSTEM_PROPERTY` | Retry multipliler | `2` |
| `kubernetes.rolling.timeout` / `KUBERNETES_ROLLING_TIMEOUT` | Rolling timeout in ms | `900000` |
| `kubernetes.logging.interval` / `KUBERNETES_LOGGING_INTERVAL` | Logging interval in ms | `20000` |
| `kubernetes.scale.timeout` / `KUBERNETES_SCALE_TIMEOUT` | Scale timeout in ms | `600000` |
Expand Down
46 changes: 43 additions & 3 deletions kubernetes-client/src/main/java/io/fabric8/kubernetes/client/Config.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,9 @@ public class Config {
public static final String KUBERNETES_WATCH_RECONNECT_LIMIT_SYSTEM_PROPERTY = "kubernetes.watch.reconnectLimit";
public static final String KUBERNETES_CONNECTION_TIMEOUT_SYSTEM_PROPERTY = "kubernetes.connection.timeout";
public static final String KUBERNETES_REQUEST_TIMEOUT_SYSTEM_PROPERTY = "kubernetes.request.timeout";
public static final String KUBERNETES_REQUEST_RETRY_BACKOFF_COUNT_SYSTEM_PROPERTY = "kubernetes.request.retry.count";
public static final String KUBERNETES_REQUEST_RETRY_BACKOFF_INITIAL_SYSTEM_PROPERTY = "kubernetes.request.retry.backoff.initial";
public static final String KUBERNETES_REQUEST_RETRY_BACKOFF_MULTIPLIER_SYSTEM_PROPERTY = "kubernetes.request.retry.backoff.multiplier";
public static final String KUBERNETES_ROLLING_TIMEOUT_SYSTEM_PROPERTY = "kubernetes.rolling.timeout";
public static final String KUBERNETES_LOGGING_INTERVAL_SYSTEM_PROPERTY = "kubernetes.logging.interval";
public static final String KUBERNETES_SCALE_TIMEOUT_SYSTEM_PROPERTY = "kubernetes.scale.timeout";
Expand Down Expand Up @@ -135,6 +138,10 @@ public class Config {
public static final Integer DEFAULT_MAX_CONCURRENT_REQUESTS = 64;
public static final Integer DEFAULT_MAX_CONCURRENT_REQUESTS_PER_HOST = 5;

public static final Integer DEFAULT_REQUEST_RETRY_BACKOFF_COUNT = 0;
public static final Integer DEFAULT_REQUEST_RETRY_BACKOFF_INITIAL = 1000;
public static final Integer DEFAULT_REQUEST_RETRY_BACKOFF_MULTIPIER = 2;

public static final String HTTP_PROTOCOL_PREFIX = "http://";
public static final String HTTPS_PROTOCOL_PREFIX = "https://";

Expand Down Expand Up @@ -173,6 +180,9 @@ public class Config {
private int watchReconnectInterval = 1000;
private int watchReconnectLimit = -1;
private int connectionTimeout = 10 * 1000;
private int requestRetryBackoffCount;
private int requestRetryBackoffInitial;
private int requestRetryBackoffMultiplier;
private int requestTimeout = 10 * 1000;
private long rollingTimeout = DEFAULT_ROLLING_TIMEOUT;
private long scaleTimeout = DEFAULT_SCALE_TIMEOUT;
Expand Down Expand Up @@ -289,11 +299,11 @@ private static String ensureHttps(String masterUrl, Config config) {

@Deprecated
public Config(String masterUrl, String apiVersion, String namespace, boolean trustCerts, boolean disableHostnameVerification, String caCertFile, String caCertData, String clientCertFile, String clientCertData, String clientKeyFile, String clientKeyData, String clientKeyAlgo, String clientKeyPassphrase, String username, String password, String oauthToken, int watchReconnectInterval, int watchReconnectLimit, int connectionTimeout, int requestTimeout, long rollingTimeout, long scaleTimeout, int loggingInterval, int maxConcurrentRequests, int maxConcurrentRequestsPerHost, String httpProxy, String httpsProxy, String[] noProxy, Map<Integer, String> errorMessages, String userAgent, TlsVersion[] tlsVersions, long websocketTimeout, long websocketPingInterval, String proxyUsername, String proxyPassword, String trustStoreFile, String trustStorePassphrase, String keyStoreFile, String keyStorePassphrase, String impersonateUsername, String[] impersonateGroups, Map<String, List<String>> impersonateExtras) {
this(masterUrl, apiVersion, namespace, trustCerts, disableHostnameVerification, caCertFile, caCertData, clientCertFile, clientCertData, clientKeyFile, clientKeyData, clientKeyAlgo, clientKeyPassphrase, username, password, oauthToken, watchReconnectInterval, watchReconnectLimit, connectionTimeout, requestTimeout, rollingTimeout, scaleTimeout, loggingInterval, maxConcurrentRequests, maxConcurrentRequestsPerHost, false, httpProxy, httpsProxy, noProxy, errorMessages, userAgent, tlsVersions, websocketTimeout, websocketPingInterval, proxyUsername, proxyPassword, trustStoreFile, trustStorePassphrase, keyStoreFile, keyStorePassphrase, impersonateUsername, impersonateGroups, impersonateExtras, null,null);
this(masterUrl, apiVersion, namespace, trustCerts, disableHostnameVerification, caCertFile, caCertData, clientCertFile, clientCertData, clientKeyFile, clientKeyData, clientKeyAlgo, clientKeyPassphrase, username, password, oauthToken, watchReconnectInterval, watchReconnectLimit, connectionTimeout, requestTimeout, rollingTimeout, scaleTimeout, loggingInterval, maxConcurrentRequests, maxConcurrentRequestsPerHost, false, httpProxy, httpsProxy, noProxy, errorMessages, userAgent, tlsVersions, websocketTimeout, websocketPingInterval, proxyUsername, proxyPassword, trustStoreFile, trustStorePassphrase, keyStoreFile, keyStorePassphrase, impersonateUsername, impersonateGroups, impersonateExtras, null,null, DEFAULT_REQUEST_RETRY_BACKOFF_COUNT, DEFAULT_REQUEST_RETRY_BACKOFF_INITIAL, DEFAULT_REQUEST_RETRY_BACKOFF_MULTIPIER);
}

@Buildable(builderPackage = "io.fabric8.kubernetes.api.builder", editableEnabled = false)
public Config(String masterUrl, String apiVersion, String namespace, boolean trustCerts, boolean disableHostnameVerification, String caCertFile, String caCertData, String clientCertFile, String clientCertData, String clientKeyFile, String clientKeyData, String clientKeyAlgo, String clientKeyPassphrase, String username, String password, String oauthToken, int watchReconnectInterval, int watchReconnectLimit, int connectionTimeout, int requestTimeout, long rollingTimeout, long scaleTimeout, int loggingInterval, int maxConcurrentRequests, int maxConcurrentRequestsPerHost, boolean http2Disable, String httpProxy, String httpsProxy, String[] noProxy, Map<Integer, String> errorMessages, String userAgent, TlsVersion[] tlsVersions, long websocketTimeout, long websocketPingInterval, String proxyUsername, String proxyPassword, String trustStoreFile, String trustStorePassphrase, String keyStoreFile, String keyStorePassphrase, String impersonateUsername, String[] impersonateGroups, Map<String, List<String>> impersonateExtras, OAuthTokenProvider oauthTokenProvider,Map<String,String> customHeaders) {
public Config(String masterUrl, String apiVersion, String namespace, boolean trustCerts, boolean disableHostnameVerification, String caCertFile, String caCertData, String clientCertFile, String clientCertData, String clientKeyFile, String clientKeyData, String clientKeyAlgo, String clientKeyPassphrase, String username, String password, String oauthToken, int watchReconnectInterval, int watchReconnectLimit, int connectionTimeout, int requestTimeout, long rollingTimeout, long scaleTimeout, int loggingInterval, int maxConcurrentRequests, int maxConcurrentRequestsPerHost, boolean http2Disable, String httpProxy, String httpsProxy, String[] noProxy, Map<Integer, String> errorMessages, String userAgent, TlsVersion[] tlsVersions, long websocketTimeout, long websocketPingInterval, String proxyUsername, String proxyPassword, String trustStoreFile, String trustStorePassphrase, String keyStoreFile, String keyStorePassphrase, String impersonateUsername, String[] impersonateGroups, Map<String, List<String>> impersonateExtras, OAuthTokenProvider oauthTokenProvider,Map<String,String> customHeaders, int requestRetryBackoffCount, int requestRetryBackoffInitial, int requestRetryBackoffMultiplier) {
this.masterUrl = masterUrl;
this.apiVersion = apiVersion;
this.namespace = namespace;
Expand All @@ -308,7 +318,7 @@ public Config(String masterUrl, String apiVersion, String namespace, boolean tru
this.clientKeyAlgo = clientKeyAlgo;
this.clientKeyPassphrase = clientKeyPassphrase;

this.requestConfig = new RequestConfig(username, password, oauthToken, watchReconnectLimit, watchReconnectInterval, connectionTimeout, rollingTimeout, requestTimeout, scaleTimeout, loggingInterval, websocketTimeout, websocketPingInterval, maxConcurrentRequests, maxConcurrentRequestsPerHost, oauthTokenProvider);
this.requestConfig = new RequestConfig(username, password, oauthToken, watchReconnectLimit, watchReconnectInterval, connectionTimeout, rollingTimeout, requestTimeout, scaleTimeout, loggingInterval, websocketTimeout, websocketPingInterval, maxConcurrentRequests, maxConcurrentRequestsPerHost, oauthTokenProvider, requestRetryBackoffCount, requestRetryBackoffInitial, requestRetryBackoffMultiplier);
this.requestConfig.setImpersonateUsername(impersonateUsername);
this.requestConfig.setImpersonateGroups(impersonateGroups);
this.requestConfig.setImpersonateExtras(impersonateExtras);
Expand Down Expand Up @@ -399,6 +409,9 @@ public static void configFromSysPropsOrEnvVars(Config config) {

config.setConnectionTimeout(Utils.getSystemPropertyOrEnvVar(KUBERNETES_CONNECTION_TIMEOUT_SYSTEM_PROPERTY, config.getConnectionTimeout()));
config.setRequestTimeout(Utils.getSystemPropertyOrEnvVar(KUBERNETES_REQUEST_TIMEOUT_SYSTEM_PROPERTY, config.getRequestTimeout()));
config.setRequestRetryBackoffCount(Utils.getSystemPropertyOrEnvVar(KUBERNETES_REQUEST_RETRY_BACKOFF_COUNT_SYSTEM_PROPERTY, config.getRequestRetryBackoffCount()));
config.setRequestRetryBackoffInitial(Utils.getSystemPropertyOrEnvVar(KUBERNETES_REQUEST_RETRY_BACKOFF_INITIAL_SYSTEM_PROPERTY, config.getRequestRetryBackoffInitial()));
config.setRequestRetryBackoffMultiplier(Utils.getSystemPropertyOrEnvVar(KUBERNETES_REQUEST_RETRY_BACKOFF_MULTIPLIER_SYSTEM_PROPERTY, config.getRequestRetryBackoffMultiplier()));

String configuredWebsocketTimeout = Utils.getSystemPropertyOrEnvVar(KUBERNETES_WEBSOCKET_TIMEOUT_SYSTEM_PROPERTY, String.valueOf(config.getWebsocketTimeout()));
if (configuredWebsocketTimeout != null) {
Expand Down Expand Up @@ -1036,6 +1049,33 @@ public void setRequestTimeout(int requestTimeout) {
this.requestConfig.setRequestTimeout(requestTimeout);
}

@JsonProperty("requestRetryBackoffCount")
public int getRequestRetryBackoffCount() {
return getRequestConfig().getRequestRetryBackoffCount();
}

public void setRequestRetryBackoffCount(int requestRetryBackoffCount) {
requestConfig.setRequestRetryBackoffCount(requestRetryBackoffCount);
}

@JsonProperty("requestRetryBackoffInitial")
public int getRequestRetryBackoffInitial() {
return getRequestConfig().getRequestRetryBackoffInitial();
}

public void setRequestRetryBackoffInitial(int requestRetryBackoffInitial) {
requestConfig.setRequestRetryBackoffInitial(requestRetryBackoffInitial);
}

@JsonProperty("requestRetryBackoffMultiplier")
public int getRequestRetryBackoffMultiplier() {
return getRequestConfig().getRequestRetryBackoffMultiplier();
}

public void setRequestRetryBackoffMultiplier(int requestRetryBackoffMultiplier) {
requestConfig.setRequestRetryBackoffMultiplier(requestRetryBackoffMultiplier);
}

@JsonProperty("rollingTimeout")
public long getRollingTimeout() {
return getRequestConfig().getRollingTimeout();
Expand Down
39 changes: 37 additions & 2 deletions kubernetes-client/src/main/java/io/fabric8/kubernetes/client/RequestConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@
import static io.fabric8.kubernetes.client.Config.DEFAULT_LOGGING_INTERVAL;
import static io.fabric8.kubernetes.client.Config.DEFAULT_MAX_CONCURRENT_REQUESTS;
import static io.fabric8.kubernetes.client.Config.DEFAULT_MAX_CONCURRENT_REQUESTS_PER_HOST;
import static io.fabric8.kubernetes.client.Config.DEFAULT_REQUEST_RETRY_BACKOFF_COUNT;
import static io.fabric8.kubernetes.client.Config.DEFAULT_REQUEST_RETRY_BACKOFF_INITIAL;
import static io.fabric8.kubernetes.client.Config.DEFAULT_REQUEST_RETRY_BACKOFF_MULTIPIER;
import static io.fabric8.kubernetes.client.Config.DEFAULT_ROLLING_TIMEOUT;
import static io.fabric8.kubernetes.client.Config.DEFAULT_SCALE_TIMEOUT;
import static io.fabric8.kubernetes.client.Config.DEFAULT_WEBSOCKET_PING_INTERVAL;
Expand All @@ -45,6 +48,9 @@ public class RequestConfig {
private int watchReconnectInterval = 1000;
private int watchReconnectLimit = -1;
private int connectionTimeout = 10 * 1000;
private int requestRetryBackoffCount = DEFAULT_REQUEST_RETRY_BACKOFF_COUNT;
private int requestRetryBackoffInitial = DEFAULT_REQUEST_RETRY_BACKOFF_INITIAL;
private int requestRetryBackoffMultiplier = DEFAULT_REQUEST_RETRY_BACKOFF_MULTIPIER;
private int requestTimeout = 10 * 1000;
private long rollingTimeout = DEFAULT_ROLLING_TIMEOUT;
private long scaleTimeout = DEFAULT_SCALE_TIMEOUT;
Expand Down Expand Up @@ -83,15 +89,17 @@ public RequestConfig(String username, String password, String oauthToken,
long websocketTimeout, long websocketPingInterval,
int maxConcurrentRequests, int maxConcurrentRequestsPerHost) {
this(username, password, oauthToken, watchReconnectLimit, watchReconnectInterval, connectionTimeout, rollingTimeout, requestTimeout, scaleTimeout, loggingInterval,
websocketTimeout, websocketPingInterval,maxConcurrentRequests, maxConcurrentRequestsPerHost, null);
websocketTimeout, websocketPingInterval,maxConcurrentRequests, maxConcurrentRequestsPerHost, null, DEFAULT_REQUEST_RETRY_BACKOFF_COUNT, DEFAULT_REQUEST_RETRY_BACKOFF_INITIAL,
DEFAULT_REQUEST_RETRY_BACKOFF_MULTIPIER);
}

@Buildable(builderPackage = "io.fabric8.kubernetes.api.builder", editableEnabled = false)
public RequestConfig(String username, String password, String oauthToken,
int watchReconnectLimit, int watchReconnectInterval,
int connectionTimeout, long rollingTimeout, int requestTimeout, long scaleTimeout, int loggingInterval,
long websocketTimeout, long websocketPingInterval,
int maxConcurrentRequests, int maxConcurrentRequestsPerHost, OAuthTokenProvider oauthTokenProvider) {
int maxConcurrentRequests, int maxConcurrentRequestsPerHost, OAuthTokenProvider oauthTokenProvider,
int requestRetryBackoffCount, int requestRetryBackoffInitial, int requestRetryBackoffMultiplier) {
this.username = username;
this.oauthToken = oauthToken;
this.password = password;
Expand All @@ -107,6 +115,9 @@ public RequestConfig(String username, String password, String oauthToken,
this.maxConcurrentRequests = maxConcurrentRequests;
this.maxConcurrentRequestsPerHost = maxConcurrentRequestsPerHost;
this.oauthTokenProvider = oauthTokenProvider;
this.requestRetryBackoffCount = requestRetryBackoffCount;
this.requestRetryBackoffInitial = requestRetryBackoffInitial;
this.requestRetryBackoffMultiplier = requestRetryBackoffMultiplier;
}

public String getUsername() {
Expand Down Expand Up @@ -168,6 +179,30 @@ public void setRequestTimeout(int requestTimeout) {
this.requestTimeout = requestTimeout;
}

public int getRequestRetryBackoffCount() {
return requestRetryBackoffCount;
}

public void setRequestRetryBackoffCount(int requestRetryBackoffCount) {
this.requestRetryBackoffCount = requestRetryBackoffCount;
}

public int getRequestRetryBackoffInitial() {
return requestRetryBackoffInitial;
}

public void setRequestRetryBackoffInitial(int requestRetryBackoffInitial) {
this.requestRetryBackoffInitial = requestRetryBackoffInitial;
}

public int getRequestRetryBackoffMultiplier() {
return requestRetryBackoffMultiplier;
}

public void setRequestRetryBackoffMultiplier(int requestRetryBackoffMultiplier) {
this.requestRetryBackoffMultiplier = requestRetryBackoffMultiplier;
}

public int getConnectionTimeout() {
return connectionTimeout;
}
Expand Down
23 changes: 22 additions & 1 deletion kubernetes-client/src/main/java/io/fabric8/kubernetes/client/dsl/base/OperationSupport.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@
import okhttp3.RequestBody;
import okhttp3.Response;
import okhttp3.ResponseBody;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.io.InputStream;
Expand All @@ -58,6 +60,7 @@ public class OperationSupport {
public static final MediaType JSON_MERGE_PATCH = MediaType.parse("application/merge-patch+json");
protected static final ObjectMapper JSON_MAPPER = Serialization.jsonMapper();
protected static final ObjectMapper YAML_MAPPER = Serialization.yamlMapper();
private static final Logger LOG = LoggerFactory.getLogger(OperationSupport.class);
private static final String CLIENT_STATUS_FLAG = "CLIENT_STATUS_FLAG";

protected OperationContext context;
Expand Down Expand Up @@ -538,7 +541,7 @@ protected <T> T handleResponse(OkHttpClient client, Request.Builder requestBuild
protected <T> T handleResponse(OkHttpClient client, Request.Builder requestBuilder, Class<T> type, Map<String, String> parameters) throws ExecutionException, InterruptedException, IOException {
VersionUsageUtils.log(this.resourceT, this.apiGroupVersion);
Request request = requestBuilder.build();
Response response = client.newCall(request).execute();
Response response = retryWithExponentialBackoff(client, request);
try (ResponseBody body = response.body()) {
assertResponseCode(request, response);
if (type != null) {
Expand All @@ -560,6 +563,24 @@ protected <T> T handleResponse(OkHttpClient client, Request.Builder requestBuild
}
}

protected Response retryWithExponentialBackoff(OkHttpClient client, Request request) throws InterruptedException, IOException {
int numRetries = config.getRequestRetryBackoffCount();
Response response;
long currentBackOff = config.getRequestRetryBackoffInitial();
boolean doRetry;
do {
response = client.newCall(request).execute();
numRetries--;
doRetry = numRetries != -1 && response.code() >= 500;
if (doRetry) {
LOG.debug("HTTP operation on url: {} should be retried as the response code was {}, retrying after {} millis", request.url(), response.code(), currentBackOff);
Thread.sleep(currentBackOff);
currentBackOff *= config.getRequestRetryBackoffMultiplier();
}
} while(doRetry);
return response;
}

/**
* Checks if the response status code is the expected and throws the appropriate KubernetesClientException if not.
*
Expand Down
Loading

0 comments on commit 11bdb2b

Please sign in to comment.