Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Retry HTTP for status >= 500 (exponential backoff) #3164

Merged
merged 3 commits into from
Jun 2, 2021
Merged

Retry HTTP for status >= 500 (exponential backoff) #3164

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e92f797
Retry HTTP for status >= 500 (exponential backoff)
attilapiros May 7, 2021
3c95a6f
Extract and reuse retry interval calculation
attilapiros May 28, 2021
278ce6d
Update README.md
attilapiros Jun 2, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
* Fix #3166: Add DSL Support for `machineconfiguration.openshift.io/v1` resources in OpenShiftClient
* Fix #3142: Add DSL support for missing resources in `operator.openshift.io` and `monitoring.coreos.com` apiGroups
* Add DSL support for missing resources in `template.openshift.io`, `helm.openshift.io`, `network.openshift.io`, `user.openshift.io` apigroups
* Fix #3087: Support HTTP operation retry with exponential backoff (for status code >= 500)

#### _**Note**_: Breaking changes in the API
##### DSL Changes:
Expand Down
2 changes: 2 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,8 @@ System properties are preferred over environment variables. The following system
| `kubernetes.watch.reconnectLimit` / `KUBERNETES_WATCH_RECONNECTLIMIT` | Number of reconnect attempts (-1 for infinite) | `-1` |
| `kubernetes.connection.timeout` / `KUBERNETES_CONNECTION_TIMEOUT` | Connection timeout in ms (0 for no timeout) | `10000` |
| `kubernetes.request.timeout` / `KUBERNETES_REQUEST_TIMEOUT` | Read timeout in ms | `10000` |
| `kubernetes.request.retry.backoffLimit` / `KUBERNETES_REQUEST_RETRY_BACKOFFLIMIT` | Number of retry attempts | `0` |
| `kubernetes.request.retry.backoffInterval` / `KUBERNETES_REQUEST_RETRY_BACKOFFINTERVAL` | Retry initial backoff interval in ms | `1000` |
| `kubernetes.rolling.timeout` / `KUBERNETES_ROLLING_TIMEOUT` | Rolling timeout in ms | `900000` |
| `kubernetes.logging.interval` / `KUBERNETES_LOGGING_INTERVAL` | Logging interval in ms | `20000` |
| `kubernetes.scale.timeout` / `KUBERNETES_SCALE_TIMEOUT` | Scale timeout in ms | `600000` |
Expand Down
33 changes: 30 additions & 3 deletions kubernetes-client/src/main/java/io/fabric8/kubernetes/client/Config.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,8 @@ public class Config {
public static final String KUBERNETES_WATCH_RECONNECT_LIMIT_SYSTEM_PROPERTY = "kubernetes.watch.reconnectLimit";
public static final String KUBERNETES_CONNECTION_TIMEOUT_SYSTEM_PROPERTY = "kubernetes.connection.timeout";
public static final String KUBERNETES_REQUEST_TIMEOUT_SYSTEM_PROPERTY = "kubernetes.request.timeout";
public static final String KUBERNETES_REQUEST_RETRY_BACKOFFLIMIT_SYSTEM_PROPERTY = "kubernetes.request.retry.backoffLimit";
public static final String KUBERNETES_REQUEST_RETRY_BACKOFFINTERVAL_SYSTEM_PROPERTY = "kubernetes.request.retry.backoffInterval";
public static final String KUBERNETES_ROLLING_TIMEOUT_SYSTEM_PROPERTY = "kubernetes.rolling.timeout";
public static final String KUBERNETES_LOGGING_INTERVAL_SYSTEM_PROPERTY = "kubernetes.logging.interval";
public static final String KUBERNETES_SCALE_TIMEOUT_SYSTEM_PROPERTY = "kubernetes.scale.timeout";
Expand Down Expand Up @@ -135,6 +137,9 @@ public class Config {
public static final Integer DEFAULT_MAX_CONCURRENT_REQUESTS = 64;
public static final Integer DEFAULT_MAX_CONCURRENT_REQUESTS_PER_HOST = 5;

public static final Integer DEFAULT_REQUEST_RETRY_BACKOFFLIMIT = 0;
public static final Integer DEFAULT_REQUEST_RETRY_BACKOFFINTERVAL = 1000;

public static final String HTTP_PROTOCOL_PREFIX = "http://";
public static final String HTTPS_PROTOCOL_PREFIX = "https://";

Expand Down Expand Up @@ -173,6 +178,8 @@ public class Config {
private int watchReconnectInterval = 1000;
private int watchReconnectLimit = -1;
private int connectionTimeout = 10 * 1000;
private int requestRetryBackoffLimit;
private int requestRetryBackoffInterval;
private int requestTimeout = 10 * 1000;
private long rollingTimeout = DEFAULT_ROLLING_TIMEOUT;
private long scaleTimeout = DEFAULT_SCALE_TIMEOUT;
Expand Down Expand Up @@ -289,11 +296,11 @@ private static String ensureHttps(String masterUrl, Config config) {

@Deprecated
public Config(String masterUrl, String apiVersion, String namespace, boolean trustCerts, boolean disableHostnameVerification, String caCertFile, String caCertData, String clientCertFile, String clientCertData, String clientKeyFile, String clientKeyData, String clientKeyAlgo, String clientKeyPassphrase, String username, String password, String oauthToken, int watchReconnectInterval, int watchReconnectLimit, int connectionTimeout, int requestTimeout, long rollingTimeout, long scaleTimeout, int loggingInterval, int maxConcurrentRequests, int maxConcurrentRequestsPerHost, String httpProxy, String httpsProxy, String[] noProxy, Map<Integer, String> errorMessages, String userAgent, TlsVersion[] tlsVersions, long websocketTimeout, long websocketPingInterval, String proxyUsername, String proxyPassword, String trustStoreFile, String trustStorePassphrase, String keyStoreFile, String keyStorePassphrase, String impersonateUsername, String[] impersonateGroups, Map<String, List<String>> impersonateExtras) {
this(masterUrl, apiVersion, namespace, trustCerts, disableHostnameVerification, caCertFile, caCertData, clientCertFile, clientCertData, clientKeyFile, clientKeyData, clientKeyAlgo, clientKeyPassphrase, username, password, oauthToken, watchReconnectInterval, watchReconnectLimit, connectionTimeout, requestTimeout, rollingTimeout, scaleTimeout, loggingInterval, maxConcurrentRequests, maxConcurrentRequestsPerHost, false, httpProxy, httpsProxy, noProxy, errorMessages, userAgent, tlsVersions, websocketTimeout, websocketPingInterval, proxyUsername, proxyPassword, trustStoreFile, trustStorePassphrase, keyStoreFile, keyStorePassphrase, impersonateUsername, impersonateGroups, impersonateExtras, null,null);
this(masterUrl, apiVersion, namespace, trustCerts, disableHostnameVerification, caCertFile, caCertData, clientCertFile, clientCertData, clientKeyFile, clientKeyData, clientKeyAlgo, clientKeyPassphrase, username, password, oauthToken, watchReconnectInterval, watchReconnectLimit, connectionTimeout, requestTimeout, rollingTimeout, scaleTimeout, loggingInterval, maxConcurrentRequests, maxConcurrentRequestsPerHost, false, httpProxy, httpsProxy, noProxy, errorMessages, userAgent, tlsVersions, websocketTimeout, websocketPingInterval, proxyUsername, proxyPassword, trustStoreFile, trustStorePassphrase, keyStoreFile, keyStorePassphrase, impersonateUsername, impersonateGroups, impersonateExtras, null,null, DEFAULT_REQUEST_RETRY_BACKOFFLIMIT, DEFAULT_REQUEST_RETRY_BACKOFFINTERVAL);
}

@Buildable(builderPackage = "io.fabric8.kubernetes.api.builder", editableEnabled = false)
public Config(String masterUrl, String apiVersion, String namespace, boolean trustCerts, boolean disableHostnameVerification, String caCertFile, String caCertData, String clientCertFile, String clientCertData, String clientKeyFile, String clientKeyData, String clientKeyAlgo, String clientKeyPassphrase, String username, String password, String oauthToken, int watchReconnectInterval, int watchReconnectLimit, int connectionTimeout, int requestTimeout, long rollingTimeout, long scaleTimeout, int loggingInterval, int maxConcurrentRequests, int maxConcurrentRequestsPerHost, boolean http2Disable, String httpProxy, String httpsProxy, String[] noProxy, Map<Integer, String> errorMessages, String userAgent, TlsVersion[] tlsVersions, long websocketTimeout, long websocketPingInterval, String proxyUsername, String proxyPassword, String trustStoreFile, String trustStorePassphrase, String keyStoreFile, String keyStorePassphrase, String impersonateUsername, String[] impersonateGroups, Map<String, List<String>> impersonateExtras, OAuthTokenProvider oauthTokenProvider,Map<String,String> customHeaders) {
public Config(String masterUrl, String apiVersion, String namespace, boolean trustCerts, boolean disableHostnameVerification, String caCertFile, String caCertData, String clientCertFile, String clientCertData, String clientKeyFile, String clientKeyData, String clientKeyAlgo, String clientKeyPassphrase, String username, String password, String oauthToken, int watchReconnectInterval, int watchReconnectLimit, int connectionTimeout, int requestTimeout, long rollingTimeout, long scaleTimeout, int loggingInterval, int maxConcurrentRequests, int maxConcurrentRequestsPerHost, boolean http2Disable, String httpProxy, String httpsProxy, String[] noProxy, Map<Integer, String> errorMessages, String userAgent, TlsVersion[] tlsVersions, long websocketTimeout, long websocketPingInterval, String proxyUsername, String proxyPassword, String trustStoreFile, String trustStorePassphrase, String keyStoreFile, String keyStorePassphrase, String impersonateUsername, String[] impersonateGroups, Map<String, List<String>> impersonateExtras, OAuthTokenProvider oauthTokenProvider,Map<String,String> customHeaders, int requestRetryBackoffLimit, int requestRetryBackoffInterval) {
this.masterUrl = masterUrl;
this.apiVersion = apiVersion;
this.namespace = namespace;
Expand All @@ -308,7 +315,7 @@ public Config(String masterUrl, String apiVersion, String namespace, boolean tru
this.clientKeyAlgo = clientKeyAlgo;
this.clientKeyPassphrase = clientKeyPassphrase;

this.requestConfig = new RequestConfig(username, password, oauthToken, watchReconnectLimit, watchReconnectInterval, connectionTimeout, rollingTimeout, requestTimeout, scaleTimeout, loggingInterval, websocketTimeout, websocketPingInterval, maxConcurrentRequests, maxConcurrentRequestsPerHost, oauthTokenProvider);
this.requestConfig = new RequestConfig(username, password, oauthToken, watchReconnectLimit, watchReconnectInterval, connectionTimeout, rollingTimeout, requestTimeout, scaleTimeout, loggingInterval, websocketTimeout, websocketPingInterval, maxConcurrentRequests, maxConcurrentRequestsPerHost, oauthTokenProvider, requestRetryBackoffLimit, requestRetryBackoffInterval);
this.requestConfig.setImpersonateUsername(impersonateUsername);
this.requestConfig.setImpersonateGroups(impersonateGroups);
this.requestConfig.setImpersonateExtras(impersonateExtras);
Expand Down Expand Up @@ -399,6 +406,8 @@ public static void configFromSysPropsOrEnvVars(Config config) {

config.setConnectionTimeout(Utils.getSystemPropertyOrEnvVar(KUBERNETES_CONNECTION_TIMEOUT_SYSTEM_PROPERTY, config.getConnectionTimeout()));
config.setRequestTimeout(Utils.getSystemPropertyOrEnvVar(KUBERNETES_REQUEST_TIMEOUT_SYSTEM_PROPERTY, config.getRequestTimeout()));
config.setRequestRetryBackoffLimit(Utils.getSystemPropertyOrEnvVar(KUBERNETES_REQUEST_RETRY_BACKOFFLIMIT_SYSTEM_PROPERTY, config.getRequestRetryBackoffLimit()));
config.setRequestRetryBackoffInterval(Utils.getSystemPropertyOrEnvVar(KUBERNETES_REQUEST_RETRY_BACKOFFINTERVAL_SYSTEM_PROPERTY, config.getRequestRetryBackoffInterval()));

String configuredWebsocketTimeout = Utils.getSystemPropertyOrEnvVar(KUBERNETES_WEBSOCKET_TIMEOUT_SYSTEM_PROPERTY, String.valueOf(config.getWebsocketTimeout()));
if (configuredWebsocketTimeout != null) {
Expand Down Expand Up @@ -1036,6 +1045,24 @@ public void setRequestTimeout(int requestTimeout) {
this.requestConfig.setRequestTimeout(requestTimeout);
}

@JsonProperty("requestRetryBackoffLimit")
public int getRequestRetryBackoffLimit() {
return getRequestConfig().getRequestRetryBackoffLimit();
}

public void setRequestRetryBackoffLimit(int requestRetryBackoffLimit) {
requestConfig.setRequestRetryBackoffLimit(requestRetryBackoffLimit);
}

@JsonProperty("requestRetryBackoffInterval")
public int getRequestRetryBackoffInterval() {
return getRequestConfig().getRequestRetryBackoffInterval();
}

public void setRequestRetryBackoffInterval(int requestRetryBackoffInterval) {
requestConfig.setRequestRetryBackoffInterval(requestRetryBackoffInterval);
}

@JsonProperty("rollingTimeout")
public long getRollingTimeout() {
return getRequestConfig().getRollingTimeout();
Expand Down
29 changes: 25 additions & 4 deletions kubernetes-client/src/main/java/io/fabric8/kubernetes/client/RequestConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
import static io.fabric8.kubernetes.client.Config.DEFAULT_LOGGING_INTERVAL;
import static io.fabric8.kubernetes.client.Config.DEFAULT_MAX_CONCURRENT_REQUESTS;
import static io.fabric8.kubernetes.client.Config.DEFAULT_MAX_CONCURRENT_REQUESTS_PER_HOST;
import static io.fabric8.kubernetes.client.Config.DEFAULT_REQUEST_RETRY_BACKOFFLIMIT;
import static io.fabric8.kubernetes.client.Config.DEFAULT_REQUEST_RETRY_BACKOFFINTERVAL;
import static io.fabric8.kubernetes.client.Config.DEFAULT_ROLLING_TIMEOUT;
import static io.fabric8.kubernetes.client.Config.DEFAULT_SCALE_TIMEOUT;
import static io.fabric8.kubernetes.client.Config.DEFAULT_WEBSOCKET_PING_INTERVAL;
Expand All @@ -45,6 +47,8 @@ public class RequestConfig {
private int watchReconnectInterval = 1000;
private int watchReconnectLimit = -1;
private int connectionTimeout = 10 * 1000;
private int requestRetryBackoffLimit = DEFAULT_REQUEST_RETRY_BACKOFFLIMIT;
private int requestRetryBackoffInterval = DEFAULT_REQUEST_RETRY_BACKOFFINTERVAL;
private int requestTimeout = 10 * 1000;
private long rollingTimeout = DEFAULT_ROLLING_TIMEOUT;
private long scaleTimeout = DEFAULT_SCALE_TIMEOUT;
Expand Down Expand Up @@ -72,8 +76,6 @@ public class RequestConfig {
* @param scaleTimeout scale timeout
* @param loggingInterval logging interval
* @param websocketTimeout web socket timeout
* @param websocketPingInterval web socket ping interval
* @param maxConcurrentRequests max concurrent requests
* @param maxConcurrentRequestsPerHost max concurrent requests per host
*/
@Deprecated
Expand All @@ -83,15 +85,16 @@ public RequestConfig(String username, String password, String oauthToken,
long websocketTimeout, long websocketPingInterval,
int maxConcurrentRequests, int maxConcurrentRequestsPerHost) {
this(username, password, oauthToken, watchReconnectLimit, watchReconnectInterval, connectionTimeout, rollingTimeout, requestTimeout, scaleTimeout, loggingInterval,
websocketTimeout, websocketPingInterval,maxConcurrentRequests, maxConcurrentRequestsPerHost, null);
websocketTimeout, websocketPingInterval,maxConcurrentRequests, maxConcurrentRequestsPerHost, null, DEFAULT_REQUEST_RETRY_BACKOFFLIMIT, DEFAULT_REQUEST_RETRY_BACKOFFINTERVAL);
}

@Buildable(builderPackage = "io.fabric8.kubernetes.api.builder", editableEnabled = false)
public RequestConfig(String username, String password, String oauthToken,
int watchReconnectLimit, int watchReconnectInterval,
int connectionTimeout, long rollingTimeout, int requestTimeout, long scaleTimeout, int loggingInterval,
long websocketTimeout, long websocketPingInterval,
int maxConcurrentRequests, int maxConcurrentRequestsPerHost, OAuthTokenProvider oauthTokenProvider) {
int maxConcurrentRequests, int maxConcurrentRequestsPerHost, OAuthTokenProvider oauthTokenProvider,
int requestRetryBackoffLimit, int requestRetryBackoffInterval) {
this.username = username;
this.oauthToken = oauthToken;
this.password = password;
Expand All @@ -107,6 +110,8 @@ public RequestConfig(String username, String password, String oauthToken,
this.maxConcurrentRequests = maxConcurrentRequests;
this.maxConcurrentRequestsPerHost = maxConcurrentRequestsPerHost;
this.oauthTokenProvider = oauthTokenProvider;
this.requestRetryBackoffLimit = requestRetryBackoffLimit;
this.requestRetryBackoffInterval = requestRetryBackoffInterval;
}

public String getUsername() {
Expand Down Expand Up @@ -168,6 +173,22 @@ public void setRequestTimeout(int requestTimeout) {
this.requestTimeout = requestTimeout;
}

public int getRequestRetryBackoffLimit() {
return requestRetryBackoffLimit;
}

public void setRequestRetryBackoffLimit(int requestRetryBackoffLimit) {
this.requestRetryBackoffLimit = requestRetryBackoffLimit;
}

public int getRequestRetryBackoffInterval() {
return requestRetryBackoffInterval;
}

public void setRequestRetryBackoffInterval(int requestRetryBackoffInterval) {
this.requestRetryBackoffInterval = requestRetryBackoffInterval;
}

public int getConnectionTimeout() {
return connectionTimeout;
}
Expand Down
36 changes: 35 additions & 1 deletion kubernetes-client/src/main/java/io/fabric8/kubernetes/client/dsl/base/OperationSupport.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import io.fabric8.kubernetes.client.Config;
import io.fabric8.kubernetes.client.KubernetesClientException;
import io.fabric8.kubernetes.client.internal.VersionUsageUtils;
import io.fabric8.kubernetes.client.utils.ExponentialBackoffIntervalCalculator;
import io.fabric8.kubernetes.client.utils.Serialization;
import io.fabric8.kubernetes.client.utils.URLUtils;
import io.fabric8.kubernetes.client.utils.Utils;
Expand All @@ -39,6 +40,8 @@
import okhttp3.RequestBody;
import okhttp3.Response;
import okhttp3.ResponseBody;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.io.InputStream;
Expand All @@ -58,7 +61,9 @@ public class OperationSupport {
public static final MediaType JSON_MERGE_PATCH = MediaType.parse("application/merge-patch+json");
protected static final ObjectMapper JSON_MAPPER = Serialization.jsonMapper();
protected static final ObjectMapper YAML_MAPPER = Serialization.yamlMapper();
private static final Logger LOG = LoggerFactory.getLogger(OperationSupport.class);
private static final String CLIENT_STATUS_FLAG = "CLIENT_STATUS_FLAG";
private static final int maxRetryIntervalExponent = 5;

protected OperationContext context;
protected final OkHttpClient client;
Expand All @@ -69,6 +74,8 @@ public class OperationSupport {
protected String apiGroupName;
protected String apiGroupVersion;
protected boolean dryRun;
private final ExponentialBackoffIntervalCalculator retryIntervalCalculator;
private final int requestRetryBackoffLimit;

public OperationSupport() {
this (new OperationContext());
Expand Down Expand Up @@ -98,6 +105,16 @@ public OperationSupport(OperationContext ctx) {
} else {
this.apiGroupVersion = "v1";
}

final int requestRetryBackoffInterval;
if (ctx.getConfig() != null) {
requestRetryBackoffInterval = ctx.getConfig().getRequestRetryBackoffInterval();
this.requestRetryBackoffLimit = ctx.getConfig().getRequestRetryBackoffLimit();
} else {
requestRetryBackoffInterval = Config.DEFAULT_REQUEST_RETRY_BACKOFFINTERVAL;
this.requestRetryBackoffLimit = Config.DEFAULT_REQUEST_RETRY_BACKOFFLIMIT;
}
this.retryIntervalCalculator = new ExponentialBackoffIntervalCalculator(requestRetryBackoffInterval, maxRetryIntervalExponent);
}

public String getAPIGroup() {
Expand Down Expand Up @@ -538,7 +555,7 @@ protected <T> T handleResponse(OkHttpClient client, Request.Builder requestBuild
protected <T> T handleResponse(OkHttpClient client, Request.Builder requestBuilder, Class<T> type, Map<String, String> parameters) throws ExecutionException, InterruptedException, IOException {
VersionUsageUtils.log(this.resourceT, this.apiGroupVersion);
Request request = requestBuilder.build();
Response response = client.newCall(request).execute();
Response response = retryWithExponentialBackoff(client, request);
try (ResponseBody body = response.body()) {
assertResponseCode(request, response);
if (type != null) {
Expand All @@ -560,6 +577,23 @@ protected <T> T handleResponse(OkHttpClient client, Request.Builder requestBuild
}
}

protected Response retryWithExponentialBackoff(OkHttpClient client, Request request) throws InterruptedException, IOException {
Response response;
boolean doRetry;
int numRetries = 0;
do {
response = client.newCall(request).execute();
doRetry = numRetries < requestRetryBackoffLimit && response.code() >= 500;
if (doRetry) {
long retryInterval= retryIntervalCalculator.getInterval(numRetries);
LOG.debug("HTTP operation on url: {} should be retried as the response code was {}, retrying after {} millis", request.url(), response.code(), retryInterval);
Thread.sleep(retryInterval);
numRetries++;
}
} while(doRetry);
return response;
}

/**
* Checks if the response status code is the expected and throws the appropriate KubernetesClientException if not.
*
Expand Down
Loading