Skip to content

Commit

Permalink
[Improve][HttpConnector]Increase custom configuration timeout. (apach…
Browse files Browse the repository at this point in the history 
…e#6223)
  • Loading branch information
@lightzhao
lightzhao authored Jan 17, 2024
1 parent c69da93 commit fa5b7d3
Show file tree
Hide file tree
Showing 5 changed files with 38 additions and 13 deletions.
2 changes: 2 additions & 0 deletions docs/en/connector-v2/sink/Http.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ They can be downloaded via install-plugin.sh or from the Maven central repositor
| retry | Int | No | - | The max retry times if request http return to `IOException` |
| retry_backoff_multiplier_ms | Int | No | 100 | The retry-backoff times(millis) multiplier if request http failed |
| retry_backoff_max_ms | Int | No | 10000 | The maximum retry-backoff times(millis) if request http failed |
| connect_timeout_ms | Int | No | 12000 | Connection timeout setting, default 12s. |
| socket_timeout_ms | Int | No | 60000 | Socket timeout setting, default 60s. |
| common-options | | No | - | Sink plugin common parameters, please refer to [Sink Common Options](common-options.md) for details |

## Example
Expand Down
2 changes: 2 additions & 0 deletions docs/en/connector-v2/source/Http.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@ They can be downloaded via install-plugin.sh or from the Maven central repositor
| retry_backoff_multiplier_ms | Int | No | 100 | The retry-backoff times(millis) multiplier if request http failed. |
| retry_backoff_max_ms | Int | No | 10000 | The maximum retry-backoff times(millis) if request http failed |
| enable_multi_lines | Boolean | No | false | |
| connect_timeout_ms | Int | No | 12000 | Connection timeout setting, default 12s. |
| socket_timeout_ms | Int | No | 60000 | Socket timeout setting, default 60s. |
| common-options | | No | - | Source plugin common parameters, please refer to [Source Common Options](common-options.md) for details |

## How to Create a Http Data Synchronization Jobs
Expand Down
25 changes: 12 additions & 13 deletions ...c/main/java/org/apache/seatunnel/connectors/seatunnel/http/client/HttpClientProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,20 +66,19 @@
public class HttpClientProvider implements AutoCloseable {
private static final String ENCODING = "UTF-8";
private static final String APPLICATION_JSON = "application/json";
private static final int CONNECT_TIMEOUT = 6000 * 2;
private static final int SOCKET_TIMEOUT = 6000 * 10;
private static final int INITIAL_CAPACITY = 16;
private static final RequestConfig REQUEST_CONFIG =
RequestConfig.custom()
.setConnectTimeout(CONNECT_TIMEOUT)
.setSocketTimeout(SOCKET_TIMEOUT)
.build();
private RequestConfig requestConfig;
private final CloseableHttpClient httpClient;
private final Retryer<CloseableHttpResponse> retryer;

public HttpClientProvider(HttpParameter httpParameter) {
this.httpClient = HttpClients.createDefault();
this.retryer = buildRetryer(httpParameter);
this.requestConfig =
RequestConfig.custom()
.setConnectTimeout(httpParameter.getConnectTimeoutMs())
.setSocketTimeout(httpParameter.getSocketTimeoutMs())
.build();
}

private Retryer<CloseableHttpResponse> buildRetryer(HttpParameter httpParameter) {
Expand Down Expand Up @@ -176,7 +175,7 @@ public HttpResponse doGet(String url, Map<String, String> headers, Map<String, S
// create a new http get
HttpGet httpGet = new HttpGet(uriBuilder.build());
// set default request config
httpGet.setConfig(REQUEST_CONFIG);
httpGet.setConfig(requestConfig);
// set request header
addHeaders(httpGet, headers);
// return http response
Expand Down Expand Up @@ -220,7 +219,7 @@ public HttpResponse doPost(String url, Map<String, String> headers, Map<String,
// create a new http get
HttpPost httpPost = new HttpPost(url);
// set default request config
httpPost.setConfig(REQUEST_CONFIG);
httpPost.setConfig(requestConfig);
// set request header
addHeaders(httpPost, headers);
// set request params
Expand Down Expand Up @@ -255,7 +254,7 @@ public HttpResponse doPost(String url, Map<String, String> headers, String body)
// create a new http post
HttpPost httpPost = new HttpPost(url);
// set default request config
httpPost.setConfig(REQUEST_CONFIG);
httpPost.setConfig(requestConfig);
// set request header
addHeaders(httpPost, headers);
// add body in request
Expand All @@ -280,7 +279,7 @@ public HttpResponse doPost(
// create a new http get
HttpPost httpPost = new HttpPost(url);
// set default request config
httpPost.setConfig(REQUEST_CONFIG);
httpPost.setConfig(requestConfig);
// set request header
addHeaders(httpPost, headers);
// set request params
Expand Down Expand Up @@ -314,7 +313,7 @@ public HttpResponse doPut(String url, Map<String, String> params) throws Excepti
// create a new http put
HttpPut httpPut = new HttpPut(url);
// set default request config
httpPut.setConfig(REQUEST_CONFIG);
httpPut.setConfig(requestConfig);
// set request params
addParameters(httpPut, params);
// return http response
Expand All @@ -332,7 +331,7 @@ public HttpResponse doDelete(String url) throws Exception {
// create a new http delete
HttpDelete httpDelete = new HttpDelete(url);
// set default request config
httpDelete.setConfig(REQUEST_CONFIG);
httpDelete.setConfig(requestConfig);
// return http response
return getResponse(httpDelete);
}
Expand Down
14 changes: 14 additions & 0 deletions ...-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/config/HttpConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ public class HttpConfig {
public static final int DEFAULT_RETRY_BACKOFF_MULTIPLIER_MS = 100;
public static final int DEFAULT_RETRY_BACKOFF_MAX_MS = 10000;
public static final boolean DEFAULT_ENABLE_MULTI_LINES = false;
public static final int DEFAULT_CONNECT_TIMEOUT_MS = 6000 * 2;
public static final int DEFAULT_SOCKET_TIMEOUT_MS = 6000 * 10;
public static final Option<String> URL =
Options.key("url").stringType().noDefaultValue().withDescription("Http request url");
public static final Option<Long> TOTAL_PAGE_SIZE =
Expand Down Expand Up @@ -112,6 +114,18 @@ public class HttpConfig {
.withDescription(
"SeaTunnel enableMultiLines.This parameter can support http splitting response text by line.");

public static final Option<Integer> CONNECT_TIMEOUT_MS =
Options.key("connect_timeout_ms")
.intType()
.defaultValue(DEFAULT_CONNECT_TIMEOUT_MS)
.withDescription("Connection timeout setting, default 12s.");

public static final Option<Integer> SOCKET_TIMEOUT_MS =
Options.key("socket_timeout_ms")
.intType()
.defaultValue(DEFAULT_SOCKET_TIMEOUT_MS)
.withDescription("Socket timeout setting, default 60s.");

public enum ResponseFormat {
JSON("json"),
TEXT("text");
Expand Down
8 changes: 8 additions & 0 deletions ...se/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/config/HttpParameter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ public class HttpParameter implements Serializable {
protected int retryBackoffMultiplierMillis = HttpConfig.DEFAULT_RETRY_BACKOFF_MULTIPLIER_MS;
protected int retryBackoffMaxMillis = HttpConfig.DEFAULT_RETRY_BACKOFF_MAX_MS;
protected boolean enableMultilines;
protected int connectTimeoutMs = HttpConfig.DEFAULT_CONNECT_TIMEOUT_MS;
protected int socketTimeoutMs = HttpConfig.DEFAULT_SOCKET_TIMEOUT_MS;

public void buildWithConfig(Config pluginConfig) {
// set url
Expand Down Expand Up @@ -85,6 +87,12 @@ public void buildWithConfig(Config pluginConfig) {
} else {
this.setEnableMultilines(HttpConfig.ENABLE_MULTI_LINES.defaultValue());
}
if (pluginConfig.hasPath(HttpConfig.CONNECT_TIMEOUT_MS.key())) {
this.setConnectTimeoutMs(pluginConfig.getInt(HttpConfig.CONNECT_TIMEOUT_MS.key()));
}
if (pluginConfig.hasPath(HttpConfig.SOCKET_TIMEOUT_MS.key())) {
this.setSocketTimeoutMs(pluginConfig.getInt(HttpConfig.SOCKET_TIMEOUT_MS.key()));
}
}

public void setRetryParameters(Config pluginConfig) {
Expand Down

0 comments on commit fa5b7d3

Please sign in to comment.