Merge: 3.5.x->master

src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/sdk/ArtifactoryClient.java

@@ -70,6 +70,7 @@
 import org.apache.http.client.methods.HttpDelete;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpRequestBase;
 import org.apache.http.entity.FileEntity;
 import org.apache.http.entity.InputStreamEntity;
 import org.apache.http.message.BasicHeader;
@@ -237,7 +238,7 @@ private <R> R getArtifactoryRespByReq(
         url = getCompleteUrl(url);
         String reqStr = "{}";
         if (reqBody != null) {
-            reqStr = JsonUtils.toJsonWithoutSkippedFields(reqBody);
+            reqStr = JsonUtils.toJson(reqBody);
         }
         String respStr = null;
         long start = System.nanoTime();
@@ -402,13 +403,13 @@ public NodeDTO getFileNode(String filePath) {
         return nodeDTO;
     }
 
-    public Pair<InputStream, Long> getFileInputStream(String filePath) throws ServiceException {
+    public Pair<InputStream, HttpRequestBase> getFileInputStream(String filePath) throws ServiceException {
         List<String> pathList = parsePath(filePath);
         return getFileInputStream(pathList.get(0), pathList.get(1), pathList.get(2));
     }
 
-    public Pair<InputStream, Long> getFileInputStream(String projectId, String repoName,
-                                                      String filePath) throws ServiceException {
+    public Pair<InputStream, HttpRequestBase> getFileInputStream(String projectId, String repoName,
+                                                                 String filePath) throws ServiceException {
         DownloadGenericFileReq req = new DownloadGenericFileReq();
         req.setProject(projectId);
         req.setRepo(repoName);
@@ -419,17 +420,10 @@ public Pair<InputStream, Long> getFileInputStream(String projectId, String repoN
         try {
             HttpMetricUtil.setHttpMetricName(CommonMetricNames.BKREPO_API_HTTP);
             HttpMetricUtil.addTagForCurrentMetric(Tag.of("api_name", "download:" + URL_DOWNLOAD_GENERIC_FILE));
-            resp = longHttpHelper.getRawResp(false, url, getJsonHeaders());
-            Header contentLengthHeader = resp.getFirstHeader("Content-Length");
-            Long contentLength = null;
-            if (contentLengthHeader != null && StringUtils.isNotBlank(contentLengthHeader.getValue())) {
-                contentLength = Long.parseLong(contentLengthHeader.getValue());
-                log.debug("Content-Length from header:{}", contentLengthHeader.getValue());
-            } else {
-                log.debug("Content-Length from header is null or blank");
-            }
+            Pair<HttpRequestBase, CloseableHttpResponse> pair = longHttpHelper.getRawResp(false, url, getJsonHeaders());
+            resp = pair.getRight();
             if (resp.getStatusLine() != null && resp.getStatusLine().getStatusCode() == 200) {
-                return Pair.of(resp.getEntity().getContent(), contentLength);
+                return Pair.of(resp.getEntity().getContent(), pair.getLeft());
             } else {
                 log.info("resp.statusLine={},resp.entity={}", resp.getStatusLine(), resp.getEntity());
                 throw new InternalException(ErrorCode.FAIL_TO_REQUEST_THIRD_FILE_SOURCE_DOWNLOAD_GENERIC_FILE);

src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/json/JsonUtils.java

@@ -51,6 +51,7 @@ public class JsonUtils {
 
     /**
      * 序列化时忽略bean中的某些字段,字段需要使用SkipLogFields注解
+     * 注意：只对第一层的字段生效，嵌套的不生效
      *
      * @param bean
      * @return

src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/Order.java

@@ -30,7 +30,7 @@
 public enum Order {
     DESCENDING(0), ASCENDING(1);
 
-    private int order;
+    private final int order;
 
     Order(int order) {
         this.order = order;

src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/BaseHttpHelper.java

@@ -35,6 +35,7 @@
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
+import org.apache.http.client.methods.HttpRequestBase;
 import org.apache.http.entity.ByteArrayEntity;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.util.EntityUtils;
@@ -57,7 +58,7 @@ CloseableHttpClient getHttpClient() {
     }
 
     @Override
-    public CloseableHttpResponse getRawResp(boolean keepAlive, String url, Header[] header) {
+    public Pair<HttpRequestBase, CloseableHttpResponse> getRawResp(boolean keepAlive, String url, Header[] header) {
         HttpGet get = new HttpGet(url);
         if (keepAlive) {
             get.setHeader("Connection", "Keep-Alive");
@@ -66,10 +67,14 @@ public CloseableHttpResponse getRawResp(boolean keepAlive, String url, Header[]
             get.setHeaders(header);
         }
         try {
-            return getHttpClient().execute(get);
+            return Pair.of(get, getHttpClient().execute(get));
         } catch (IOException e) {
             log.error("Get request fail", e);
             throw new InternalException(e, ErrorCode.API_ERROR);
+        } finally {
+            if (log.isDebugEnabled()) {
+                log.debug("getRawResp,url={},headers={}", url, header);
+            }
         }
     }
 
@@ -82,13 +87,28 @@ public Pair<Integer, String> get(boolean keepAlive, String url, Header[] header)
         if (header != null && header.length > 0) {
             get.setHeaders(header);
         }
+        int httpStatusCode = -1;
+        String respStr = null;
         try (CloseableHttpResponse response = getHttpClient().execute(get)) {
-            int httpStatusCode = response.getStatusLine().getStatusCode();
+            httpStatusCode = response.getStatusLine().getStatusCode();
             HttpEntity entity = response.getEntity();
-            return Pair.of(httpStatusCode, EntityUtils.toString(entity, CHARSET));
+            respStr = EntityUtils.toString(entity, CHARSET);
+            return Pair.of(httpStatusCode, respStr);
         } catch (IOException e) {
             log.error("Get request fail", e);
             throw new InternalException(e, ErrorCode.API_ERROR);
+        } finally {
+            get.releaseConnection();
+            if (log.isDebugEnabled()) {
+                log.debug(
+                    "get:keepAlive={},url={},headers={},httpStatusCode={},respStr={}",
+                    keepAlive,
+                    url,
+                    header,
+                    httpStatusCode,
+                    respStr
+                );
+            }
         }
     }
 
@@ -99,30 +119,43 @@ public Pair<Integer, byte[]> post(String url, HttpEntity requestEntity, Header..
         post.setHeader("Connection", "Keep-Alive");
         post.setHeaders(headers);
         post.setEntity(requestEntity);
+        int httpStatusCode = -1;
+        String respStr = null;
         try (CloseableHttpResponse httpResponse = getHttpClient().execute(post)) {
-            int statusCode = httpResponse.getStatusLine().getStatusCode();
-            if (statusCode != HttpStatus.SC_OK) {
+            httpStatusCode = httpResponse.getStatusLine().getStatusCode();
+            if (httpStatusCode != HttpStatus.SC_OK) {
                 String message = httpResponse.getStatusLine().getReasonPhrase();
                 HttpEntity entity = httpResponse.getEntity();
-                String content = "";
                 if (entity != null && entity.getContent() != null) {
-                    content = new String(EntityUtils.toByteArray(entity), CHARSET);
+                    respStr = new String(EntityUtils.toByteArray(entity), CHARSET);
                 }
                 log.warn(
-                    "Post request fail, statusCode={}, errorReason={}, body={}, url={}, headers={}",
-                    statusCode,
+                    "Post request fail, httpStatusCode={}, errorReason={}, body={}, url={}, headers={}",
+                    httpStatusCode,
                     message,
-                    content,
+                    respStr,
                     url,
                     headers
                 );
                 throw new InternalException(message, ErrorCode.API_ERROR);
             }
             HttpEntity entity = httpResponse.getEntity();
-            return Pair.of(statusCode, EntityUtils.toByteArray(entity));
+            return Pair.of(httpStatusCode, EntityUtils.toByteArray(entity));
         } catch (IOException e) {
             log.error("Post request fail", e);
             throw new InternalException(e, ErrorCode.API_ERROR);
+        } finally {
+            post.releaseConnection();
+            if (log.isDebugEnabled()) {
+                log.debug(
+                    "post:url={},headers={},requestEntity={},httpStatusCode={},respStr={}",
+                    url,
+                    headers,
+                    requestEntity,
+                    httpStatusCode,
+                    respStr
+                );
+            }
         }
     }
 
@@ -133,19 +166,38 @@ public Pair<Integer, String> put(String url, HttpEntity requestEntity, Header...
         put.setHeader("Connection", "Keep-Alive");
         put.setHeaders(headers);
         put.setEntity(requestEntity);
+        int httpStatusCode = -1;
+        String respStr = null;
         try (CloseableHttpResponse httpResponse = getHttpClient().execute(put)) {
-            int statusCode = httpResponse.getStatusLine().getStatusCode();
+            httpStatusCode = httpResponse.getStatusLine().getStatusCode();
             HttpEntity entity = httpResponse.getEntity();
-            String content = new String(EntityUtils.toByteArray(entity), CHARSET);
-            if (statusCode != HttpStatus.SC_OK) {
+            respStr = new String(EntityUtils.toByteArray(entity), CHARSET);
+            if (httpStatusCode != HttpStatus.SC_OK) {
                 String message = httpResponse.getStatusLine().getReasonPhrase();
-                log.info("Put request fail, statusCode={}, errorReason={}, content={}", statusCode, message, content);
+                log.warn(
+                    "Put request fail, httpStatusCode={}, errorReason={}, respStr={}",
+                    httpStatusCode,
+                    message,
+                    respStr
+                );
                 throw new InternalException(message, ErrorCode.API_ERROR);
             }
-            return Pair.of(statusCode, content);
+            return Pair.of(httpStatusCode, respStr);
         } catch (IOException e) {
             log.error("Put request fail", e);
             throw new InternalException(e, ErrorCode.API_ERROR);
+        } finally {
+            put.releaseConnection();
+            if (log.isDebugEnabled()) {
+                log.debug(
+                    "put:url={},headers={},requestEntity={},httpStatusCode={},respStr={}",
+                    url,
+                    headers,
+                    requestEntity,
+                    httpStatusCode,
+                    respStr
+                );
+            }
         }
     }
 
@@ -163,23 +215,39 @@ public Pair<Integer, String> delete(String url, String content, Header... header
             delete.setEntity(requestEntity);
         }
         delete.setHeaders(headers);
+        int httpStatusCode = -1;
+        String respStr = null;
         try (CloseableHttpResponse httpResponse = getHttpClient().execute(delete)) {
-            int statusCode = httpResponse.getStatusLine().getStatusCode();
-            if (statusCode != HttpStatus.SC_OK) {
+            httpStatusCode = httpResponse.getStatusLine().getStatusCode();
+            if (httpStatusCode != HttpStatus.SC_OK) {
                 String message = httpResponse.getStatusLine().getReasonPhrase();
-                log.info("Delete request fail, url={}, statusCode={}, errorReason={}", url, statusCode, message);
-                throw new InternalException(String.format("url=%s,statusCode=%s" +
-                    "，message=%s", url, statusCode, message), ErrorCode.API_ERROR);
+                log.info("Delete request fail, url={}, httpStatusCode={}, errorReason={}", url, httpStatusCode,
+                    message);
+                throw new InternalException(String.format("url=%s,httpStatusCode=%s" +
+                    "，message=%s", url, httpStatusCode, message), ErrorCode.API_ERROR);
             }
             HttpEntity entity = httpResponse.getEntity();
             byte[] respBytes = EntityUtils.toByteArray(entity);
             if (respBytes == null) {
                 return null;
             }
-            return Pair.of(statusCode, new String(respBytes, CHARSET));
+            respStr = new String(respBytes, CHARSET);
+            return Pair.of(httpStatusCode, respStr);
         } catch (IOException e) {
             log.error("Delete request fail", e);
             throw new InternalException(e, ErrorCode.API_ERROR);
+        } finally {
+            delete.releaseConnection();
+            if (log.isDebugEnabled()) {
+                log.debug(
+                    "delete:url={},headers={},body={},httpStatusCode={},respStr={}",
+                    url,
+                    headers,
+                    content,
+                    httpStatusCode,
+                    respStr
+                );
+            }
         }
     }
 

src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/ExtHttpHelper.java

@@ -27,9 +27,11 @@
 import com.tencent.bk.job.common.constant.ErrorCode;
 import com.tencent.bk.job.common.exception.InternalException;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.tuple.Pair;
 import org.apache.http.Header;
 import org.apache.http.HttpEntity;
 import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpRequestBase;
 import org.apache.http.entity.ByteArrayEntity;
 import org.apache.http.message.BasicHeader;
 
@@ -138,7 +140,7 @@ public String get(boolean keepAlive, String url, Header[] header) {
         return httpHelper.get(keepAlive, url, header).getRight();
     }
 
-    public CloseableHttpResponse getRawResp(boolean keepAlive, String url, Header[] header) {
+    public Pair<HttpRequestBase, CloseableHttpResponse> getRawResp(boolean keepAlive, String url, Header[] header) {
         return httpHelper.getRawResp(keepAlive, url, header);
     }
 

src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpHelper.java

@@ -28,10 +28,11 @@
 import org.apache.http.Header;
 import org.apache.http.HttpEntity;
 import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpRequestBase;
 
 public interface HttpHelper {
 
-    CloseableHttpResponse getRawResp(boolean keepAlive, String url, Header[] header);
+    Pair<HttpRequestBase, CloseableHttpResponse> getRawResp(boolean keepAlive, String url, Header[] header);
 
     Pair<Integer, String> get(boolean keepAlive, String url, Header[] header);
 

src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/WatchableHttpHelper.java

@@ -31,6 +31,7 @@
 import org.apache.http.Header;
 import org.apache.http.HttpEntity;
 import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpRequestBase;
 
 import java.util.AbstractList;
 import java.util.concurrent.TimeUnit;
@@ -49,7 +50,7 @@ public WatchableHttpHelper(HttpHelper httpHelper, MeterRegistry meterRegistry) {
     }
 
     @Override
-    public CloseableHttpResponse getRawResp(boolean keepAlive, String url, Header[] header) {
+    public Pair<HttpRequestBase, CloseableHttpResponse> getRawResp(boolean keepAlive, String url, Header[] header) {
         return httpHelper.getRawResp(keepAlive, url, header);
     }
 

src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/api/web/impl/WebBackupResourceImpl.java

@@ -59,6 +59,7 @@
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.tuple.Pair;
+import org.apache.http.client.methods.HttpRequestBase;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.MediaType;
@@ -197,7 +198,7 @@ private Pair<Long, StreamingResponseBody> getFileSizeAndStreamFromArtifactory(St
             throw new InternalException(ErrorCode.FAIL_TO_GET_NODE_INFO_FROM_ARTIFACTORY);
         }
         try {
-            Pair<InputStream, Long> pair = artifactoryClient.getFileInputStream(
+            Pair<InputStream, HttpRequestBase> pair = artifactoryClient.getFileInputStream(
                 artifactoryConfig.getArtifactoryJobProject(),
                 backupStorageConfig.getBackupRepo(),
                 fileName

src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/executor/ImportJobExecutor.java

@@ -72,6 +72,7 @@
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.tuple.Pair;
+import org.apache.http.client.methods.HttpRequestBase;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.i18n.LocaleContextHolder;
 import org.springframework.stereotype.Service;
@@ -201,7 +202,7 @@ private void processImportJob(String jobId) {
             // 下载文件
             if (!importFileDirectory.exists()) {
                 log.debug("begin to download from artifactory:{}", importJob.getFileName());
-                Pair<InputStream, Long> pair = artifactoryClient.getFileInputStream(
+                Pair<InputStream, HttpRequestBase> pair = artifactoryClient.getFileInputStream(
                     artifactoryConfig.getArtifactoryJobProject(),
                     backupStorageConfig.getBackupRepo(),
                     importJob.getFileName()

src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskLogResourceImpl.java

@@ -49,6 +49,7 @@
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.tuple.Pair;
+import org.apache.http.client.methods.HttpRequestBase;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.MediaType;
@@ -222,7 +223,7 @@ private Pair<Long, StreamingResponseBody> getFileSizeAndStreamFromArtifactory(
         }
         try {
             log.debug("get {} fileInputStream from artifactory", exportInfo.getZipFileName());
-            Pair<InputStream, Long> pair = artifactoryClient.getFileInputStream(
+            Pair<InputStream, HttpRequestBase> pair = artifactoryClient.getFileInputStream(
                 artifactoryConfig.getArtifactoryJobProject(),
                 logExportConfig.getLogExportRepo(),
                 exportInfo.getZipFileName()