diff --git a/docs/apidoc/bk-api-gateway/v3/zh/batch_get_job_instance_ip_log.md b/docs/apidoc/bk-api-gateway/v3/zh/batch_get_job_instance_ip_log.md index ef4a783d10..2abd98dbdb 100644 --- a/docs/apidoc/bk-api-gateway/v3/zh/batch_get_job_instance_ip_log.md +++ b/docs/apidoc/bk-api-gateway/v3/zh/batch_get_job_instance_ip_log.md @@ -109,7 +109,10 @@ }, "dest_path": "/tmp/1.log", "status": 4, - "log_content": "[2021-06-28 11:32:16] FileName: /tmp/1.log FileSize: 9.0 Bytes State: dest agent success download file Speed: 1 KB/s Progress: 100% StatusDesc: dest agent success download file Detail: success" + "log_content": "[2021-06-28 11:32:16] FileName: /tmp/1.log FileSize: 9.0 Bytes State: dest agent success download file Speed: 1 KB/s Progress: 100% StatusDesc: dest agent success download file Detail: success", + "size": "1.0 Bytes", + "speed": "0 KB/s", + "process": "100%" } ] }, @@ -127,7 +130,10 @@ }, "src_path": "/data/1.log", "status": 4, - "log_content": "[2021-06-28 11:32:16] FileName: /data/1.log FileSize: 9.0 Bytes State: source agent success upload file Speed: 1 KB/s Progress: 100% StatusDesc: source agent success upload file Detail: success upload" + "log_content": "[2021-06-28 11:32:16] FileName: /data/1.log FileSize: 9.0 Bytes State: source agent success upload file Speed: 1 KB/s Progress: 100% StatusDesc: source agent success upload file Detail: success upload", + "size": "1.0 Bytes", + "speed": "0 KB/s", + "process": "100%" } ] } @@ -173,15 +179,18 @@ ##### file_log -| 字段 | 类型 | 描述 | -|-----------|-----------|-----------| -| mode | 分发模式 | 0:上传;1:下载| -| src_ip | object |文件源主机IP。定义见ip | -| src_path | string | 源文件路径 | -| dest_ip | object | 分发目标主机IP，mode=1时有值。定义见ip | -| dest_path | string | 目标路径，mode=1时有值 | -| status | int | 任务状态。1-等待开始;2-上传中;3-下载中;4-成功;5-失败 | -| log_content | string | 文件分发日志内容 | +| 字段 | 类型 | 描述 | +|--------------|-----------|-----------| +| mode | 分发模式 | 0:上传;1:下载| +| src_ip | object |文件源主机IP。定义见ip | +| src_path | string | 源文件路径 | +| dest_ip | object | 分发目标主机IP，mode=1时有值。定义见ip | +| dest_path | string | 目标路径，mode=1时有值 | +| status | int | 任务状态。1-等待开始;2-上传中;3-下载中;4-成功;5-失败 | +| log_content | string | 文件分发日志内容 | +| size | string | 文件大小 | +| speed | string | 文件传输速率 | +| process | string | 文件传输进度 | ##### ip diff --git a/docs/apidoc/bk-api-gateway/v3/zh/get_step_instance_status.md b/docs/apidoc/bk-api-gateway/v3/zh/get_step_instance_status.md index d3c88bbe47..04542c04a0 100644 --- a/docs/apidoc/bk-api-gateway/v3/zh/get_step_instance_status.md +++ b/docs/apidoc/bk-api-gateway/v3/zh/get_step_instance_status.md @@ -54,34 +54,52 @@ "start_time": 1605064271000, "end_time": 1605064272000, "total_time": 1000, - "step_host_result_list": [ + "step_result_group_list": [ { - "bk_host_id": 101, - "ip": "127.0.0.1", - "ipv6": null, - "bk_cloud_id": 0, - "status": 9, - "status_desc": "执行成功", + "result_type": 9, + "result_type_desc": "执行成功", "tag": "tag1", - "group_key": "9_tag1", - "exit_code": 0, - "start_time": 1605064271000, - "end_time": 1605064272000, - "total_time": 1000 + "host_size": 2, + "host_result_list": [ + { + "bk_host_id": 101, + "ip": "127.0.0.1", + "ipv6": null, + "bk_agent_id": null, + "bk_cloud_id": 0, + "bk_cloud_name": "Default Area", + "status": 9, + "status_desc": "执行成功", + "tag": "tag1", + "exit_code": 0, + "start_time": 1605064271000, + "end_time": 1605064272000, + "total_time": 1000 + } + ] }, { - "bk_host_id": 102, - "ip": "127.0.0.2", - "ipv6": null, - "bk_cloud_id": 0, - "status": 9, - "status_desc": "执行成功", + "result_type": 9, + "result_type_desc": "执行成功", "tag": "tag2", - "group_key": "9_tag2", - "exit_code": 0, - "start_time": 1605064271000, - "end_time": 1605064272000, - "total_time": 1000 + "host_size": 2, + "host_result_list": [ + { + "bk_host_id": 102, + "ip": "127.0.0.2", + "ipv6": null, + "bk_agent_id": null, + "bk_cloud_id": 0, + "bk_cloud_name": "Default Area", + "status": 9, + "status_desc": "执行成功", + "tag": "tag2", + "exit_code": 0, + "start_time": 1605064271000, + "end_time": 1605064272000, + "total_time": 1000 + } + ] } ] } @@ -99,33 +117,45 @@ ##### data -| 字段 | 类型 | 是否一定存在 | 描述 | -|-----------------------|--------------------------|------------|-----------| -| step_instance_id | long | 是 | 作业步骤实例ID | -| execute_count | int | 是 | 步骤重试次数 | -| name | string | 是 | 步骤名称 | -| type | int | 是 | 步骤类型：1-脚本步骤；2-文件步骤；4-SQL步骤 | -| status | int | 是 | 作业步骤状态码: 1-未执行，2-正在执行，3-执行成功，4-执行失败，5-跳过，6-忽略错误，7-等待用户，8-手动结束，9-状态异常，10-步骤强制终止中，11-步骤强制终止成功，12-步骤强制终止失败 | -| create_time | long | 是 | 作业步骤实例创建时间，Unix时间戳，单位毫秒 | -| start_time | long | 是 | 开始执行时间，Unix时间戳，单位毫秒 | -| end_time | long | 是 | 执行结束时间，Unix时间戳，单位毫秒 | -| total_time | int | 是 | 总耗时，单位毫秒 | -| step_host_result_list | list | 是 | 每个主机的任务执行结果，定义见step_host_result | - - -##### step_host_result - -| 字段 | 类型 | 是否一定存在 | 描述 | -|-------------|-----------|------------|-----------| -| bk_host_id | long | 是 | 主机ID | -| ip | string | 否 | IP | -| ipv6 | string | 否 | IPv6 | -| bk_cloud_id | long | 否 | 管控区域ID | -| status | int | 是 | 任务状态：0-未知错误，1-Agent异常，2-无效主机，3-上次已成功，5-等待执行，7-正在执行，9-执行成功，11-执行失败，12-任务下发失败，13-任务超时，15-任务日志错误，16-GSE脚本日志超时，17-GSE文件日志超时，101-脚本执行失败，102-脚本执行超时，103-脚本执行被终止，104-脚本返回码非零，202-文件传输失败，203-源文件不存在，301-文件任务系统错误-未分类的，303-文件任务超时，310-Agent异常，311-用户名不存在，312-用户密码错误，320-文件获取失败，321-文件超出限制，329-文件传输错误，399-任务执行出错，403-任务强制终止成功，404-任务强制终止失败，500-未知状态 | -| status_desc | string | 是 | 任务状态描述 | -| tag | string | 否 | 用户通过job_success/job_fail函数模板自定义输出的结果。仅脚本任务存在该参数 | -| group_key | string | 否 | 基于status与tag字段的分组键，仅用于调用方验证分组内数据数量是否正确，请勿强依赖该字段 | -| exit_code | int | 否 | 脚本任务exit code | -| start_time | long | 是 | 开始执行时间，Unix时间戳，单位毫秒 | -| end_time | long | 是 | 执行结束时间，Unix时间戳，单位毫秒 | -| total_time | int | 是 | 总耗时，单位毫秒 | +| 字段 | 类型 | 是否一定存在 | 描述 | +|------------------------|--------------------------|------------|-----------| +| step_instance_id | long | 是 | 作业步骤实例ID | +| execute_count | int | 是 | 步骤重试次数 | +| name | string | 是 | 步骤名称 | +| type | int | 是 | 步骤类型：1-脚本步骤；2-文件步骤；4-SQL步骤 | +| status | int | 是 | 作业步骤状态码: 1-未执行，2-正在执行，3-执行成功，4-执行失败，5-跳过，6-忽略错误，7-等待用户，8-手动结束，9-状态异常，10-步骤强制终止中，11-步骤强制终止成功，12-步骤强制终止失败 | +| create_time | long | 是 | 作业步骤实例创建时间，Unix时间戳，单位毫秒 | +| start_time | long | 是 | 开始执行时间，Unix时间戳，单位毫秒 | +| end_time | long | 是 | 执行结束时间，Unix时间戳，单位毫秒 | +| total_time | int | 是 | 总耗时，单位毫秒 | +| step_result_group_list | list | 是 | 任务执行结果分组列表，元素定义见step_result_group | + + +##### step_result_group + +| 字段 | 类型 | 是否一定存在 | 描述 | +|--------------------|--------------------|------------|-------------| +| result_type | int | 是 | 分组类型 | +| result_type_desc | string | 是 | 分组类型描述 | +| tag | string | 是 | 分组标签 | +| host_size | int | 是 | 分组内主机总量 | +| host_result_list | list | 是 | 每个分组内的主机任务执行结果列表，元素定义见host_result | + + +##### host_result + +| 字段 | 类型 | 是否一定存在 | 描述 | +|--------------------|-----------|------------|-----------| +| bk_host_id | long | 是 | 主机ID | +| ip | string | 否 | IP | +| ipv6 | string | 否 | IPv6 | +| bk_agent_id | string | 否 | AgentId | +| bk_cloud_id | long | 否 | 管控区域ID | +| bk_cloud_name | string | 否 | 管控区域名称 | +| status | int | 是 | 任务状态：0-未知错误，1-Agent异常，2-无效主机，3-上次已成功，5-等待执行，7-正在执行，9-执行成功，11-执行失败，12-任务下发失败，13-任务超时，15-任务日志错误，16-GSE脚本日志超时，17-GSE文件日志超时，101-脚本执行失败，102-脚本执行超时，103-脚本执行被终止，104-脚本返回码非零，202-文件传输失败，203-源文件不存在，301-文件任务系统错误-未分类的，303-文件任务超时，310-Agent异常，311-用户名不存在，312-用户密码错误，320-文件获取失败，321-文件超出限制，329-文件传输错误，399-任务执行出错，403-任务强制终止成功，404-任务强制终止失败，500-未知状态 | +| status_desc | string | 是 | 任务状态描述 | +| tag | string | 否 | 用户通过job_success/job_fail函数模板自定义输出的结果。仅脚本任务存在该参数 | +| exit_code | int | 否 | 脚本任务exit code | +| start_time | long | 是 | 开始执行时间，Unix时间戳，单位毫秒 | +| end_time | long | 是 | 执行结束时间，Unix时间戳，单位毫秒 | +| total_time | int | 是 | 总耗时，单位毫秒 | diff --git a/docs/apidoc/bk-api-gateway/v3/zh/query_agent_info.md b/docs/apidoc/bk-api-gateway/v3/zh/query_agent_info.md new file mode 100644 index 0000000000..a6f64272f2 --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/query_agent_info.md @@ -0,0 +1,88 @@ +### 功能描述 + +查询主机Agent在作业平台中的信息（可执行状态、版本等） + +### 请求参数说明 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Body参数 + +| 字段 | 类型 | 必选 | 描述 | +|------------------------|--------------|-------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID，与bk_scope_type对应, 表示业务ID或者业务集ID | +| host_id_list | list | 是 | 主机ID数组，单次查询主机数量不可超过5000 | + + +### 请求参数示例 + +- POST +```json +{ + "bk_scope_type": "biz", + "bk_scope_id": "1", + "host_id_list": [1,2,3] +} +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "", + "data": { + "agent_info_list": [ + { + "bk_host_id": 1, + "status": 0, + "version": "2.1.4" + }, + { + "bk_host_id": 2, + "status": 1, + "version": "2.1.5" + }, + { + "bk_host_id": 3, + "status": 1, + "version": "2.1.6" + } + ] + } +} +``` +### 返回结果说明 + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +##### data + +| 字段 | 类型 | 描述 | +|-----------------------|----------------|-----------| +| agent_info_list | list | 元素为Agent信息，详情见agent_info对象定义，若传入的host_id在返回结果的列表中不存在，则表示未查询到该主机的agent信息 | + + +##### agent_info + +| 字段 | 类型 | 描述 | +|-----------------|------------------|-----------| +| bk_host_id | long | 主机ID | +| status | int | Agent状态：0-异常，1-正常 | +| version | string | Agent的版本 | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/batch_get_job_instance_ip_log.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/batch_get_job_instance_ip_log.md index 82b827c9fb..6076c6465d 100644 --- a/docs/apidoc/esb/jobv3-confapis/apidocs/en/batch_get_job_instance_ip_log.md +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/batch_get_job_instance_ip_log.md @@ -102,7 +102,10 @@ Bulk query of job execution logs by host list }, "dest_path": "/tmp/1.log", "status": 4, - "log_content": "[2021-06-28 11:32:16] FileName: /tmp/1.log FileSize: 9.0 Bytes State: dest agent success download file Speed: 1 KB/s Progress: 100% StatusDesc: dest agent success download file Detail: success" + "log_content": "[2021-06-28 11:32:16] FileName: /tmp/1.log FileSize: 9.0 Bytes State: dest agent success download file Speed: 1 KB/s Progress: 100% StatusDesc: dest agent success download file Detail: success", + "size": "1.0 Bytes", + "speed": "0 KB/s", + "process": "100%" } ] }, @@ -120,7 +123,10 @@ Bulk query of job execution logs by host list }, "src_path": "/data/1.log", "status": 4, - "log_content": "[2021-06-28 11:32:16] FileName: /data/1.log FileSize: 9.0 Bytes State: source agent success upload file Speed: 1 KB/s Progress: 100% StatusDesc: source agent success upload file Detail: success upload" + "log_content": "[2021-06-28 11:32:16] FileName: /data/1.log FileSize: 9.0 Bytes State: source agent success upload file Speed: 1 KB/s Progress: 100% StatusDesc: source agent success upload file Detail: success upload", + "size": "1.0 Bytes", + "speed": "0 KB/s", + "process": "100%" } ] } @@ -176,15 +182,18 @@ Bulk query of job execution logs by host list #### file_log -| Fields | Type | Description | -|-----------|-----------|-----------| -| mode | int | Distribution mode. 0: Upload; 1: Download | -| src_ip | object |File source host IP. see ip for definition | -| src_path | string | Source file paths | -| dest_ip | object | Distribute the target host IP, with value for mode=1. See ip for definition. | -| dest_path | string | Target path, with value for mode=1 | -| status | int | Task status. 1-Waiting; 2-Uploading; 3-Downloading; 4- Success; 5- Failure | +| Fields | Type | Description | +|-------------|-----------|-----------| +| mode | int | Distribution mode. 0: Upload; 1: Download | +| src_ip | object |File source host IP. see ip for definition | +| src_path | string | Source file paths | +| dest_ip | object | Distribute the target host IP, with value for mode=1. See ip for definition. | +| dest_path | string | Target path, with value for mode=1 | +| status | int | Task status. 1-Waiting; 2-Uploading; 3-Downloading; 4- Success; 5- Failure | | log_content | string | File distribution log contents | +| size | string | File size | +| speed | string | File transfer speed | +| process | string | File transfer process | #### ip diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/batch_get_job_instance_ip_log.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/batch_get_job_instance_ip_log.md index fcef5cc8bb..906bf51387 100644 --- a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/batch_get_job_instance_ip_log.md +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/batch_get_job_instance_ip_log.md @@ -103,7 +103,10 @@ }, "dest_path": "/tmp/1.log", "status": 4, - "log_content": "[2021-06-28 11:32:16] FileName: /tmp/1.log FileSize: 9.0 Bytes State: dest agent success download file Speed: 1 KB/s Progress: 100% StatusDesc: dest agent success download file Detail: success" + "log_content": "[2021-06-28 11:32:16] FileName: /tmp/1.log FileSize: 9.0 Bytes State: dest agent success download file Speed: 1 KB/s Progress: 100% StatusDesc: dest agent success download file Detail: success", + "size": "1.0 Bytes", + "speed": "0 KB/s", + "process": "100%" } ] }, @@ -121,7 +124,10 @@ }, "src_path": "/data/1.log", "status": 4, - "log_content": "[2021-06-28 11:32:16] FileName: /data/1.log FileSize: 9.0 Bytes State: source agent success upload file Speed: 1 KB/s Progress: 100% StatusDesc: source agent success upload file Detail: success upload" + "log_content": "[2021-06-28 11:32:16] FileName: /data/1.log FileSize: 9.0 Bytes State: source agent success upload file Speed: 1 KB/s Progress: 100% StatusDesc: source agent success upload file Detail: success upload", + "size": "1.0 Bytes", + "speed": "0 KB/s", + "process": "100%" } ] } @@ -176,15 +182,18 @@ #### file_log -| 字段 | 类型 | 描述 | -|-----------|-----------|-----------| -| mode | int | 分发模式。0:上传;1:下载| -| src_ip | object |文件源主机IP。定义见ip | -| src_path | string | 源文件路径 | -| dest_ip | object | 分发目标主机IP，mode=1时有值。定义见ip | -| dest_path | string | 目标路径，mode=1时有值 | -| status | int | 任务状态。1-等待开始;2-上传中;3-下载中;4-成功;5-失败 | -| log_content | string | 文件分发日志内容 | +| 字段 | 类型 | 描述 | +|--------------|-----------|-----------| +| mode | 分发模式 | 0:上传;1:下载| +| src_ip | object |文件源主机IP。定义见ip | +| src_path | string | 源文件路径 | +| dest_ip | object | 分发目标主机IP，mode=1时有值。定义见ip | +| dest_path | string | 目标路径，mode=1时有值 | +| status | int | 任务状态。1-等待开始;2-上传中;3-下载中;4-成功;5-失败 | +| log_content | string | 文件分发日志内容 | +| size | string | 文件大小 | +| speed | string | 文件传输速率 | +| process | string | 文件传输进度 | #### ip diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/sdk/ArtifactoryClient.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/sdk/ArtifactoryClient.java index 0f60289b8c..1f749b7802 100644 --- a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/sdk/ArtifactoryClient.java +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/sdk/ArtifactoryClient.java @@ -51,15 +51,17 @@ import com.tencent.bk.job.common.artifactory.model.req.Sort; import com.tencent.bk.job.common.artifactory.model.req.UploadGenericFileReq; import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.HttpMethodEnum; import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.exception.NotImplementedException; import com.tencent.bk.job.common.exception.ServiceException; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.util.Base64Util; import com.tencent.bk.job.common.util.StringUtil; -import com.tencent.bk.job.common.util.http.ExtHttpHelper; +import com.tencent.bk.job.common.util.http.HttpHelper; import com.tencent.bk.job.common.util.http.HttpHelperFactory; import com.tencent.bk.job.common.util.http.HttpMetricUtil; +import com.tencent.bk.job.common.util.http.HttpRequest; import com.tencent.bk.job.common.util.json.JsonUtils; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Tag; @@ -111,8 +113,8 @@ public class ArtifactoryClient { private final String password; private final MeterRegistry meterRegistry; - private final ExtHttpHelper httpHelper = HttpHelperFactory.getDefaultHttpHelper(); - private final ExtHttpHelper longHttpHelper = HttpHelperFactory.getLongRetryableHttpHelper(); + private final HttpHelper httpHelper = HttpHelperFactory.getDefaultHttpHelper(); + private final HttpHelper longHttpHelper = HttpHelperFactory.getLongRetryableHttpHelper(); public ArtifactoryClient(String baseUrl, String username, String password, MeterRegistry meterRegistry) { this.baseUrl = StringUtil.removeSuffix(baseUrl, "/"); @@ -171,28 +173,30 @@ private Header[] getUploadFileHeaders() { return headerList.toArray(headers); } - private String doHttpGet(String url, ArtifactoryReq reqBody, ExtHttpHelper httpHelper) { - if (null == reqBody) { - return httpHelper.get(url, getJsonHeaders()); - } else { - return httpHelper.get(url + reqBody.toUrlParams(), getJsonHeaders()); - } + private String doHttpGet(String url, ArtifactoryReq reqBody, HttpHelper httpHelper) { + return httpHelper.request( + HttpRequest.builder(HttpMethodEnum.GET, reqBody == null ? url : url + reqBody.toUrlParams()) + .setHeaders(getJsonHeaders()) + .build()) + .getEntity(); } - private String doHttpPost(String url, ArtifactoryReq reqBody, ExtHttpHelper httpHelper) { - if (null == reqBody) { - return httpHelper.post(url, "{}", getJsonHeaders()); - } else { - return httpHelper.post(url, JsonUtils.toJson(reqBody), getJsonHeaders()); - } + private String doHttpPost(String url, ArtifactoryReq reqBody, HttpHelper httpHelper) { + return httpHelper.request( + HttpRequest.builder(HttpMethodEnum.POST, url) + .setStringEntity(reqBody == null ? "{}" : JsonUtils.toJson(reqBody)) + .setHeaders(getJsonHeaders()) + .build()) + .getEntity(); } - private String doHttpDelete(String url, ArtifactoryReq reqBody, ExtHttpHelper httpHelper) { - if (null == reqBody) { - return httpHelper.delete(url, "{}", getJsonHeaders()); - } else { - return httpHelper.delete(url + reqBody.toUrlParams(), JsonUtils.toJson(reqBody), getJsonHeaders()); - } + private String doHttpDelete(String url, ArtifactoryReq reqBody, HttpHelper httpHelper) { + return httpHelper.request( + HttpRequest.builder(HttpMethodEnum.DELETE, reqBody == null ? url : url + reqBody.toUrlParams()) + .setStringEntity(reqBody == null ? "{}" : JsonUtils.toJson(reqBody)) + .setHeaders(getJsonHeaders()) + .build()) + .getEntity(); } @SuppressWarnings("unchecked") @@ -237,7 +241,7 @@ private R getArtifactoryRespByReq( String urlTemplate, ArtifactoryReq reqBody, TypeReference typeReference, - ExtHttpHelper httpHelper + HttpHelper httpHelper ) throws ServiceException { // URL模板变量替换 String url = StringUtil.replacePathVariables(urlTemplate, reqBody); @@ -552,7 +556,13 @@ private NodeDTO uploadGenericFileWithEntity( try { HttpMetricUtil.setHttpMetricName(CommonMetricNames.BKREPO_API_HTTP); HttpMetricUtil.addTagForCurrentMetric(Tag.of("api_name", "upload:" + URL_UPLOAD_GENERIC_FILE)); - respStr = longHttpHelper.put(url, reqEntity, getUploadFileHeaders()); + + respStr = longHttpHelper.request( + HttpRequest.builder(HttpMethodEnum.PUT, url) + .setHttpEntity(reqEntity) + .setHeaders(getUploadFileHeaders()) + .build()) + .getEntity(); if (log.isDebugEnabled()) { log.debug("respStr={}", getSimplifiedStrForLog(respStr)); } diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/CcHostInfoDTO.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/CcHostInfoDTO.java index a8f1f622e9..69b226f036 100644 --- a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/CcHostInfoDTO.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/CcHostInfoDTO.java @@ -48,8 +48,14 @@ public class CcHostInfoDTO { private String hostName; @JsonProperty("bk_os_name") private String os; + @JsonProperty("bk_os_type") + private String osType; @JsonProperty("bk_cloud_id") private Long cloudId; + @JsonProperty("bk_cloud_vendor") + private String cloudVendorId; + @JsonProperty("last_time") + private String lastTime; @JsonIgnore public String getFirstIp() { diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/ListHostsWithoutBizReq.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/ListHostsWithoutBizReq.java index 26fda06ea8..f43b83e8c5 100644 --- a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/ListHostsWithoutBizReq.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/ListHostsWithoutBizReq.java @@ -48,7 +48,10 @@ public class ListHostsWithoutBizReq extends EsbReq { "bk_agent_id", "bk_host_name", "bk_os_name", - "bk_cloud_id" + "bk_os_type", + "bk_cloud_id", + "bk_cloud_vendor", + "last_time" ); private PageDTO page; diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/HostEventDetail.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/HostEventDetail.java index 6fe0af503d..bd0a0efc01 100644 --- a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/HostEventDetail.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/HostEventDetail.java @@ -30,6 +30,7 @@ import com.tencent.bk.job.common.util.StringUtil; import com.tencent.bk.job.common.util.TimeUtil; import lombok.Data; +import org.apache.commons.lang3.StringUtils; import java.util.List; @@ -82,7 +83,11 @@ public static ApplicationHostDTO toHostInfoDTO(HostEventDetail eventDetail) { hostInfoDTO.setOsType(eventDetail.osType); hostInfoDTO.setCloudAreaId(Long.parseLong(eventDetail.getCloudId())); hostInfoDTO.setCloudVendorId(eventDetail.cloudVendorId); - hostInfoDTO.setLastTime(TimeUtil.parseIsoZonedTimeToMillis(eventDetail.getLastTime())); + Long lastTimeMills = null; + if (StringUtils.isNotBlank(eventDetail.getLastTime())) { + lastTimeMills = TimeUtil.parseIsoZonedTimeToMillis(eventDetail.getLastTime()); + } + hostInfoDTO.setLastTime(lastTimeMills); return hostInfoDTO; } } diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/HostProp.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/HostProp.java index feceb805f9..0010e03efa 100644 --- a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/HostProp.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/HostProp.java @@ -32,6 +32,7 @@ import lombok.Getter; import lombok.Setter; import lombok.ToString; +import org.apache.commons.lang3.StringUtils; import java.util.List; @@ -79,7 +80,9 @@ public ApplicationHostDTO toApplicationHostDTO() { applicationHostDTO.setOsType(StringUtil.substring(osType, osTypeNameMaxLength)); applicationHostDTO.setCloudAreaId(cloudAreaId); applicationHostDTO.setCloudVendorId(cloudVendorId); - applicationHostDTO.setLastTime(TimeUtil.parseIsoZonedTimeToMillis(lastTime)); + if (StringUtils.isNotBlank(lastTime)) { + applicationHostDTO.setLastTime(TimeUtil.parseIsoZonedTimeToMillis(lastTime)); + } return applicationHostDTO; } } diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/BaseCmdbApiClient.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/BaseCmdbApiClient.java index 67fa8286da..d3933ab5c2 100644 --- a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/BaseCmdbApiClient.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/BaseCmdbApiClient.java @@ -31,16 +31,17 @@ import com.tencent.bk.job.common.constant.HttpMethodEnum; import com.tencent.bk.job.common.esb.config.AppProperties; import com.tencent.bk.job.common.esb.config.EsbProperties; -import com.tencent.bk.job.common.esb.model.ApiRequestInfo; import com.tencent.bk.job.common.esb.model.BkApiAuthorization; import com.tencent.bk.job.common.esb.model.EsbReq; import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.common.esb.model.OpenApiRequestInfo; import com.tencent.bk.job.common.esb.sdk.AbstractBkApiClient; import com.tencent.bk.job.common.exception.InternalCmdbException; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.util.ApiUtil; import com.tencent.bk.job.common.util.FlowController; -import com.tencent.bk.job.common.util.http.ExtHttpHelper; +import com.tencent.bk.job.common.util.http.HttpHelper; +import com.tencent.bk.job.common.util.http.HttpHelperFactory; import com.tencent.bk.job.common.util.http.HttpMetricUtil; import com.tencent.bk.job.common.util.json.JsonUtils; import io.micrometer.core.instrument.MeterRegistry; @@ -110,7 +111,12 @@ protected BaseCmdbApiClient(FlowController flowController, CmdbConfig cmdbConfig, MeterRegistry meterRegistry, String lang) { - super(meterRegistry, CmdbMetricNames.CMDB_API_PREFIX, esbProperties.getService().getUrl(), lang); + super(meterRegistry, + CmdbMetricNames.CMDB_API_PREFIX, + esbProperties.getService().getUrl(), + HttpHelperFactory.getRetryableHttpHelper(), + lang + ); this.globalFlowController = flowController; this.cmdbConfig = cmdbConfig; this.cmdbSupplierAccount = cmdbConfig.getDefaultSupplierAccount(); @@ -129,13 +135,12 @@ protected EsbResp requestCmdbApi(HttpMethodEnum method, TypeReference> typeReference) { return requestCmdbApi(method, uri, queryParams, reqBody, typeReference, null); } - protected EsbResp requestCmdbApi(HttpMethodEnum method, String uri, String queryParams, EsbReq reqBody, TypeReference> typeReference, - ExtHttpHelper httpHelper) { + HttpHelper httpHelper) { String apiName = ApiUtil.getApiNameByUri(interfaceNameMap, uri); if (cmdbConfig != null && cmdbConfig.getEnableFlowControl()) { @@ -156,7 +161,7 @@ protected EsbResp requestCmdbApi(HttpMethodEnum method, try { HttpMetricUtil.setHttpMetricName(CommonMetricNames.ESB_CMDB_API_HTTP); HttpMetricUtil.addTagForCurrentMetric(Tag.of("api_name", apiName)); - ApiRequestInfo requestInfo = ApiRequestInfo + OpenApiRequestInfo requestInfo = OpenApiRequestInfo .builder() .method(method) .uri(uri) diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/BizCmdbClient.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/BizCmdbClient.java index 34c1b0f5c9..4071a3a2a9 100644 --- a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/BizCmdbClient.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/BizCmdbClient.java @@ -97,6 +97,7 @@ import com.tencent.bk.job.common.util.FlowController; import com.tencent.bk.job.common.util.JobContextUtil; import com.tencent.bk.job.common.util.ThreadUtils; +import com.tencent.bk.job.common.util.TimeUtil; import com.tencent.bk.job.common.util.Utils; import com.tencent.bk.job.common.util.http.HttpHelperFactory; import com.tencent.bk.job.common.util.json.JsonUtils; @@ -614,6 +615,13 @@ private ApplicationHostDTO convertHost(long bizId, CcHostInfoDTO ccHostInfo) { hostDTO.setAgentId(ccHostInfo.getAgentId()); hostDTO.setBizId(bizId); hostDTO.setHostName(ccHostInfo.getHostName()); + hostDTO.setOsType(ccHostInfo.getOsType()); + hostDTO.setCloudVendorId(ccHostInfo.getCloudVendorId()); + Long lastTimeMills = null; + if (StringUtils.isNotBlank(ccHostInfo.getLastTime())) { + lastTimeMills = TimeUtil.parseIsoZonedTimeToMillis(ccHostInfo.getLastTime()); + } + hostDTO.setLastTime(lastTimeMills); return hostDTO; } diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/IBizCmdbClient.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/IBizCmdbClient.java index bcbf3abac2..dd7a44f9e6 100644 --- a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/IBizCmdbClient.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/IBizCmdbClient.java @@ -182,7 +182,7 @@ public interface IBizCmdbClient { List listHostsByCloudIpv6s(List cloudIpv6s); /** - * 根据IP批量获取主机 + * 根据HostId批量获取主机 * * @param hostIds 主机ID列表 * @return 主机列表 diff --git a/src/backend/commons/common-i18n/src/main/java/com/tencent/bk/job/common/i18n/locale/BkConsts.java b/src/backend/commons/common-i18n/src/main/java/com/tencent/bk/job/common/i18n/locale/BkConsts.java new file mode 100644 index 0000000000..13603e7f0f --- /dev/null +++ b/src/backend/commons/common-i18n/src/main/java/com/tencent/bk/job/common/i18n/locale/BkConsts.java @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.i18n.locale; + +/** + * 蓝鲸公共协议涉及的常量 + */ +public class BkConsts { + + // 国际化语言Header取值 + public static final String HEADER_VALUE_LANG_ZH = "zh"; + public static final String HEADER_VALUE_LANG_ZH_CN = "zh-cn"; + public static final String HEADER_VALUE_LANG_EN = "en"; + public static final String HEADER_VALUE_LANG_EN_US = "en-us"; + +} diff --git a/src/backend/commons/common-i18n/src/main/java/com/tencent/bk/job/common/i18n/locale/LocaleUtils.java b/src/backend/commons/common-i18n/src/main/java/com/tencent/bk/job/common/i18n/locale/LocaleUtils.java index 9ea4b7ff19..f3bc1e2112 100644 --- a/src/backend/commons/common-i18n/src/main/java/com/tencent/bk/job/common/i18n/locale/LocaleUtils.java +++ b/src/backend/commons/common-i18n/src/main/java/com/tencent/bk/job/common/i18n/locale/LocaleUtils.java @@ -38,6 +38,7 @@ public class LocaleUtils { public static final String LANG_ZH = "zh"; public static final String LANG_EN = "en"; public static final String LANG_EN_US = "en_US"; + /** * 蓝鲸通用的LANG HEADER */ @@ -50,19 +51,28 @@ public class LocaleUtils { private static final Map localeMap = new HashMap<>(); static { - localeMap.put("zh", LANG_ZH); - localeMap.put("zh-cn", LANG_ZH_CN); + localeMap.put(BkConsts.HEADER_VALUE_LANG_ZH, LANG_ZH); + localeMap.put(BkConsts.HEADER_VALUE_LANG_ZH_CN, LANG_ZH_CN); localeMap.put("zh_cn", LANG_ZH_CN); - localeMap.put("en", LANG_EN); - localeMap.put("en-us", LANG_EN_US); + localeMap.put(BkConsts.HEADER_VALUE_LANG_EN, LANG_EN); + localeMap.put(BkConsts.HEADER_VALUE_LANG_EN_US, LANG_EN_US); localeMap.put("en_us", LANG_EN_US); } + private static final Map bkLangMap = new HashMap<>(); + + static { + bkLangMap.put(LANG_ZH, BkConsts.HEADER_VALUE_LANG_ZH); + bkLangMap.put(LANG_ZH_CN, BkConsts.HEADER_VALUE_LANG_ZH_CN); + bkLangMap.put(LANG_EN, BkConsts.HEADER_VALUE_LANG_EN); + bkLangMap.put(LANG_EN_US, BkConsts.HEADER_VALUE_LANG_EN_US); + } + /** * 对不同形式的language Header值进行统一 * - * @param lang - * @return + * @param lang 语言Header取值 + * @return 标准化后的语言Header取值 */ public static String getNormalLang(String lang) { lang = lang.toLowerCase(); @@ -72,6 +82,19 @@ public static String getNormalLang(String lang) { return lang; } + /** + * 根据标准语言取值获取蓝鲸协议语言参数取值 + * + * @param normalLang 标准语言取值 + * @return 蓝鲸协议语言参数取值 + */ + public static String getBkLang(String normalLang) { + if (bkLangMap.containsKey(normalLang)) { + return bkLangMap.get(normalLang); + } + return null; + } + public static Locale getLocale(String lang) { if (StringUtils.isBlank(lang)) { return Locale.SIMPLIFIED_CHINESE; diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message.properties index 4f00837874..4e4ebc7734 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message.properties @@ -77,6 +77,10 @@ #第三方API请求错误 1216001=第三方 API 请求错误 +#消息通知中心异常 +1217001=消息通知中心API不存在:{0} +1217002=消息通知中心接口数据异常 + ##业务错误-通用 1241001=请求参数缺失 @@ -204,6 +208,7 @@ 1260016=文件源ID与标识至少指定一个 1260017=文件源[id={0}]不在业务/业务集下 1260018=接入点响应异常，详情：{0} +1260019=文件源code不可为空 ##业务错误-文件Worker服务(job-file-worker) 1262001=第三方文件源响应异常：ListBucket，详情：{0} diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en.properties index 66a819b817..ce7cf7bfb3 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en.properties @@ -77,6 +77,10 @@ #第三方API请求错误 1216001=Third API data unexpected +#消息通知中心异常 +1217001=BK-Notice API not found:{0} +1217002=BK-Notice API data unexpected + ## Business error - common 1241001=Missing request parameters 1241002=Invalid request parameters @@ -204,6 +208,7 @@ 1260016=At least one of id/code required 1260017=FileSource[id={0}] does not belong to specified business 1260018=FileWorker response error, Detail:{0} +1260019=File source code can not be empty ## Business error (job-file-worker) 1262001=Third file source response error:ListBucket, Detail:{0} diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en_US.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en_US.properties index 66a819b817..ce7cf7bfb3 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en_US.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en_US.properties @@ -77,6 +77,10 @@ #第三方API请求错误 1216001=Third API data unexpected +#消息通知中心异常 +1217001=BK-Notice API not found:{0} +1217002=BK-Notice API data unexpected + ## Business error - common 1241001=Missing request parameters 1241002=Invalid request parameters @@ -204,6 +208,7 @@ 1260016=At least one of id/code required 1260017=FileSource[id={0}] does not belong to specified business 1260018=FileWorker response error, Detail:{0} +1260019=File source code can not be empty ## Business error (job-file-worker) 1262001=Third file source response error:ListBucket, Detail:{0} diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh.properties index 4f00837874..4e4ebc7734 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh.properties @@ -77,6 +77,10 @@ #第三方API请求错误 1216001=第三方 API 请求错误 +#消息通知中心异常 +1217001=消息通知中心API不存在:{0} +1217002=消息通知中心接口数据异常 + ##业务错误-通用 1241001=请求参数缺失 @@ -204,6 +208,7 @@ 1260016=文件源ID与标识至少指定一个 1260017=文件源[id={0}]不在业务/业务集下 1260018=接入点响应异常，详情：{0} +1260019=文件源code不可为空 ##业务错误-文件Worker服务(job-file-worker) 1262001=第三方文件源响应异常：ListBucket，详情：{0} diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh_CN.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh_CN.properties index 6ed124262b..b058a7153e 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh_CN.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh_CN.properties @@ -77,6 +77,10 @@ #第三方API请求错误 1216001=第三方 API 请求错误 +#消息通知中心异常 +1217001=消息通知中心API不存在:{0} +1217002=消息通知中心接口数据异常 + ##业务错误-通用 1241001=请求参数缺失 @@ -205,6 +209,7 @@ 1260016=文件源ID与标识至少指定一个 1260017=文件源[id={0}]不在业务/业务集下 1260018=接入点响应异常，详情：{0} +1260019=文件源code不可为空 ##业务错误-文件Worker服务(job-file-worker) 1262001=第三方文件源响应异常：ListBucket，详情：{0} diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages.properties index 092b94e838..55e374504e 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages.properties @@ -72,3 +72,6 @@ validation.constraints.AccountAlias_tooLong.message=账号别名不能超过255 validation.constraints.AccountDescription_tooLong.message=账号描述不能超过1024字符 validation.constraints.InvalidJobInstanceId.message=任务实例ID必须为正整数 validation.constraints.InvalidStepInstanceId.message=步骤实例ID必须为正整数 +validation.constraints.queryAgentInfoHostIds_tooMany.message=单次查询主机数量不可超过5000 +validation.constraints.InvalidAnnouncementsOffset.message=公告偏移量必须大于等于0 +validation.constraints.InvalidAnnouncementsLimit.message=公告获取数量必须大于等于1 diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_en.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_en.properties index 0b9f15e3c2..c908964129 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_en.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_en.properties @@ -71,3 +71,6 @@ validation.constraints.AccountAlias_tooLong.message=Account alias length cannot validation.constraints.AccountDescription_tooLong.message=Account description length cannot be larger than 1024 validation.constraints.InvalidJobInstanceId.message=Job instance id must be a positive number validation.constraints.InvalidStepInstanceId.message=Step instance id must be a positive number +validation.constraints.queryAgentInfoHostIds_tooMany.message=HostIds size can not be over 5000 +validation.constraints.InvalidAnnouncementsOffset.message=Announcements offset must be equal or greater than 0 +validation.constraints.InvalidAnnouncementsLimit.message=Announcements limit must be equal or greater than 1 diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_en_US.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_en_US.properties index 0b9f15e3c2..c908964129 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_en_US.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_en_US.properties @@ -71,3 +71,6 @@ validation.constraints.AccountAlias_tooLong.message=Account alias length cannot validation.constraints.AccountDescription_tooLong.message=Account description length cannot be larger than 1024 validation.constraints.InvalidJobInstanceId.message=Job instance id must be a positive number validation.constraints.InvalidStepInstanceId.message=Step instance id must be a positive number +validation.constraints.queryAgentInfoHostIds_tooMany.message=HostIds size can not be over 5000 +validation.constraints.InvalidAnnouncementsOffset.message=Announcements offset must be equal or greater than 0 +validation.constraints.InvalidAnnouncementsLimit.message=Announcements limit must be equal or greater than 1 diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_zh.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_zh.properties index 5e708c3336..3340848b6b 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_zh.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_zh.properties @@ -71,3 +71,6 @@ validation.constraints.AccountAlias_tooLong.message=账号别名不能超过255 validation.constraints.AccountDescription_tooLong.message=账号描述不能超过1024字符 validation.constraints.InvalidJobInstanceId.message=任务实例ID必须为正整数 validation.constraints.InvalidStepInstanceId.message=步骤实例ID必须为正整数 +validation.constraints.queryAgentInfoHostIds_tooMany.message=单次查询主机数量不可超过5000 +validation.constraints.InvalidAnnouncementsOffset.message=公告偏移量必须大于等于0 +validation.constraints.InvalidAnnouncementsLimit.message=公告获取数量必须大于等于1 diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_zh_CN.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_zh_CN.properties index 977234aa7e..e62ef5c1c1 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_zh_CN.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_zh_CN.properties @@ -71,3 +71,6 @@ validation.constraints.AccountAlias_tooLong.message=账号别名不能超过255 validation.constraints.AccountDescription_tooLong.message=账号描述不能超过1024字符 validation.constraints.InvalidJobInstanceId.message=任务实例ID必须为正整数 validation.constraints.InvalidStepInstanceId.message=步骤实例ID必须为正整数 +validation.constraints.queryAgentInfoHostIds_tooMany.message=单次查询主机数量不可超过5000 +validation.constraints.InvalidAnnouncementsOffset.message=公告偏移量必须大于等于0 +validation.constraints.InvalidAnnouncementsLimit.message=公告获取数量必须大于等于1 diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/client/EsbIamClient.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/client/EsbIamClient.java index e5ce660e44..790fc805af 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/client/EsbIamClient.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/client/EsbIamClient.java @@ -29,10 +29,10 @@ import com.tencent.bk.job.common.constant.HttpMethodEnum; import com.tencent.bk.job.common.esb.config.AppProperties; import com.tencent.bk.job.common.esb.config.EsbProperties; -import com.tencent.bk.job.common.esb.model.ApiRequestInfo; import com.tencent.bk.job.common.esb.model.BkApiAuthorization; import com.tencent.bk.job.common.esb.model.EsbReq; import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.common.esb.model.OpenApiRequestInfo; import com.tencent.bk.job.common.esb.sdk.AbstractBkApiClient; import com.tencent.bk.job.common.exception.InternalIamException; import com.tencent.bk.job.common.iam.dto.AuthByPathReq; @@ -47,6 +47,7 @@ import com.tencent.bk.job.common.iam.dto.GetApplyUrlResponse; import com.tencent.bk.job.common.iam.dto.RegisterResourceRequest; import com.tencent.bk.job.common.metrics.CommonMetricNames; +import com.tencent.bk.job.common.util.http.HttpHelperFactory; import com.tencent.bk.job.common.util.http.HttpMetricUtil; import com.tencent.bk.sdk.iam.constants.SystemId; import com.tencent.bk.sdk.iam.dto.action.ActionDTO; @@ -78,7 +79,7 @@ public class EsbIamClient extends AbstractBkApiClient implements IIamClient { public EsbIamClient(MeterRegistry meterRegistry, AppProperties appProperties, EsbProperties esbProperties) { - super(meterRegistry, IAM_API, esbProperties.getService().getUrl()); + super(meterRegistry, IAM_API, esbProperties.getService().getUrl(), HttpHelperFactory.getDefaultHttpHelper()); this.authorization = BkApiAuthorization.appAuthorization(appProperties.getCode(), appProperties.getSecret(), "admin"); } @@ -181,7 +182,7 @@ private EsbResp requestIamApi(HttpMethodEnum method, try { HttpMetricUtil.setHttpMetricName(CommonMetricNames.IAM_API_HTTP); HttpMetricUtil.addTagForCurrentMetric(Tag.of("api_name", uri)); - ApiRequestInfo requestInfo = ApiRequestInfo + OpenApiRequestInfo requestInfo = OpenApiRequestInfo .builder() .method(method) .uri(uri) diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/http/IamHttpClientServiceImpl.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/http/IamHttpClientServiceImpl.java index 1158e47bde..d5c78a219a 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/http/IamHttpClientServiceImpl.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/http/IamHttpClientServiceImpl.java @@ -25,11 +25,13 @@ package com.tencent.bk.job.common.iam.http; import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.HttpMethodEnum; import com.tencent.bk.job.common.exception.InternalIamException; import com.tencent.bk.job.common.metrics.CommonMetricNames; -import com.tencent.bk.job.common.util.http.ExtHttpHelper; +import com.tencent.bk.job.common.util.http.HttpHelper; import com.tencent.bk.job.common.util.http.HttpHelperFactory; import com.tencent.bk.job.common.util.http.HttpMetricUtil; +import com.tencent.bk.job.common.util.http.HttpRequest; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.sdk.iam.config.IamConfiguration; import com.tencent.bk.sdk.iam.constants.HttpHeader; @@ -39,14 +41,11 @@ import org.apache.http.Header; import org.apache.http.message.BasicHeader; -import java.util.ArrayList; -import java.util.List; - @Slf4j public class IamHttpClientServiceImpl implements HttpClientService { private final String DEFAULT_CHARSET = "UTF-8"; - private final ExtHttpHelper httpHelper = HttpHelperFactory.getDefaultHttpHelper(); + private final HttpHelper httpHelper = HttpHelperFactory.getDefaultHttpHelper(); private final IamConfiguration iamConfiguration; public IamHttpClientServiceImpl(IamConfiguration iamConfiguration) { @@ -59,7 +58,11 @@ public String doHttpGet(String uri) { try { HttpMetricUtil.setHttpMetricName(CommonMetricNames.IAM_API_HTTP); HttpMetricUtil.addTagForCurrentMetric(Tag.of("api_name", uri)); - return httpHelper.get(buildUrl(uri), buildAuthHeader()); + return httpHelper.request( + HttpRequest.builder(HttpMethodEnum.GET, buildUrl(uri)) + .setHeaders(buildAuthHeader()) + .build()) + .getEntity(); } catch (Exception e) { throw new InternalIamException(e, ErrorCode.IAM_API_DATA_ERROR, null); } finally { @@ -72,7 +75,12 @@ public String doHttpPost(String uri, Object body) { try { HttpMetricUtil.setHttpMetricName(CommonMetricNames.IAM_API_HTTP); HttpMetricUtil.addTagForCurrentMetric(Tag.of("api_name", uri)); - return httpHelper.post(buildUrl(uri), DEFAULT_CHARSET, JsonUtils.toJson(body), buildAuthHeader()); + return httpHelper.request( + HttpRequest.builder(HttpMethodEnum.POST, buildUrl(uri)) + .setHeaders(buildAuthHeader()) + .setStringEntity(JsonUtils.toJson(body)) + .build()) + .getEntity(); } catch (Exception e) { log.error("Fail to request IAM", e); throw new InternalIamException(e, ErrorCode.IAM_API_DATA_ERROR, null); @@ -86,7 +94,12 @@ public String doHttpPut(String uri, Object body) { try { HttpMetricUtil.setHttpMetricName(CommonMetricNames.IAM_API_HTTP); HttpMetricUtil.addTagForCurrentMetric(Tag.of("api_name", uri)); - return httpHelper.put(buildUrl(uri), DEFAULT_CHARSET, JsonUtils.toJson(body), buildAuthHeader()); + return httpHelper.request( + HttpRequest.builder(HttpMethodEnum.PUT, buildUrl(uri)) + .setHeaders(buildAuthHeader()) + .setStringEntity(JsonUtils.toJson(body)) + .build()) + .getEntity(); } catch (Exception e) { log.error("Fail to request IAM", e); throw new InternalIamException(e, ErrorCode.IAM_API_DATA_ERROR, null); @@ -100,7 +113,11 @@ public String doHttpDelete(String uri) { try { HttpMetricUtil.setHttpMetricName(CommonMetricNames.IAM_API_HTTP); HttpMetricUtil.addTagForCurrentMetric(Tag.of("api_name", uri)); - return httpHelper.delete(buildUrl(uri), buildAuthHeader()); + return httpHelper.request( + HttpRequest.builder(HttpMethodEnum.DELETE, buildUrl(uri)) + .setHeaders(buildAuthHeader()) + .build()) + .getEntity(); } catch (Exception e) { throw new InternalIamException(e, ErrorCode.IAM_API_DATA_ERROR, null); } finally { @@ -112,11 +129,11 @@ private String buildUrl(String uri) { return iamConfiguration.getIamBaseUrl() + uri; } - private List
buildAuthHeader() { - List
headerList = new ArrayList<>(); - headerList.add(new BasicHeader(HttpHeader.BK_APP_CODE, iamConfiguration.getAppCode())); - headerList.add(new BasicHeader(HttpHeader.BK_APP_SECRET, iamConfiguration.getAppSecret())); - return headerList; + private Header[] buildAuthHeader() { + Header[] headers = new Header[2]; + headers[0] = new BasicHeader(HttpHeader.BK_APP_CODE, iamConfiguration.getAppCode()); + headers[1] = new BasicHeader(HttpHeader.BK_APP_SECRET, iamConfiguration.getAppSecret()); + return headers; } } diff --git a/src/backend/commons/common-service/src/main/resources/logback-default.xml b/src/backend/commons/common-service/src/main/resources/logback-default.xml index 97bf863197..93935e0c53 100644 --- a/src/backend/commons/common-service/src/main/resources/logback-default.xml +++ b/src/backend/commons/common-service/src/main/resources/logback-default.xml @@ -126,6 +126,7 @@ Default job logback configuration provided for import + ${AUDIT_EVENT_LOG_FILE} diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/exception/handler/EsbExceptionControllerAdvice.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/exception/handler/EsbExceptionControllerAdvice.java index 0b9b0631b3..716b80febc 100644 --- a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/exception/handler/EsbExceptionControllerAdvice.java +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/exception/handler/EsbExceptionControllerAdvice.java @@ -32,6 +32,7 @@ import com.tencent.bk.job.common.exception.FailedPreconditionException; import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.exception.InvalidParamException; +import com.tencent.bk.job.common.exception.MissingParameterException; import com.tencent.bk.job.common.exception.NotFoundException; import com.tencent.bk.job.common.exception.ResourceExhaustedException; import com.tencent.bk.job.common.exception.ServiceException; @@ -160,6 +161,15 @@ ResponseEntity handleUnauthenticatedException(HttpServletRequest request, Una return new ResponseEntity<>(EsbResp.buildCommonFailResp(ex), HttpStatus.OK); } + @ExceptionHandler({MissingParameterException.class}) + @ResponseBody + ResponseEntity handleMissingParameterException (HttpServletRequest request, + MissingParameterException ex) { + String errorMsg = "Handle MissingParameterException , uri: " + request.getRequestURI(); + log.warn(errorMsg, ex); + return new ResponseEntity<>(EsbResp.buildCommonFailResp(ex), HttpStatus.BAD_REQUEST); + } + @Override @SuppressWarnings("all") protected ResponseEntity handleMethodArgumentNotValid(MethodArgumentNotValidException ex, diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ErrorCode.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ErrorCode.java index 0ee8347888..d4ce0f7006 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ErrorCode.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ErrorCode.java @@ -38,6 +38,90 @@ public class ErrorCode { */ public static final int RESULT_OK = 0; + // ==================================== 系统级错误 ================================================// + + // ======= 系统错误-平台服务错误 =======// + // LICENSE 不可用 + public static final int LICENSE_ERROR = 1210101; + // GSE 不可用 + public static final int GSE_ERROR = 1210001; + // GSE数据异常：{0} + public static final int GSE_API_DATA_ERROR = 1210002; + + // CMDB错误 + // CMDB服务状态不可达 - 地址配置错误或者地址无法正确解析 + public static final int CMDB_UNREACHABLE_SERVER = 1211001; + // CMDB接口返回数据结构异常- 一般是被网关防火墙重定向返回统一登录页面 + public static final int CMDB_API_DATA_ERROR = 1211002; + // 根据动态分组ID查找主机失败，动态分组ID：{0}，原因：{1}，请确认指定的动态分组在业务下是否存在 + public static final int FAIL_TO_FIND_HOST_BY_DYNAMIC_GROUP = 1211003; + // 根据业务ID查找动态分组失败，业务ID：{0}，原因：{1}，请确认指定的业务是否存在动态分组 + public static final int FAIL_TO_FIND_DYNAMIC_GROUP_BY_BIZ = 1211004; + + // PaaS异常 + // CMSI接口访问异常 + public static final int CMSI_API_ACCESS_ERROR = 1213001; + // 用户管理接口访问异常 + public static final int USER_MANAGE_API_ACCESS_ERROR = 1213002; + // 调用CMSI接口获取通知渠道数据异常 + public static final int CMSI_MSG_CHANNEL_DATA_ERROR = 1213003; + // 调用CMSI接口发送通知失败，错误码：{0}，错误信息：{1} + public static final int CMSI_FAIL_TO_SEND_MSG = 1213004; + + // 制品库异常 + // Artifactory接口返回数据结构异常 + public static final int ARTIFACTORY_API_DATA_ERROR = 1214001; + // 制品库中找不到节点:{0}，请到制品库核实 + public static final int CAN_NOT_FIND_NODE_IN_ARTIFACTORY = 1214002; + + // IAM接口数据异常- 一般是被网关防火墙重定向返回统一登录页面 + public static final int IAM_API_DATA_ERROR = 1215001; + // 用户({0})账号已被冻结，请处理后再重试 + public static final int IAM_USER_ACCOUNT_FROZEN = 1215002; + + // 第三方API请求错误 + public static final int API_ERROR = 1216001; + + // 消息通知中心异常 + // 消息通知中心API不存在:{0} + public static final int BK_NOTICE_API_NOT_FOUND = 1217001; + // 消息通知中心接口数据异常 + public static final int BK_NOTICE_API_DATA_ERROR = 1217002; + + // ======== 系统错误-权限错误 ==================// + // 用户({0})权限不足，请前往权限中心确认并申请补充后重试 + public static final int PERMISSION_DENIED = 1238001; + // 蓝鲸统一权限错误码，用户({0})权限不足，请前往权限中心确认并申请补充后重试 + public static final int BK_PERMISSION_DENIED = 9900403; + + // ========= 系统错误-请求 ====================// + // 内部服务异常 + public static final int INTERNAL_ERROR = 1240002; + // 错误的请求 + public static final int BAD_REQUEST = 1240003; + // Cookie过期或者不存在 + public static final int COOKIE_ILLEGAL = 1240004; + // 服务不可用 + public static final int SERVICE_UNAVAILABLE = 1240001; + // 服务认证失败 + public static final int SERVICE_AUTH_FAIL = 1240005; + // 配置异常：{0} + public static final int INVALID_CONFIG = 1240006; + // 命令行参数异常：{0} + public static final int INVALID_CMD_ARGS = 1240007; + + // ======= 系统错误-公共组件错误 =======// + // Redis服务不可用，连接不上 - IP不存在或者配置错误 + public static final int REDIS_CONNECT_FAIL = 1250001; + // Redis服务内存满或者其他问题 - 内存不足够 + public static final int REDIS_DATA_EXCEPTION = 1250002; + // DB 不可用 + public static final int DB_ERROR = 1252001; + // MQ 不可用 + public static final int MQ_ERROR = 1255001; + // NFS存储 不可用 + public static final int NFS_ERROR = 1259001; + // ==================================== 业务级错误 ================================================// /* * 业务通用-1241xxx @@ -291,6 +375,10 @@ public class ErrorCode { public static final int FAIL_TO_DOWNLOAD_NODE_FROM_ARTIFACTORY = 1249002; // 备份服务 end + // ======= 系统错误-公共组件错误 =======// + // 1250xxx-1259xxx被【系统错误-公共组件错误】占用 + // ======= 系统错误-公共组件错误 =======// + // 文件网关 start // 文件源不存在:{0} public static final int FILE_SOURCE_NOT_EXIST = 1260001; @@ -329,6 +417,8 @@ public class ErrorCode { public static final int FILE_SOURCE_ID_NOT_IN_BIZ = 1260017; // 接入点响应异常，详情：{0} public static final int FAIL_TO_REQUEST_FILE_WORKER_WITH_REASON = 1260018; + // 文件源code不可为空 + public static final int FILE_SOURCE_CODE_CAN_NOT_BE_EMPTY = 1260019; // 文件网关 end // 文件代理 start @@ -352,81 +442,4 @@ public class ErrorCode { // 迁移失败，任务: {0}, 详情: {1} public static final int MIGRATION_FAIL = 1263001; - // ==================================== 系统级错误 ================================================// - // ======== 系统错误-权限错误 ==================// - // 用户({0})权限不足，请前往权限中心确认并申请补充后重试 - public static final int PERMISSION_DENIED = 1238001; - // 蓝鲸统一权限错误码，用户({0})权限不足，请前往权限中心确认并申请补充后重试 - public static final int BK_PERMISSION_DENIED = 9900403; - - // ========= 系统错误-请求 ====================// - // 内部服务异常 - public static final int INTERNAL_ERROR = 1240002; - // 错误的请求 - public static final int BAD_REQUEST = 1240003; - // Cookie过期或者不存在 - public static final int COOKIE_ILLEGAL = 1240004; - // 服务不可用 - public static final int SERVICE_UNAVAILABLE = 1240001; - // 服务认证失败 - public static final int SERVICE_AUTH_FAIL = 1240005; - // 配置异常：{0} - public static final int INVALID_CONFIG = 1240006; - // 命令行参数异常：{0} - public static final int INVALID_CMD_ARGS = 1240007; - - // ======= 系统错误-公共组件错误 =======// - // Redis服务不可用，连接不上 - IP不存在或者配置错误 - public static final int REDIS_CONNECT_FAIL = 1250001; - // Redis服务内存满或者其他问题 - 内存不足够 - public static final int REDIS_DATA_EXCEPTION = 1250002; - // NFS存储 不可用 - public static final int NFS_ERROR = 1259001; - // DB 不可用 - public static final int DB_ERROR = 1252001; - // MQ 不可用 - public static final int MQ_ERROR = 1255001; - - // ======= 系统错误-平台服务错误 =======// - // LICENSE 不可用 - public static final int LICENSE_ERROR = 1210101; - // GSE 不可用 - public static final int GSE_ERROR = 1210001; - // GSE数据异常：{0} - public static final int GSE_API_DATA_ERROR = 1210002; - - // CMDB错误 - // CMDB服务状态不可达 - 地址配置错误或者地址无法正确解析 - public static final int CMDB_UNREACHABLE_SERVER = 1211001; - // CMDB接口返回数据结构异常- 一般是被网关防火墙重定向返回统一登录页面 - public static final int CMDB_API_DATA_ERROR = 1211002; - // 根据动态分组ID查找主机失败，动态分组ID：{0}，原因：{1}，请确认指定的动态分组在业务下是否存在 - public static final int FAIL_TO_FIND_HOST_BY_DYNAMIC_GROUP = 1211003; - // 根据业务ID查找动态分组失败，业务ID：{0}，原因：{1}，请确认指定的业务是否存在动态分组 - public static final int FAIL_TO_FIND_DYNAMIC_GROUP_BY_BIZ = 1211004; - - // PaaS异常 - // CMSI接口访问异常 - public static final int CMSI_API_ACCESS_ERROR = 1213001; - // 用户管理接口访问异常 - public static final int USER_MANAGE_API_ACCESS_ERROR = 1213002; - // 调用CMSI接口获取通知渠道数据异常 - public static final int CMSI_MSG_CHANNEL_DATA_ERROR = 1213003; - // 调用CMSI接口发送通知失败，错误码：{0}，错误信息：{1} - public static final int CMSI_FAIL_TO_SEND_MSG = 1213004; - - // 制品库异常 - // Artifactory接口返回数据结构异常 - public static final int ARTIFACTORY_API_DATA_ERROR = 1214001; - // 制品库中找不到节点:{0}，请到制品库核实 - public static final int CAN_NOT_FIND_NODE_IN_ARTIFACTORY = 1214002; - - // IAM接口数据异常- 一般是被网关防火墙重定向返回统一登录页面 - public static final int IAM_API_DATA_ERROR = 1215001; - // 用户({0})账号已被冻结，请处理后再重试 - public static final int IAM_USER_ACCOUNT_FROZEN = 1215002; - - // 第三方API请求错误 - public static final int API_ERROR = 1216001; - } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/HttpStatusException.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/HttpStatusException.java new file mode 100644 index 0000000000..3c51b3d38f --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/HttpStatusException.java @@ -0,0 +1,46 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.exception; + +import lombok.Getter; +import lombok.ToString; + +/** + * 服务异常 + */ +@Getter +@ToString +public class HttpStatusException extends RuntimeException { + private final String uri; + private final int httpStatus; + private final String reasonPhrase; + + public HttpStatusException(String uri, int httpStatus, String reasonPhrase) { + super("http status not ok, uri=" + uri + ", statusCode=" + httpStatus + ", reasonPhrase=" + reasonPhrase); + this.uri = uri; + this.httpStatus = httpStatus; + this.reasonPhrase = reasonPhrase; + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/MissingParameterException.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/MissingParameterException.java new file mode 100644 index 0000000000..684ce859d8 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/MissingParameterException.java @@ -0,0 +1,79 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.exception; + +import com.tencent.bk.job.common.model.ValidateResult; +import com.tencent.bk.job.common.model.error.ErrorType; +import lombok.Getter; +import lombok.ToString; + +/** + * 请求参数为空异常 + */ +@Getter +@ToString +public class MissingParameterException extends ServiceException { + public MissingParameterException(Integer errorCode) { + super(ErrorType.INVALID_PARAM, errorCode); + } + + public MissingParameterException(Integer errorCode, Object[] errorParams) { + super(ErrorType.INVALID_PARAM, errorCode, errorParams); + } + + public MissingParameterException(Integer errorCode, Object errorParam) { + super(ErrorType.INVALID_PARAM, errorCode, new Object[]{errorParam}); + } + + public MissingParameterException(String message, Integer errorCode) { + super(message, ErrorType.INVALID_PARAM, errorCode); + } + + public MissingParameterException(String message, Integer errorCode, Object[] errorParams) { + super(message, ErrorType.INVALID_PARAM, errorCode, errorParams); + } + + public MissingParameterException(Throwable cause, Integer errorCode) { + super(cause, ErrorType.INVALID_PARAM, errorCode); + } + + public MissingParameterException(Throwable cause, Integer errorCode, Object[] errorParams) { + super(cause, ErrorType.INVALID_PARAM, errorCode, errorParams); + } + + public MissingParameterException(String message, Throwable cause, Integer errorCode) { + super(message, cause, ErrorType.INVALID_PARAM, errorCode); + } + + public MissingParameterException(String message, Throwable cause, Integer errorCode, + Object[] errorParams) { + super(message, cause, ErrorType.INVALID_PARAM, errorCode, errorParams); + } + + public MissingParameterException(ValidateResult validateResult) { + super(ErrorType.INVALID_PARAM, validateResult.getErrorCode(), validateResult.getErrorParams()); + } + +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/metrics/CommonMetricNames.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/metrics/CommonMetricNames.java index 771c68f787..0fba47495d 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/metrics/CommonMetricNames.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/metrics/CommonMetricNames.java @@ -76,6 +76,15 @@ public class CommonMetricNames { */ public static final String IAM_API = "job.client.iam.api"; + /** + * 仅统计调用消息通知中心后台 API的HTTP请求过程 + */ + public static final String BK_NOTICE_API_HTTP = "job.client.bknotice.api.http"; + /** + * 统计调用消息通知中心后台 API整个过程，含反序列化 + */ + public static final String BK_NOTICE_API = "job.client.bknotice.api"; + /** * 被调用ESB API diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/BaseHttpHelper.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/BaseHttpHelper.java index 62042779ab..9cc6b30d31 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/BaseHttpHelper.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/BaseHttpHelper.java @@ -25,11 +25,16 @@ package com.tencent.bk.job.common.util.http; import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.HttpMethodEnum; +import com.tencent.bk.job.common.exception.HttpStatusException; import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.common.exception.NotImplementedException; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.http.Header; import org.apache.http.HttpEntity; +import org.apache.http.HttpEntityEnclosingRequest; import org.apache.http.HttpStatus; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; @@ -38,10 +43,11 @@ import org.apache.http.client.methods.HttpRequestBase; import org.apache.http.entity.ByteArrayEntity; import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.protocol.HttpContext; +import org.apache.http.protocol.HttpCoreContext; import org.apache.http.util.EntityUtils; import java.io.IOException; -import java.io.UnsupportedEncodingException; @Slf4j public class BaseHttpHelper implements HttpHelper { @@ -53,10 +59,6 @@ public BaseHttpHelper(CloseableHttpClient httpClient) { this.httpClient = httpClient; } - CloseableHttpClient getHttpClient() { - return httpClient; - } - @Override public Pair getRawResp(boolean keepAlive, String url, Header[] header) { HttpGet get = new HttpGet(url); @@ -67,7 +69,7 @@ public Pair getRawResp(boolean keepAlive get.setHeaders(header); } try { - return Pair.of(get, getHttpClient().execute(get)); + return Pair.of(get, httpClient.execute(get)); } catch (IOException e) { log.error("Get request fail", e); throw new InternalException(e, ErrorCode.API_ERROR); @@ -79,169 +81,124 @@ public Pair getRawResp(boolean keepAlive } @Override - public Pair get(boolean keepAlive, String url, Header[] header) { - HttpGet get = new HttpGet(url); - if (keepAlive) { - get.setHeader("Connection", "Keep-Alive"); + public HttpResponse request(HttpRequest request) { + HttpMethodEnum method = request.getMethod(); + HttpContext httpContext = buildHttpContext(request); + HttpRequestBase httpClientRequest; + switch (method) { + case GET: + httpClientRequest = buildHttpGet(request); + break; + case POST: + httpClientRequest = buildHttpPost(request); + break; + case PUT: + httpClientRequest = buildHttpPut(request); + break; + case DELETE: + httpClientRequest = buildHttpDelete(request); + break; + default: + log.warn("Unsupported http method : {}", method); + throw new NotImplementedException(ErrorCode.API_ERROR); } - if (header != null && header.length > 0) { - get.setHeaders(header); + return execute(httpClientRequest, httpContext); + } + + private HttpGet buildHttpGet(HttpRequest request) { + HttpGet get = new HttpGet(request.getUrl()); + setCommonHttpClientRequest(request, get); + + return get; + } + + private void setCommonHttpClientRequest(HttpRequest request, HttpRequestBase httpClientRequest) { + setConnectionKeepAlive(request, httpClientRequest); + if (request.getHeaders() != null && request.getHeaders().length > 0) { + httpClientRequest.setHeaders(request.getHeaders()); } - int httpStatusCode = -1; - String respStr = null; - try (CloseableHttpResponse response = getHttpClient().execute(get)) { - httpStatusCode = response.getStatusLine().getStatusCode(); - HttpEntity entity = response.getEntity(); - respStr = EntityUtils.toString(entity, CHARSET); - return Pair.of(httpStatusCode, respStr); - } catch (IOException e) { - log.error("Get request fail", e); - throw new InternalException(e, ErrorCode.API_ERROR); - } finally { - get.releaseConnection(); - if (log.isDebugEnabled()) { - log.debug( - "get:keepAlive={},url={},httpStatusCode={},respStr={}", - keepAlive, - url, - httpStatusCode, - respStr - ); + } + + private HttpPost buildHttpPost(HttpRequest request) { + HttpPost post = new HttpPost(request.getUrl()); + setCommonHttpClientRequest(request, post); + setEntity(post, request); + return post; + } + + private HttpPut buildHttpPut(HttpRequest request) { + HttpPut put = new HttpPut(request.getUrl()); + setCommonHttpClientRequest(request, put); + setEntity(put, request); + return put; + } + + private FakeHttpDelete buildHttpDelete(HttpRequest request) { + FakeHttpDelete delete = new FakeHttpDelete(request.getUrl()); + setCommonHttpClientRequest(request, delete); + setEntity(delete, request); + return delete; + } + + private void setEntity(HttpEntityEnclosingRequest request, HttpRequest httpRequest) { + if (httpRequest.getHttpEntity() != null) { + request.setEntity(httpRequest.getHttpEntity()); + } else if (StringUtils.isNotBlank(httpRequest.getStringEntity())) { + try { + request.setEntity(new ByteArrayEntity(httpRequest.getStringEntity().getBytes(CHARSET))); + } catch (IOException e) { + throw new InternalException(e, ErrorCode.API_ERROR); } } } - @Override - public Pair post(String url, HttpEntity requestEntity, Header... headers) { - HttpPost post = new HttpPost(url); - // 设置为长连接，服务端判断有此参数就不关闭连接。 - post.setHeader("Connection", "Keep-Alive"); - post.setHeaders(headers); - post.setEntity(requestEntity); - int httpStatusCode = -1; - String respStr = null; - try (CloseableHttpResponse httpResponse = getHttpClient().execute(post)) { - httpStatusCode = httpResponse.getStatusLine().getStatusCode(); - if (httpStatusCode != HttpStatus.SC_OK) { - String message = httpResponse.getStatusLine().getReasonPhrase(); - HttpEntity entity = httpResponse.getEntity(); - if (entity != null && entity.getContent() != null) { - respStr = new String(EntityUtils.toByteArray(entity), CHARSET); - } - log.warn( - "Post request fail, httpStatusCode={}, errorReason={}, body={}, url={}", - httpStatusCode, - message, - respStr, - url - ); - throw new InternalException(message, ErrorCode.API_ERROR); - } - HttpEntity entity = httpResponse.getEntity(); - return Pair.of(httpStatusCode, EntityUtils.toByteArray(entity)); - } catch (IOException e) { - log.error("Post request fail", e); - throw new InternalException(e, ErrorCode.API_ERROR); - } finally { - post.releaseConnection(); - if (log.isDebugEnabled()) { - log.debug( - "post:url={},headers={},requestEntity={},httpStatusCode={},respStr={}", - url, - headers, - requestEntity, - httpStatusCode, - respStr - ); - } + private HttpContext buildHttpContext(HttpRequest request) { + HttpCoreContext httpContext = HttpCoreContext.create(); + httpContext.setAttribute(HttpContextAttributeNames.RETRY_MODE, request.getRetryMode().getValue()); + return httpContext; + } + + private void setConnectionKeepAlive(HttpRequest request, HttpRequestBase httpRequestBase) { + if (request.isKeepAlive()) { + // 设置为长连接，服务端判断有此参数就不关闭连接。 + httpRequestBase.setHeader("Connection", "Keep-Alive"); } } - @Override - public Pair put(String url, HttpEntity requestEntity, Header... headers) { - HttpPut put = new HttpPut(url); - // 设置为长连接，服务端判断有此参数就不关闭连接。 - put.setHeader("Connection", "Keep-Alive"); - put.setHeaders(headers); - put.setEntity(requestEntity); + private HttpResponse execute(HttpRequestBase httpClientRequest, HttpContext context) { int httpStatusCode = -1; String respStr = null; - try (CloseableHttpResponse httpResponse = getHttpClient().execute(put)) { + try (CloseableHttpResponse httpResponse = httpClient.execute(httpClientRequest, context)) { httpStatusCode = httpResponse.getStatusLine().getStatusCode(); HttpEntity entity = httpResponse.getEntity(); - respStr = new String(EntityUtils.toByteArray(entity), CHARSET); - if (httpStatusCode != HttpStatus.SC_OK) { - String message = httpResponse.getStatusLine().getReasonPhrase(); - log.warn( - "Put request fail, httpStatusCode={}, errorReason={}, respStr={}", - httpStatusCode, - message, - respStr - ); - throw new InternalException(message, ErrorCode.API_ERROR); + if (entity != null && entity.getContent() != null) { + respStr = new String(EntityUtils.toByteArray(entity), CHARSET); } - return Pair.of(httpStatusCode, respStr); - } catch (IOException e) { - log.error("Put request fail", e); - throw new InternalException(e, ErrorCode.API_ERROR); - } finally { - put.releaseConnection(); - if (log.isDebugEnabled()) { - log.debug( - "put:url={},headers={},requestEntity={},httpStatusCode={},respStr={}", - url, - headers, - requestEntity, + // 状态码>=400判定为失败 + if (httpStatusCode >= HttpStatus.SC_BAD_REQUEST) { + String reasonPhrase = httpResponse.getStatusLine().getReasonPhrase(); + log.warn( + "Request fail, method: {}, url={}, httpStatusCode={}, errorReason={}, body={},", + httpClientRequest.getMethod(), + httpClientRequest.getURI().getPath(), httpStatusCode, + reasonPhrase, respStr ); + throw new HttpStatusException(httpClientRequest.getURI().toString(), httpStatusCode, reasonPhrase); + } else { + return new HttpResponse(httpStatusCode, respStr); } - } - } - - @Override - public Pair delete(String url, String content, Header... headers) { - FakeHttpDelete delete = new FakeHttpDelete(url); - if (content != null) { - HttpEntity requestEntity; - try { - requestEntity = new ByteArrayEntity(content.getBytes(CHARSET)); - } catch (UnsupportedEncodingException e) { - log.error("Fail to get ByteArrayEntity", e); - throw new InternalException(e, ErrorCode.INTERNAL_ERROR); - } - delete.setEntity(requestEntity); - } - delete.setHeaders(headers); - int httpStatusCode = -1; - String respStr = null; - try (CloseableHttpResponse httpResponse = getHttpClient().execute(delete)) { - httpStatusCode = httpResponse.getStatusLine().getStatusCode(); - if (httpStatusCode != HttpStatus.SC_OK) { - String message = httpResponse.getStatusLine().getReasonPhrase(); - log.info("Delete request fail, url={}, httpStatusCode={}, errorReason={}", url, httpStatusCode, - message); - throw new InternalException(String.format("url=%s,httpStatusCode=%s" + - "，message=%s", url, httpStatusCode, message), ErrorCode.API_ERROR); - } - HttpEntity entity = httpResponse.getEntity(); - byte[] respBytes = EntityUtils.toByteArray(entity); - if (respBytes == null) { - return null; - } - respStr = new String(respBytes, CHARSET); - return Pair.of(httpStatusCode, respStr); } catch (IOException e) { - log.error("Delete request fail", e); + log.error("Request fail", e); throw new InternalException(e, ErrorCode.API_ERROR); } finally { - delete.releaseConnection(); + httpClientRequest.releaseConnection(); if (log.isDebugEnabled()) { log.debug( - "delete:url={},headers={},body={},httpStatusCode={},respStr={}", - url, - headers, - content, + "Request done, method: {}, url={}, httpStatusCode={}, respStr={}", + httpClientRequest.getMethod(), + httpClientRequest.getURI().getPath(), httpStatusCode, respStr ); diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/ExtHttpHelper.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/ExtHttpHelper.java deleted file mode 100644 index d05fab8442..0000000000 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/ExtHttpHelper.java +++ /dev/null @@ -1,207 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.util.http; - -import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.exception.InternalException; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.http.Header; -import org.apache.http.HttpEntity; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpRequestBase; -import org.apache.http.entity.ByteArrayEntity; -import org.apache.http.message.BasicHeader; - -import java.io.IOException; -import java.util.List; - -@Slf4j -public class ExtHttpHelper { - - private final HttpHelper httpHelper; - - public ExtHttpHelper(HttpHelper httpHelper) { - this.httpHelper = httpHelper; - } - - /** - * 支持自定义头的POST请求 - * - * @param url 提交的地址 - * @param content 提交的内容字符串 - * @param headers 自定义请求头 - * @return - */ - public String post(String url, String content, Header... headers) { - String CHARSET = "UTF-8"; - return post(url, CHARSET, content, headers); - } - - public String post(String url, String charset, String content, List
headerList) throws Exception { - Header[] headers = new Header[headerList.size()]; - return post(url, charset, content, headerList.toArray(headers)); - } - - - /** - * 支持自定义头的POST请求 - * - * @param url 提交的地址 - * @param charset 字符集，用于解析返回的字符串 - * @param content 提交的内容字符串 - * @param headers 自定义请求头 - * @return - */ - public String post(String url, String charset, String content, Header... headers) { - try { - byte[] resp = post(url, - new ByteArrayEntity(content == null ? "".getBytes() : content.getBytes(charset)), - headers); - if (null == resp) { - return null; - } - return new String(resp, charset); - } catch (IOException e) { - log.error("Post request fail", e); - throw new InternalException(e, ErrorCode.API_ERROR); - } - } - - /** - * 提交POST请求，并返回字节数组 - * - * @param url 提交的地址 - * @param content 提交的内容字节数据 - * @param contentType 默认传null则为"application/x-www-form-urlencoded" - * @return 返回字节数组 - */ - public byte[] post(String url, byte[] content, String contentType) { - return post(url, new ByteArrayEntity(content), contentType); - } - - /** - * 提交POST请求，并返回字节数组 - * - * @param url 提交的地址 - * @param requestEntity 封装好的请求实体 - * @param contentType 默认传null则为"application/x-www-form-urlencoded" - * @return 返回字节数组 - */ - public byte[] post(String url, HttpEntity requestEntity, String contentType) { - return post(url, requestEntity, - new BasicHeader("Content-Type", contentType == null ? "application/x-www-form-urlencoded" : contentType)); - } - - public byte[] post(String url, HttpEntity requestEntity, Header... headers) { - return httpHelper.post(url, requestEntity, headers).getRight(); - } - - /** - * GET请求，并返回字符串 - * - * @param url 提交的地址 - * @return - */ - public String get(String url) { - return get(url, (Header[]) null); - } - - public String get(String url, List
headerList) { - Header[] headers = new Header[headerList.size()]; - return get(url, headerList.toArray(headers)); - } - - public String get(String url, Header[] header) { - return get(true, url, header); - } - - public String get(boolean keepAlive, String url, Header[] header) { - return httpHelper.get(keepAlive, url, header).getRight(); - } - - public Pair getRawResp(boolean keepAlive, String url, Header[] header) { - return httpHelper.getRawResp(keepAlive, url, header); - } - - public String put(String url, HttpEntity requestEntity, Header... headers) { - return httpHelper.put(url, requestEntity, headers).getRight(); - } - - /** - * 发起PUT请求并获取响应 - * - * @param url 地址 - * @param charset 字符集名称 - * @param content body内容 - * @param headerList 请求头列表 - * @return 响应字符串 - */ - public String put(String url, String charset, String content, List
headerList) { - Header[] headers = new Header[headerList.size()]; - try { - return put(url, new ByteArrayEntity(content.getBytes(charset)), headerList.toArray(headers)); - } catch (IOException e) { - log.error("Put request fail", e); - throw new InternalException(e, ErrorCode.API_ERROR); - } - } - - /** - * 发起DELETE请求并获取响应 - * - * @param url 地址 - * @param content body内容 - * @param headers 请求头数组 - * @return 响应字符串 - */ - public String delete(String url, String content, Header... headers) { - return httpHelper.delete(url, content, headers).getRight(); - } - - /** - * 发起DELETE请求并获取响应 - * - * @param url 地址 - * @param content body内容 - * @param headerList 请求头列表 - * @return 响应字符串 - */ - public String delete(String url, String content, List
headerList) { - Header[] headers = new Header[headerList.size()]; - return httpHelper.delete(url, content, headerList.toArray(headers)).getRight(); - } - - /** - * 发起DELETE请求并获取响应 - * - * @param url 地址 - * @param headerList 请求头列表 - * @return 响应字符串 - */ - public String delete(String url, List
headerList) { - return delete(url, null, headerList); - } -} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpContextAttributeNames.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpContextAttributeNames.java new file mode 100644 index 0000000000..855e127226 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpContextAttributeNames.java @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util.http; + +/** + * HTTP Client Context 属性名称定义 + */ +public interface HttpContextAttributeNames { + /** + * 重试模式 + */ + String RETRY_MODE = "RETRY_MODE"; + /** + * 请求是否幂等 + */ + String IS_IDEMPOTENT = "IS_IDEMPOTENT"; +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpHelper.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpHelper.java index 02c66923c2..ca9815e24e 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpHelper.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpHelper.java @@ -26,19 +26,21 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.http.Header; -import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpRequestBase; +/** + * Job http 调用基础实现 + */ public interface HttpHelper { Pair getRawResp(boolean keepAlive, String url, Header[] header); - Pair get(boolean keepAlive, String url, Header[] header); - - Pair post(String url, HttpEntity requestEntity, Header... headers); - - Pair put(String url, HttpEntity requestEntity, Header... headers); - - Pair delete(String url, String content, Header... headers); + /** + * 发起 http 请求 + * + * @param request 请求 + * @return 响应 + */ + HttpResponse request(HttpRequest request); } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpHelperFactory.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpHelperFactory.java index fec007edea..92d4a7c672 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpHelperFactory.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpHelperFactory.java @@ -2,21 +2,10 @@ import io.micrometer.core.instrument.MeterRegistry; import lombok.extern.slf4j.Slf4j; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.conn.socket.LayeredConnectionSocketFactory; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.conn.ssl.TrustSelfSignedStrategy; +import org.apache.http.client.HttpRequestRetryHandler; import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.impl.client.StandardHttpRequestRetryHandler; -import org.apache.http.ssl.SSLContexts; import org.springframework.stereotype.Service; -import java.security.KeyManagementException; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.util.concurrent.TimeUnit; - /** * Http请求基础工厂类 */ @@ -30,106 +19,81 @@ public class HttpHelperFactory { private static final CloseableHttpClient RETRYABLE_HTTP_CLIENT; private static final CloseableHttpClient LONG_RETRYABLE_HTTP_CLIENT; - private static CloseableHttpClient getHttpClient(@SuppressWarnings("SameParameterValue") int connRequestTimeout, - @SuppressWarnings("SameParameterValue") int connTimeout, - int socketTimeout, - boolean canRetry) { - return getHttpClient( - connRequestTimeout, - connTimeout, - socketTimeout, + static { + DEFAULT_HTTP_CLIENT = JobHttpClientFactory.createHttpClient( + 15000, + 15000, + 15000, 500, 1000, - canRetry - ); - } - - private static CloseableHttpClient getHttpClient(int connRequestTimeout, - int connTimeout, - int socketTimeout, - @SuppressWarnings("SameParameterValue") int maxConnPerRoute, - @SuppressWarnings("SameParameterValue") int maxConnTotal, - boolean canRetry) { - HttpClientBuilder httpClientBuilder = HttpClientBuilder.create() - .setDefaultRequestConfig( - RequestConfig.custom() - .setConnectionRequestTimeout(connRequestTimeout) - .setConnectTimeout(connTimeout) - .setSocketTimeout(socketTimeout) - .build() - ) - .evictExpiredConnections() - .evictIdleConnections(5, TimeUnit.SECONDS) - .disableAutomaticRetries() - .disableAuthCaching() - .disableCookieManagement(); - if (canRetry) { - httpClientBuilder.setRetryHandler(new StandardHttpRequestRetryHandler()); - } - CloseableHttpClient httpClient; - LayeredConnectionSocketFactory sslSocketFactory = null; - try { - sslSocketFactory = new SSLConnectionSocketFactory( - SSLContexts.custom() - .loadTrustMaterial(null, new TrustSelfSignedStrategy()) - .build() - ); - } catch (NoSuchAlgorithmException | KeyManagementException | KeyStoreException e) { - log.error("", e); - } - httpClientBuilder.setConnectionManager( - // esb的keep-alive时间为90s，需要<90s,防止连接超时抛出org.apache.http.NoHttpResponseException: - // The target server failed to respond - // 因此，timeToLive使用34s - JobHttpClientConnectionManagerFactory.createWatchableConnectionManager( - maxConnPerRoute, - maxConnTotal, - 34, - TimeUnit.SECONDS, - sslSocketFactory - )); - httpClient = httpClientBuilder.build(); - return httpClient; - } - - static { - DEFAULT_HTTP_CLIENT = getHttpClient( - 15000, 15000, 15000, false - ); - RETRYABLE_HTTP_CLIENT = getHttpClient( - 15000, 15000, 15000, true - ); - LONG_RETRYABLE_HTTP_CLIENT = getHttpClient( - 15000, 15000, 30000, true - ); + 60, + false, + null); + RETRYABLE_HTTP_CLIENT = JobHttpClientFactory.createHttpClient( + 15000, + 15000, + 15000, + 500, + 1000, + 60, + true, + new JobHttpRequestRetryHandler()); + LONG_RETRYABLE_HTTP_CLIENT = JobHttpClientFactory.createHttpClient( + 15000, + 15000, + 30000, + 500, + 1000, + 60, + true, + new JobHttpRequestRetryHandler()); } public static void setMeterRegistry(MeterRegistry meterRegistry) { HttpHelperFactory.meterRegistry = meterRegistry; } - private static HttpHelper getWatchableHttpHelper(HttpHelper httpHelper) { + private static WatchableHttpHelper getWatchableHttpHelper(HttpHelper httpHelper) { return new WatchableHttpHelper(httpHelper, meterRegistry); } - private static ExtHttpHelper getWatchableExtHelper(HttpHelper httpHelper) { - HttpHelper watchableHttpHelper = getWatchableHttpHelper(httpHelper); - return new ExtHttpHelper(watchableHttpHelper); + private static WatchableHttpHelper getWatchableExtHelper(HttpHelper httpHelper) { + return getWatchableHttpHelper(httpHelper); } - public static ExtHttpHelper getDefaultHttpHelper() { + public static WatchableHttpHelper getDefaultHttpHelper() { HttpHelper baseHttpHelper = new BaseHttpHelper(DEFAULT_HTTP_CLIENT); return getWatchableExtHelper(baseHttpHelper); } @SuppressWarnings("unused") - public static ExtHttpHelper getRetryableHttpHelper() { + public static WatchableHttpHelper getRetryableHttpHelper() { HttpHelper baseHttpHelper = new BaseHttpHelper(RETRYABLE_HTTP_CLIENT); return getWatchableExtHelper(baseHttpHelper); } - public static ExtHttpHelper getLongRetryableHttpHelper() { + public static WatchableHttpHelper getLongRetryableHttpHelper() { HttpHelper baseHttpHelper = new BaseHttpHelper(LONG_RETRYABLE_HTTP_CLIENT); return getWatchableExtHelper(baseHttpHelper); } + + public static HttpHelper createHttpHelper(int connRequestTimeout, + int connTimeout, + int socketTimeout, + int maxConnPerRoute, + int maxConnTotal, + int timeToLive, + boolean allowRetry, + HttpRequestRetryHandler retryHandler) { + CloseableHttpClient httpClient = JobHttpClientFactory.createHttpClient( + connRequestTimeout, + connTimeout, + socketTimeout, + maxConnPerRoute, + maxConnTotal, + timeToLive, + allowRetry, + retryHandler); + return new BaseHttpHelper(httpClient); + } } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpRequest.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpRequest.java new file mode 100644 index 0000000000..54ae31c8c2 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpRequest.java @@ -0,0 +1,128 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util.http; + +import com.tencent.bk.job.common.constant.HttpMethodEnum; +import lombok.Data; +import org.apache.http.Header; +import org.apache.http.HttpEntity; + +/** + * Http 请求 + */ +@Data +public class HttpRequest { + private HttpMethodEnum method; + private String url; + /** + * 文本格式的 entity + */ + private String stringEntity; + /** + * 标准 httpEntity，优先级最高 + */ + private HttpEntity httpEntity; + private Header[] headers; + /** + * 请求重试模式 + */ + private RetryModeEnum retryMode; + /** + * 请求是否幂等 + */ + private final Boolean idempotent; + + private boolean keepAlive; + + public HttpRequest(Builder builder) { + this.method = builder.method; + this.url = builder.url; + this.stringEntity = builder.stringEntity; + this.httpEntity = builder.httpEntity; + this.headers = builder.headers; + this.retryMode = builder.retryMode; + this.idempotent = builder.idempotent; + this.keepAlive = builder.keepAlive; + } + + public static Builder builder(HttpMethodEnum method, String url) { + return new Builder(method, url); + } + + public static class Builder { + private final HttpMethodEnum method; + private final String url; + private String stringEntity; + private HttpEntity httpEntity; + private Header[] headers; + private RetryModeEnum retryMode = RetryModeEnum.SAFE_GUARANTEED; + private Boolean idempotent; + /** + * 按照 http1.1 之后的标准，默认都使用长连接 + */ + private boolean keepAlive = true; + + public Builder(HttpMethodEnum method, String url) { + this.method = method; + this.url = url; + } + + public Builder setStringEntity(String stringEntity) { + this.stringEntity = stringEntity; + return this; + } + + public Builder setHttpEntity(HttpEntity httpEntity) { + this.httpEntity = httpEntity; + return this; + } + + public Builder setHeaders(Header[] headers) { + this.headers = headers; + return this; + } + + public Builder setRetryMode(RetryModeEnum retryMode) { + this.retryMode = retryMode; + return this; + } + + public Builder setIdempotent(Boolean idempotent) { + this.idempotent = idempotent; + return this; + } + + public Builder setKeepAlive(boolean keepAlive) { + this.keepAlive = keepAlive; + return this; + } + + public HttpRequest build() { + return new HttpRequest(this); + } + } + + +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpResponse.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpResponse.java new file mode 100644 index 0000000000..7e48ee01bb --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpResponse.java @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util.http; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@NoArgsConstructor +@Getter +@Setter +public class HttpResponse { + /** + * http status + */ + private int statusCode; + /** + * response entity + */ + private String entity; + + public HttpResponse(int statusCode, String entity) { + this.statusCode = statusCode; + this.entity = entity; + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/JobHttpClientFactory.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/JobHttpClientFactory.java new file mode 100644 index 0000000000..073caf9f2e --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/JobHttpClientFactory.java @@ -0,0 +1,141 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util.http; + +import lombok.extern.slf4j.Slf4j; +import org.apache.http.client.HttpRequestRetryHandler; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.conn.socket.LayeredConnectionSocketFactory; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.conn.ssl.TrustSelfSignedStrategy; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.StandardHttpRequestRetryHandler; +import org.apache.http.ssl.SSLContexts; + +import java.security.KeyManagementException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.util.concurrent.TimeUnit; + +/** + * Job http client 工厂 + */ +@Slf4j +public class JobHttpClientFactory { + /** + * 创建 HttpClient + * + * @param connRequestTimeout 从连接池获取到连接的超时时间，单位秒 + * @param connTimeout 建立连接的超时时间，单位秒 + * @param socketTimeout 客户端与服务端进行交互的时间，单位秒 + * @param maxConnPerRoute 每个路由的最大连接 + * @param maxConnTotal 连接池最大连接数 + * @param timeToLive 连接存活时间 + * @param retryHandler 请求重试Handler; 如果传入 null，表示不需要重试 + * @return HttpClient + */ + public static CloseableHttpClient createHttpClient(int connRequestTimeout, + int connTimeout, + int socketTimeout, + int maxConnPerRoute, + int maxConnTotal, + int timeToLive, + boolean allowRetry, + HttpRequestRetryHandler retryHandler) { + return createHttpClient( + connRequestTimeout, + connTimeout, + socketTimeout, + maxConnPerRoute, + maxConnTotal, + timeToLive, + allowRetry, + retryHandler, + (httpClientBuilder) -> { + // do nothing + }); + } + + public static CloseableHttpClient createHttpClient(int connRequestTimeout, + int connTimeout, + int socketTimeout, + int maxConnPerRoute, + int maxConnTotal, + int timeToLive, + boolean allowRetry, + HttpRequestRetryHandler retryHandler, + HttpClientCustomizer customizer) { + HttpClientBuilder httpClientBuilder = HttpClientBuilder.create() + .setDefaultRequestConfig( + RequestConfig.custom() + .setConnectionRequestTimeout(connRequestTimeout) + .setConnectTimeout(connTimeout) + .setSocketTimeout(socketTimeout) + .build() + ) + .evictExpiredConnections() + .evictIdleConnections(5, TimeUnit.SECONDS) + .disableAuthCaching() + .disableCookieManagement(); + if (!allowRetry) { + httpClientBuilder.disableAutomaticRetries(); + } else { + if (retryHandler != null) { + httpClientBuilder.setRetryHandler(retryHandler); + } else { + httpClientBuilder.setRetryHandler(new StandardHttpRequestRetryHandler()); + } + } + + LayeredConnectionSocketFactory sslSocketFactory = null; + try { + sslSocketFactory = new SSLConnectionSocketFactory( + SSLContexts.custom() + .loadTrustMaterial(null, new TrustSelfSignedStrategy()) + .build() + ); + } catch (NoSuchAlgorithmException | KeyManagementException | KeyStoreException e) { + log.error("Create HttpClient exception", e); + } + httpClientBuilder.setConnectionManager( + JobHttpClientConnectionManagerFactory.createWatchableConnectionManager( + maxConnPerRoute, + maxConnTotal, + timeToLive, + TimeUnit.SECONDS, + sslSocketFactory + )); + + if (customizer != null) { + customizer.customize(httpClientBuilder); + } + return httpClientBuilder.build(); + } + + public interface HttpClientCustomizer { + void customize(HttpClientBuilder httpClientBuilder); + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/JobHttpRequestRetryHandler.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/JobHttpRequestRetryHandler.java new file mode 100644 index 0000000000..d1d0bc3983 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/JobHttpRequestRetryHandler.java @@ -0,0 +1,169 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util.http; + +import lombok.extern.slf4j.Slf4j; +import org.apache.http.HttpRequest; +import org.apache.http.NoHttpResponseException; +import org.apache.http.client.HttpRequestRetryHandler; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.conn.ConnectTimeoutException; +import org.apache.http.protocol.HttpContext; + +import java.io.IOException; +import java.net.SocketTimeoutException; +import java.util.HashSet; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import static com.tencent.bk.job.common.util.http.RetryModeEnum.SAFE_GUARANTEED; + +/** + * Job 第三方请求调用重试 Handler + */ +@Slf4j +public class JobHttpRequestRetryHandler implements HttpRequestRetryHandler { + + private final int retryCount; + + private final Map retryMethods; + + private final Set> retryableExceptions; + + public JobHttpRequestRetryHandler() { + this.retryCount = 3; + this.retryMethods = new ConcurrentHashMap<>(); + // 蓝鲸各平台仅能保证 GET、DELETE 是幂等的，跟 RESTful API 规范有差异 + this.retryMethods.put("GET", Boolean.TRUE); + this.retryMethods.put("DELETE", Boolean.TRUE); + retryableExceptions = new HashSet<>(); + // ConnectTimeoutException 建立连接超时/从连接池获取连接超时 + // SocketTimeoutException Socket连接/读取超时 + // NoHttpResponseException 服务器端关闭连接或者服务端负载太高无法响应，nginx reload，以及网络连接问题 + // httpclient 异常处理参考 https://hc.apache.org/httpclient-legacy/exception-handling.html 的说明 + retryableExceptions.add(ConnectTimeoutException.class); + retryableExceptions.add(SocketTimeoutException.class); + retryableExceptions.add(NoHttpResponseException.class); + } + + @Override + public boolean retryRequest(IOException exception, + int executionCount, + HttpContext context) { + if (isExceedMaxRetry(executionCount)) { + log.info("[{}] Retry rejected. Http request exceed max retry times : {}", + exception.getClass().getSimpleName(), this.retryCount); + return false; + } + + if (!isExceptionRetryable(exception)) { + log.info("[{}] Retry rejected. Exception is not retryable", exception.getClass().getSimpleName()); + return false; + } + + return checkByRetryMode(context, exception); + + } + + private boolean isExceptionRetryable(IOException exception) { + if (log.isDebugEnabled()) { + log.debug("Http exception : {}", exception.getClass().getName()); + } + return retryableExceptions.contains(exception.getClass()); + } + + private boolean checkByRetryMode(HttpContext context, IOException exception) { + RetryModeEnum retryMode = getRetryMode(context); + + switch (retryMode) { + case ALWAYS: + log.info("[{}] Retry accepted. Retry mode is ALWAYS", exception.getClass().getSimpleName()); + return true; + case NEVER: + log.info("[{}] Retry rejected. Retry mode is NEVER", exception.getClass().getSimpleName()); + return false; + case SAFE_GUARANTEED: + // 判断重试请求是否安全 + boolean isRetryable = checkIsRetrySafe(context, exception); + if (isRetryable) { + log.info("[{}] Retry accepted. Retry is safe", exception.getClass().getSimpleName()); + return true; + } else { + log.info("[{}] Retry rejected. Retry is not safe", exception.getClass().getSimpleName()); + return false; + } + default: + return false; + } + + } + + private boolean checkIsRetrySafe(HttpContext context, IOException exception) { + boolean isRequestIdempotent = isRequestIdempotent(context); + if (isRequestIdempotent) { + return true; + } + // ConnectTimeoutException/NoHttpResponseException，服务端并没有开始接受和正式处理请求，可以重试 + // SocketTimeoutException 可能服务端已经接受处理了请求，如果二次请求可能引发问题 + return exception instanceof ConnectTimeoutException || exception instanceof NoHttpResponseException; + } + + private boolean isRequestIdempotent(HttpContext context) { + // 判断方法使用幂等 + Object isIdempotentAttrVal = context.getAttribute(HttpContextAttributeNames.IS_IDEMPOTENT); + boolean isIdempotent = isIdempotentAttrVal != null && (boolean) isIdempotentAttrVal; + if (isIdempotent) { + // 如果上下文主动设置了该请求的幂等参数，那么优先使用 + return true; + } + + // 上下文没有主动设置幂等参数，那么按照 http method 的实际幂等情况判断 + final HttpClientContext clientContext = HttpClientContext.adapt(context); + final HttpRequest request = clientContext.getRequest(); + final String method = request.getRequestLine().getMethod().toUpperCase(Locale.ROOT); + final Boolean b = this.retryMethods.get(method); + return b != null && b; + } + + + private RetryModeEnum getRetryMode(HttpContext context) { + Object retryModeObj = context.getAttribute(HttpContextAttributeNames.RETRY_MODE); + RetryModeEnum retryMode = SAFE_GUARANTEED; + if (retryModeObj != null) { + retryMode = RetryModeEnum.valOf((int) retryModeObj); + } + if (log.isDebugEnabled()) { + log.debug("RetryMode : {}", retryMode); + } + return retryMode; + } + + private boolean isExceedMaxRetry(int executionCount) { + return executionCount > this.retryCount; + } + +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/RetryModeEnum.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/RetryModeEnum.java new file mode 100644 index 0000000000..42f9507151 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/RetryModeEnum.java @@ -0,0 +1,62 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util.http; + +/** + * http 请求重试方式 + */ +public enum RetryModeEnum { + /** + * 总是重试 + */ + ALWAYS(1), + /** + * 禁止重试 + */ + NEVER(2), + /** + * 在保证安全的情况下尽可能重试 + */ + SAFE_GUARANTEED(3); + + private final int value; + + RetryModeEnum(int value) { + this.value = value; + } + + public static RetryModeEnum valOf(int val) { + for (RetryModeEnum modeEnum : values()) { + if (modeEnum.value == val) { + return modeEnum; + } + } + throw new IllegalArgumentException("No RetryModeEnum constant: " + val); + } + + public int getValue() { + return value; + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/WatchableHttpHelper.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/WatchableHttpHelper.java index fc3515b1df..61c0dfd368 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/WatchableHttpHelper.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/WatchableHttpHelper.java @@ -29,7 +29,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.http.Header; -import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpRequestBase; @@ -55,99 +54,14 @@ public Pair getRawResp(boolean keepAlive } @Override - public Pair get(boolean keepAlive, String url, Header[] header) { + public HttpResponse request(HttpRequest request) { String httpMetricName = HttpMetricUtil.getHttpMetricName(); long start = System.nanoTime(); String httpStatus = null; try { - Pair pair = httpHelper.get(keepAlive, url, header); - if (pair != null) { - httpStatus = "" + pair.getLeft(); - } else { - httpStatus = "null"; - } - return pair; - } catch (Throwable t) { - httpStatus = "error"; - throw t; - } finally { - long end = System.nanoTime(); - AbstractList httpMetricTags = HttpMetricUtil.getCurrentMetricTags(); - httpMetricTags.add(Tag.of("http_status", httpStatus)); - if (meterRegistry != null && StringUtils.isNotBlank(httpMetricName)) { - meterRegistry.timer(httpMetricName, httpMetricTags) - .record(end - start, TimeUnit.NANOSECONDS); - } - } - } - - @Override - public Pair post(String url, HttpEntity requestEntity, Header... headers) { - String httpMetricName = HttpMetricUtil.getHttpMetricName(); - long start = System.nanoTime(); - String httpStatus = null; - try { - Pair pair = httpHelper.post(url, requestEntity, headers); - if (pair != null) { - httpStatus = "" + pair.getLeft(); - } else { - httpStatus = "null"; - } - return pair; - } catch (Throwable t) { - httpStatus = "error"; - throw t; - } finally { - long end = System.nanoTime(); - AbstractList httpMetricTags = HttpMetricUtil.getCurrentMetricTags(); - httpMetricTags.add(Tag.of("http_status", httpStatus)); - if (meterRegistry != null && StringUtils.isNotBlank(httpMetricName)) { - meterRegistry.timer(httpMetricName, httpMetricTags) - .record(end - start, TimeUnit.NANOSECONDS); - } - } - } - - @Override - public Pair put(String url, HttpEntity requestEntity, Header... headers) { - String httpMetricName = HttpMetricUtil.getHttpMetricName(); - long start = System.nanoTime(); - String httpStatus = null; - try { - Pair pair = httpHelper.put(url, requestEntity, headers); - if (pair != null) { - httpStatus = "" + pair.getLeft(); - } else { - httpStatus = "null"; - } - return pair; - } catch (Throwable t) { - httpStatus = "error"; - throw t; - } finally { - long end = System.nanoTime(); - AbstractList httpMetricTags = HttpMetricUtil.getCurrentMetricTags(); - httpMetricTags.add(Tag.of("http_status", httpStatus)); - if (meterRegistry != null && StringUtils.isNotBlank(httpMetricName)) { - meterRegistry.timer(httpMetricName, httpMetricTags) - .record(end - start, TimeUnit.NANOSECONDS); - } - } - } - - @Override - public Pair delete(String url, String content, Header... headers) { - String httpMetricName = HttpMetricUtil.getHttpMetricName(); - long start = System.nanoTime(); - String httpStatus = null; - try { - Pair pair = httpHelper.delete(url, content, headers); - if (pair != null) { - httpStatus = "" + pair.getLeft(); - } else { - httpStatus = "null"; - } - return pair; + HttpResponse response = httpHelper.request(request); + httpStatus = "" + response.getStatusCode(); + return response; } catch (Throwable t) { httpStatus = "error"; throw t; diff --git a/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/http/BaseHttpHelperTest.java b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/http/BaseHttpHelperTest.java new file mode 100644 index 0000000000..1e29843650 --- /dev/null +++ b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/http/BaseHttpHelperTest.java @@ -0,0 +1,67 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util.http; + +import com.tencent.bk.job.common.constant.HttpMethodEnum; +import com.tencent.bk.job.common.exception.InternalException; +import org.apache.http.NoHttpResponseException; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.protocol.HttpContext; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class BaseHttpHelperTest { + @Test + @DisplayName("测试 BaseHttpHelper 异常重试") + void whenGetThrowSocketTimeoutExceptionThenRetry() { + JobHttpRequestRetryHandler mockedRetryHandler = Mockito.spy(new JobHttpRequestRetryHandler()); + + CloseableHttpClient retryableHttpClient = JobHttpClientFactory.createHttpClient( + 15000, + 15000, + 15000, + 1, + 2, + 60, + true, + mockedRetryHandler, + (httpClientBuilder -> { + httpClientBuilder.addInterceptorFirst((org.apache.http.HttpRequest request, HttpContext context) -> { + throw new NoHttpResponseException(""); + }); + })); + HttpHelper httpHelper = new BaseHttpHelper(retryableHttpClient); + assertThrows(InternalException.class, + () -> httpHelper.request(HttpRequest.builder(HttpMethodEnum.GET, "http://localhost:8080/test") + .build())); + + // GET + NoHttpResponseException 会被重试 + Mockito.verify(mockedRetryHandler, Mockito.times(4)) + .retryRequest(Mockito.any(), Mockito.anyInt(), Mockito.any()); + } +} diff --git a/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/http/JobHttpRequestRetryHandlerTest.java b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/http/JobHttpRequestRetryHandlerTest.java new file mode 100644 index 0000000000..f6262f85d1 --- /dev/null +++ b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/http/JobHttpRequestRetryHandlerTest.java @@ -0,0 +1,274 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util.http; + +import org.apache.http.NoHttpResponseException; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.conn.ConnectTimeoutException; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.protocol.HttpContext; +import org.apache.http.protocol.HttpCoreContext; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.net.SocketTimeoutException; +import java.net.UnknownHostException; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Http Client 组件调用测试 + */ +public class JobHttpRequestRetryHandlerTest { + @Test + @DisplayName("验证ConnectTimeoutException在非幂等请求下能够重试") + void givenNotIdempotentRequestThrowConnectionTimeoutExceptionThenRetry() { + JobHttpRequestRetryHandler mockedRetryHandler = Mockito.spy(new JobHttpRequestRetryHandler()); + + CloseableHttpClient retryableHttpClient = JobHttpClientFactory.createHttpClient( + 15000, + 15000, + 15000, + 1, + 2, + 60, + true, + mockedRetryHandler, + (httpClientBuilder -> { + httpClientBuilder.addInterceptorFirst((org.apache.http.HttpRequest request, HttpContext context) -> { + throw new ConnectTimeoutException(); + }); + })); + + HttpPost post = new HttpPost("http://127.0.0.1:8080/test"); + assertThrows(ConnectTimeoutException.class, () -> retryableHttpClient.execute(post)); + + Mockito.verify(mockedRetryHandler, Mockito.times(4)) + .retryRequest(Mockito.any(), Mockito.anyInt(), Mockito.any()); + } + + @Test + @DisplayName("验证NoHttpResponseException在非幂等请求下能够重试") + void givenNotIdempotentRequestThrowNoHttpResponseExceptionThenRetry() { + JobHttpRequestRetryHandler mockedRetryHandler = Mockito.spy(new JobHttpRequestRetryHandler()); + + CloseableHttpClient retryableHttpClient = JobHttpClientFactory.createHttpClient( + 15000, + 15000, + 15000, + 1, + 2, + 60, + true, + mockedRetryHandler, + (httpClientBuilder -> { + httpClientBuilder.addInterceptorFirst((org.apache.http.HttpRequest request, HttpContext context) -> { + throw new NoHttpResponseException(""); + }); + })); + + HttpPost post = new HttpPost("http://127.0.0.1:8080/test"); + assertThrows(NoHttpResponseException.class, () -> retryableHttpClient.execute(post)); + + Mockito.verify(mockedRetryHandler, Mockito.times(4)) + .retryRequest(Mockito.any(), Mockito.anyInt(), Mockito.any()); + } + + @Test + @DisplayName("验证SocketTimeoutException在非幂等请求下不会被重试") + void givenNotIdempotentRequestThrowSocketTimeoutExceptionThenDoNotRetry() { + JobHttpRequestRetryHandler mockedRetryHandler = Mockito.spy(new JobHttpRequestRetryHandler()); + + CloseableHttpClient retryableHttpClient = JobHttpClientFactory.createHttpClient( + 15000, + 15000, + 15000, + 1, + 2, + 60, + true, + mockedRetryHandler, + (httpClientBuilder -> { + httpClientBuilder.addInterceptorFirst((org.apache.http.HttpRequest request, HttpContext context) -> { + throw new SocketTimeoutException(); + }); + })); + + HttpPost post = new HttpPost("http://127.0.0.1:8080/test"); + assertThrows(SocketTimeoutException.class, () -> retryableHttpClient.execute(post)); + + Mockito.verify(mockedRetryHandler, Mockito.times(1)) + .retryRequest(Mockito.any(), Mockito.anyInt(), Mockito.any()); + } + + @Test + @DisplayName("验证抛出其他不可重试的异常，不会被重试") + void givenThrowNotRetryableExceptionThenDoNotRetry() { + JobHttpRequestRetryHandler mockedRetryHandler = Mockito.spy(new JobHttpRequestRetryHandler()); + + CloseableHttpClient retryableHttpClient = JobHttpClientFactory.createHttpClient( + 15000, + 15000, + 15000, + 1, + 2, + 60, + true, + mockedRetryHandler, + (httpClientBuilder -> { + httpClientBuilder.addInterceptorFirst((org.apache.http.HttpRequest request, HttpContext context) -> { + throw new UnknownHostException(""); + }); + })); + + HttpGet get = new HttpGet("http://127.0.0.1:8080/test"); + assertThrows(UnknownHostException.class, () -> retryableHttpClient.execute(get)); + + Mockito.verify(mockedRetryHandler, Mockito.times(1)) + .retryRequest(Mockito.any(), Mockito.anyInt(), Mockito.any()); + } + + @Test + @DisplayName("测试指定重试模式为RetryModeEnum.ALWAYS, 虽然 POST 不是幂等方法，但是仍然能够重试") + void givenRetryModeAlwaysAndPostThenRetry() { + JobHttpRequestRetryHandler mockedRetryHandler = Mockito.spy(new JobHttpRequestRetryHandler()); + + CloseableHttpClient retryableHttpClient = JobHttpClientFactory.createHttpClient( + 15000, + 15000, + 15000, + 1, + 2, + 60, + true, + mockedRetryHandler, + (httpClientBuilder -> { + httpClientBuilder.addInterceptorFirst((org.apache.http.HttpRequest request, HttpContext context) -> { + throw new SocketTimeoutException(); + }); + })); + + HttpPost post = new HttpPost("http://127.0.0.1:8080/test"); + HttpCoreContext httpContext = HttpCoreContext.create(); + httpContext.setAttribute(HttpContextAttributeNames.RETRY_MODE, RetryModeEnum.ALWAYS.getValue()); + assertThrows(SocketTimeoutException.class, () -> retryableHttpClient.execute(post, httpContext)); + + + // 虽然 POST 不是幂等的方法，但是通过设置重试模式为RetryModeEnum.ALWAYS，依然会被重试3 次，共计调用方法 4 次 + Mockito.verify(mockedRetryHandler, Mockito.times(4)) + .retryRequest(Mockito.any(), Mockito.anyInt(), Mockito.any()); + } + + @Test + @DisplayName("测试指定重试模式为RetryModeEnum.NEVER, 虽然 GET 是幂等方法，但是仍然不能够重试") + void givenRetryModeNeverAndPostThenRetry() { + JobHttpRequestRetryHandler mockedRetryHandler = Mockito.spy(new JobHttpRequestRetryHandler()); + + CloseableHttpClient retryableHttpClient = JobHttpClientFactory.createHttpClient( + 15000, + 15000, + 15000, + 1, + 2, + 60, + true, + mockedRetryHandler, + (httpClientBuilder -> { + httpClientBuilder.addInterceptorFirst((org.apache.http.HttpRequest request, HttpContext context) -> { + throw new SocketTimeoutException(); + }); + })); + + HttpGet get = new HttpGet("http://127.0.0.1:8080/test"); + HttpCoreContext httpContext = HttpCoreContext.create(); + httpContext.setAttribute(HttpContextAttributeNames.RETRY_MODE, RetryModeEnum.NEVER.getValue()); + assertThrows(SocketTimeoutException.class, () -> retryableHttpClient.execute(get, httpContext)); + + + // 虽然 GET 是幂等的方法，但是通过设置重试模式为RetryModeEnum.NEVER，不会被重试 + Mockito.verify(mockedRetryHandler, Mockito.times(1)) + .retryRequest(Mockito.any(), Mockito.anyInt(), Mockito.any()); + } + + @Test + @DisplayName("测试使用默认的重试模式RetryModeEnum.SAFE_GUARANTEED，非幂等方法比如 POST 不会被重试") + void givenNotIdempotentRequestThenDoNotRetry() { + JobHttpRequestRetryHandler mockedRetryHandler = Mockito.spy(new JobHttpRequestRetryHandler()); + + CloseableHttpClient retryableHttpClient = JobHttpClientFactory.createHttpClient( + 15000, + 15000, + 15000, + 1, + 2, + 60, + true, + mockedRetryHandler, + (httpClientBuilder -> { + httpClientBuilder.addInterceptorFirst((org.apache.http.HttpRequest request, HttpContext context) -> { + throw new SocketTimeoutException(); + }); + })); + HttpPost post = new HttpPost("http://127.0.0.1:8080/test"); + HttpCoreContext httpContext = HttpCoreContext.create(); + assertThrows(SocketTimeoutException.class, () -> retryableHttpClient.execute(post, httpContext)); + + + // POST 不幂等的方法，所以不重试，只会调用一次 + Mockito.verify(mockedRetryHandler, Mockito.times(1)) + .retryRequest(Mockito.any(), Mockito.anyInt(), Mockito.any()); + } + + @Test + @DisplayName("测试使用默认的重试模式RetryModeEnum.SAFE_GUARANTEED，幂等方法比如 GET 会被重试") + void givenIdempotentRequestThenRetry() { + JobHttpRequestRetryHandler mockedRetryHandler = Mockito.spy(new JobHttpRequestRetryHandler()); + + CloseableHttpClient retryableHttpClient = JobHttpClientFactory.createHttpClient( + 15000, + 15000, + 15000, + 1, + 2, + 60, + true, + mockedRetryHandler, + (httpClientBuilder -> { + httpClientBuilder.addInterceptorFirst((org.apache.http.HttpRequest request, HttpContext context) -> { + throw new SocketTimeoutException(); + }); + })); + HttpGet get = new HttpGet("http://127.0.0.1:8080/test"); + HttpCoreContext httpContext = HttpCoreContext.create(); + assertThrows(SocketTimeoutException.class, () -> retryableHttpClient.execute(get, httpContext)); + + + // GET 幂等的方法，所以会重试 + Mockito.verify(mockedRetryHandler, Mockito.times(4)) + .retryRequest(Mockito.any(), Mockito.anyInt(), Mockito.any()); + } +} diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/config/BkApiGatewayProperties.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/config/BkApiGatewayProperties.java index eb68b7452b..896e487106 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/config/BkApiGatewayProperties.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/config/BkApiGatewayProperties.java @@ -35,12 +35,14 @@ public class BkApiGatewayProperties { - private GseApiGwConfig gse; + private ApiGwConfig gse; + + private ApiGwConfig bkNotice; @Getter @Setter @ToString - public static class GseApiGwConfig { + public static class ApiGwConfig { /** * 蓝鲸Api Gateway url */ diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/EsbResp.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/EsbResp.java index c232135c87..a6722407e1 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/EsbResp.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/EsbResp.java @@ -137,6 +137,6 @@ public static EsbResp buildValidateFailResp(ErrorDetailDTO errorDetail) { @JsonIgnore public boolean isSuccess() { - return this.code == ErrorCode.RESULT_OK; + return (this.result != null && this.result) || (this.code != null && this.code == ErrorCode.RESULT_OK); } } diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/ApiRequestInfo.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/OpenApiRequestInfo.java similarity index 85% rename from src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/ApiRequestInfo.java rename to src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/OpenApiRequestInfo.java index 5280a14d97..67b4b57bf1 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/ApiRequestInfo.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/OpenApiRequestInfo.java @@ -25,6 +25,7 @@ package com.tencent.bk.job.common.esb.model; import com.tencent.bk.job.common.constant.HttpMethodEnum; +import com.tencent.bk.job.common.util.http.RetryModeEnum; import lombok.Data; import org.apache.commons.lang3.StringUtils; @@ -33,8 +34,13 @@ import java.util.HashMap; import java.util.Map; +/** + * Open API 请求封装 + * + * @param + */ @Data -public class ApiRequestInfo { +public class OpenApiRequestInfo { private final HttpMethodEnum method; private final String uri; /** @@ -47,14 +53,24 @@ public class ApiRequestInfo { private Map queryParamsMap; private final T body; private final BkApiAuthorization authorization; + /** + * 请求重试模式 + */ + private final RetryModeEnum retryMode; + /** + * 请求是否幂等 + */ + private final Boolean idempotent; - public ApiRequestInfo(Builder builder) { + public OpenApiRequestInfo(Builder builder) { this.method = builder.method; this.uri = builder.uri; this.queryParams = builder.queryParams; this.queryParamsMap = builder.queryParamsMap; this.body = builder.body; this.authorization = builder.authorization; + this.retryMode = builder.retryMode; + this.idempotent = builder.idempotent; } public static Builder builder() { @@ -68,6 +84,8 @@ public static class Builder { private Map queryParamsMap; private T body; private BkApiAuthorization authorization; + private RetryModeEnum retryMode = RetryModeEnum.SAFE_GUARANTEED; + private Boolean idempotent; public Builder method(HttpMethodEnum method) { this.method = method; @@ -117,8 +135,18 @@ public Builder addQueryParams(Map paramsMap) { return this; } - public ApiRequestInfo build() { - return new ApiRequestInfo<>(this); + public Builder setRetryMode(RetryModeEnum retryMode) { + this.retryMode = retryMode; + return this; + } + + public Builder setIdempotent(Boolean idempotent) { + this.idempotent = idempotent; + return this; + } + + public OpenApiRequestInfo build() { + return new OpenApiRequestInfo<>(this); } } @@ -158,6 +186,4 @@ private String urlEncode(String str) { throw new RuntimeException("encode failed"); } } - - } diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/sdk/AbstractBkApiClient.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/sdk/AbstractBkApiClient.java index 746e0c7aeb..21e2ec5f5d 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/sdk/AbstractBkApiClient.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/sdk/AbstractBkApiClient.java @@ -29,13 +29,14 @@ import com.tencent.bk.job.common.constant.HttpMethodEnum; import com.tencent.bk.job.common.esb.constants.EsbLang; import com.tencent.bk.job.common.esb.metrics.EsbMetricTags; -import com.tencent.bk.job.common.esb.model.ApiRequestInfo; import com.tencent.bk.job.common.esb.model.BkApiAuthorization; import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.common.esb.model.OpenApiRequestInfo; import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.util.JobContextUtil; -import com.tencent.bk.job.common.util.http.ExtHttpHelper; -import com.tencent.bk.job.common.util.http.HttpHelperFactory; +import com.tencent.bk.job.common.util.http.HttpHelper; +import com.tencent.bk.job.common.util.http.HttpRequest; +import com.tencent.bk.job.common.util.http.HttpResponse; import com.tencent.bk.job.common.util.json.JsonMapper; import com.tencent.bk.job.common.util.json.JsonUtils; import io.micrometer.core.instrument.MeterRegistry; @@ -49,7 +50,6 @@ import org.slf4j.LoggerFactory; import javax.servlet.http.HttpServletRequest; -import java.util.Arrays; import java.util.Collection; import java.util.concurrent.TimeUnit; @@ -64,7 +64,7 @@ public abstract class AbstractBkApiClient { private String lang; private final Logger log = LoggerFactory.getLogger(this.getClass()); private final String baseAccessUrl; - private final ExtHttpHelper defaultHttpHelper = HttpHelperFactory.getDefaultHttpHelper(); + private final HttpHelper defaultHttpHelper; private final MeterRegistry meterRegistry; private static final String BK_API_AUTH_HEADER = "X-Bkapi-Authorization"; /** @@ -74,25 +74,27 @@ public abstract class AbstractBkApiClient { private JsonMapper jsonMapper = JsonMapper.nonNullMapper(); /** - * @param meterRegistry MeterRegistry - * @param metricName API http 请求指标名称 - * @param baseAccessUrl API 服务访问地址 + * @param meterRegistry MeterRegistry + * @param metricName API http 请求指标名称 + * @param baseAccessUrl API 服务访问地址 + * @param defaultHttpHelper http 请求处理客户端 */ public AbstractBkApiClient(MeterRegistry meterRegistry, String metricName, - String baseAccessUrl) { + String baseAccessUrl, + HttpHelper defaultHttpHelper) { this.meterRegistry = meterRegistry; this.metricName = metricName; this.baseAccessUrl = baseAccessUrl; + this.defaultHttpHelper = defaultHttpHelper; } public AbstractBkApiClient(MeterRegistry meterRegistry, String metricName, String baseAccessUrl, + HttpHelper defaultHttpHelper, String lang) { - this.meterRegistry = meterRegistry; - this.metricName = metricName; - this.baseAccessUrl = baseAccessUrl; + this(meterRegistry, metricName, baseAccessUrl, defaultHttpHelper); this.lang = lang; } @@ -105,27 +107,21 @@ public void setJsonMapper(JsonMapper jsonMapper) { this.jsonMapper = jsonMapper; } - public EsbResp doRequest(ApiRequestInfo requestInfo, - TypeReference> typeReference) { - return doRequest(requestInfo, typeReference, defaultHttpHelper, null); - } - - public EsbResp doRequest(ApiRequestInfo requestInfo, + public EsbResp doRequest(OpenApiRequestInfo requestInfo, TypeReference> typeReference, - ExtHttpHelper httpHelper) { - return doRequest(requestInfo, typeReference, httpHelper, null); + HttpHelper httpHelper) { + return doRequest(requestInfo, typeReference, null, httpHelper); } - public EsbResp doRequest(ApiRequestInfo requestInfo, - TypeReference> typeReference, - BkApiLogStrategy logStrategy) { - return doRequest(requestInfo, typeReference, defaultHttpHelper, logStrategy); + public EsbResp doRequest(OpenApiRequestInfo requestInfo, + TypeReference> typeReference) { + return doRequest(requestInfo, typeReference, null); } - public EsbResp doRequest(ApiRequestInfo requestInfo, + public EsbResp doRequest(OpenApiRequestInfo requestInfo, TypeReference> typeReference, - ExtHttpHelper httpHelper, - BkApiLogStrategy logStrategy) { + BkApiLogStrategy logStrategy, + HttpHelper httpHelper) { HttpMethodEnum httpMethod = requestInfo.getMethod(); BkApiContext apiContext = new BkApiContext<>(httpMethod.name(), requestInfo.getUri(), requestInfo.getBody(), null, null, 0, false); @@ -155,10 +151,10 @@ public EsbResp doRequest(ApiRequestInfo requestInfo, } } - private EsbResp requestApiAndWrapResponse(ApiRequestInfo requestInfo, + private EsbResp requestApiAndWrapResponse(OpenApiRequestInfo requestInfo, BkApiContext apiContext, TypeReference> typeReference, - ExtHttpHelper httpHelper) { + HttpHelper httpHelper) { String uri = apiContext.getUri(); EsbResp esbResp; String respStr; @@ -166,7 +162,7 @@ private EsbResp requestApiAndWrapResponse(ApiRequestInfo requestInf HttpMethodEnum httpMethod = requestInfo.getMethod(); long start = System.currentTimeMillis(); try { - respStr = requestApi(requestInfo, httpHelper); + respStr = requestApi(httpHelper, requestInfo); apiContext.setOriginResp(respStr); if (StringUtils.isBlank(respStr)) { @@ -234,77 +230,24 @@ private Iterable buildMetricTags(String uri, String status) { return tags; } - private String requestApi(ApiRequestInfo requestInfo, - ExtHttpHelper httpHelper) { - String respStr = null; - HttpMethodEnum httpMethod = requestInfo.getMethod(); + private String requestApi(HttpHelper httpHelper, OpenApiRequestInfo requestInfo) { String url = buildApiUrl(requestInfo.buildFinalUri()); - switch (httpMethod) { - case POST: - respStr = postForString(url, requestInfo.getBody(), - requestInfo.getAuthorization(), httpHelper); - break; - case PUT: - respStr = putForString(url, requestInfo.getBody(), - requestInfo.getAuthorization(), httpHelper); - break; - case GET: - respStr = getForString(url, requestInfo.getAuthorization(), httpHelper); - break; - case DELETE: - respStr = deleteForString(url, requestInfo.getAuthorization(), httpHelper); - break; - default: - log.warn("Unimplemented http method: {}", httpMethod.name()); - break; - } - return respStr; - } - private String postForString(String url, - T body, - BkApiAuthorization authorization, - ExtHttpHelper httpHelper) { - if (httpHelper == null) { - httpHelper = defaultHttpHelper; - } - String responseBody; - Header[] header = buildBkApiRequestHeaders(authorization); - responseBody = httpHelper.post(url, jsonMapper.toJson(body), header); - return responseBody; - } + Header[] headers = buildBkApiRequestHeaders(requestInfo.getAuthorization()); + HttpRequest httpRequest = HttpRequest.builder(requestInfo.getMethod(), url) + .setHeaders(headers) + .setKeepAlive(true) + .setRetryMode(requestInfo.getRetryMode()) + .setIdempotent(requestInfo.getIdempotent()) + .setStringEntity(requestInfo.getBody() != null ? jsonMapper.toJson(requestInfo.getBody()) : null) + .build(); - private String putForString(String url, - T body, - BkApiAuthorization authorization, - ExtHttpHelper httpHelper) { - if (httpHelper == null) { - httpHelper = defaultHttpHelper; - } - String responseBody; - Header[] header = buildBkApiRequestHeaders(authorization); - responseBody = httpHelper.put(url, "UTF-8", jsonMapper.toJson(body), Arrays.asList(header)); - return responseBody; + HttpResponse httpResponse = chooseHttpHelper(httpHelper).request(httpRequest); + return httpResponse.getEntity(); } - private String getForString(String url, - BkApiAuthorization authorization, - ExtHttpHelper httpHelper) { - if (httpHelper == null) { - httpHelper = defaultHttpHelper; - } - Header[] header = buildBkApiRequestHeaders(authorization); - return httpHelper.get(url, header); - } - - private String deleteForString(String url, - BkApiAuthorization authorization, - ExtHttpHelper httpHelper) { - if (httpHelper == null) { - httpHelper = defaultHttpHelper; - } - Header[] header = buildBkApiRequestHeaders(authorization); - return httpHelper.delete(url, Arrays.asList(header)); + private HttpHelper chooseHttpHelper(HttpHelper httpHelper) { + return httpHelper != null ? httpHelper : defaultHttpHelper; } private String buildApiUrl(String uri) { diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/GseClient.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/GseClient.java index f191fc4fc4..886ae6c48d 100644 --- a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/GseClient.java +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/GseClient.java @@ -1,3 +1,27 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + package com.tencent.bk.job.common.gse; import com.tencent.bk.job.common.constant.ErrorCode; @@ -18,8 +42,10 @@ import com.tencent.bk.job.common.gse.v2.model.resp.AgentState; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.springframework.util.CollectionUtils; import org.springframework.util.StopWatch; +import java.util.Collections; import java.util.List; @@ -60,6 +86,10 @@ public ScriptTaskResult getExecuteScriptResult(GetExecuteScriptResultRequest req public List listAgentState(ListAgentStateReq req) { StopWatch watch = new StopWatch("listAgentState"); List agentIdList = req.getAgentIdList(); + if (CollectionUtils.isEmpty(agentIdList)) { + log.info("agentIdList is empty"); + return Collections.emptyList(); + } String firstAgentId = agentIdList.get(0); watch.start("chooseGseApiClientByAgentId"); diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/AbstractAgentStateClientImpl.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/AbstractAgentStateClientImpl.java index 36cf305187..b2803b1b1d 100644 --- a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/AbstractAgentStateClientImpl.java +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/AbstractAgentStateClientImpl.java @@ -24,14 +24,14 @@ package com.tencent.bk.job.common.gse.service; -import com.tencent.bk.job.common.gse.GseClient; +import com.tencent.bk.job.common.gse.IGseClient; import com.tencent.bk.job.common.gse.config.AgentStateQueryConfig; -import com.tencent.bk.job.common.gse.constants.AgentAliveStatusEnum; import com.tencent.bk.job.common.gse.v2.model.req.ListAgentStateReq; import com.tencent.bk.job.common.gse.v2.model.resp.AgentState; import com.tencent.bk.job.common.util.ConcurrencyUtil; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.slf4j.helpers.FormattingTuple; import org.slf4j.helpers.MessageFormatter; import org.springframework.util.StopWatch; @@ -48,11 +48,11 @@ public abstract class AbstractAgentStateClientImpl implements AgentStateClient { private final AgentStateQueryConfig agentStateQueryConfig; - private final GseClient gseClient; + private final IGseClient gseClient; private final ThreadPoolExecutor threadPoolExecutor; public AbstractAgentStateClientImpl(AgentStateQueryConfig agentStateQueryConfig, - GseClient gseClient, + IGseClient gseClient, ThreadPoolExecutor threadPoolExecutor) { this.agentStateQueryConfig = agentStateQueryConfig; this.gseClient = gseClient; @@ -60,6 +60,9 @@ public AbstractAgentStateClientImpl(AgentStateQueryConfig agentStateQueryConfig, } protected AgentState getAgentState(String agentId) { + if (StringUtils.isBlank(agentId)) { + return null; + } ListAgentStateReq req = new ListAgentStateReq(); req.setAgentIdList(Collections.singletonList(agentId)); List agentStateList = gseClient.listAgentState(req); @@ -82,7 +85,7 @@ protected AgentState getAgentState(String agentId) { return agentStateList.get(0); } - protected Map batchGetAgentStateConcurrent(List agentIdList) { + public Map batchGetAgentStateConcurrent(List agentIdList) { StopWatch watch = new StopWatch("batchGetAgentStateConcurrent"); watch.start("splitToBatch"); @@ -133,17 +136,6 @@ protected Map batchGetAgentStateConcurrent(List agen return resultMap; } - protected Map batchGetAgentAliveStatus(Map agentStateMap) { - Map agentAliveStatusMap = new HashMap<>(); - for (Map.Entry entry : agentStateMap.entrySet()) { - String agentId = entry.getKey(); - AgentState agentState = entry.getValue(); - agentAliveStatusMap.put(agentId, - AgentAliveStatusEnum.fromAgentState(agentState) == AgentAliveStatusEnum.ALIVE); - } - return agentAliveStatusMap; - } - private Map batchGetAgentStatusWithoutLimit(List agentIdList) { StopWatch watch = new StopWatch("batchGetAgentStatusWithoutLimit"); diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/GseV1AgentStateClientImpl.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/GseV1AgentStateClientImpl.java new file mode 100644 index 0000000000..18240710ca --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/GseV1AgentStateClientImpl.java @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.service; + +import com.tencent.bk.job.common.gse.config.AgentStateQueryConfig; +import com.tencent.bk.job.common.gse.service.model.HostAgentStateQuery; +import com.tencent.bk.job.common.gse.v1.GseV1ApiClient; +import lombok.extern.slf4j.Slf4j; + +import java.util.concurrent.ThreadPoolExecutor; + +@Slf4j +public class GseV1AgentStateClientImpl extends SingleChannelAgentStateClientImpl { + + public GseV1AgentStateClientImpl(AgentStateQueryConfig agentStateQueryConfig, + GseV1ApiClient gseV1ApiClient, + ThreadPoolExecutor threadPoolExecutor) { + super(agentStateQueryConfig, gseV1ApiClient, threadPoolExecutor); + } + + @Override + public String getEffectiveAgentId(HostAgentStateQuery hostAgentStateQuery) { + return hostAgentStateQuery.getCloudIp(); + } +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/GseV2AgentStateClientImpl.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/GseV2AgentStateClientImpl.java new file mode 100644 index 0000000000..0d32cdfa42 --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/GseV2AgentStateClientImpl.java @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.service; + +import com.tencent.bk.job.common.gse.config.AgentStateQueryConfig; +import com.tencent.bk.job.common.gse.service.model.HostAgentStateQuery; +import com.tencent.bk.job.common.gse.v2.GseV2ApiClient; +import lombok.extern.slf4j.Slf4j; + +import java.util.concurrent.ThreadPoolExecutor; + +@Slf4j +public class GseV2AgentStateClientImpl extends SingleChannelAgentStateClientImpl { + + public GseV2AgentStateClientImpl(AgentStateQueryConfig agentStateQueryConfig, + GseV2ApiClient gseV2ApiClient, + ThreadPoolExecutor threadPoolExecutor) { + super(agentStateQueryConfig, gseV2ApiClient, threadPoolExecutor); + } + + @Override + public String getEffectiveAgentId(HostAgentStateQuery hostAgentStateQuery) { + return hostAgentStateQuery.getAgentId(); + } +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/PreferV2AgentStateClientImpl.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/PreferV2AgentStateClientImpl.java index 3e05e0a5d0..3c4d015060 100644 --- a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/PreferV2AgentStateClientImpl.java +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/PreferV2AgentStateClientImpl.java @@ -27,6 +27,7 @@ import com.tencent.bk.job.common.gse.GseClient; import com.tencent.bk.job.common.gse.config.AgentStateQueryConfig; import com.tencent.bk.job.common.gse.service.model.HostAgentStateQuery; +import com.tencent.bk.job.common.gse.util.AgentStateUtil; import com.tencent.bk.job.common.gse.util.AgentUtils; import com.tencent.bk.job.common.gse.v2.model.resp.AgentState; import lombok.extern.slf4j.Slf4j; @@ -60,6 +61,9 @@ public String getEffectiveAgentId(HostAgentStateQuery hostAgentStateQuery) { @Override public AgentState getAgentState(HostAgentStateQuery hostAgentStateQuery) { String finalAgentId = getEffectiveAgentId(hostAgentStateQuery); + if (StringUtils.isBlank(finalAgentId)) { + return null; + } return getAgentState(finalAgentId); } @@ -93,6 +97,6 @@ private Pair, List> classifyGseAgentIds(List agentI @Override public Map batchGetAgentAliveStatus(List hostAgentStateQueryList) { Map agentStateMap = batchGetAgentState(hostAgentStateQueryList); - return batchGetAgentAliveStatus(agentStateMap); + return AgentStateUtil.batchGetAgentAliveStatus(agentStateMap); } } diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/SingleChannelAgentStateClientImpl.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/SingleChannelAgentStateClientImpl.java new file mode 100644 index 0000000000..184f756322 --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/SingleChannelAgentStateClientImpl.java @@ -0,0 +1,70 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.service; + +import com.tencent.bk.job.common.gse.IGseClient; +import com.tencent.bk.job.common.gse.config.AgentStateQueryConfig; +import com.tencent.bk.job.common.gse.service.model.HostAgentStateQuery; +import com.tencent.bk.job.common.gse.util.AgentStateUtil; +import com.tencent.bk.job.common.gse.v2.model.resp.AgentState; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.stream.Collectors; + +@Slf4j +public abstract class SingleChannelAgentStateClientImpl extends AbstractAgentStateClientImpl { + + public SingleChannelAgentStateClientImpl(AgentStateQueryConfig agentStateQueryConfig, + IGseClient gseClient, + ThreadPoolExecutor threadPoolExecutor) { + super(agentStateQueryConfig, gseClient, threadPoolExecutor); + } + + @Override + public AgentState getAgentState(HostAgentStateQuery hostAgentStateQuery) { + String finalAgentId = getEffectiveAgentId(hostAgentStateQuery); + return getAgentState(finalAgentId); + } + + @Override + public Map batchGetAgentState(List hostAgentStateQueryList) { + List queryAgentIds = hostAgentStateQueryList.stream() + .map(this::getEffectiveAgentId) + .filter(StringUtils::isNotEmpty) + .collect(Collectors.toList()); + + return batchGetAgentStateConcurrent(queryAgentIds); + } + + @Override + public Map batchGetAgentAliveStatus(List hostAgentStateQueryList) { + Map agentStateMap = batchGetAgentState(hostAgentStateQueryList); + return AgentStateUtil.batchGetAgentAliveStatus(agentStateMap); + } +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/UseV2ByFeatureAgentStateClientImpl.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/UseV2ByFeatureAgentStateClientImpl.java index f8c2c98980..4069853613 100644 --- a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/UseV2ByFeatureAgentStateClientImpl.java +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/UseV2ByFeatureAgentStateClientImpl.java @@ -25,9 +25,8 @@ package com.tencent.bk.job.common.gse.service; import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; -import com.tencent.bk.job.common.gse.GseClient; -import com.tencent.bk.job.common.gse.config.AgentStateQueryConfig; import com.tencent.bk.job.common.gse.service.model.HostAgentStateQuery; +import com.tencent.bk.job.common.gse.util.AgentStateUtil; import com.tencent.bk.job.common.gse.v2.model.resp.AgentState; import com.tencent.bk.job.common.model.dto.ResourceScope; import com.tencent.bk.job.common.util.feature.FeatureExecutionContext; @@ -45,19 +44,20 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.ThreadPoolExecutor; import java.util.stream.Collectors; @Slf4j -public class UseV2ByFeatureAgentStateClientImpl extends AbstractAgentStateClientImpl { +public class UseV2ByFeatureAgentStateClientImpl implements AgentStateClient { private final BizHostInfoQueryService bizHostInfoQueryService; + private final SingleChannelAgentStateClientImpl gseV1AgentStateClient; + private final SingleChannelAgentStateClientImpl gseV2AgentStateClient; - public UseV2ByFeatureAgentStateClientImpl(AgentStateQueryConfig agentStateQueryConfig, - GseClient gseClient, - BizHostInfoQueryService bizHostInfoQueryService, - ThreadPoolExecutor threadPoolExecutor) { - super(agentStateQueryConfig, gseClient, threadPoolExecutor); + public UseV2ByFeatureAgentStateClientImpl(GseV1AgentStateClientImpl gseV1AgentStateClient, + GseV2AgentStateClientImpl gseV2AgentStateClient, + BizHostInfoQueryService bizHostInfoQueryService) { + this.gseV1AgentStateClient = gseV1AgentStateClient; + this.gseV2AgentStateClient = gseV2AgentStateClient; this.bizHostInfoQueryService = bizHostInfoQueryService; } @@ -76,8 +76,15 @@ public String getEffectiveAgentId(HostAgentStateQuery hostAgentStateQuery) { @Override public AgentState getAgentState(HostAgentStateQuery hostAgentStateQuery) { + // 填充需要的字段 String effectiveAgentId = getEffectiveAgentId(hostAgentStateQuery); - return getAgentState(effectiveAgentId); + if (StringUtils.isBlank(effectiveAgentId)) { + return null; + } + if (needToUseGseV2(hostAgentStateQuery)) { + return gseV2AgentStateClient.getAgentState(hostAgentStateQuery); + } + return gseV1AgentStateClient.getAgentState(hostAgentStateQuery); } private void fillBizIdByHostId(HostAgentStateQuery hostAgentStateQuery) { @@ -185,8 +192,8 @@ public Map batchGetAgentState(List host cloudIpList.size(), agentIdList.size() ); - Map agentStateMap = batchGetAgentStateConcurrent(cloudIpList); - agentStateMap.putAll(batchGetAgentStateConcurrent(agentIdList)); + Map agentStateMap = gseV1AgentStateClient.batchGetAgentStateConcurrent(cloudIpList); + agentStateMap.putAll(gseV2AgentStateClient.batchGetAgentStateConcurrent(agentIdList)); if (log.isDebugEnabled()) { log.debug("agentStateMap={}", JsonUtils.toJson(agentStateMap)); } @@ -372,6 +379,6 @@ private Map queryAgentIdsByHostId(List hostIdList) { @Override public Map batchGetAgentAliveStatus(List hostAgentStateQueryList) { Map agentStateMap = batchGetAgentState(hostAgentStateQueryList); - return batchGetAgentAliveStatus(agentStateMap); + return AgentStateUtil.batchGetAgentAliveStatus(agentStateMap); } } diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/util/AgentStateUtil.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/util/AgentStateUtil.java new file mode 100644 index 0000000000..a7f2004e55 --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/util/AgentStateUtil.java @@ -0,0 +1,46 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.util; + +import com.tencent.bk.job.common.gse.constants.AgentAliveStatusEnum; +import com.tencent.bk.job.common.gse.v2.model.resp.AgentState; + +import java.util.HashMap; +import java.util.Map; + +public class AgentStateUtil { + + public static Map batchGetAgentAliveStatus(Map agentStateMap) { + Map agentAliveStatusMap = new HashMap<>(); + for (Map.Entry entry : agentStateMap.entrySet()) { + String agentId = entry.getKey(); + AgentState agentState = entry.getValue(); + agentAliveStatusMap.put(agentId, + AgentAliveStatusEnum.fromAgentState(agentState) == AgentAliveStatusEnum.ALIVE); + } + return agentAliveStatusMap; + } + +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/GseV1ApiClient.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/GseV1ApiClient.java index 77151f4fff..70ab1201ef 100644 --- a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/GseV1ApiClient.java +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/GseV1ApiClient.java @@ -1,3 +1,27 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + package com.tencent.bk.job.common.gse.v1; import com.tencent.bk.gse.cacheapi.AgentStatusRequestInfo; @@ -313,6 +337,10 @@ public List listAgentState(ListAgentStateReq req) { watch.start("queryAgentStatusFromCacheApi"); List cloudIps = req.getAgentIdList(); + if (CollectionUtils.isEmpty(cloudIps)) { + log.info("cloudIps is empty"); + return Collections.emptyList(); + } AgentStatusResponse agentStatusResponse = queryAgentStatusFromCacheApi(cloudIps); watch.stop(); @@ -368,7 +396,7 @@ private AgentStatusResponse queryAgentStatusFromCacheApi(Collection agen watch.start("quireAgentStatus"); AgentStatusResponse response = gseClient.getCacheClient().quireAgentStatus(request); watch.stop(); - log.debug("QueryAgentStatus response: {}", response); + log.debug("QueryAgentStatus response: {}", getResponseLogStr(response)); return response; } catch (Throwable e) { log.error("QueryAgentStatus error", e); @@ -423,6 +451,10 @@ private AgentStatusRequestInfo buildQueryAgentStatusRequest(Collection a return request; } + private String getResponseLogStr(AgentStatusResponse response) { + return response == null ? null : response.toString().replace("\n", ""); + } + @Override public GseTaskResponse asyncTransferFile(TransferFileRequest request) { List copyFileV1Request = toV1CopyFileInfoRequest(request); diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/GseV2ApiClient.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/GseV2ApiClient.java index 56c17dda9e..5531bf93a1 100644 --- a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/GseV2ApiClient.java +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/GseV2ApiClient.java @@ -1,12 +1,36 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + package com.tencent.bk.job.common.gse.v2; import com.fasterxml.jackson.core.type.TypeReference; import com.tencent.bk.job.common.constant.HttpMethodEnum; import com.tencent.bk.job.common.esb.config.AppProperties; import com.tencent.bk.job.common.esb.config.BkApiGatewayProperties; -import com.tencent.bk.job.common.esb.model.ApiRequestInfo; import com.tencent.bk.job.common.esb.model.BkApiAuthorization; import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.common.esb.model.OpenApiRequestInfo; import com.tencent.bk.job.common.esb.sdk.AbstractBkApiClient; import com.tencent.bk.job.common.esb.sdk.BkApiContext; import com.tencent.bk.job.common.esb.sdk.BkApiLogStrategy; @@ -25,12 +49,15 @@ import com.tencent.bk.job.common.gse.v2.model.req.ListAgentStateReq; import com.tencent.bk.job.common.gse.v2.model.resp.AgentState; import com.tencent.bk.job.common.util.StringUtil; +import com.tencent.bk.job.common.util.http.HttpHelperFactory; +import com.tencent.bk.job.common.util.http.JobHttpRequestRetryHandler; import com.tencent.bk.job.common.util.json.JsonUtils; import io.micrometer.core.instrument.MeterRegistry; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.slf4j.Logger; +import java.util.Collections; import java.util.List; @Slf4j @@ -46,9 +73,21 @@ public class GseV2ApiClient extends AbstractBkApiClient implements IGseClient { public GseV2ApiClient(MeterRegistry meterRegistry, AppProperties appProperties, BkApiGatewayProperties bkApiGatewayProperties) { + super(meterRegistry, GseMetricNames.GSE_V2_API_METRICS_NAME_PREFIX, - bkApiGatewayProperties.getGse().getUrl()); + bkApiGatewayProperties.getGse().getUrl(), + HttpHelperFactory.createHttpHelper( + 15000, + 15000, + 15000, + 1000, + 2000, + 60, + true, + new JobHttpRequestRetryHandler() + ) + ); gseBkApiAuthorization = BkApiAuthorization.appAuthorization(appProperties.getCode(), appProperties.getSecret()); log.info("Init GseV2ApiClient, bkGseApiGatewayUrl: {}, appCode: {}", bkApiGatewayProperties.getGse().getUrl(), appProperties.getCode()); @@ -62,7 +101,8 @@ public GseTaskResponse asyncExecuteScript(ExecuteScriptRequest request) { request, new TypeReference>() { }, - null); + null, + false); return buildGseTaskResponse(resp); } @@ -70,15 +110,17 @@ public GseTaskResponse asyncExecuteScript(ExecuteScriptRequest request) { private EsbResp requestGseApi(String uri, Object reqBody, TypeReference> typeReference, - BkApiLogStrategy logStrategy) { - ApiRequestInfo requestInfo = ApiRequestInfo + BkApiLogStrategy logStrategy, + boolean isRequestIdempotent) { + OpenApiRequestInfo requestInfo = OpenApiRequestInfo .builder() .method(HttpMethodEnum.POST) .uri(uri) .body(reqBody) .authorization(gseBkApiAuthorization) + .setIdempotent(isRequestIdempotent) .build(); - return doRequest(requestInfo, typeReference, logStrategy); + return doRequest(requestInfo, typeReference, logStrategy, null); } private GseTaskResponse buildGseTaskResponse(EsbResp resp) { @@ -112,19 +154,25 @@ public void logResp(Logger log, BkApiContext context) { StringUtil.substring(context.getOriginResp(), 10000)); } } - }); + }, + true); return resp.getData(); } @Override public List listAgentState(ListAgentStateReq req) { + if (CollectionUtils.isEmpty(req.getAgentIdList())) { + log.info("agentIdList is empty"); + return Collections.emptyList(); + } EsbResp> resp = requestGseApi( URI_LIST_AGENT_STATE, req, new TypeReference>>() { }, - null); + null, + true); return resp.getData(); } @@ -136,7 +184,8 @@ public GseTaskResponse asyncTransferFile(TransferFileRequest request) { request, new TypeReference>() { }, - null); + null, + false); return buildGseTaskResponse(resp); } @@ -149,7 +198,8 @@ public FileTaskResult getTransferFileResult(GetTransferFileResultRequest request request, new TypeReference>() { }, - null); + null, + true); FileTaskResult fileTaskResult = resp.getData(); if (fileTaskResult != null && CollectionUtils.isNotEmpty(fileTaskResult.getAtomicFileTaskResults())) { fileTaskResult.getAtomicFileTaskResults().forEach(atomicFileTaskResult -> { @@ -169,7 +219,8 @@ public GseTaskResponse terminateGseFileTask(TerminateGseTaskRequest request) { request, new TypeReference>() { }, - null); + null, + true); return buildGseTaskResponse(resp); } @@ -180,7 +231,8 @@ public GseTaskResponse terminateGseScriptTask(TerminateGseTaskRequest request) { request, new TypeReference>() { }, - null); + null, + true); return buildGseTaskResponse(resp); } } diff --git a/src/backend/commons/notice-sdk/build.gradle b/src/backend/commons/notice-sdk/build.gradle new file mode 100644 index 0000000000..907fb546e8 --- /dev/null +++ b/src/backend/commons/notice-sdk/build.gradle @@ -0,0 +1,41 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +dependencies { + api project(':commons:common') + api project(':commons:esb-sdk') + api project(':commons:common-i18n') + implementation 'com.fasterxml.jackson.core:jackson-core' + implementation 'com.fasterxml.jackson.core:jackson-databind' + implementation 'com.fasterxml.jackson.core:jackson-annotations' + implementation 'org.apache.commons:commons-lang3' + implementation "net.sf.dozer:dozer" + implementation 'io.micrometer:micrometer-registry-prometheus' + implementation 'org.apache.commons:commons-collections4' + implementation 'org.apache.httpcomponents:httpclient' + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' + testImplementation 'org.junit.jupiter:junit-jupiter' + testImplementation 'org.springframework.boot:spring-boot-starter-test' +} diff --git a/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/IBkNoticeClient.java b/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/IBkNoticeClient.java new file mode 100644 index 0000000000..3b14bbab83 --- /dev/null +++ b/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/IBkNoticeClient.java @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.notice; + +import com.tencent.bk.job.common.notice.model.AnnouncementDTO; +import com.tencent.bk.job.common.notice.model.BkNoticeApp; + +import java.util.List; + +public interface IBkNoticeClient { + + BkNoticeApp registerApplication(); + + List getCurrentAnnouncements(String bkLanguage, Integer offset, Integer limit); + +} diff --git a/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/config/BkNoticeProperties.java b/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/config/BkNoticeProperties.java new file mode 100644 index 0000000000..60abda52fd --- /dev/null +++ b/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/config/BkNoticeProperties.java @@ -0,0 +1,41 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.notice.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * 消息通知中心配置 + */ +@ConfigurationProperties(prefix = "bk-notice") +@Getter +@Setter +public class BkNoticeProperties { + + private boolean enabled = true; + +} diff --git a/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/config/NoticeAutoConfiguration.java b/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/config/NoticeAutoConfiguration.java new file mode 100644 index 0000000000..a51a34904a --- /dev/null +++ b/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/config/NoticeAutoConfiguration.java @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.notice.config; + +import com.tencent.bk.job.common.esb.config.AppProperties; +import com.tencent.bk.job.common.esb.config.BkApiGatewayProperties; +import com.tencent.bk.job.common.notice.impl.BkNoticeClient; +import io.micrometer.core.instrument.MeterRegistry; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Slf4j +@Configuration(proxyBeanMethods = false) +@EnableConfigurationProperties({BkNoticeProperties.class}) +public class NoticeAutoConfiguration { + + @Bean + public BkNoticeClient bkNoticeClient(MeterRegistry meterRegistry, + AppProperties appProperties, + BkApiGatewayProperties bkApiGatewayProperties) { + return new BkNoticeClient(meterRegistry, appProperties, bkApiGatewayProperties); + } + +} diff --git a/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/exception/BkNoticeException.java b/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/exception/BkNoticeException.java new file mode 100644 index 0000000000..14addf24b7 --- /dev/null +++ b/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/exception/BkNoticeException.java @@ -0,0 +1,25 @@ +package com.tencent.bk.job.common.notice.exception; + +import com.tencent.bk.job.common.exception.InternalException; +import lombok.Getter; +import lombok.ToString; + +/** + * 调用蓝鲸消息通知中心异常 + */ +@Getter +@ToString +public class BkNoticeException extends InternalException { + + public BkNoticeException(Throwable cause, Integer errorCode, Object[] errorParams) { + super(cause, errorCode, errorParams); + } + + public BkNoticeException(String message, Integer errorCode) { + super(message, errorCode); + } + + public BkNoticeException(String message, Throwable cause, Integer errorCode) { + super(message, cause, errorCode); + } +} diff --git a/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/impl/BkNoticeClient.java b/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/impl/BkNoticeClient.java new file mode 100644 index 0000000000..dc8913520e --- /dev/null +++ b/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/impl/BkNoticeClient.java @@ -0,0 +1,170 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.notice.impl; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.HttpMethodEnum; +import com.tencent.bk.job.common.esb.config.AppProperties; +import com.tencent.bk.job.common.esb.config.BkApiGatewayProperties; +import com.tencent.bk.job.common.esb.model.BkApiAuthorization; +import com.tencent.bk.job.common.esb.model.EsbReq; +import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.common.esb.model.OpenApiRequestInfo; +import com.tencent.bk.job.common.esb.sdk.AbstractBkApiClient; +import com.tencent.bk.job.common.exception.HttpStatusException; +import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.common.metrics.CommonMetricNames; +import com.tencent.bk.job.common.notice.IBkNoticeClient; +import com.tencent.bk.job.common.notice.exception.BkNoticeException; +import com.tencent.bk.job.common.notice.model.AnnouncementDTO; +import com.tencent.bk.job.common.notice.model.BkNoticeApp; +import com.tencent.bk.job.common.util.http.HttpHelperFactory; +import com.tencent.bk.job.common.util.http.HttpMetricUtil; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpStatus; + +import java.util.List; + +@SuppressWarnings("SameParameterValue") +public class BkNoticeClient extends AbstractBkApiClient implements IBkNoticeClient { + + private static final String URI_REGISTER_APPLICATION = "/apigw/v1/register/"; + private static final String URI_GET_CURRENT_ANNOUNCEMENTS = "/apigw/v1/announcement/get_current_announcements/"; + + private final AppProperties appProperties; + private final BkApiAuthorization authorization; + + public BkNoticeClient(MeterRegistry meterRegistry, + AppProperties appProperties, + BkApiGatewayProperties bkApiGatewayProperties) { + super( + meterRegistry, + CommonMetricNames.BK_NOTICE_API, + getBkNoticeUrlSafely(bkApiGatewayProperties), + HttpHelperFactory.getDefaultHttpHelper() + ); + this.appProperties = appProperties; + authorization = BkApiAuthorization.appAuthorization(appProperties.getCode(), appProperties.getSecret()); + } + + private static String getBkNoticeUrlSafely(BkApiGatewayProperties bkApiGatewayProperties) { + if (bkApiGatewayProperties == null || bkApiGatewayProperties.getBkNotice() == null) { + return null; + } + return bkApiGatewayProperties.getBkNotice().getUrl(); + } + + @Override + public BkNoticeApp registerApplication() { + EsbResp resp = requestBkNoticeApi( + HttpMethodEnum.POST, + URI_REGISTER_APPLICATION, + null, + new TypeReference>() { + }, + true + ); + return resp.getData(); + } + + @Override + public List getCurrentAnnouncements(String bkLanguage, Integer offset, Integer limit) { + EsbResp> resp = requestBkNoticeApi( + HttpMethodEnum.GET, + buildUriWithParams(bkLanguage, offset, limit), + null, + new TypeReference>>() { + }, + true + ); + return resp.getData(); + } + + private String buildUriWithParams(String bkLanguage, Integer offset, Integer limit) { + StringBuilder sb = new StringBuilder(); + sb.append(URI_GET_CURRENT_ANNOUNCEMENTS); + sb.append("?platform="); + sb.append(appProperties.getCode()); + if (StringUtils.isNotBlank(bkLanguage)) { + sb.append("&language="); + sb.append(bkLanguage); + } + if (offset != null) { + sb.append("&offset="); + sb.append(offset); + } + if (limit != null) { + sb.append("&limit="); + sb.append(limit); + } + return sb.toString(); + } + + /** + * 通过ESB请求消息通知中心API的统一入口，监控数据埋点位置 + * + * @param method Http方法 + * @param uri 请求地址 + * @param reqBody 请求体内容 + * @param typeReference 指定了返回值类型的EsbResp TypeReference对象 + * @param 泛型：返回值类型 + * @return 返回值类型实例 + */ + private EsbResp requestBkNoticeApi(HttpMethodEnum method, + String uri, + EsbReq reqBody, + TypeReference> typeReference, + Boolean idempotent) { + try { + HttpMetricUtil.setHttpMetricName(CommonMetricNames.BK_NOTICE_API_HTTP); + HttpMetricUtil.addTagForCurrentMetric(Tag.of("api_name", uri)); + OpenApiRequestInfo requestInfo = OpenApiRequestInfo + .builder() + .method(method) + .uri(uri) + .body(reqBody) + .authorization(authorization) + .setIdempotent(idempotent) + .build(); + return doRequest(requestInfo, typeReference); + } catch (InternalException e) { + // 接口不存在的场景需要使用指定错误码以便前端兼容处理 + if (e.getCause() instanceof HttpStatusException) { + HttpStatusException httpStatusException = (HttpStatusException) e.getCause(); + if (httpStatusException.getHttpStatus() == HttpStatus.SC_NOT_FOUND) { + throw new BkNoticeException(e, ErrorCode.BK_NOTICE_API_NOT_FOUND, new String[]{uri}); + } + } + throw new BkNoticeException(e, ErrorCode.BK_NOTICE_API_DATA_ERROR, null); + } catch (Exception e) { + throw new BkNoticeException(e, ErrorCode.BK_NOTICE_API_DATA_ERROR, null); + } finally { + HttpMetricUtil.clearHttpMetric(); + } + } +} diff --git a/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/model/AnnouncementDTO.java b/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/model/AnnouncementDTO.java new file mode 100644 index 0000000000..fc02fe7e37 --- /dev/null +++ b/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/model/AnnouncementDTO.java @@ -0,0 +1,89 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.notice.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.util.List; + +@Slf4j +@NoArgsConstructor +@Data +public class AnnouncementDTO { + + private Long id; + + /** + * 标题 + */ + private String title; + + /** + * 内容 + */ + private String content; + + /** + * 内容列表（国际化，各种语言的内容） + */ + @JsonProperty("content_list") + private List contentList; + + /** + * 公告类型: event(活动通知)/announce(平台公告) + */ + @JsonProperty("announce_type") + private String announceType; + + /** + * 开始时间 + */ + @JsonProperty("start_time") + private String startTime; + + /** + * 结束时间 + */ + @JsonProperty("end_time") + private String endTime; + + @NoArgsConstructor + @Data + public static class ContentWithLanguage { + /** + * 内容 + */ + private String content; + + /** + * 语言 + */ + private String language; + } + +} diff --git a/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/model/BkNoticeApp.java b/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/model/BkNoticeApp.java new file mode 100644 index 0000000000..6942ad961d --- /dev/null +++ b/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/model/BkNoticeApp.java @@ -0,0 +1,37 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.notice.model; + +import lombok.Data; +import lombok.NoArgsConstructor; + +@NoArgsConstructor +@Data +public class BkNoticeApp { + + private String code; + + private String name; +} diff --git a/src/backend/commons/notice-sdk/src/main/resources/META-INF/spring.factories b/src/backend/commons/notice-sdk/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..dde166dd38 --- /dev/null +++ b/src/backend/commons/notice-sdk/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +com.tencent.bk.job.common.notice.config.NoticeAutoConfiguration diff --git a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/cmsi/CmsiApiClient.java b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/cmsi/CmsiApiClient.java index 8a9755d091..336fe2e07e 100644 --- a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/cmsi/CmsiApiClient.java +++ b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/cmsi/CmsiApiClient.java @@ -30,10 +30,10 @@ import com.tencent.bk.job.common.esb.config.AppProperties; import com.tencent.bk.job.common.esb.config.EsbProperties; import com.tencent.bk.job.common.esb.metrics.EsbMetricTags; -import com.tencent.bk.job.common.esb.model.ApiRequestInfo; import com.tencent.bk.job.common.esb.model.BkApiAuthorization; import com.tencent.bk.job.common.esb.model.EsbReq; import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.common.esb.model.OpenApiRequestInfo; import com.tencent.bk.job.common.esb.sdk.AbstractBkApiClient; import com.tencent.bk.job.common.exception.InternalCmsiException; import com.tencent.bk.job.common.metrics.CommonMetricNames; @@ -41,6 +41,7 @@ import com.tencent.bk.job.common.paas.exception.PaasException; import com.tencent.bk.job.common.paas.model.EsbNotifyChannelDTO; import com.tencent.bk.job.common.paas.model.PostSendMsgReq; +import com.tencent.bk.job.common.util.http.HttpHelperFactory; import com.tencent.bk.job.common.util.http.HttpMetricUtil; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Tag; @@ -66,7 +67,8 @@ public class CmsiApiClient extends AbstractBkApiClient { public CmsiApiClient(EsbProperties esbProperties, AppProperties appProperties, MeterRegistry meterRegistry) { - super(meterRegistry, ESB_CMSI_API, esbProperties.getService().getUrl()); + super(meterRegistry, ESB_CMSI_API, esbProperties.getService().getUrl(), + HttpHelperFactory.getDefaultHttpHelper()); this.authorization = BkApiAuthorization.appAuthorization(appProperties.getCode(), appProperties.getSecret(), "admin"); } @@ -78,7 +80,7 @@ public List getNotifyChannelList() { Tag.of(EsbMetricTags.KEY_API_NAME, API_GET_NOTIFY_CHANNEL_LIST) ); EsbResp> esbResp = doRequest( - ApiRequestInfo.builder() + OpenApiRequestInfo.builder() .method(HttpMethodEnum.GET) .uri(API_GET_NOTIFY_CHANNEL_LIST) .authorization(authorization) @@ -107,7 +109,7 @@ public void sendMsg(String msgType, HttpMetricUtil.setHttpMetricName(CommonMetricNames.ESB_CMSI_API_HTTP); HttpMetricUtil.addTagForCurrentMetric(Tag.of(EsbMetricTags.KEY_API_NAME, uri)); EsbResp esbResp = doRequest( - ApiRequestInfo.builder() + OpenApiRequestInfo.builder() .method(HttpMethodEnum.POST) .uri(uri) .body(req) diff --git a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/login/StandardLoginClient.java b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/login/StandardLoginClient.java index 9337972caa..7445a75687 100644 --- a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/login/StandardLoginClient.java +++ b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/login/StandardLoginClient.java @@ -29,15 +29,16 @@ import com.tencent.bk.job.common.constant.HttpMethodEnum; import com.tencent.bk.job.common.esb.config.AppProperties; import com.tencent.bk.job.common.esb.config.EsbProperties; -import com.tencent.bk.job.common.esb.model.ApiRequestInfo; import com.tencent.bk.job.common.esb.model.BkApiAuthorization; import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.common.esb.model.OpenApiRequestInfo; import com.tencent.bk.job.common.esb.sdk.AbstractBkApiClient; import com.tencent.bk.job.common.exception.InternalUserManageException; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.dto.BkUserDTO; import com.tencent.bk.job.common.paas.exception.AppPermissionDeniedException; import com.tencent.bk.job.common.paas.model.EsbUserDto; +import com.tencent.bk.job.common.util.http.HttpHelperFactory; import com.tencent.bk.job.common.util.http.HttpMetricUtil; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Tag; @@ -60,7 +61,8 @@ public class StandardLoginClient extends AbstractBkApiClient implements ILoginCl private final AppProperties appProperties; public StandardLoginClient(EsbProperties esbProperties, AppProperties appProperties, MeterRegistry meterRegistry) { - super(meterRegistry, ESB_BK_LOGIN_API, esbProperties.getService().getUrl()); + super(meterRegistry, ESB_BK_LOGIN_API, esbProperties.getService().getUrl(), + HttpHelperFactory.getDefaultHttpHelper()); this.appProperties = appProperties; } @@ -78,7 +80,7 @@ public BkUserDTO getUserInfoByToken(String bkToken) { Tag.of("api_name", API_GET_USER_INFO) ); EsbResp esbResp = doRequest( - ApiRequestInfo.builder() + OpenApiRequestInfo.builder() .method(HttpMethodEnum.GET) .uri(API_GET_USER_INFO) .addQueryParam("bk_token", bkToken) diff --git a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/user/UserMgrApiClient.java b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/user/UserMgrApiClient.java index c8ef1165de..a2401eddea 100644 --- a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/user/UserMgrApiClient.java +++ b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/user/UserMgrApiClient.java @@ -31,15 +31,16 @@ import com.tencent.bk.job.common.esb.config.EsbProperties; import com.tencent.bk.job.common.esb.constants.EsbLang; import com.tencent.bk.job.common.esb.metrics.EsbMetricTags; -import com.tencent.bk.job.common.esb.model.ApiRequestInfo; import com.tencent.bk.job.common.esb.model.BkApiAuthorization; import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.common.esb.model.OpenApiRequestInfo; import com.tencent.bk.job.common.esb.sdk.AbstractBkApiClient; import com.tencent.bk.job.common.exception.InternalUserManageException; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.dto.BkUserDTO; import com.tencent.bk.job.common.paas.model.EsbListUsersResult; import com.tencent.bk.job.common.paas.model.GetUserListReq; +import com.tencent.bk.job.common.util.http.HttpHelperFactory; import com.tencent.bk.job.common.util.http.HttpMetricUtil; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Tag; @@ -66,7 +67,12 @@ public class UserMgrApiClient extends AbstractBkApiClient { public UserMgrApiClient(EsbProperties esbProperties, AppProperties appProperties, MeterRegistry meterRegistry) { - super(meterRegistry, ESB_USER_MANAGE_API, esbProperties.getService().getUrl(), EsbLang.EN); + super(meterRegistry, + ESB_USER_MANAGE_API, + esbProperties.getService().getUrl(), + HttpHelperFactory.getRetryableHttpHelper(), + EsbLang.EN + ); this.authorization = BkApiAuthorization.appAuthorization(appProperties.getCode(), appProperties.getSecret(), "admin"); } @@ -82,7 +88,7 @@ public List getAllUserList() { Tag.of(EsbMetricTags.KEY_API_NAME, API_GET_USER_LIST) ); EsbResp> esbResp = doRequest( - ApiRequestInfo.builder() + OpenApiRequestInfo.builder() .method(HttpMethodEnum.GET) .uri(API_GET_USER_LIST) .queryParams(req.toUrlParams()) diff --git a/src/backend/job-assemble/src/test/resources/application-test.yml b/src/backend/job-assemble/src/test/resources/application-test.yml index ab8ea613d1..b818e915d9 100644 --- a/src/backend/job-assemble/src/test/resources/application-test.yml +++ b/src/backend/job-assemble/src/test/resources/application-test.yml @@ -143,6 +143,8 @@ app: bk-api-gateway: gse: url: gse.apigw.com + bkNotice: + url: bk-notice.apigw.com esb: service: url: esb.service diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbFileLogV3DTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbFileLogV3DTO.java index 7f146d81f0..3d4e612fdf 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbFileLogV3DTO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbFileLogV3DTO.java @@ -55,6 +55,15 @@ public class EsbFileLogV3DTO { @JsonProperty("log_content") private String logContent; + @JsonProperty("size") + private String size; + + @JsonProperty("speed") + private String speed; + + @JsonProperty("process") + private String process; + } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbStepInstanceStatusV3DTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbStepInstanceStatusV3DTO.java index 18f0c9e9f0..e31346e5c3 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbStepInstanceStatusV3DTO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbStepInstanceStatusV3DTO.java @@ -83,9 +83,27 @@ public class EsbStepInstanceStatusV3DTO { @JsonProperty("total_time") private Long totalTime; - @JsonProperty("step_host_result_list") + @JsonProperty("step_result_group_list") @JsonInclude(JsonInclude.Include.NON_EMPTY) - private List stepHostResultList; + private List stepResultGroupList; + + @Setter + @Getter + public static class StepResultGroup { + @JsonProperty("result_type") + private Integer resultType; + + @JsonProperty("result_type_desc") + private String resultTypeDesc; + + private String tag; + + @JsonProperty("host_size") + private Integer hostSize; + + @JsonProperty("host_result_list") + private List hostResultList; + } @Setter @Getter @@ -97,9 +115,15 @@ public static class HostResult { private String ipv6; + @JsonProperty("bk_agent_id") + private String agentId; + @JsonProperty("bk_cloud_id") private Long cloudAreaId; + @JsonProperty("bk_cloud_name") + private String cloudAreaName; + private Integer status; @JsonProperty("status_desc") @@ -107,9 +131,6 @@ public static class HostResult { private String tag; - @JsonProperty("group_key") - private String groupKey; - @JsonProperty("exit_code") private Integer exitCode; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBatchGetJobInstanceIpLogV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBatchGetJobInstanceIpLogV3ResourceImpl.java index fb5f81e14d..bee44adb6e 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBatchGetJobInstanceIpLogV3ResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBatchGetJobInstanceIpLogV3ResourceImpl.java @@ -220,6 +220,9 @@ private EsbFileLogV3DTO toEsbFileLogV3DTO(ServiceFileTaskLogDTO fileTaskLog) { } fileLog.setLogContent(fileTaskLog.getContent()); + fileLog.setSize(fileTaskLog.getSize()); + fileLog.setSpeed(fileTaskLog.getSpeed()); + fileLog.setProcess(fileTaskLog.getProcess()); fileLog.setStatus(fileTaskLog.getStatus()); return fileLog; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceStatusV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceStatusV3ResourceImpl.java index 3a6f723bd1..e9f5b4ce69 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceStatusV3ResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceStatusV3ResourceImpl.java @@ -34,8 +34,9 @@ import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.ValidateResult; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.service.AppScopeMappingService; -import com.tencent.bk.job.execute.engine.model.ExecuteObject; +import com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum; import com.tencent.bk.job.execute.model.ExecuteObjectTask; import com.tencent.bk.job.execute.model.ResultGroupDTO; import com.tencent.bk.job.execute.model.StepExecutionDetailDTO; @@ -83,32 +84,45 @@ private EsbStepInstanceStatusV3DTO buildEsbStepInstanceStatusV3DTO(StepExecution stepInst.setEndTime(stepInstance.getEndTime()); stepInst.setTotalTime(stepInstance.getTotalTime()); - List stepHostResults = new ArrayList<>(); + List stepResultGroupList = new ArrayList<>(); List resultGroups = executionResult.getResultGroups(); for (ResultGroupDTO resultGroup : resultGroups) { - List agentTaskList = resultGroup.getExecuteObjectTasks(); - if (CollectionUtils.isEmpty(agentTaskList)) { + List executeObjectTasks = resultGroup.getExecuteObjectTasks(); + if (CollectionUtils.isEmpty(executeObjectTasks)) { continue; } - for (ExecuteObjectTask agentTask : agentTaskList) { + EsbStepInstanceStatusV3DTO.StepResultGroup stepResultGroup = + new EsbStepInstanceStatusV3DTO.StepResultGroup(); + stepResultGroup.setResultType(resultGroup.getStatus()); + ExecuteObjectTaskStatusEnum taskStatusEnum = ExecuteObjectTaskStatusEnum.valueOf(resultGroup.getStatus()); + if (taskStatusEnum != null) { + stepResultGroup.setResultTypeDesc(messageI18nService.getI18n(taskStatusEnum.getI18nKey())); + } + stepResultGroup.setTag(resultGroup.getTag()); + stepResultGroup.setHostSize(resultGroup.getTotal()); + List hostResults = new ArrayList<>(); + for (ExecuteObjectTask executeObjectTask : executeObjectTasks) { EsbStepInstanceStatusV3DTO.HostResult stepHostResult = new EsbStepInstanceStatusV3DTO.HostResult(); - ExecuteObject executeObject = agentTask.getExecuteObject(); - stepHostResult.setHostId(agentTask.getHostId()); - stepHostResult.setIp(executeObject.getHost().getIp()); - stepHostResult.setIpv6(executeObject.getHost().getIpv6()); - stepHostResult.setCloudAreaId(executeObject.getHost().getBkCloudId()); - stepHostResult.setStatus(agentTask.getStatus().getValue()); - stepHostResult.setStatusDesc(messageI18nService.getI18n(agentTask.getStatus().getI18nKey())); - stepHostResult.setTag(agentTask.getTag()); - stepHostResult.setGroupKey(resultGroup.getGroupKey()); - stepHostResult.setExitCode(agentTask.getExitCode()); - stepHostResult.setStartTime(agentTask.getStartTime()); - stepHostResult.setEndTime(agentTask.getEndTime()); - stepHostResult.setTotalTime(agentTask.getTotalTime()); - stepHostResults.add(stepHostResult); + HostDTO host = executeObjectTask.getExecuteObject().getHost(); + stepHostResult.setHostId(host.getHostId()); + stepHostResult.setIp(host.getIp()); + stepHostResult.setIpv6(host.getIpv6()); + stepHostResult.setCloudAreaId(host.getBkCloudId()); + stepHostResult.setAgentId(host.getAgentId()); + stepHostResult.setCloudAreaName(host.getBkCloudName()); + stepHostResult.setStatus(executeObjectTask.getStatus().getValue()); + stepHostResult.setStatusDesc(messageI18nService.getI18n(executeObjectTask.getStatus().getI18nKey())); + stepHostResult.setTag(executeObjectTask.getTag()); + stepHostResult.setExitCode(executeObjectTask.getExitCode()); + stepHostResult.setStartTime(executeObjectTask.getStartTime()); + stepHostResult.setEndTime(executeObjectTask.getEndTime()); + stepHostResult.setTotalTime(executeObjectTask.getTotalTime()); + hostResults.add(stepHostResult); } + stepResultGroup.setHostResultList(hostResults); + stepResultGroupList.add(stepResultGroup); } - stepInst.setStepHostResultList(stepHostResults); + stepInst.setStepResultGroupList(stepResultGroupList); return stepInst; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/GseConfig.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/GseConfig.java index 08f0c4e07a..7a719dc371 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/GseConfig.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/GseConfig.java @@ -1,12 +1,40 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + package com.tencent.bk.job.execute.config; -import com.tencent.bk.job.common.gse.GseClient; import com.tencent.bk.job.common.gse.config.AgentStateQueryConfig; import com.tencent.bk.job.common.gse.constants.DefaultBeanNames; import com.tencent.bk.job.common.gse.service.AgentStateClient; import com.tencent.bk.job.common.gse.service.AutoChoosingAgentStateClientImpl; import com.tencent.bk.job.common.gse.service.BizHostInfoQueryService; +import com.tencent.bk.job.common.gse.service.GseV1AgentStateClientImpl; +import com.tencent.bk.job.common.gse.service.GseV2AgentStateClientImpl; import com.tencent.bk.job.common.gse.service.UseV2ByFeatureAgentStateClientImpl; +import com.tencent.bk.job.common.gse.v1.GseV1ApiClient; +import com.tencent.bk.job.common.gse.v2.GseV2ApiClient; +import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -18,25 +46,50 @@ public class GseConfig { public static final String EXECUTE_BEAN_PREFIX = "jobExecute"; + public static final String EXECUTE_BEAN_GSE_V1_AGENT_STATE_CLIENT = EXECUTE_BEAN_PREFIX + "GseV1AgentStateClient"; + public static final String EXECUTE_BEAN_GSE_V2_AGENT_STATE_CLIENT = EXECUTE_BEAN_PREFIX + "GseV2AgentStateClient"; public static final String EXECUTE_BEAN_AGENT_STATE_CLIENT = EXECUTE_BEAN_PREFIX + "AgentStateClient"; public static final String EXECUTE_BEAN_USE_V2_BY_FEATURE_AGENT_STATE_CLIENT = EXECUTE_BEAN_PREFIX + DefaultBeanNames.USE_V2_BY_FEATURE_AGENT_STATE_CLIENT; - @Bean(EXECUTE_BEAN_USE_V2_BY_FEATURE_AGENT_STATE_CLIENT) - public AgentStateClient useV2ByFeatureAgentStateClient(AgentStateQueryConfig agentStateQueryConfig, - GseClient gseClient, - @Qualifier("jobExecuteBizHostInfoQueryService") - BizHostInfoQueryService bizHostInfoQueryService, + @Bean(EXECUTE_BEAN_GSE_V1_AGENT_STATE_CLIENT) + public GseV1AgentStateClientImpl gseV1AgentStateClient(AgentStateQueryConfig agentStateQueryConfig, + ObjectProvider gseV1ApiClient, @Qualifier(DefaultBeanNames.AGENT_STATUS_QUERY_THREAD_POOL_EXECUTOR) ThreadPoolExecutor threadPoolExecutor) { - return new UseV2ByFeatureAgentStateClientImpl( + return new GseV1AgentStateClientImpl( agentStateQueryConfig, - gseClient, - bizHostInfoQueryService, + gseV1ApiClient.getIfAvailable(), threadPoolExecutor ); } + @Bean(EXECUTE_BEAN_GSE_V2_AGENT_STATE_CLIENT) + public GseV2AgentStateClientImpl gseV2AgentStateClient(AgentStateQueryConfig agentStateQueryConfig, + ObjectProvider gseV2ApiClient, + @Qualifier(DefaultBeanNames.AGENT_STATUS_QUERY_THREAD_POOL_EXECUTOR) + ThreadPoolExecutor threadPoolExecutor) { + return new GseV2AgentStateClientImpl( + agentStateQueryConfig, + gseV2ApiClient.getIfAvailable(), + threadPoolExecutor + ); + } + + @Bean(EXECUTE_BEAN_USE_V2_BY_FEATURE_AGENT_STATE_CLIENT) + public AgentStateClient useV2ByFeatureAgentStateClient(@Qualifier(EXECUTE_BEAN_GSE_V1_AGENT_STATE_CLIENT) + GseV1AgentStateClientImpl gseV1AgentStateClient, + @Qualifier(EXECUTE_BEAN_GSE_V2_AGENT_STATE_CLIENT) + GseV2AgentStateClientImpl gseV2AgentStateClient, + @Qualifier("jobExecuteBizHostInfoQueryService") + BizHostInfoQueryService bizHostInfoQueryService) { + return new UseV2ByFeatureAgentStateClientImpl( + gseV1AgentStateClient, + gseV2AgentStateClient, + bizHostInfoQueryService + ); + } + @Primary @Bean(EXECUTE_BEAN_AGENT_STATE_CLIENT) public AgentStateClient AutoChoosingAgentStateClientImpl( diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/esb/EsbFileSourceV3ResourceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/esb/EsbFileSourceV3ResourceImpl.java index 593c93aa00..f7e34dd4d8 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/esb/EsbFileSourceV3ResourceImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/esb/EsbFileSourceV3ResourceImpl.java @@ -6,6 +6,7 @@ import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.common.exception.FailedPreconditionException; import com.tencent.bk.job.common.exception.InvalidParamException; +import com.tencent.bk.job.common.exception.MissingParameterException; import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.file_gateway.consts.WorkerSelectModeEnum; import com.tencent.bk.job.file_gateway.consts.WorkerSelectScopeEnum; @@ -93,6 +94,9 @@ private void checkCreateParam(EsbCreateOrUpdateFileSourceV3Req req) { private Integer checkUpdateParamAndGetId(EsbCreateOrUpdateFileSourceV3Req req) { Long appId = req.getAppId(); String code = req.getCode(); + if (StringUtils.isBlank(code)) { + throw new MissingParameterException(ErrorCode.FILE_SOURCE_CODE_CAN_NOT_BE_EMPTY); + } Integer id = fileSourceService.getFileSourceIdByCode(appId, code); if (id == null) { throw new FailedPreconditionException(ErrorCode.FAIL_TO_FIND_FILE_SOURCE_BY_CODE, new String[]{code}); diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileSourceDAOImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileSourceDAOImpl.java index d660a2a18d..d740366be6 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileSourceDAOImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileSourceDAOImpl.java @@ -198,7 +198,9 @@ private void saveFileSourceShareInfo(Integer fileSourceId, FileSourceDTO fileSou public int updateFileSource(FileSourceDTO fileSourceDTO) { val query = dslContext.update(defaultTable); var updateSetStep = query.set(defaultTable.APP_ID, fileSourceDTO.getAppId()); - updateSetStep = updateSetStep.set(defaultTable.CODE, fileSourceDTO.getCode()); + if (StringUtils.isNotBlank(fileSourceDTO.getCode())) { + updateSetStep = updateSetStep.set(defaultTable.CODE, fileSourceDTO.getCode()); + } if (StringUtils.isNotBlank(fileSourceDTO.getAlias())) { updateSetStep = updateSetStep.set(defaultTable.ALIAS, fileSourceDTO.getAlias()); } @@ -498,22 +500,15 @@ public boolean existsFileSourceUsingCredential(Long appId, String credentialId) @Override public Integer getFileSourceIdByCode(Long appId, String code) { List conditions = new ArrayList<>(); - if (appId != null) { - conditions.add(defaultTable.APP_ID.eq(appId)); - } - if (code != null) { - conditions.add(defaultTable.CODE.eq(code)); - } + conditions.add(defaultTable.APP_ID.eq(appId)); + conditions.add(defaultTable.CODE.eq(code)); val query = dslContext.select( defaultTable.ID ).from(defaultTable) .where(conditions); - val result = query.fetch(); - if (result.size() > 0) { - if (result.size() > 1) { - log.warn("{} records found when get id by code, use first one", result.size()); - } - return result.get(0).get(defaultTable.ID); + val result = query.fetchOne(); + if (result != null) { + return result.get(defaultTable.ID); } return null; } diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileSourceServiceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileSourceServiceImpl.java index 2070652a62..6d453f91a2 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileSourceServiceImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileSourceServiceImpl.java @@ -185,7 +185,7 @@ private void authManage(String username, long appId, int fileSourceId) { instance = @AuditInstanceRecord( resourceType = ResourceTypeId.FILE_SOURCE, instanceIds = "#fileSource?.id", - instanceNames = "$fileSource?.alias" + instanceNames = "$?.alias" ), content = EventContentConstants.EDIT_FILE_SOURCE ) diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/OpService.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/OpService.java index 70101885e4..daffe4b53f 100644 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/OpService.java +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/OpService.java @@ -24,10 +24,12 @@ package com.tencent.bk.job.file.worker.service; +import com.tencent.bk.job.common.constant.HttpMethodEnum; import com.tencent.bk.job.common.model.http.HttpReq; -import com.tencent.bk.job.common.util.http.ExtHttpHelper; +import com.tencent.bk.job.common.util.http.HttpHelper; import com.tencent.bk.job.common.util.http.HttpHelperFactory; import com.tencent.bk.job.common.util.http.HttpReqGenUtil; +import com.tencent.bk.job.common.util.http.HttpRequest; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.file.worker.config.WorkerConfig; import com.tencent.bk.job.file.worker.task.heartbeat.HeartBeatTask; @@ -45,7 +47,7 @@ @Service public class OpService { - private final ExtHttpHelper httpHelper = HttpHelperFactory.getDefaultHttpHelper(); + private final HttpHelper httpHelper = HttpHelperFactory.getDefaultHttpHelper(); private final WorkerConfig workerConfig; private final FileTaskService fileTaskService; private final GatewayInfoService gatewayInfoService; @@ -81,7 +83,12 @@ public List offLine() { HttpReq req = HttpReqGenUtil.genSimpleJsonReq(url, offLineReq); String respStr; try { - respStr = httpHelper.post(url, req.getBody(), req.getHeaders()); + respStr = httpHelper.request( + HttpRequest.builder(HttpMethodEnum.POST, url) + .setStringEntity(req.getBody()) + .setHeaders(req.getHeaders()) + .build()) + .getEntity(); log.info(String.format("respStr=%s", respStr)); // 停止任务 Integer allStoppedFileCount = fileTaskService.stopTasksAtOnce(runningTaskIdList, diff --git a/src/backend/job-manage/api-job-manage/build.gradle b/src/backend/job-manage/api-job-manage/build.gradle index 191505f4fd..6f876179b8 100644 --- a/src/backend/job-manage/api-job-manage/build.gradle +++ b/src/backend/job-manage/api-job-manage/build.gradle @@ -26,6 +26,8 @@ dependencies { api project(':commons:common') api project(':commons:common-i18n') api project(':commons:common-iam') + api project(':commons:cmdb-sdk') + api project(':commons:notice-sdk') api project(':job-execute:api-job-execute') api(project(":commons:common-api")) implementation "org.springframework:spring-web" diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbAgentInfoV3Resource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbAgentInfoV3Resource.java new file mode 100644 index 0000000000..f8151a2eb7 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbAgentInfoV3Resource.java @@ -0,0 +1,57 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.esb.v3; + +import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; +import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbQueryAgentInfoV3Req; +import com.tencent.bk.job.manage.model.esb.v3.response.EsbQueryAgentInfoV3Resp; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +/** + * 查询Agent信息API-V3 + */ +@RequestMapping("/esb/api/v3") +@RestController +@Validated +@EsbAPI +public interface EsbAgentInfoV3Resource { + + @PostMapping("/query_agent_info") + EsbResp queryAgentInfo( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestBody + @Validated + EsbQueryAgentInfoV3Req req + ); + +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/op/EventReplayOpResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/op/EventReplayOpResource.java new file mode 100644 index 0000000000..26cc271b3e --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/op/EventReplayOpResource.java @@ -0,0 +1,53 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.op; + +import com.tencent.bk.job.common.cc.model.result.HostEventDetail; +import com.tencent.bk.job.common.cc.model.result.ResourceEvent; +import com.tencent.bk.job.common.model.Response; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@Api(tags = {"job-manage:api:EventReplay-OP"}) +@RequestMapping("/op/eventReplay") +@RestController +public interface EventReplayOpResource { + + @ApiOperation(value = "重放主机事件", produces = "application/json") + @PostMapping("/host") + Response replayHostEvent( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") String username, + @ApiParam(value = "主机事件", required = true) + @RequestBody ResourceEvent event + ); + +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebNoticeResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebNoticeResource.java new file mode 100644 index 0000000000..229e428fd4 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebNoticeResource.java @@ -0,0 +1,67 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.web; + +import com.tencent.bk.job.common.annotation.WebAPI; +import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.manage.model.web.vo.notice.AnnouncementVO; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import javax.validation.constraints.Min; +import java.util.List; + +/** + * 对接蓝鲸消息中心相关的 WEB API + */ +@Validated +@Api(tags = {"job-manage:web:Notice"}) +@RequestMapping("/web/notice") +@RestController +@WebAPI +public interface WebNoticeResource { + + @ApiOperation(value = "获获取公告列表（返回码1217001表示消息通知中心API不存在，该环境未对接消息通知中心，需要兼容处理）", produces = "application/json") + @GetMapping(value = {"/announcement/currentAnnouncements"}) + Response> getCurrentAnnouncements( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @Min(message = "{validation.constraints.InvalidAnnouncementsOffset.message}", value = 0L) + @RequestParam(value = "offset", required = false) + Integer offset, + @Min(message = "{validation.constraints.InvalidAnnouncementsLimit.message}", value = 1L) + @RequestParam(value = "limit", required = false) + Integer limit + ); + +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/globalsetting/GlobalSettingKeys.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/globalsetting/GlobalSettingKeys.java index ae6a211a3b..19e90d6638 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/globalsetting/GlobalSettingKeys.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/globalsetting/GlobalSettingKeys.java @@ -54,4 +54,8 @@ public class GlobalSettingKeys { public static final String KEY_IS_BIZSET_MIGRATED_TO_CMDB = "IS_BIZSET_MIGRATED_TO_CMDB"; // 助手信息 public static final String KEY_BK_HELPER = "BK_HELPER"; + // 是否对接消息通知中心 + public static final String KEY_ENABLE_BK_NOTICE = "ENABLE_BK_NOTICE"; + // 消息通知中心是否注册成功 + public static final String KEY_BK_NOTICE_REGISTERED_SUCCESS = "BK_NOTICE_REGISTERED_SUCCESS"; } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbQueryAgentInfoV3Req.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbQueryAgentInfoV3Req.java new file mode 100644 index 0000000000..8f6cfc0cc4 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbQueryAgentInfoV3Req.java @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.esb.v3.request; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.esb.model.EsbAppScopeReq; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import javax.validation.constraints.Size; +import java.util.List; + +/** + * 查询Agent信息请求 + */ +@EqualsAndHashCode(callSuper = true) +@Data +public class EsbQueryAgentInfoV3Req extends EsbAppScopeReq { + + /** + * 主机ID列表 + */ + @Size(max = 5000, message = "{validation.constraints.queryAgentInfoHostIds_tooMany.message}") + @JsonProperty(value = "host_id_list") + private List hostIdList; + +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbAgentInfoV3DTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbAgentInfoV3DTO.java new file mode 100644 index 0000000000..66245876eb --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbAgentInfoV3DTO.java @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.esb.v3.response; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode +public class EsbAgentInfoV3DTO { + + @JsonPropertyDescription("Host ID") + @JsonProperty(value = "bk_host_id") + private Long hostId; + + @JsonPropertyDescription("Agent status: 0 - abnormal, 1 - normal") + private int status; + + /** + * Agent版本 + */ + @JsonPropertyDescription("Agent Version") + private String version; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbQueryAgentInfoV3Resp.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbQueryAgentInfoV3Resp.java new file mode 100644 index 0000000000..e9d37a21cd --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbQueryAgentInfoV3Resp.java @@ -0,0 +1,41 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.esb.v3.response; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import java.util.List; + +@Data +@EqualsAndHashCode +public class EsbQueryAgentInfoV3Resp { + + @JsonPropertyDescription("Agent info list") + @JsonProperty(value = "agent_info_list") + private List agentInfoList; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notice/AnnouncementVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notice/AnnouncementVO.java new file mode 100644 index 0000000000..4a71aed891 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notice/AnnouncementVO.java @@ -0,0 +1,74 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.vo.notice; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.notice.model.AnnouncementDTO; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@NoArgsConstructor +@AllArgsConstructor +@ApiModel("公告内容") +@Data +public class AnnouncementVO { + + @ApiModelProperty("ID") + private Long id; + + @ApiModelProperty("标题") + private String title; + + @ApiModelProperty("内容") + private String content; + + @ApiModelProperty("公告类型: event(活动通知)/announce(平台公告)") + @JsonProperty("announce_type") + private String announceType; + + @ApiModelProperty("开始时间") + @JsonProperty("start_time") + private String startTime; + + @ApiModelProperty("结束时间") + @JsonProperty("end_time") + private String endTime; + + public static AnnouncementVO fromDTO(AnnouncementDTO announcementDTO) { + AnnouncementVO announcementVO = new AnnouncementVO(); + announcementVO.setId(announcementDTO.getId()); + announcementVO.setTitle(announcementDTO.getTitle()); + announcementVO.setContent(announcementDTO.getContent()); + announcementVO.setAnnounceType(announcementDTO.getAnnounceType()); + announcementVO.setStartTime(announcementDTO.getStartTime()); + announcementVO.setEndTime(announcementDTO.getEndTime()); + return announcementVO; + } +} diff --git a/src/backend/job-manage/boot-job-manage/src/main/resources/logback-spring.xml b/src/backend/job-manage/boot-job-manage/src/main/resources/logback-spring.xml index 2ac5449b8f..3f0df06198 100644 --- a/src/backend/job-manage/boot-job-manage/src/main/resources/logback-spring.xml +++ b/src/backend/job-manage/boot-job-manage/src/main/resources/logback-spring.xml @@ -8,6 +8,7 @@ + @@ -25,26 +26,60 @@ + + + ${NOTICE_LOG_FILE} + + ${NOTICE_LOG_FILE}-%d{yyyyMMdd}.log.%i + 1GB + 7 + 2GB + + + ${FILE_LOG_PATTERN} + UTF-8 + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/backend/job-manage/boot-job-manage/src/test/resources/application-test.yml b/src/backend/job-manage/boot-job-manage/src/test/resources/application-test.yml index c3815727d0..324219d057 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/resources/application-test.yml +++ b/src/backend/job-manage/boot-job-manage/src/test/resources/application-test.yml @@ -29,6 +29,8 @@ job: bk-api-gateway: gse: url: gse.apigw.com + bkNotice: + url: bk-notice.apigw.com esb: service: url: esb.service diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbAgentInfoResourceV3Impl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbAgentInfoResourceV3Impl.java new file mode 100644 index 0000000000..8fcc48e39f --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbAgentInfoResourceV3Impl.java @@ -0,0 +1,177 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.esb.impl.v3; + +import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.common.gse.constants.AgentStateStatusEnum; +import com.tencent.bk.job.common.gse.v2.model.req.ListAgentStateReq; +import com.tencent.bk.job.common.gse.v2.model.resp.AgentState; +import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.tencent.bk.job.common.model.dto.ResourceScope; +import com.tencent.bk.job.manage.api.esb.v3.EsbAgentInfoV3Resource; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbQueryAgentInfoV3Req; +import com.tencent.bk.job.manage.model.esb.v3.response.EsbAgentInfoV3DTO; +import com.tencent.bk.job.manage.model.esb.v3.response.EsbQueryAgentInfoV3Resp; +import com.tencent.bk.job.manage.service.agent.status.ScopeAgentStatusService; +import com.tencent.bk.job.manage.service.host.HostService; +import com.tencent.bk.job.manage.service.host.ScopeHostService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; +import org.jooq.tools.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RestController; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +@RestController +@Slf4j +public class EsbAgentInfoResourceV3Impl implements EsbAgentInfoV3Resource { + + private final HostService hostService; + private final ScopeAgentStatusService scopeAgentStatusService; + private final ScopeHostService scopeHostService; + + @Autowired + public EsbAgentInfoResourceV3Impl(HostService hostService, + ScopeAgentStatusService scopeAgentStatusService, + ScopeHostService scopeHostService) { + this.hostService = hostService; + this.scopeAgentStatusService = scopeAgentStatusService; + this.scopeHostService = scopeHostService; + } + + @Override + public EsbResp queryAgentInfo(String username, + String appCode, + EsbQueryAgentInfoV3Req req) { + ResourceScope resourceScope = new ResourceScope(req.getScopeType(), req.getScopeId()); + boolean needToUseGseV2 = scopeAgentStatusService.needToUseGseV2(resourceScope); + + List hostIdList = req.getHostIdList(); + List validHostIdList = scopeHostService.filterScopeHostIds( + req.getAppResourceScope(), + new HashSet<>(hostIdList) + ); + hostIdList.removeAll(validHostIdList); + if (CollectionUtils.isNotEmpty(hostIdList)) { + log.warn("Ignore hostIds not in {}:{}", resourceScope, hostIdList); + } + Map hostIdAgentIdMap = generateHostIdAgentIdMap(needToUseGseV2, validHostIdList); + + ListAgentStateReq listAgentStateReq = new ListAgentStateReq(); + listAgentStateReq.setAgentIdList(new ArrayList<>(new HashSet<>(hostIdAgentIdMap.values()))); + List agentStateList = scopeAgentStatusService.listAgentState(resourceScope, listAgentStateReq); + + List agentInfoList = buildEsbAgentInfoList( + agentStateList, + validHostIdList, + hostIdAgentIdMap, + needToUseGseV2 + ); + + EsbQueryAgentInfoV3Resp resp = new EsbQueryAgentInfoV3Resp(); + resp.setAgentInfoList(agentInfoList); + return EsbResp.buildSuccessResp(resp); + } + + private Map generateHostIdAgentIdMap(boolean needToUseGseV2, List hostIdList) { + Map hostInfoMap = hostService.listHostsByHostIds(hostIdList); + Map hostIdAgentIdMap = new HashMap<>(); + if (needToUseGseV2) { + // 使用AgentId + for (Long hostId : hostIdList) { + ApplicationHostDTO hostDTO = hostInfoMap.get(hostId); + if (hostDTO != null) { + String agentId = hostDTO.getAgentId(); + if (!StringUtils.isBlank(agentId)) { + hostIdAgentIdMap.put(hostId, agentId); + } else { + log.info("AgentId of host({}) is blank, ignore", hostDTO); + } + } else { + log.info("Cannot find host by hostId:{}", hostId); + } + } + return hostIdAgentIdMap; + } + // 使用CloudIp + for (Long hostId : hostIdList) { + ApplicationHostDTO hostDTO = hostInfoMap.get(hostId); + if (hostDTO != null) { + String cloudIp = hostDTO.getCloudIp(); + if (!StringUtils.isBlank(cloudIp)) { + hostIdAgentIdMap.put(hostId, cloudIp); + } else { + log.info("cloudIp of host({}) is blank, ignore", hostDTO); + } + } else { + log.info("Cannot find host by hostId:{}", hostId); + } + } + return hostIdAgentIdMap; + } + + private List buildEsbAgentInfoList(List agentStateList, + List hostIdList, + Map hostIdAgentIdMap, + boolean needToUseGseV2) { + Map agentStateMap = new HashMap<>(); + for (AgentState agentState : agentStateList) { + agentStateMap.put(agentState.getAgentId(), agentState); + } + List agentInfoList = new ArrayList<>(); + for (Long hostId : hostIdList) { + String agentId = hostIdAgentIdMap.get(hostId); + if (StringUtils.isBlank(agentId)) { + continue; + } + AgentState agentState = agentStateMap.get(agentId); + EsbAgentInfoV3DTO agentInfoV3DTO = new EsbAgentInfoV3DTO(); + agentInfoV3DTO.setHostId(hostId); + agentInfoV3DTO.setStatus(getEsbAgentStatus(agentState)); + String agentVersion = agentState.getVersion(); + if (StringUtils.isBlank(agentVersion)) { + if (!needToUseGseV2) { + agentInfoV3DTO.setVersion("V1"); + } + } else { + agentInfoV3DTO.setVersion(agentVersion); + } + agentInfoList.add(agentInfoV3DTO); + } + return agentInfoList; + } + + private Integer getEsbAgentStatus(AgentState agentState) { + if (agentState.getStatusCode() == AgentStateStatusEnum.RUNNING.getValue()) { + return 1; + } + return 0; + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/op/impl/EventReplayOpResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/op/impl/EventReplayOpResourceImpl.java new file mode 100644 index 0000000000..66c85b0008 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/op/impl/EventReplayOpResourceImpl.java @@ -0,0 +1,53 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.op.impl; + +import com.tencent.bk.job.common.cc.model.result.HostEventDetail; +import com.tencent.bk.job.common.cc.model.result.ResourceEvent; +import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.manage.api.op.EventReplayOpResource; +import com.tencent.bk.job.manage.service.impl.sync.HostEventWatcher; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RestController; + +@Slf4j +@RestController +public class EventReplayOpResourceImpl implements EventReplayOpResource { + + private final HostEventWatcher hostEventWatcher; + + @Autowired + public EventReplayOpResourceImpl(HostEventWatcher hostEventWatcher) { + this.hostEventWatcher = hostEventWatcher; + } + + @Override + public Response replayHostEvent(String username, ResourceEvent event) { + hostEventWatcher.handleEvent(event); + return Response.buildSuccessResp(null); + } + +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebNoticeResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebNoticeResourceImpl.java new file mode 100644 index 0000000000..3c77f1097d --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebNoticeResourceImpl.java @@ -0,0 +1,73 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.web.impl; + +import com.tencent.bk.job.common.i18n.locale.BkConsts; +import com.tencent.bk.job.common.i18n.locale.LocaleUtils; +import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.common.notice.IBkNoticeClient; +import com.tencent.bk.job.common.notice.config.BkNoticeProperties; +import com.tencent.bk.job.common.util.JobContextUtil; +import com.tencent.bk.job.manage.api.web.WebNoticeResource; +import com.tencent.bk.job.manage.model.web.vo.notice.AnnouncementVO; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RestController; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +@RestController +@Slf4j +public class WebNoticeResourceImpl implements WebNoticeResource { + + private final IBkNoticeClient bkNoticeClient; + private final BkNoticeProperties bkNoticeProperties; + + @Autowired + public WebNoticeResourceImpl(@Autowired(required = false) IBkNoticeClient bkNoticeClient, + BkNoticeProperties bkNoticeProperties) { + this.bkNoticeClient = bkNoticeClient; + this.bkNoticeProperties = bkNoticeProperties; + } + + @Override + public Response> getCurrentAnnouncements(String username, Integer offset, Integer limit) { + if (!bkNoticeProperties.isEnabled()) { + log.info("bkNotice not enabled, please check config value: bkNotice.enabled"); + return Response.buildSuccessResp(Collections.emptyList()); + } + String userLang = JobContextUtil.getUserLang(); + String bkLang = LocaleUtils.getBkLang(userLang); + if (bkLang == null) { + bkLang = BkConsts.HEADER_VALUE_LANG_EN; + } + List resultList = bkNoticeClient.getCurrentAnnouncements(bkLang, offset, limit).stream() + .map(AnnouncementVO::fromDTO) + .collect(Collectors.toList()); + return Response.buildSuccessResp(resultList); + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/ExecutorConfiguration.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/ExecutorConfiguration.java index c23617d22c..c2a21982f3 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/ExecutorConfiguration.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/ExecutorConfiguration.java @@ -129,6 +129,19 @@ public ThreadPoolExecutor adminAuthExecutor(MeterRegistry meterRegistry) { ); } + @Bean("initRunnerExecutor") + public ThreadPoolExecutor initRunnerExecutor(MeterRegistry meterRegistry) { + return new WatchableThreadPoolExecutor( + meterRegistry, + "initRunnerExecutor", + 0, + 5, + 1, + TimeUnit.SECONDS, + new LinkedBlockingQueue<>() + ); + } + private ThreadFactory getThreadFactoryByNameAndSeq(String namePrefix, AtomicInteger seq) { return r -> { Thread t = new Thread(Thread.currentThread().getThreadGroup(), r, diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/GseConfig.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/GseConfig.java index 198b810b3e..93b0117b30 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/GseConfig.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/GseConfig.java @@ -24,13 +24,17 @@ package com.tencent.bk.job.manage.config; -import com.tencent.bk.job.common.gse.GseClient; import com.tencent.bk.job.common.gse.config.AgentStateQueryConfig; import com.tencent.bk.job.common.gse.constants.DefaultBeanNames; import com.tencent.bk.job.common.gse.service.AgentStateClient; import com.tencent.bk.job.common.gse.service.AutoChoosingAgentStateClientImpl; import com.tencent.bk.job.common.gse.service.BizHostInfoQueryService; +import com.tencent.bk.job.common.gse.service.GseV1AgentStateClientImpl; +import com.tencent.bk.job.common.gse.service.GseV2AgentStateClientImpl; import com.tencent.bk.job.common.gse.service.UseV2ByFeatureAgentStateClientImpl; +import com.tencent.bk.job.common.gse.v1.GseV1ApiClient; +import com.tencent.bk.job.common.gse.v2.GseV2ApiClient; +import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -43,24 +47,49 @@ public class GseConfig { public static final String MANAGE_BEAN_PREFIX = "jobManage"; public static final String MANAGE_BEAN_AGENT_STATE_CLIENT = MANAGE_BEAN_PREFIX + "AgentStateClient"; + public static final String MANAGE_BEAN_GSE_V1_AGENT_STATE_CLIENT = MANAGE_BEAN_PREFIX + "GseV1AgentStateClient"; + public static final String MANAGE_BEAN_GSE_V2_AGENT_STATE_CLIENT = MANAGE_BEAN_PREFIX + "GseV2AgentStateClient"; public static final String MANAGE_BEAN_USE_V2_BY_FEATURE_AGENT_STATE_CLIENT = MANAGE_BEAN_PREFIX + DefaultBeanNames.USE_V2_BY_FEATURE_AGENT_STATE_CLIENT; - @Bean(MANAGE_BEAN_USE_V2_BY_FEATURE_AGENT_STATE_CLIENT) - public AgentStateClient useV2ByFeatureAgentStateClient(AgentStateQueryConfig agentStateQueryConfig, - GseClient gseClient, - @Qualifier("jobManageBizHostInfoQueryService") - BizHostInfoQueryService bizHostInfoQueryService, + @Bean(MANAGE_BEAN_GSE_V1_AGENT_STATE_CLIENT) + public GseV1AgentStateClientImpl gseV1AgentStateClient(AgentStateQueryConfig agentStateQueryConfig, + ObjectProvider gseV1ApiClient, @Qualifier(DefaultBeanNames.AGENT_STATUS_QUERY_THREAD_POOL_EXECUTOR) ThreadPoolExecutor threadPoolExecutor) { - return new UseV2ByFeatureAgentStateClientImpl( + return new GseV1AgentStateClientImpl( + agentStateQueryConfig, + gseV1ApiClient.getIfAvailable(), + threadPoolExecutor + ); + } + + @Bean(MANAGE_BEAN_GSE_V2_AGENT_STATE_CLIENT) + public GseV2AgentStateClientImpl gseV2AgentStateClient(AgentStateQueryConfig agentStateQueryConfig, + ObjectProvider gseV2ApiClient, + @Qualifier(DefaultBeanNames.AGENT_STATUS_QUERY_THREAD_POOL_EXECUTOR) + ThreadPoolExecutor threadPoolExecutor) { + return new GseV2AgentStateClientImpl( agentStateQueryConfig, - gseClient, - bizHostInfoQueryService, + gseV2ApiClient.getIfAvailable(), threadPoolExecutor ); } + @Bean(MANAGE_BEAN_USE_V2_BY_FEATURE_AGENT_STATE_CLIENT) + public AgentStateClient useV2ByFeatureAgentStateClient(@Qualifier(MANAGE_BEAN_GSE_V1_AGENT_STATE_CLIENT) + GseV1AgentStateClientImpl gseV1AgentStateClient, + @Qualifier(MANAGE_BEAN_GSE_V2_AGENT_STATE_CLIENT) + GseV2AgentStateClientImpl gseV2AgentStateClient, + @Qualifier("jobManageBizHostInfoQueryService") + BizHostInfoQueryService bizHostInfoQueryService) { + return new UseV2ByFeatureAgentStateClientImpl( + gseV1AgentStateClient, + gseV2AgentStateClient, + bizHostInfoQueryService + ); + } + @Primary @Bean(MANAGE_BEAN_AGENT_STATE_CLIENT) public AgentStateClient AutoChoosingAgentStateClientImpl( diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ApplicationHostDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ApplicationHostDAO.java index 975cef1eea..76145bc482 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ApplicationHostDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ApplicationHostDAO.java @@ -178,6 +178,8 @@ Long countHostInfoByMultiKeys(Collection bizIds, int batchInsertHost(List applicationHostDTOList); + int updateHostAttrsByHostId(ApplicationHostDTO applicationHostDTO); + int updateHostAttrsBeforeLastTime(ApplicationHostDTO applicationHostDTO); int batchUpdateHostsBeforeLastTime(List applicationHostDTOList); @@ -186,7 +188,7 @@ Long countHostInfoByMultiKeys(Collection bizIds, // 删除类操作 - int deleteHostBeforeLastTime(Long bizId, Long hostId, Long lastTime); + int deleteHostBeforeOrEqualLastTime(Long bizId, Long hostId, Long lastTime); /** * 根据传入的主机ID批量删除主机 diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ApplicationHostDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ApplicationHostDAOImpl.java index fc6ba5bdcb..3d18ed5d69 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ApplicationHostDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ApplicationHostDAOImpl.java @@ -895,8 +895,24 @@ public boolean existAppHostInfoByHostId(Long hostId) { } } + @Override + public int updateHostAttrsByHostId(ApplicationHostDTO applicationHostDTO) { + checkHostId(applicationHostDTO); + List conditions = new ArrayList<>(); + conditions.add(TABLE.HOST_ID.eq(ULong.valueOf(applicationHostDTO.getHostId()))); + return updateHostAttrsByConditions(applicationHostDTO, conditions); + } + @Override public int updateHostAttrsBeforeLastTime(ApplicationHostDTO applicationHostDTO) { + checkHostId(applicationHostDTO); + List conditions = new ArrayList<>(); + conditions.add(TABLE.HOST_ID.eq(ULong.valueOf(applicationHostDTO.getHostId()))); + conditions.add(TABLE.LAST_TIME.lessThan(applicationHostDTO.getLastTime())); + return updateHostAttrsByConditions(applicationHostDTO, conditions); + } + + private void checkHostId(ApplicationHostDTO applicationHostDTO) { Long hostId = applicationHostDTO.getHostId(); if (hostId == null || hostId <= 0) { FormattingTuple msg = MessageFormatter.format( @@ -906,9 +922,9 @@ public int updateHostAttrsBeforeLastTime(ApplicationHostDTO applicationHostDTO) log.error(msg.getMessage()); throw new InternalException(msg.getMessage(), ErrorCode.INTERNAL_ERROR); } - List conditions = new ArrayList<>(); - conditions.add(TABLE.HOST_ID.eq(ULong.valueOf(applicationHostDTO.getHostId()))); - conditions.add(TABLE.LAST_TIME.lessThan(applicationHostDTO.getLastTime())); + } + + public int updateHostAttrsByConditions(ApplicationHostDTO applicationHostDTO, Collection conditions) { val query = context.update(TABLE) .set(TABLE.CLOUD_AREA_ID, ULong.valueOf(applicationHostDTO.getCloudAreaId())) .set(TABLE.IP, applicationHostDTO.getIp()) @@ -984,7 +1000,7 @@ public int batchUpdateHostsBeforeLastTime(List hostList) { @Transactional(value = "jobManageTransactionManager") @Override - public int deleteHostBeforeLastTime(Long bizId, Long hostId, Long lastTime) { + public int deleteHostBeforeOrEqualLastTime(Long bizId, Long hostId, Long lastTime) { int affectedNum; List conditions = new ArrayList<>(); if (bizId != null) { diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/EsbUserInfoDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/EsbUserInfoDAO.java index 495c3056a5..4eb88a3fd5 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/EsbUserInfoDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/EsbUserInfoDAO.java @@ -41,4 +41,6 @@ public interface EsbUserInfoDAO { List listEsbUserInfo(Collection userNames); + List listExistUserName(Collection userNames); + } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/EsbUserInfoDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/EsbUserInfoDAOImpl.java index 2d760e6459..7f5db83781 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/EsbUserInfoDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/EsbUserInfoDAOImpl.java @@ -31,6 +31,7 @@ import lombok.val; import org.jooq.Condition; import org.jooq.DSLContext; +import org.jooq.Record1; import org.jooq.Result; import org.jooq.conf.ParamType; import org.jooq.types.ULong; @@ -39,9 +40,11 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; +import org.springframework.util.CollectionUtils; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; @Repository @@ -118,6 +121,20 @@ public List listEsbUserInfo(Collection userNames) { return listEsbUserInfoByConditions(dslContext, conditions, null); } + @Override + public List listExistUserName(Collection userNames) { + if (CollectionUtils.isEmpty(userNames)) { + return Collections.emptyList(); + } + List conditions = new ArrayList<>(); + conditions.add(defaultTable.USERNAME.in(userNames)); + val baseQuery = dslContext.select(defaultTable.USERNAME) + .from(defaultTable) + .where(conditions); + Result> records = baseQuery.fetch(); + return records.map(record -> record.get(defaultTable.USERNAME)); + } + private List listEsbUserInfoByConditions(DSLContext dslContext, List conditions, Long limit) { val baseQuery = dslContext.selectFrom(defaultTable) diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/HostTopoDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/HostTopoDTO.java index e3ca841bf7..57d2b001c3 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/HostTopoDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/HostTopoDTO.java @@ -30,6 +30,7 @@ import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; +import org.apache.commons.lang3.StringUtils; /** * 主机拓扑 @@ -61,12 +62,16 @@ public class HostTopoDTO { private Long lastTime; public static HostTopoDTO fromHostRelationEvent(HostRelationEventDetail eventDetail) { + Long lastTimeMills = null; + if (StringUtils.isNotBlank(eventDetail.getLastTime())) { + lastTimeMills = TimeUtil.parseIsoZonedTimeToMillis(eventDetail.getLastTime()); + } return new HostTopoDTO( eventDetail.getHostId(), eventDetail.getBizId(), eventDetail.getSetId(), eventDetail.getModuleId(), - TimeUtil.parseIsoZonedTimeToMillis(eventDetail.getLastTime()) + lastTimeMills ); } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/runner/ClearDeletedHostsRunner.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/runner/ClearDeletedHostsRunner.java index f8aeceb892..b01fe48024 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/runner/ClearDeletedHostsRunner.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/runner/ClearDeletedHostsRunner.java @@ -27,9 +27,12 @@ import com.tencent.bk.job.manage.task.ClearDeletedHostsTask; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.CommandLineRunner; import org.springframework.stereotype.Component; +import java.util.concurrent.ThreadPoolExecutor; + /** * 进程启动时立即执行一次无效主机清理 */ @@ -38,14 +41,17 @@ public class ClearDeletedHostsRunner implements CommandLineRunner { private final ClearDeletedHostsTask clearDeletedHostsTask; + private final ThreadPoolExecutor initRunnerExecutor; @Autowired - public ClearDeletedHostsRunner(ClearDeletedHostsTask clearDeletedHostsTask) { + public ClearDeletedHostsRunner(ClearDeletedHostsTask clearDeletedHostsTask, + @Qualifier("initRunnerExecutor") ThreadPoolExecutor initRunnerExecutor) { this.clearDeletedHostsTask = clearDeletedHostsTask; + this.initRunnerExecutor = initRunnerExecutor; } @Override public void run(String... args) { - new Thread(clearDeletedHostsTask::execute).start(); + initRunnerExecutor.submit(clearDeletedHostsTask::execute); } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/runner/InitArtifactoryDataRunner.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/runner/InitArtifactoryDataRunner.java index fdbadf374f..ea712a93dd 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/runner/InitArtifactoryDataRunner.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/runner/InitArtifactoryDataRunner.java @@ -30,27 +30,37 @@ import com.tencent.bk.job.manage.config.LocalFileConfigForManage; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.CommandLineRunner; import org.springframework.stereotype.Component; +import java.util.concurrent.ThreadPoolExecutor; + @Slf4j @Component("jobManageInitArtifactoryDataRunner") public class InitArtifactoryDataRunner implements CommandLineRunner { private final ArtifactoryConfig artifactoryConfig; private final LocalFileConfigForManage localFileConfigForManage; + private final ThreadPoolExecutor initRunnerExecutor; @Autowired public InitArtifactoryDataRunner( ArtifactoryConfig artifactoryConfig, - LocalFileConfigForManage localFileConfigForManage + LocalFileConfigForManage localFileConfigForManage, + @Qualifier("initRunnerExecutor") ThreadPoolExecutor initRunnerExecutor ) { this.artifactoryConfig = artifactoryConfig; this.localFileConfigForManage = localFileConfigForManage; + this.initRunnerExecutor = initRunnerExecutor; } @Override public void run(String... args) { + initRunnerExecutor.submit(this::initArtifactoryData); + } + + public void initArtifactoryData() { if (!JobConstants.FILE_STORAGE_BACKEND_ARTIFACTORY.equals(localFileConfigForManage.getStorageBackend())) { //不使用制品库作为后端存储时不初始化 return; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/runner/RegisterBkNoticeRunner.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/runner/RegisterBkNoticeRunner.java new file mode 100644 index 0000000000..09f77f36ff --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/runner/RegisterBkNoticeRunner.java @@ -0,0 +1,107 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.runner; + +import com.tencent.bk.job.common.notice.IBkNoticeClient; +import com.tencent.bk.job.common.notice.model.BkNoticeApp; +import com.tencent.bk.job.common.util.ThreadUtils; +import com.tencent.bk.job.common.util.TimeUtil; +import com.tencent.bk.job.manage.common.consts.globalsetting.GlobalSettingKeys; +import com.tencent.bk.job.manage.dao.globalsetting.GlobalSettingDAO; +import com.tencent.bk.job.manage.model.dto.GlobalSettingDTO; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.helpers.MessageFormatter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.CommandLineRunner; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import java.util.concurrent.ThreadPoolExecutor; + +/** + * 进程启动时向消息中心注册平台信息（幂等操作） + */ +@SuppressWarnings("ConstantConditions") +@Slf4j +@Component +@ConditionalOnProperty(value = "bkNotice.enabled", havingValue = "true", matchIfMissing = true) +public class RegisterBkNoticeRunner implements CommandLineRunner { + + private final IBkNoticeClient bkNoticeClient; + private final ThreadPoolExecutor initRunnerExecutor; + private final GlobalSettingDAO globalSettingDAO; + + @Autowired + public RegisterBkNoticeRunner(IBkNoticeClient bkNoticeClient, + @Qualifier("initRunnerExecutor") ThreadPoolExecutor initRunnerExecutor, + GlobalSettingDAO globalSettingDAO) { + this.bkNoticeClient = bkNoticeClient; + this.initRunnerExecutor = initRunnerExecutor; + this.globalSettingDAO = globalSettingDAO; + } + + @Override + public void run(String... args) { + initRunnerExecutor.submit(() -> { + boolean registerSuccess = false; + // 最多重试30min，覆盖整个蓝鲸部署时间 + int maxRetryTimes = 180; + int retryTimes = 0; + while (!registerSuccess && retryTimes < maxRetryTimes) { + try { + BkNoticeApp bkNoticeApp = bkNoticeClient.registerApplication(); + log.info("registerApplication result:{}", bkNoticeApp); + registerSuccess = true; + } catch (Exception e) { + retryTimes++; + if (retryTimes < maxRetryTimes) { + String msg = MessageFormatter.format( + "Fail to registerApplication, retry {}", + retryTimes + ).getMessage(); + log.warn(msg, e); + ThreadUtils.sleep(10000); + } else { + log.warn("Fail to registerApplication finally", e); + } + } + } + // 将注册结果写入DB中 + if (registerSuccess) { + int affectedNum = globalSettingDAO.upsertGlobalSetting(buildRegisterResult(registerSuccess)); + log.info("Write to db, registerSuccess={}, affectedNum={}", registerSuccess, affectedNum); + } + }); + } + + private GlobalSettingDTO buildRegisterResult(boolean registerSuccess) { + GlobalSettingDTO globalSettingDTO = new GlobalSettingDTO(); + globalSettingDTO.setKey(GlobalSettingKeys.KEY_BK_NOTICE_REGISTERED_SUCCESS); + globalSettingDTO.setValue(String.valueOf(registerSuccess)); + globalSettingDTO.setDescription("Updated at " + TimeUtil.getCurrentTimeStr("yyyy-MM-dd HH:mm:ss.SSS")); + return globalSettingDTO; + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/agent/status/ScopeAgentStatusService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/agent/status/ScopeAgentStatusService.java new file mode 100644 index 0000000000..a78c32e640 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/agent/status/ScopeAgentStatusService.java @@ -0,0 +1,15 @@ +package com.tencent.bk.job.manage.service.agent.status; + +import com.tencent.bk.job.common.gse.v2.model.req.ListAgentStateReq; +import com.tencent.bk.job.common.gse.v2.model.resp.AgentState; +import com.tencent.bk.job.common.model.dto.ResourceScope; + +import java.util.List; + +public interface ScopeAgentStatusService { + + boolean needToUseGseV2(ResourceScope resourceScope); + + List listAgentState(ResourceScope resourceScope, ListAgentStateReq req); + +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/agent/status/impl/ScopeAgentStatusServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/agent/status/impl/ScopeAgentStatusServiceImpl.java new file mode 100644 index 0000000000..27d0b598aa --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/agent/status/impl/ScopeAgentStatusServiceImpl.java @@ -0,0 +1,54 @@ +package com.tencent.bk.job.manage.service.agent.status.impl; + +import com.tencent.bk.job.common.gse.IGseClient; +import com.tencent.bk.job.common.gse.v1.GseV1ApiClient; +import com.tencent.bk.job.common.gse.v2.GseV2ApiClient; +import com.tencent.bk.job.common.gse.v2.model.req.ListAgentStateReq; +import com.tencent.bk.job.common.gse.v2.model.resp.AgentState; +import com.tencent.bk.job.common.model.dto.ResourceScope; +import com.tencent.bk.job.common.util.feature.FeatureExecutionContext; +import com.tencent.bk.job.common.util.feature.FeatureIdConstants; +import com.tencent.bk.job.common.util.feature.FeatureToggle; +import com.tencent.bk.job.common.util.feature.ToggleStrategyContextParams; +import com.tencent.bk.job.manage.service.agent.status.ScopeAgentStatusService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Slf4j +@Service +public class ScopeAgentStatusServiceImpl implements ScopeAgentStatusService { + + private final GseV1ApiClient gseV1ApiClient; + private final GseV2ApiClient gseV2ApiClient; + + public ScopeAgentStatusServiceImpl(@Autowired(required = false) GseV1ApiClient gseV1ApiClient, + @Autowired(required = false) GseV2ApiClient gseV2ApiClient) { + this.gseV1ApiClient = gseV1ApiClient; + this.gseV2ApiClient = gseV2ApiClient; + } + + @Override + public boolean needToUseGseV2(ResourceScope resourceScope) { + FeatureExecutionContext featureExecutionContext = + FeatureExecutionContext.builder() + .addContextParam(ToggleStrategyContextParams.CTX_PARAM_RESOURCE_SCOPE, resourceScope); + return FeatureToggle.checkFeature( + FeatureIdConstants.FEATURE_AGENT_STATUS_GSE_V2, + featureExecutionContext + ); + } + + @Override + public List listAgentState(ResourceScope resourceScope, ListAgentStateReq req) { + IGseClient gseClient = chooseGseApiClientByResourceScope(resourceScope); + return gseClient.listAgentState(req); + } + + private IGseClient chooseGseApiClientByResourceScope(ResourceScope resourceScope) { + return needToUseGseV2(resourceScope) ? gseV2ApiClient : gseV1ApiClient; + } + +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/BizHostService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/BizHostService.java index 333b80fc4c..0c79a84bec 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/BizHostService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/BizHostService.java @@ -86,6 +86,16 @@ public interface BizHostService { List getHostsByBizAndHostIds(Collection bizIds, Collection hostIds); + /** + * 根据 bizIds 过滤出在这些业务下的hostIds + * + * @param bizIds 业务 ID集合 + * @param hostIds 主机 ID集合 + * @return 主机ID列表 + */ + List filterHostIdsByBiz(Collection bizIds, + Collection hostIds); + /** * 根据 bizId、ip集合查询主机信息 * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/HostService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/HostService.java index f744b13089..75fb7cbc29 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/HostService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/HostService.java @@ -30,6 +30,7 @@ import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.model.dto.HostSimpleDTO; import com.tencent.bk.job.manage.model.inner.ServiceListAppHostResultDTO; +import org.apache.commons.lang3.tuple.Pair; import java.util.Collection; import java.util.List; @@ -62,9 +63,11 @@ public interface HostService { * 创建或更新主机（仅更新时间戳在当前数据之前的数据） * * @param hostInfoDTO 主机信息 - * @return 新增主机数量 + * @return Pair<是否创建 ， 受影响主机数量> */ - int createOrUpdateHostBeforeLastTime(ApplicationHostDTO hostInfoDTO); + Pair createOrUpdateHostBeforeLastTime(ApplicationHostDTO hostInfoDTO); + + int updateHostAttrsByHostId(ApplicationHostDTO hostInfoDTO); /** * 更新主机属性（仅更新时间戳在当前数据之前的数据） @@ -75,11 +78,11 @@ public interface HostService { int updateHostAttrsBeforeLastTime(ApplicationHostDTO hostInfoDTO); /** - * 删除主机（仅删除时间戳在当前数据之前的数据） + * 删除主机（仅删除时间戳等于或在当前数据之前的数据） * * @param hostInfoDTO 主机信息 */ - void deleteHostBeforeLastTime(ApplicationHostDTO hostInfoDTO); + int deleteHostBeforeOrEqualLastTime(ApplicationHostDTO hostInfoDTO); long countHostsByOsType(String osType); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/ScopeHostService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/ScopeHostService.java index ddaccd7649..6175d3a6ab 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/ScopeHostService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/ScopeHostService.java @@ -37,6 +37,16 @@ */ public interface ScopeHostService { + /** + * 根据 资源范围 过滤出在该资源范围下的hostIds + * + * @param appResourceScope 资源范围 + * @param hostIds 主机ID集合 + * @return 主机ID列表 + */ + List filterScopeHostIds(AppResourceScope appResourceScope, + Collection hostIds); + /** * 根据 HostId 列表查询主机信息 * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/BizHostServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/BizHostServiceImpl.java index 4e558cf3c1..05138a7773 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/BizHostServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/BizHostServiceImpl.java @@ -105,6 +105,14 @@ public List getHostsByBizAndHostIds(Collection bizIds, return applicationHostDAO.listHostInfoByHostIds(hostIdsInBiz); } + @Override + public List filterHostIdsByBiz(Collection bizIds, Collection hostIds) { + if (CollectionUtils.isEmpty(bizIds) || CollectionUtils.isEmpty(hostIds)) { + return Collections.emptyList(); + } + return hostTopoDAO.listHostIdByBizAndHostIds(bizIds, hostIds); + } + @Override public List getHostsByBizAndIps(Collection bizIds, Collection ips) { return applicationHostDAO.listHostInfoByBizAndIps(bizIds, ips); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/HostServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/HostServiceImpl.java index 14b911c0f1..a60a6a8df7 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/HostServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/HostServiceImpl.java @@ -185,21 +185,23 @@ public int updateHostsStatus(List simpleHostList) { @Transactional(value = "jobManageTransactionManager") @Override - public int createOrUpdateHostBeforeLastTime(ApplicationHostDTO hostInfoDTO) { + public Pair createOrUpdateHostBeforeLastTime(ApplicationHostDTO hostInfoDTO) { + boolean needToCreate = false; try { if (applicationHostDAO.existAppHostInfoByHostId(hostInfoDTO.getHostId())) { // 只更新事件中的主机属性与agent状态 - applicationHostDAO.updateHostAttrsBeforeLastTime(hostInfoDTO); - return 0; + int affectedNum = applicationHostDAO.updateHostAttrsBeforeLastTime(hostInfoDTO); + return Pair.of(needToCreate, affectedNum); } else { + needToCreate = true; hostInfoDTO.setBizId(JobConstants.PUBLIC_APP_ID); int affectedNum = applicationHostDAO.insertHostWithoutTopo(hostInfoDTO); log.info("insert host: id={}, affectedNum={}", hostInfoDTO.getHostId(), affectedNum); - return affectedNum; + return Pair.of(needToCreate, affectedNum); } } catch (Throwable t) { log.error("createOrUpdateHostBeforeLastTime fail", t); - return 0; + return Pair.of(needToCreate, 0); } finally { // 从拓扑表向主机表同步拓扑数据 int affectedNum = applicationHostDAO.syncHostTopo(hostInfoDTO.getHostId()); @@ -209,13 +211,17 @@ public int createOrUpdateHostBeforeLastTime(ApplicationHostDTO hostInfoDTO) { } } + public int updateHostAttrsByHostId(ApplicationHostDTO hostInfoDTO) { + return applicationHostDAO.updateHostAttrsByHostId(hostInfoDTO); + } + public int updateHostAttrsBeforeLastTime(ApplicationHostDTO hostInfoDTO) { return applicationHostDAO.updateHostAttrsBeforeLastTime(hostInfoDTO); } @Override - public void deleteHostBeforeLastTime(ApplicationHostDTO hostInfoDTO) { - int affectedRowNum = applicationHostDAO.deleteHostBeforeLastTime( + public int deleteHostBeforeOrEqualLastTime(ApplicationHostDTO hostInfoDTO) { + int affectedRowNum = applicationHostDAO.deleteHostBeforeOrEqualLastTime( null, hostInfoDTO.getHostId(), hostInfoDTO.getLastTime() @@ -229,6 +235,7 @@ public void deleteHostBeforeLastTime(ApplicationHostDTO hostInfoDTO) { if (affectedRowNum > 0) { hostCache.deleteHost(hostInfoDTO); } + return affectedRowNum; } private void updateHostCache(ApplicationHostDTO hostInfoDTO) { diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/ScopeHostServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/ScopeHostServiceImpl.java index 79e42b0e9d..d65e59acf4 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/ScopeHostServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/ScopeHostServiceImpl.java @@ -46,6 +46,7 @@ import org.springframework.stereotype.Service; import org.springframework.util.StopWatch; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -74,6 +75,24 @@ public ScopeHostServiceImpl(ApplicationService applicationService, this.bizTopoService = bizTopoService; } + @Override + public List filterScopeHostIds(AppResourceScope appResourceScope, + Collection hostIds) { + ApplicationDTO applicationDTO = applicationService.getAppByScope(appResourceScope); + if (applicationDTO.isAllBizSet()) { + // 全业务 + return new ArrayList<>(hostIds); + } else if (applicationDTO.isBizSet()) { + // 业务集 + List bizIds = applicationDTO.getSubBizIds(); + return bizHostService.filterHostIdsByBiz(bizIds, hostIds); + } else { + // 普通业务 + Long bizId = Long.parseLong(applicationDTO.getScope().getId()); + return bizHostService.filterHostIdsByBiz(Collections.singletonList(bizId), hostIds); + } + } + @Override public List getScopeHostsByIds(AppResourceScope appResourceScope, Collection hostIds) { diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/GlobalSettingsServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/GlobalSettingsServiceImpl.java index 1494877092..c0577cb9c2 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/GlobalSettingsServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/GlobalSettingsServiceImpl.java @@ -33,6 +33,7 @@ import com.tencent.bk.job.common.i18n.locale.LocaleUtils; import com.tencent.bk.job.common.i18n.service.MessageI18nService; import com.tencent.bk.job.common.iam.constant.ActionId; +import com.tencent.bk.job.common.notice.config.BkNoticeProperties; import com.tencent.bk.job.common.util.JobContextUtil; import com.tencent.bk.job.common.util.StringUtil; import com.tencent.bk.job.common.util.TimeUtil; @@ -126,6 +127,7 @@ public class GlobalSettingsServiceImpl implements GlobalSettingsService { private final MessageI18nService i18nService; private final JobManageConfig jobManageConfig; private final LocalFileConfigForManage localFileConfigForManage; + private final BkNoticeProperties bkNoticeProperties; private final NotifyTemplateConverter notifyTemplateConverter; private final BuildProperties buildProperties; @Value("${job.manage.upload.filesize.max:5GB}") @@ -142,6 +144,7 @@ public GlobalSettingsServiceImpl(NotifyEsbChannelDAO notifyEsbChannelDAO, MessageI18nService i18nService, JobManageConfig jobManageConfig, LocalFileConfigForManage localFileConfigForManage, + BkNoticeProperties bkNoticeProperties, NotifyTemplateConverter notifyTemplateConverter, BuildProperties buildProperties) { this.notifyEsbChannelDAO = notifyEsbChannelDAO; @@ -154,6 +157,7 @@ public GlobalSettingsServiceImpl(NotifyEsbChannelDAO notifyEsbChannelDAO, this.i18nService = i18nService; this.jobManageConfig = jobManageConfig; this.localFileConfigForManage = localFileConfigForManage; + this.bkNoticeProperties = bkNoticeProperties; this.notifyTemplateConverter = notifyTemplateConverter; this.buildProperties = buildProperties; } @@ -670,12 +674,26 @@ private void addEnableUploadToArtifactoryConfig(Map configMap) { ); } + private void addEnableBkNoticeConfig(Map configMap) { + configMap.put( + GlobalSettingKeys.KEY_ENABLE_BK_NOTICE, + bkNoticeProperties.isEnabled() && bkNoticeRegisteredSuccess() + ); + } + + private boolean bkNoticeRegisteredSuccess() { + GlobalSettingDTO globalSettingDTO = + globalSettingDAO.getGlobalSetting(GlobalSettingKeys.KEY_BK_NOTICE_REGISTERED_SUCCESS); + return globalSettingDTO != null && "true".equals(globalSettingDTO.getValue().toLowerCase()); + } + @Override public Map getJobConfig(String username) { Map configMap = new HashMap<>(); addFileUploadConfig(configMap); addEnableFeatureFileManageConfig(configMap); addEnableUploadToArtifactoryConfig(configMap); + addEnableBkNoticeConfig(configMap); return configMap; } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/IndexServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/IndexServiceImpl.java index e0eaed4eb3..4f4c301da8 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/IndexServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/IndexServiceImpl.java @@ -50,6 +50,7 @@ import com.tencent.bk.job.manage.model.web.vo.index.JobAndScriptStatistics; import com.tencent.bk.job.manage.model.web.vo.task.TaskTemplateVO; import com.tencent.bk.job.manage.service.IndexService; +import com.tencent.bk.job.manage.service.host.HostDetailService; import com.tencent.bk.job.manage.service.template.TaskTemplateService; import lombok.extern.slf4j.Slf4j; import lombok.val; @@ -74,6 +75,7 @@ public class IndexServiceImpl implements IndexService { private final TaskTemplateService taskTemplateService; private final TaskTemplateDAO taskTemplateDAO; private final ScriptDAO scriptDAO; + private final HostDetailService hostDetailService; @Autowired public IndexServiceImpl(IndexGreetingDAO indexGreetingDAO, @@ -82,7 +84,8 @@ public IndexServiceImpl(IndexGreetingDAO indexGreetingDAO, TopologyHelper topologyHelper, TaskTemplateService taskTemplateService, TaskTemplateDAO taskTemplateDAO, - ScriptDAO scriptDAO) { + ScriptDAO scriptDAO, + HostDetailService hostDetailService) { this.indexGreetingDAO = indexGreetingDAO; this.applicationDAO = applicationDAO; this.applicationHostDAO = applicationHostDAO; @@ -90,6 +93,7 @@ public IndexServiceImpl(IndexGreetingDAO indexGreetingDAO, this.taskTemplateService = taskTemplateService; this.taskTemplateDAO = taskTemplateDAO; this.scriptDAO = scriptDAO; + this.hostDetailService = hostDetailService; } @Override @@ -187,6 +191,7 @@ public PageData listHostsByAgentStatus(String username, List hostInfoVOList; val hosts = applicationHostDAO.listHostInfoBySearchContents( bizIds, null, null, null, status, start, pageSize); + hostDetailService.fillDetailForApplicationHosts(hosts); Long count = applicationHostDAO.countHostInfoBySearchContents( bizIds, null, null, null, status); hostInfoVOList = hosts.stream() diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/SendNotifyTask.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/SendNotifyTask.java index 1409cb46da..397deaa618 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/SendNotifyTask.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/SendNotifyTask.java @@ -24,15 +24,16 @@ package com.tencent.bk.job.manage.service.impl.notify; +import com.tencent.bk.job.common.util.ThreadUtils; import com.tencent.bk.job.manage.dao.notify.EsbUserInfoDAO; -import com.tencent.bk.job.manage.model.dto.notify.EsbUserInfoDTO; import com.tencent.bk.job.manage.service.impl.WatchableSendMsgService; import lombok.Builder; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; -import org.slf4j.helpers.FormattingTuple; import org.slf4j.helpers.MessageFormatter; +import java.util.Collection; +import java.util.HashSet; import java.util.List; import java.util.Set; @@ -54,6 +55,8 @@ public class SendNotifyTask implements Runnable { private final String title; private final String content; + private Set validReceivers; + public void bindService(WatchableSendMsgService watchableSendMsgService, EsbUserInfoDAO esbUserInfoDAO) { this.watchableSendMsgService = watchableSendMsgService; @@ -62,8 +65,13 @@ public void bindService(WatchableSendMsgService watchableSendMsgService, @Override public void run() { - if (CollectionUtils.isEmpty(receivers)) { - logInvalidReceivers(); + pickValidReceivers(); + Collection removedReceivers = CollectionUtils.subtract(receivers, validReceivers); + if (CollectionUtils.isNotEmpty(removedReceivers)) { + log.info("Invalid user removed:{}, ignore to send notify to them", removedReceivers); + } + if (CollectionUtils.isEmpty(validReceivers)) { + logValidReceiversEmpty(); return; } try { @@ -78,8 +86,13 @@ public void run() { } } - private void logInvalidReceivers() { - log.warn("receivers is null or empty, skip, msgType={},title={}", msgType, title); + private void pickValidReceivers() { + List existUserNameList = esbUserInfoDAO.listExistUserName(receivers); + validReceivers = new HashSet<>(existUserNameList); + } + + private void logValidReceiversEmpty() { + log.info("valid receivers is null or empty, skip, msgType={},title={}", msgType, title); } private boolean sendMsgWithRetry() { @@ -93,47 +106,67 @@ private boolean sendMsgWithRetry() { createTimeMillis, msgType, sender, - receivers, + validReceivers, title, content ); result = true; } catch (Exception e) { - log.error("Fail to sendMsg", e); + if (count < NOTIFY_MAX_RETRY_COUNT) { + long sleepMills = count * 1000; + String msg = MessageFormatter.format( + "Fail to sendMsg, sleep {}ms and retry {}", + sleepMills, + count + ).getMessage(); + log.warn(msg, e); + ThreadUtils.sleep(sleepMills); + } else { + log.error("Fail to sendMsg", e); + } } } return result; } private void logSendSuccess() { - log.info("Success to send notify:({},{},{})", String.join(",", receivers), msgType, title); + log.info("Success to send notify:({},{},{})", String.join(",", validReceivers), msgType, title); } private void handleSendFail(Exception e) { - List validUsers = esbUserInfoDAO.listEsbUserInfo(receivers); - if (validUsers.isEmpty()) { - // 收信人已全部离职/某些渠道不支持平台用户 - logIgnoreToSend(); - return; - } - FormattingTuple msg = MessageFormatter.arrayFormat( - "Fail to send notify:({},{},{},{})", - new Object[]{ - String.join(",", receivers), - msgType, - title, - content - } - ); - if (e != null) { - log.error(msg.getMessage(), e); + int titleMaxLength = 32; + int contentMaxLength = 200; + String msg; + if (log.isDebugEnabled()) { + msg = MessageFormatter.arrayFormat( + "Fail to send notify:({},{},{},{})", + new Object[]{ + String.join(",", validReceivers), + msgType, + title, + content + } + ).getMessage(); + log.debug(msg, e); } else { - log.error(msg.getMessage()); + msg = MessageFormatter.arrayFormat( + "Fail to send notify:({},{},{},{})", + new Object[]{ + String.join(",", validReceivers), + msgType, + buildLogContent(title, titleMaxLength), + buildLogContent(content, contentMaxLength) + } + ).getMessage(); + log.error(msg, e); } } - private void logIgnoreToSend() { - log.info("Ignore to send notify:({},{},{})", String.join(",", receivers), msgType, title); + private String buildLogContent(String content, int maxLength) { + if (content != null && content.length() > maxLength) { + return content.substring(0, maxLength) + "..."; + } + return content; } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/AbstractCmdbResourceEventWatcher.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/AbstractCmdbResourceEventWatcher.java index 509279d741..ef0c6388fb 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/AbstractCmdbResourceEventWatcher.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/AbstractCmdbResourceEventWatcher.java @@ -274,7 +274,7 @@ protected boolean isWatchingEnabled() { * * @param event cmdb事件 */ - protected abstract void handleEvent(ResourceEvent event); + public abstract void handleEvent(ResourceEvent event); /** * 获取CMDB事件指标标签 diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizEventWatcher.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizEventWatcher.java index 20aa149ae2..0652f955fa 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizEventWatcher.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizEventWatcher.java @@ -86,7 +86,7 @@ protected ResourceWatchResult fetchEventsByStartTime(Long startT } @Override - protected void handleEvent(ResourceEvent event) { + public void handleEvent(ResourceEvent event) { String eventType = event.getEventType(); ApplicationDTO newestApp = BizEventDetail.toAppInfoDTO(event.getDetail()); ApplicationDTO cachedApp = null; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSetEventWatcher.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSetEventWatcher.java index d06a3a1401..2ea5e7d352 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSetEventWatcher.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSetEventWatcher.java @@ -76,7 +76,7 @@ protected ResourceWatchResult fetchEventsByStartTime(Long sta } @Override - protected void handleEvent(ResourceEvent event) { + public void handleEvent(ResourceEvent event) { log.info("Handle BizSetEvent: {}", event); ApplicationDTO latestApp = event.getDetail().toApplicationDTO(); String eventType = event.getEventType(); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSetRelationEventWatcher.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSetRelationEventWatcher.java index 1ce49f2db3..d3767bddf2 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSetRelationEventWatcher.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSetRelationEventWatcher.java @@ -80,7 +80,7 @@ protected ResourceWatchResult fetchEventsByStartTime( } @Override - protected void handleEvent(ResourceEvent event) { + public void handleEvent(ResourceEvent event) { log.info("Handle BizSetRelationEvent: {}", event); String eventType = event.getEventType(); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostEventHandler.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostEventHandler.java index 4f352d05c9..89f07d3060 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostEventHandler.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostEventHandler.java @@ -38,11 +38,15 @@ import com.tencent.bk.job.manage.service.host.HostService; import io.micrometer.core.instrument.Tags; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.tuple.Pair; import org.slf4j.helpers.FormattingTuple; import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.cloud.sleuth.Tracer; +import java.util.Collections; +import java.util.List; import java.util.concurrent.BlockingQueue; @Slf4j @@ -108,10 +112,36 @@ private void handleOneEventIndeed(ResourceEvent event) { // 尝试设置Agent状态 tryToUpdateAgentStatus(hostInfoDTO); // 更新DB与缓存中的主机数据 - hostService.createOrUpdateHostBeforeLastTime(hostInfoDTO); + Pair pair = hostService.createOrUpdateHostBeforeLastTime(hostInfoDTO); + int affectedNum = pair.getRight(); + if (affectedNum == 0) { + log.info( + "no host affected after handle according to lastTime, " + + "try to query latest host from cmdb and update" + ); + // 从CMDB查询最新主机信息 + List hostList = hostService.listHostsFromCmdbByHostIds( + Collections.singletonList(hostInfoDTO.getHostId()) + ); + if (CollectionUtils.isNotEmpty(hostList)) { + hostInfoDTO = hostList.get(0); + affectedNum = hostService.updateHostAttrsByHostId(hostInfoDTO); + log.info("update host attrs:{}, affectedNum={}", hostInfoDTO, affectedNum); + } else { + // 机器在CMDB中已不存在，忽略 + log.info("host not exist in cmdb:{}, ignore", hostInfoDTO); + } + } else { + log.info( + "{} host affected, created:{}", + affectedNum, + pair.getLeft() + ); + } break; case ResourceWatchReq.EVENT_TYPE_DELETE: - hostService.deleteHostBeforeLastTime(hostInfoDTO); + int deletedNum = hostService.deleteHostBeforeOrEqualLastTime(hostInfoDTO); + log.info("delete host:{}, deletedNum={}", hostInfoDTO, deletedNum); break; default: break; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostEventWatcher.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostEventWatcher.java index a5efb51b59..37482e7484 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostEventWatcher.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostEventWatcher.java @@ -112,7 +112,7 @@ protected ResourceWatchResult fetchEventsByStartTime(Long start } @Override - protected void handleEvent(ResourceEvent event) { + public void handleEvent(ResourceEvent event) { dispatchEventToHandler(event); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostRelationEventWatcher.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostRelationEventWatcher.java index 34f6bcaca8..3e91e25abb 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostRelationEventWatcher.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostRelationEventWatcher.java @@ -122,7 +122,7 @@ protected ResourceWatchResult fetchEventsByStartTime(Lo } @Override - protected void handleEvent(ResourceEvent event) { + public void handleEvent(ResourceEvent event) { dispatchEventToHandler(event); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostSyncService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostSyncService.java index aa4b89a8d6..86042ca153 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostSyncService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostSyncService.java @@ -40,11 +40,12 @@ import com.tencent.bk.job.manage.model.dto.HostTopoDTO; import com.tencent.bk.job.manage.service.host.HostService; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.lang3.tuple.Triple; import org.slf4j.helpers.FormattingTuple; import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.util.Pair; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; import org.springframework.util.StopWatch; @@ -182,7 +183,10 @@ private boolean needToUpdate(HostProp hostProp, if (!localHostIds.contains(hostProp.getHostId())) { return false; } - Long lastTime = TimeUtil.parseIsoZonedTimeToMillis(hostProp.getLastTime()); + Long lastTime = null; + if (StringUtils.isNotBlank(hostProp.getLastTime())) { + lastTime = TimeUtil.parseIsoZonedTimeToMillis(hostProp.getLastTime()); + } if (lastTime == null || lastTime < 0) { lastTime = cmdbHostsFetchTimeMills; log.warn( @@ -233,9 +237,13 @@ private Set refreshBizHostAndRelations(Long bizId, if (hostProp != null) { hostPropList.add(hostProp); cmdbHostIds.add(hostProp.getHostId()); + Long lastTimeMills = null; + if (StringUtils.isNotBlank(hostProp.getLastTime())) { + lastTimeMills = TimeUtil.parseIsoZonedTimeToMillis(hostProp.getLastTime()); + } BasicHostDTO cmdbBasicHost = new BasicHostDTO( hostProp.getHostId(), - TimeUtil.parseIsoZonedTimeToMillis(hostProp.getLastTime()) + lastTimeMills ); cmdbBasicHosts.add(cmdbBasicHost); } @@ -304,8 +312,8 @@ private Set refreshBizHostAndRelations(Long bizId, // 刷新主机数据 Pair refreshHostResult = refreshHosts(insertHostList, updateHostList, watch); - int insertedHostNum = refreshHostResult.getFirst(); - int updatedHostNum = refreshHostResult.getSecond(); + int insertedHostNum = refreshHostResult.getLeft(); + int updatedHostNum = refreshHostResult.getRight(); logRefreshResult( bizId, @@ -656,12 +664,16 @@ private String buildTopoKey(Long bizId, HostProp host, ModuleProp moduleProp) { } private HostTopoDTO buildHostTopo(Long bizId, HostProp host, ModuleProp moduleProp) { + Long lastTimeMills = null; + if (StringUtils.isNotBlank(moduleProp.getLastTime())) { + lastTimeMills = TimeUtil.parseIsoZonedTimeToMillis(moduleProp.getLastTime()); + } return new HostTopoDTO( host.getHostId(), bizId, moduleProp.getSetId(), moduleProp.getModuleId(), - TimeUtil.parseIsoZonedTimeToMillis(moduleProp.getLastTime()) + lastTimeMills ); } @@ -694,7 +706,8 @@ private int applyInsertHostList(List insertHostList, private int tryToInsertOneHost(ApplicationHostDTO hostDTO) { try { - return hostService.createOrUpdateHostBeforeLastTime(hostDTO); + Pair pair = hostService.createOrUpdateHostBeforeLastTime(hostDTO); + return pair.getRight(); } catch (Exception e) { FormattingTuple msg = MessageFormatter.format("Fail to insert host={}", hostDTO); log.error(msg.getMessage(), e); diff --git a/src/backend/settings.gradle b/src/backend/settings.gradle index 58340448d1..700490ecef 100644 --- a/src/backend/settings.gradle +++ b/src/backend/settings.gradle @@ -30,6 +30,7 @@ include 'commons:esb-sdk' include 'commons:cmdb-sdk' include 'commons:paas-sdk' include 'commons:gse-sdk' +include 'commons:notice-sdk' include 'commons:common-iam' include 'commons:common-jwt' include 'commons:common-utils' diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/AbstractHttpClient.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/AbstractHttpClient.java index cbe74265c4..7f7800f878 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/AbstractHttpClient.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/AbstractHttpClient.java @@ -24,12 +24,15 @@ package com.tencent.bk.job.upgrader.client; +import com.tencent.bk.job.common.constant.HttpMethodEnum; import com.tencent.bk.job.common.util.http.BasicHttpReq; -import com.tencent.bk.job.common.util.http.ExtHttpHelper; +import com.tencent.bk.job.common.util.http.HttpHelper; import com.tencent.bk.job.common.util.http.HttpHelperFactory; +import com.tencent.bk.job.common.util.http.HttpRequest; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.upgrader.model.IamReq; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; import org.apache.http.Header; import org.apache.http.message.BasicHeader; @@ -40,7 +43,7 @@ @Slf4j public abstract class AbstractHttpClient { private String hostUrl; - private ExtHttpHelper defaultHttpHelper = HttpHelperFactory.getDefaultHttpHelper(); + private HttpHelper defaultHttpHelper = HttpHelperFactory.getDefaultHttpHelper(); public AbstractHttpClient(String hostUrl) { this.hostUrl = hostUrl; @@ -52,7 +55,7 @@ public String doHttpGet(String uri, IamReq params) throws Exception { abstract List
getBasicHeaders(); - public String doHttpGet(String uri, BasicHttpReq params, ExtHttpHelper httpHelper) throws Exception { + public String doHttpGet(String uri, BasicHttpReq params, HttpHelper httpHelper) throws Exception { if (params == null) { params = new BasicHttpReq(); } @@ -62,14 +65,15 @@ public String doHttpGet(String uri, BasicHttpReq params, ExtHttpHelper httpHelpe boolean error = false; long start = System.currentTimeMillis(); String responseBody = null; - String url = hostUrl; + String url; try { if (!hostUrl.endsWith("/") && !uri.startsWith("/")) { url = hostUrl + "/" + uri + params.toUrlParams(); } else { url = hostUrl + uri + params.toUrlParams(); } - responseBody = httpHelper.get(url, getBasicHeaders()); + responseBody = httpHelper.request(HttpRequest.builder(HttpMethodEnum.GET, url) + .setHeaders(toHeaderArray(getBasicHeaders())).build()).getEntity(); return responseBody; } catch (Exception e) { log.error("Get url {}| params={}| exception={}", hostUrl + uri, @@ -83,13 +87,24 @@ public String doHttpGet(String uri, BasicHttpReq params, ExtHttpHelper httpHelpe } } + private Header[] toHeaderArray(List
headerList) { + if (CollectionUtils.isEmpty(headerList)) { + return new Header[0]; + } + Header[] headerArray = new Header[headerList.size()]; + for (int i = 0; i < headerList.size(); i++) { + headerArray[i] = headerList.get(i); + } + return headerArray; + } + protected String doHttpPost(String uri, T params) throws Exception { return doHttpPost(uri, params, defaultHttpHelper); } protected String doHttpPost( String uri, T params, - ExtHttpHelper httpHelper + HttpHelper httpHelper ) throws Exception { if (httpHelper == null) { httpHelper = defaultHttpHelper; @@ -106,7 +121,8 @@ protected String doHttpPost( } List
headerList = getBasicHeaders(); headerList.add(new BasicHeader(HDR_CONTENT_TYPE, "application/json")); - responseBody = httpHelper.post(url, "UTF-8", buildPostBody(params), headerList); + responseBody = httpHelper.request(HttpRequest.builder(HttpMethodEnum.POST, url) + .setStringEntity(buildPostBody(params)).setHeaders(toHeaderArray(headerList)).build()).getEntity(); return responseBody; } catch (Exception e) { log.warn("Post url {}| params={}| exception={}", uri, JsonUtils.toJsonWithoutSkippedFields(params), diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/IamClient.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/IamClient.java index 9208d9e2a7..4f411163dd 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/IamClient.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/IamClient.java @@ -28,7 +28,7 @@ import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.exception.InternalIamException; import com.tencent.bk.job.common.util.StringUtil; -import com.tencent.bk.job.common.util.http.ExtHttpHelper; +import com.tencent.bk.job.common.util.http.HttpHelper; import com.tencent.bk.job.common.util.json.JsonMapper; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.upgrader.model.ActionPolicies; @@ -70,7 +70,7 @@ private R getIamRespByReq(String method, String uri, IamReq reqBody, @SuppressWarnings("all") private R getIamRespByReq(String method, String uri, IamReq reqBody, TypeReference typeReference, - ExtHttpHelper httpHelper) throws RuntimeException { + HttpHelper httpHelper) throws RuntimeException { // URL模板变量替换 uri = StringUtil.replacePathVariables(uri, reqBody); String reqStr = JsonUtils.toJsonWithoutSkippedFields(reqBody); diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/JobClient.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/JobClient.java index 314dcef788..f5bbb9b27e 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/JobClient.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/JobClient.java @@ -30,7 +30,7 @@ import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.common.util.StringUtil; import com.tencent.bk.job.common.util.http.BasicHttpReq; -import com.tencent.bk.job.common.util.http.ExtHttpHelper; +import com.tencent.bk.job.common.util.http.HttpHelper; import com.tencent.bk.job.common.util.json.JsonMapper; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.upgrader.model.AppInfo; @@ -77,7 +77,7 @@ private R getJobRespByReq( String uri, BasicHttpReq reqBody, TypeReference typeReference, - ExtHttpHelper httpHelper + HttpHelper httpHelper ) { // URL模板变量替换 uri = StringUtil.replacePathVariables(uri, reqBody); diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/BaseUpgradeTask.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/BaseUpgradeTask.java index 5fb6461937..1f822de7aa 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/BaseUpgradeTask.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/BaseUpgradeTask.java @@ -26,12 +26,14 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.HttpMethodEnum; import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.common.util.Base64Util; import com.tencent.bk.job.common.util.http.BaseHttpHelper; -import com.tencent.bk.job.common.util.http.ExtHttpHelper; +import com.tencent.bk.job.common.util.http.HttpHelper; +import com.tencent.bk.job.common.util.http.HttpRequest; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.upgrader.anotation.UpgradeTask; import com.tencent.bk.job.upgrader.task.param.ParamNameConsts; @@ -58,10 +60,10 @@ public abstract class BaseUpgradeTask implements IUpgradeTask { private Properties properties; - private static final ExtHttpHelper HTTP_HELPER; + private static final HttpHelper HTTP_HELPER; static { - HTTP_HELPER = new ExtHttpHelper(new BaseHttpHelper(getHttpClient())); + HTTP_HELPER = new BaseHttpHelper(getHttpClient()); } BaseUpgradeTask() { @@ -97,7 +99,9 @@ public Response post(String url, String content) throws InternalException headers[1] = new BasicHeader("Content-Type", "application/json"); try { - String respStr = HTTP_HELPER.post(url, content, headers); + String respStr = HTTP_HELPER.request( + HttpRequest.builder(HttpMethodEnum.POST, url).setStringEntity(content).setHeaders(headers).build()) + .getEntity(); log.info("Post {}, content: {}, response: {}", url, content, respStr); if (StringUtils.isBlank(respStr)) { String errorMsg = diff --git a/src/frontend/package.json b/src/frontend/package.json index 5eb36df872..a1b754f0bd 100644 --- a/src/frontend/package.json +++ b/src/frontend/package.json @@ -15,6 +15,7 @@ "license": "ISC", "dependencies": { "@blueking/ip-selector": "0.2.0-beta", + "@blueking/notice-component-vue2": "2.0.0-beta.2", "@blueking/paas-login": "0.0.11", "@blueking/user-selector": "1.0.11", "@vue/babel-preset-jsx": "1.4.0", diff --git a/src/frontend/src/components/jb-router-view/permission.vue b/src/frontend/src/components/jb-router-view/permission.vue index b6ad631f5c..f20b53fce2 100644 --- a/src/frontend/src/components/jb-router-view/permission.vue +++ b/src/frontend/src/components/jb-router-view/permission.vue @@ -51,10 +51,7 @@