diff --git a/package.sh b/package.sh index 8c83efda31..879ae47406 100644 --- a/package.sh +++ b/package.sh @@ -152,6 +152,7 @@ fi cp -r support-files/bk-cmdb/ release/job/support-files/ cp -r support-files/bkiam/ release/job/support-files/ cp -r support-files/dependJarInfo/ release/job/support-files/ +cp support-files/javaagent/* release/job/backend/ # Package dependJarLists if [[ -d "support-files/dependJarLists/" ]]; then cp -r support-files/dependJarLists/ release/job/support-files/ diff --git a/src/backend/build.gradle b/src/backend/build.gradle index 3545fd29b3..31ee52ac29 100644 --- a/src/backend/build.gradle +++ b/src/backend/build.gradle @@ -70,6 +70,7 @@ ext { set("springBootVersion", "2.4.6") // https://mvnrepository.com/artifact/org.springframework.cloud/spring-cloud-dependencies set('springCloudVersion', "2020.0.3") + set('springCloudOtelVersion', "1.0.0-M13") // https://mvnrepository.com/artifact/io.springfox/springfox-swagger-ui set('swaggerVersion', "2.9.2") set('junitVersion', "5.5.2") @@ -154,6 +155,12 @@ allprojects { mavenLocal() maven { url mavenRepoUrl } maven { url "https://plugins.gradle.org/m2/" } + maven { + url "https://repo.spring.io/snapshot" + } + maven { + url "https://repo.spring.io/milestone" + } mavenCentral() } } @@ -198,6 +205,7 @@ subprojects { imports { mavenBom "org.springframework.boot:spring-boot-dependencies:$springBootVersion" mavenBom "org.springframework.cloud:spring-cloud-dependencies:${springCloudVersion}" + mavenBom "org.springframework.cloud:spring-cloud-sleuth-otel-dependencies:${springCloudOtelVersion}" } dependency "org.junit.jupiter:junit-jupiter:$junitVersion" dependency "org.springframework.boot:spring-boot-starter-web:$springBootVersion" @@ -262,6 +270,7 @@ subprojects { configurations { all*.exclude group: 'junit', module: 'junit' all*.exclude group: 'org.junit.vintage', module: 'junit-vintage-engine' + all*.exclude group: 'org.springframework.cloud', module: 'spring-cloud-sleuth-brave' } } apply plugin: TaskTreePlugin diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/BusinessInfoDTO.java b/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/BusinessInfoDTO.java index a264ea114a..bfc3b8c029 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/BusinessInfoDTO.java +++ b/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/BusinessInfoDTO.java @@ -30,29 +30,45 @@ import lombok.ToString; /** - * @description - * @date 2019/3/4 + * 业务DTO */ @Getter @Setter @ToString public class BusinessInfoDTO { + /** + * 业务ID + */ @JsonProperty("bk_biz_id") private Long bizId; + /** + * 业务名称 + */ @JsonProperty("bk_biz_name") private String bizName; + /** + * 业务运维 + */ @JsonProperty("bk_biz_maintainer") private String maintainers; + /** + * 开发商账号 + */ @JsonProperty("bk_supplier_account") private String supplierAccount; + /** + * 表示业务类型 + */ @JsonProperty("default") private Integer defaultApp; + /** + * 业务所在时区 + */ @JsonProperty("time_zone") private String timezone; - @JsonProperty("bk_operate_dept_id") - private Long operateDeptId; - @JsonProperty("bk_operate_dept_name") - private String operateDeptName; + /** + * 业务语言 + */ @JsonProperty("language") private String language; } diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BasicBizSet.java b/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BasicBizSet.java deleted file mode 100644 index b2c0d8e6f8..0000000000 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BasicBizSet.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.tencent.bk.job.common.cc.model.bizset; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.ToString; - -/** - * 基础业务集信息，用于生成更新CMDB业务集ID的Json数据文件 - */ -@Setter -@Getter -@AllArgsConstructor -@NoArgsConstructor -@ToString -public class BasicBizSet { - @JsonProperty("biz_set_id") - private Long id; - - @JsonProperty("biz_set_name") - private String name; -} diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BizSetInfo.java b/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BizSetInfo.java index 6c10bc515a..d99c0513ef 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BizSetInfo.java +++ b/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BizSetInfo.java @@ -81,12 +81,6 @@ public class BizSetInfo { @JsonProperty("bk_supplier_account") private String supplierAccount; - /** - * 运维部门ID - */ - @JsonProperty("bk_operate_dept_id") - private Long operateDeptId; - /** * 时区 */ diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/SearchBizSetReq.java b/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/SearchBizSetReq.java index 11a062d875..a2ebcf6e96 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/SearchBizSetReq.java +++ b/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/SearchBizSetReq.java @@ -51,7 +51,7 @@ public class SearchBizSetReq extends EsbReq { * 查询字段 */ private List fields = Arrays.asList("bk_biz_set_id", "bk_biz_set_name", "bk_biz_set_desc", - "bk_biz_maintainer", "bk_supplier_account", "bk_operate_dept_id", "create_time", "last_time", "time_zone", + "bk_biz_maintainer", "bk_supplier_account", "create_time", "last_time", "time_zone", "language", "bk_scope"); /** diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/BizEventDetail.java b/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/BizEventDetail.java index 1d15f167ff..2205294bb0 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/BizEventDetail.java +++ b/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/BizEventDetail.java @@ -25,8 +25,6 @@ package com.tencent.bk.job.common.cc.model.result; import com.fasterxml.jackson.annotation.JsonProperty; -import com.tencent.bk.job.common.cc.util.VersionCompatUtil; -import com.tencent.bk.job.common.constant.AppTypeEnum; import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; import com.tencent.bk.job.common.model.dto.ApplicationDTO; import com.tencent.bk.job.common.model.dto.ResourceScope; @@ -45,16 +43,10 @@ public class BizEventDetail { private Long bizId; @JsonProperty("bk_biz_name") private String appName; - @JsonProperty("bk_biz_maintainer") - private String maintainers; @JsonProperty("bk_supplier_account") private String supplierAccount; @JsonProperty("time_zone") private String timezone; - @JsonProperty("bk_operate_dept_id") - private Long operateDeptId; - @JsonProperty("bk_operate_dept_name") - private String operateDeptName; @JsonProperty("language") private String language; @@ -62,12 +54,9 @@ public static ApplicationDTO toAppInfoDTO(BizEventDetail bizEventDetail) { ApplicationDTO applicationDTO = new ApplicationDTO(); applicationDTO.setScope(new ResourceScope(ResourceScopeTypeEnum.BIZ, String.valueOf(bizEventDetail.getBizId()))); - applicationDTO.setAppType(AppTypeEnum.NORMAL); applicationDTO.setName(bizEventDetail.getAppName()); - applicationDTO.setMaintainers(VersionCompatUtil.convertMaintainers(bizEventDetail.getMaintainers())); applicationDTO.setBkSupplierAccount(bizEventDetail.getSupplierAccount()); applicationDTO.setTimeZone(bizEventDetail.getTimezone()); - applicationDTO.setOperateDeptId(bizEventDetail.getOperateDeptId()); applicationDTO.setLanguage(bizEventDetail.getLanguage()); return applicationDTO; } diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/BizSetEventDetail.java b/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/BizSetEventDetail.java index f7669a01a0..1db627a2fa 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/BizSetEventDetail.java +++ b/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/BizSetEventDetail.java @@ -26,8 +26,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; -import com.tencent.bk.job.common.cc.util.VersionCompatUtil; -import com.tencent.bk.job.common.constant.AppTypeEnum; import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; import com.tencent.bk.job.common.model.dto.ApplicationDTO; import com.tencent.bk.job.common.model.dto.ResourceScope; @@ -79,10 +77,8 @@ public ApplicationDTO toApplicationDTO() { ApplicationDTO applicationDTO = new ApplicationDTO(); ResourceScope resourceScope = new ResourceScope(ResourceScopeTypeEnum.BIZ_SET, String.valueOf(bizSetId)); applicationDTO.setScope(resourceScope); - applicationDTO.setAppType(AppTypeEnum.APP_SET); applicationDTO.setName(bizSetName); applicationDTO.setBkSupplierAccount(supplierAccount); - applicationDTO.setMaintainers(VersionCompatUtil.convertMaintainers(maintainers)); applicationDTO.setTimeZone(timezone); applicationDTO.setLanguage(language); return applicationDTO; diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/BizCmdbClient.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/BizCmdbClient.java index fb1653226b..a4a5d24bbb 100644 --- a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/BizCmdbClient.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/BizCmdbClient.java @@ -77,8 +77,6 @@ import com.tencent.bk.job.common.cc.model.result.SearchCloudAreaResult; import com.tencent.bk.job.common.cc.model.result.SearchDynamicGroupResult; import com.tencent.bk.job.common.cc.util.TopologyUtil; -import com.tencent.bk.job.common.cc.util.VersionCompatUtil; -import com.tencent.bk.job.common.constant.AppTypeEnum; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; import com.tencent.bk.job.common.esb.config.EsbConfig; @@ -90,7 +88,7 @@ import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.dto.ApplicationDTO; import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.model.dto.PageDTO; import com.tencent.bk.job.common.model.dto.ResourceScope; import com.tencent.bk.job.common.model.error.ErrorType; @@ -777,12 +775,9 @@ public List getAllBizApps() { private ApplicationDTO convertToAppInfo(BusinessInfoDTO businessInfo) { ApplicationDTO appInfo = new ApplicationDTO(); appInfo.setName(businessInfo.getBizName()); - appInfo.setMaintainers(VersionCompatUtil.convertMaintainers(businessInfo.getMaintainers())); appInfo.setBkSupplierAccount(businessInfo.getSupplierAccount()); appInfo.setTimeZone(businessInfo.getTimezone()); appInfo.setScope(new ResourceScope(ResourceScopeTypeEnum.BIZ, businessInfo.getBizId().toString())); - appInfo.setAppType(AppTypeEnum.NORMAL); - appInfo.setOperateDeptId(businessInfo.getOperateDeptId()); appInfo.setLanguage(businessInfo.getLanguage()); return appInfo; } @@ -963,17 +958,17 @@ public List getCloudAreaList() { } @Override - public List listBizHosts(long bizId, Collection ipList) { + public List listBizHosts(long bizId, Collection ipList) { if (ipList == null || ipList.isEmpty()) { return Collections.emptyList(); } List appHosts = getHostByIp(new GetHostByIpInput(bizId, null, null, - ipList.stream().map(IpDTO::getIp).collect(Collectors.toList()))); + ipList.stream().map(HostDTO::getIp).collect(Collectors.toList()))); if (appHosts == null || appHosts.isEmpty()) { return Collections.emptyList(); } return appHosts.stream().filter(host -> - ipList.contains(new IpDTO(host.getCloudAreaId(), host.getIp()))).collect(Collectors.toList()); + ipList.contains(new HostDTO(host.getCloudAreaId(), host.getIp()))).collect(Collectors.toList()); } @Override @@ -1055,12 +1050,12 @@ public ApplicationHostDTO getHostByIp(Long cloudAreaId, String ip) { } @Override - public List listHostsByIps(List hostIps) { + public List listHostsByIps(List cloudIps) { ListHostsWithoutBizReq req = makeBaseReq(ListHostsWithoutBizReq.class, defaultUin, defaultSupplierAccount); PropertyFilterDTO condition = new PropertyFilterDTO(); condition.setCondition("OR"); - Map> hostGroups = groupHostsByCloudAreaId(hostIps); - hostGroups.forEach((bkCloudId, hosts) -> { + Map> hostGroups = groupHostsByBkCloudId(cloudIps); + hostGroups.forEach((bkCloudId, ips) -> { ComposeRuleDTO hostRule = new ComposeRuleDTO(); hostRule.setCondition("AND"); BaseRuleDTO bkCloudIdRule = new BaseRuleDTO(); @@ -1072,7 +1067,7 @@ public List listHostsByIps(List hostIps) { BaseRuleDTO ipRule = new BaseRuleDTO(); ipRule.setField("bk_host_innerip"); ipRule.setOperator("in"); - ipRule.setValue(hosts.stream().map(IpDTO::getIp).collect(Collectors.toList())); + ipRule.setValue(ips); hostRule.addRule(ipRule); condition.addRule(hostRule); @@ -1128,8 +1123,21 @@ private void setBizRelationInfo(List hosts) { }); } - private Map> groupHostsByCloudAreaId(List hostIps) { - return hostIps.stream().collect(Collectors.groupingBy(IpDTO::getCloudAreaId)); + private Map> groupHostsByBkCloudId(List cloudIps) { + Map> hostGroup = new HashMap<>(); + cloudIps.forEach(cloudIp -> { + String[] cloudIdAndIp = cloudIp.split(":"); + Long bkCloudId = Long.valueOf(cloudIdAndIp[0]); + String ip = cloudIdAndIp[1]; + List ipList = hostGroup.computeIfAbsent(bkCloudId, (k) -> new ArrayList<>()); + ipList.add(ip); + }); + return hostGroup; + } + + @Override + public List listHostsByHostIds(List hostIds) { + return null; } @Override @@ -1144,7 +1152,7 @@ public List getObjAttributeList(String objId) { } @Override - public Set getAppUsersByRole(Long bizId, String role) { + public Set listUsersByRole(Long bizId, String role) { CcCountInfo searchResult; GetAppReq req = makeBaseReqByWeb(GetAppReq.class, null, defaultUin, defaultSupplierAccount); Map condition = new HashMap<>(); @@ -1188,7 +1196,7 @@ public Set getAppUsersByRole(Long bizId, String role) { } @Override - public List getAppRoleList() { + public List listRoles() { List esbObjAttributeDTO = getObjAttributeList("biz"); return esbObjAttributeDTO.stream().filter(it -> it.getBkPropertyGroup().equals("role") @@ -1286,8 +1294,8 @@ public ResourceWatchResult getHostRelationEvents(Long s public ResourceWatchResult getAppEvents(Long startTime, String cursor) { ResourceWatchReq req = makeBaseReqByWeb( ResourceWatchReq.class, null, defaultUin, defaultSupplierAccount); - req.setFields(Arrays.asList("bk_biz_id", "bk_biz_name", "bk_biz_maintainer", "bk_supplier_account", - "time_zone", "bk_operate_dept_id", "bk_operate_dept_name", "language")); + req.setFields(Arrays.asList("bk_biz_id", "bk_biz_name", "bk_supplier_account", + "time_zone", "language")); req.setResource("biz"); req.setCursor(cursor); req.setStartTime(startTime); diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/BizSetCmdbClient.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/BizSetCmdbClient.java index 147c32c044..e1094d8c5e 100644 --- a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/BizSetCmdbClient.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/BizSetCmdbClient.java @@ -27,9 +27,11 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.tencent.bk.job.common.cc.config.CmdbConfig; import com.tencent.bk.job.common.cc.model.bizset.BizInfo; +import com.tencent.bk.job.common.cc.model.bizset.BizSetFilter; import com.tencent.bk.job.common.cc.model.bizset.BizSetInfo; import com.tencent.bk.job.common.cc.model.bizset.BizSetScope; import com.tencent.bk.job.common.cc.model.bizset.Page; +import com.tencent.bk.job.common.cc.model.bizset.Rule; import com.tencent.bk.job.common.cc.model.bizset.SearchBizInBusinessReq; import com.tencent.bk.job.common.cc.model.bizset.SearchBizInBusinessSetResp; import com.tencent.bk.job.common.cc.model.bizset.SearchBizSetReq; @@ -46,11 +48,16 @@ import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.util.http.HttpHelperFactory; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.http.client.methods.HttpPost; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; /** * cmdb API Client - 业务集相关 @@ -116,7 +123,7 @@ public List searchAllBizSet() { int start = 0; List bizSetInfoList = new ArrayList<>(); while (start < bizSetCount) { - bizSetInfoList.addAll(searchBizSet(start, limit)); + bizSetInfoList.addAll(searchBizSet(null, start, limit)); start += limit; } return bizSetInfoList; @@ -125,16 +132,19 @@ public List searchAllBizSet() { /** * 查询业务集信息 * + * @param filter 查询条件 + * @param start 分页起始 + * @param limit 每页大小 * @return 业务集信息列表 */ - private List searchBizSet(int start, int limit) { + private List searchBizSet(BizSetFilter filter, int start, int limit) { SearchBizSetReq req = makeCmdbBaseReq(SearchBizSetReq.class); Page page = new Page(); page.setEnableCount(false); page.setStart(start); page.setLimit(limit); req.setPage(page); - req.setFilter(null); + req.setFilter(filter); try { EsbResp resp = getEsbRespByReq( HttpPost.METHOD_NAME, @@ -225,7 +235,7 @@ private List searchBizInBizSet(long bizSetId, int start, int limit) { } @Override - public List getAllBizSetApps() { + public List listAllBizSets() { List bizSetInfoList = searchAllBizSet(); bizSetInfoList.parallelStream().forEach(bizSetInfo -> { // 查询业务集下包含的子业务(全业务除外) @@ -287,4 +297,43 @@ public ResourceWatchResult getBizSetRelationEvents(Lo throw new InternalException(e, ErrorCode.CMDB_API_DATA_ERROR, null); } } + + @Override + public Set listUsersByRole(Long bizSetId, String role) { + if (!"bk_biz_maintainer".equals(role)) { + log.warn("Unavailable role for biz set! role: {}", role); + return Collections.emptySet(); + } + + BizSetInfo bizSet = queryBizSet(bizSetId); + if (bizSet == null) { + log.warn("BizSet: {} is not exist", bizSetId); + return Collections.emptySet(); + } + + Set userSet = new HashSet<>(); + String maintainers = bizSet.getMaintainer(); + if (StringUtils.isNotEmpty(maintainers)) { + for (String user : maintainers.split(",")) { + if (StringUtils.isNotBlank(user)) { + userSet.add(user); + } + } + } + + return userSet; + } + + @Override + public BizSetInfo queryBizSet(Long bizSetId) { + BizSetFilter filter = new BizSetFilter(); + filter.setCondition("AND"); + Rule bizSetIdRule = new Rule(); + bizSetIdRule.setField("bk_biz_set_id"); + bizSetIdRule.setOperator("equal"); + bizSetIdRule.setValue(bizSetId); + filter.setRules(Collections.singletonList(bizSetIdRule)); + List results = searchBizSet(filter, 0, 1); + return CollectionUtils.isEmpty(results) ? null : results.get(0); + } } diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/IBizCmdbClient.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/IBizCmdbClient.java index c38ace09b4..e91c0141ac 100644 --- a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/IBizCmdbClient.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/IBizCmdbClient.java @@ -40,7 +40,7 @@ import com.tencent.bk.job.common.cc.model.result.ResourceWatchResult; import com.tencent.bk.job.common.model.dto.ApplicationDTO; import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import java.util.Collection; import java.util.List; @@ -158,10 +158,18 @@ public interface IBizCmdbClient { /** * 根据IP批量获取主机 * - * @param hostIps 云区域+IP列表 + * @param cloudIps 云区域+IP列表 * @return 主机列表 */ - List listHostsByIps(List hostIps); + List listHostsByIps(List cloudIps); + + /** + * 根据IP批量获取主机 + * + * @param hostIds 主机ID列表 + * @return 主机列表 + */ + List listHostsByHostIds(List hostIds); /** * 批量获取业务下的主机列表 @@ -170,7 +178,7 @@ public interface IBizCmdbClient { * @param ipList 主机IP列表 * @return 主机 */ - List listBizHosts(long bizId, Collection ipList); + List listBizHosts(long bizId, Collection ipList); List findHostBizRelations(String uin, List hostIdList); @@ -184,15 +192,15 @@ public interface IBizCmdbClient { /** * 根据cmdb业务角色获取人员 * - * @param appId 业务ID + * @param bizId cmdb业务ID * @param role 业务角色 */ - Set getAppUsersByRole(Long appId, String role); + Set listUsersByRole(Long bizId, String role); /** * 获取CMDB业务角色列表 */ - List getAppRoleList(); + List listRoles(); /** * 批量获取topo节点层级 diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/IBizSetCmdbClient.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/IBizSetCmdbClient.java index 78223bfd04..898b138465 100644 --- a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/IBizSetCmdbClient.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/IBizSetCmdbClient.java @@ -30,18 +30,27 @@ import com.tencent.bk.job.common.cc.model.result.ResourceWatchResult; import java.util.List; +import java.util.Set; /** * CMDB业务集相关接口 */ public interface IBizSetCmdbClient { + /** + * 查询业务集 + * + * @param bizSetId 业务集ID + * @return 业务集 + */ + BizSetInfo queryBizSet(Long bizSetId); + /** * 从CC获取所有业务集信息 * * @return 业务集列表 */ - List getAllBizSetApps(); + List listAllBizSets(); /** * 根据游标获取业务集事件 @@ -60,4 +69,12 @@ public interface IBizSetCmdbClient { * @return 事件 */ ResourceWatchResult getBizSetRelationEvents(Long startTime, String cursor); + + /** + * 根据cmdb业务角色获取人员 + * + * @param bizId cmdb业务ID + * @param role 业务角色 + */ + Set listUsersByRole(Long bizId, String role); } diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/common/message.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/common/message.properties index 56f1fb4124..8d91002035 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/common/message.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/common/message.properties @@ -109,8 +109,3 @@ permission.action.name.global_settings=全局设置管理 # 高危语句 permission.action.name.high-risk_detect_rule=高危语句规则管理 permission.action.name.high-risk_detect_record=高危语句拦截记录查看 - -job.resource.status.online=已上线 -job.resource.status.draft=未上线 -job.resource.status.offline=已下线 -job.resource.status.disabled=禁用 diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/common/message_en.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/common/message_en.properties index 2a9c45da61..1bbf203034 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/common/message_en.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/common/message_en.properties @@ -109,9 +109,3 @@ permission.action.name.global_settings=Manage Global Settings # 高危语句 permission.action.name.high-risk_detect_rule=High-risk Detect Rule Setting permission.action.name.high-risk_detect_record=High-risk Detect Record - - -job.resource.status.online=Online -job.resource.status.draft=Stand-by -job.resource.status.offline=Offline -job.resource.status.disabled=Banned diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/common/message_en_US.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/common/message_en_US.properties index 62fa981f12..1bbf203034 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/common/message_en_US.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/common/message_en_US.properties @@ -109,8 +109,3 @@ permission.action.name.global_settings=Manage Global Settings # 高危语句 permission.action.name.high-risk_detect_rule=High-risk Detect Rule Setting permission.action.name.high-risk_detect_record=High-risk Detect Record - -job.resource.status.online=Online -job.resource.status.draft=Stand-by -job.resource.status.offline=Offline -job.resource.status.disabled=Banned diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/common/message_zh.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/common/message_zh.properties index 56f1fb4124..8d91002035 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/common/message_zh.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/common/message_zh.properties @@ -109,8 +109,3 @@ permission.action.name.global_settings=全局设置管理 # 高危语句 permission.action.name.high-risk_detect_rule=高危语句规则管理 permission.action.name.high-risk_detect_record=高危语句拦截记录查看 - -job.resource.status.online=已上线 -job.resource.status.draft=未上线 -job.resource.status.offline=已下线 -job.resource.status.disabled=禁用 diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/common/message_zh_CN.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/common/message_zh_CN.properties index 56f1fb4124..8d91002035 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/common/message_zh_CN.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/common/message_zh_CN.properties @@ -109,8 +109,3 @@ permission.action.name.global_settings=全局设置管理 # 高危语句 permission.action.name.high-risk_detect_rule=高危语句规则管理 permission.action.name.high-risk_detect_record=高危语句拦截记录查看 - -job.resource.status.online=已上线 -job.resource.status.draft=未上线 -job.resource.status.offline=已下线 -job.resource.status.disabled=禁用 diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message.properties index 83df1ee47f..0f710bd543 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message.properties @@ -72,8 +72,11 @@ 1211002=CMDB API返回数据异常 1211003=根据动态分组ID查找主机失败，动态分组ID：{0}，原因：{1}，请确认指定的动态分组在业务下是否存在 1211004=根据业务ID查找动态分组失败，业务ID：{0}，原因：{1}，请确认指定的业务是否存在动态分组 -1213001=PAAS 服务不可用 -1213002=PAAS API返回数据异常 +1213001=CMSI接口访问异常 +1213002=用户管理接口访问异常 +1213003=调用CMSI接口获取通知渠道数据异常 +1213004=调用CMSI接口发送通知失败，错误码：{0}，错误信息：{1} + 1214001=ARTIFACTORY API返回数据异常 1214002=制品库中找不到节点:{0}，请到制品库核实 @@ -143,11 +146,10 @@ 1243048=凭证不存在 1243049=业务/业务集不存在 1243050=文件后缀不允许 -1243051=资源范围不存在:{0} ##业务错误-作业执行(job-execute) 1244001=主机为空 -1244002=主机未注册:{0} +1244002=主机[{0}]无效，请检查源或目标主机是否存在于配置平台；另外，主机跨业务需要联系作业平台管理员先将其添加到IP白名单。 1244003=账号[{0}]不存在 1244004=没有账号[{0}]的权限 1244005=启动作业失败 @@ -171,6 +173,8 @@ 1244023=本地文件{0}在后台不存在 1244024=保存文件到本地失败 1244025=任务被丢弃 +1244026=非法的滚动策略 +1244027=滚动批次不能大于{0} ## 业务错误-定时任务(job-crontab) 1245006=定时任务执行时间已失效 @@ -196,6 +200,9 @@ 1260012=无法根据标识{0}找到文件源 1260013=文件源服务异常 1260014=通过[{0}]找不到file-worker +1260015=文件源标识已存在：{0} +1260016=文件源ID与标识至少指定一个 +1260017=文件源[id={0}]不在业务/业务集下 ##业务错误-文件Worker服务(job-file-worker) 1262001=第三方文件源响应异常：ListBucket，详情：{0} diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en.properties index 96262944b9..6138db5d90 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en.properties @@ -72,8 +72,11 @@ 1211002=CMDB API returned data exception 1211003=Fail to find host by dynamic group, id:{0}, reason:{1}, please confirm the specified dynamic group in business 1211004=Fail to find dynamic group by biz, id:{0}, reason:{1}, please confirm dynamic group in the specified business -1213001=PAAS service is unavailable -1213002=PAAS API returned data exception +1213001=Fail to request CMSI API +1213002=Fail to request UserManage API +1213003=CMSI exception when get notify channels +1213004=CMSI exception when send notify, error_code={0}, error_msg={1} + 1214001=ARTIFACTORY API returned data exception 1214002=Cannot find node in bkrepo:{0}, please check in bkrepo @@ -143,11 +146,10 @@ 1243048=Credential not exist 1243049=Business/Business Set not exist 1243050=File suffix not allow -1243051=Scope not exist:{0} ## Business error - job-execute 1244001=Host is empty -1244002=The host is not registered:{0} +1244002=Hosts invalid:{0}, please check them in CMDB, which can come from file source servers or target servers. Besides, please contact JOB administrator to add them to ip whitelist if you have to use hosts between businesses. 1244003=Account [{0}] does not exist 1244004=No permission to use this account[{0}] 1244005=Fail to start job @@ -171,6 +173,8 @@ 1244023=The local file {0} not exists in backend 1244024=Fail to save file to local 1244025=Task abandoned +1244026=Invalid rolling strategy +1244027=Rolling batch can not be greater than {0} ## Business error - job-crontab 1245006=Cron job execution time already passed @@ -196,6 +200,9 @@ 1260012=Cannot find file source by code:{0} 1260013=File source service invalid 1260014=Cannot find file-worker by {0} +1260015=FileSource code already exists:{0} +1260016=At least one of id/code required +1260017=FileSource[id={0}] does not belong to specified business ## Business error (job-file-worker) 1262001=Third file source response error:ListBucket, Detail:{0} diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en_US.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en_US.properties index 96262944b9..6138db5d90 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en_US.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en_US.properties @@ -72,8 +72,11 @@ 1211002=CMDB API returned data exception 1211003=Fail to find host by dynamic group, id:{0}, reason:{1}, please confirm the specified dynamic group in business 1211004=Fail to find dynamic group by biz, id:{0}, reason:{1}, please confirm dynamic group in the specified business -1213001=PAAS service is unavailable -1213002=PAAS API returned data exception +1213001=Fail to request CMSI API +1213002=Fail to request UserManage API +1213003=CMSI exception when get notify channels +1213004=CMSI exception when send notify, error_code={0}, error_msg={1} + 1214001=ARTIFACTORY API returned data exception 1214002=Cannot find node in bkrepo:{0}, please check in bkrepo @@ -143,11 +146,10 @@ 1243048=Credential not exist 1243049=Business/Business Set not exist 1243050=File suffix not allow -1243051=Scope not exist:{0} ## Business error - job-execute 1244001=Host is empty -1244002=The host is not registered:{0} +1244002=Hosts invalid:{0}, please check them in CMDB, which can come from file source servers or target servers. Besides, please contact JOB administrator to add them to ip whitelist if you have to use hosts between businesses. 1244003=Account [{0}] does not exist 1244004=No permission to use this account[{0}] 1244005=Fail to start job @@ -171,6 +173,8 @@ 1244023=The local file {0} not exists in backend 1244024=Fail to save file to local 1244025=Task abandoned +1244026=Invalid rolling strategy +1244027=Rolling batch can not be greater than {0} ## Business error - job-crontab 1245006=Cron job execution time already passed @@ -196,6 +200,9 @@ 1260012=Cannot find file source by code:{0} 1260013=File source service invalid 1260014=Cannot find file-worker by {0} +1260015=FileSource code already exists:{0} +1260016=At least one of id/code required +1260017=FileSource[id={0}] does not belong to specified business ## Business error (job-file-worker) 1262001=Third file source response error:ListBucket, Detail:{0} diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh.properties index 83df1ee47f..0f710bd543 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh.properties @@ -72,8 +72,11 @@ 1211002=CMDB API返回数据异常 1211003=根据动态分组ID查找主机失败，动态分组ID：{0}，原因：{1}，请确认指定的动态分组在业务下是否存在 1211004=根据业务ID查找动态分组失败，业务ID：{0}，原因：{1}，请确认指定的业务是否存在动态分组 -1213001=PAAS 服务不可用 -1213002=PAAS API返回数据异常 +1213001=CMSI接口访问异常 +1213002=用户管理接口访问异常 +1213003=调用CMSI接口获取通知渠道数据异常 +1213004=调用CMSI接口发送通知失败，错误码：{0}，错误信息：{1} + 1214001=ARTIFACTORY API返回数据异常 1214002=制品库中找不到节点:{0}，请到制品库核实 @@ -143,11 +146,10 @@ 1243048=凭证不存在 1243049=业务/业务集不存在 1243050=文件后缀不允许 -1243051=资源范围不存在:{0} ##业务错误-作业执行(job-execute) 1244001=主机为空 -1244002=主机未注册:{0} +1244002=主机[{0}]无效，请检查源或目标主机是否存在于配置平台；另外，主机跨业务需要联系作业平台管理员先将其添加到IP白名单。 1244003=账号[{0}]不存在 1244004=没有账号[{0}]的权限 1244005=启动作业失败 @@ -171,6 +173,8 @@ 1244023=本地文件{0}在后台不存在 1244024=保存文件到本地失败 1244025=任务被丢弃 +1244026=非法的滚动策略 +1244027=滚动批次不能大于{0} ## 业务错误-定时任务(job-crontab) 1245006=定时任务执行时间已失效 @@ -196,6 +200,9 @@ 1260012=无法根据标识{0}找到文件源 1260013=文件源服务异常 1260014=通过[{0}]找不到file-worker +1260015=文件源标识已存在：{0} +1260016=文件源ID与标识至少指定一个 +1260017=文件源[id={0}]不在业务/业务集下 ##业务错误-文件Worker服务(job-file-worker) 1262001=第三方文件源响应异常：ListBucket，详情：{0} diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh_CN.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh_CN.properties index 83df1ee47f..0f710bd543 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh_CN.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh_CN.properties @@ -72,8 +72,11 @@ 1211002=CMDB API返回数据异常 1211003=根据动态分组ID查找主机失败，动态分组ID：{0}，原因：{1}，请确认指定的动态分组在业务下是否存在 1211004=根据业务ID查找动态分组失败，业务ID：{0}，原因：{1}，请确认指定的业务是否存在动态分组 -1213001=PAAS 服务不可用 -1213002=PAAS API返回数据异常 +1213001=CMSI接口访问异常 +1213002=用户管理接口访问异常 +1213003=调用CMSI接口获取通知渠道数据异常 +1213004=调用CMSI接口发送通知失败，错误码：{0}，错误信息：{1} + 1214001=ARTIFACTORY API返回数据异常 1214002=制品库中找不到节点:{0}，请到制品库核实 @@ -143,11 +146,10 @@ 1243048=凭证不存在 1243049=业务/业务集不存在 1243050=文件后缀不允许 -1243051=资源范围不存在:{0} ##业务错误-作业执行(job-execute) 1244001=主机为空 -1244002=主机未注册:{0} +1244002=主机[{0}]无效，请检查源或目标主机是否存在于配置平台；另外，主机跨业务需要联系作业平台管理员先将其添加到IP白名单。 1244003=账号[{0}]不存在 1244004=没有账号[{0}]的权限 1244005=启动作业失败 @@ -171,6 +173,8 @@ 1244023=本地文件{0}在后台不存在 1244024=保存文件到本地失败 1244025=任务被丢弃 +1244026=非法的滚动策略 +1244027=滚动批次不能大于{0} ## 业务错误-定时任务(job-crontab) 1245006=定时任务执行时间已失效 @@ -196,6 +200,9 @@ 1260012=无法根据标识{0}找到文件源 1260013=文件源服务异常 1260014=通过[{0}]找不到file-worker +1260015=文件源标识已存在：{0} +1260016=文件源ID与标识至少指定一个 +1260017=文件源[id={0}]不在业务/业务集下 ##业务错误-文件Worker服务(job-file-worker) 1262001=第三方文件源响应异常：ListBucket，详情：{0} diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages.properties index 3f0c902eb5..05af8d1cd0 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages.properties @@ -44,4 +44,3 @@ validation.constraints.InvalidBkScopeType.message=非法的资源范围类型 validation.constraints.InvalidBkScopeId.message=非法的资源范围ID validation.constraints.InvalidUploadFileSuffix.message=非法的上传文件后缀 validation.constraints.InvalidUploadFileRestrictMode.message=非法的上传文件限制模式 -validation.constraints.InvalidScriptId_empty.message=脚本ID不能为空 diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_en.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_en.properties index 4abe87e0e3..840353f9f8 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_en.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_en.properties @@ -44,4 +44,3 @@ validation.constraints.InvalidBkScopeType.message=Invalid bk_scope_type validation.constraints.InvalidBkScopeId.message=Invalid bk_scope_id validation.constraints.InvalidUploadFileSuffix.message=Invalid file suffix validation.constraints.InvalidUploadFileRestrictMode.message=Invalid restrict mode -validation.constraints.InvalidScriptId_empty.message=Script id cannot be empty diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_en_US.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_en_US.properties index eeec7f7801..16f479f74a 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_en_US.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_en_US.properties @@ -44,4 +44,3 @@ validation.constraints.InvalidBkScopeType.message=Invalid bk_scope_type validation.constraints.InvalidBkScopeId.message=Invalid bk_scope_id validation.constraints.InvalidUploadFileSuffix.message=Invalid file suffix validation.constraints.InvalidUploadFileRestrictMode.message=Invalid restrict mode -validation.constraints.InvalidScriptId_empty.message=Script id cannot be empty diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_zh.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_zh.properties index 3f0c902eb5..05af8d1cd0 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_zh.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_zh.properties @@ -44,4 +44,3 @@ validation.constraints.InvalidBkScopeType.message=非法的资源范围类型 validation.constraints.InvalidBkScopeId.message=非法的资源范围ID validation.constraints.InvalidUploadFileSuffix.message=非法的上传文件后缀 validation.constraints.InvalidUploadFileRestrictMode.message=非法的上传文件限制模式 -validation.constraints.InvalidScriptId_empty.message=脚本ID不能为空 diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_zh_CN.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_zh_CN.properties index 3f0c902eb5..05af8d1cd0 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_zh_CN.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_zh_CN.properties @@ -44,4 +44,3 @@ validation.constraints.InvalidBkScopeType.message=非法的资源范围类型 validation.constraints.InvalidBkScopeId.message=非法的资源范围ID validation.constraints.InvalidUploadFileSuffix.message=非法的上传文件后缀 validation.constraints.InvalidUploadFileRestrictMode.message=非法的上传文件限制模式 -validation.constraints.InvalidScriptId_empty.message=脚本ID不能为空 diff --git a/src/backend/commons/common-service/build.gradle b/src/backend/commons/common-service/build.gradle index ad98439519..67ca78de5f 100644 --- a/src/backend/commons/common-service/build.gradle +++ b/src/backend/commons/common-service/build.gradle @@ -54,5 +54,9 @@ dependencies { api 'io.jsonwebtoken:jjwt' api 'io.prometheus:simpleclient_pushgateway' api 'io.micrometer:micrometer-registry-prometheus' + api "org.springframework.cloud:spring-cloud-sleuth-api" + api "org.springframework.cloud:spring-cloud-sleuth-otel" + api "org.springframework.cloud:spring-cloud-sleuth-otel-autoconfigure" + api "io.opentelemetry:opentelemetry-exporter-otlp-trace" testImplementation 'org.springframework.boot:spring-boot-starter-test' } diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/trace/executors/TraceCallable.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/trace/executors/TraceCallable.java index bd07576979..65ac6e9b17 100644 --- a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/trace/executors/TraceCallable.java +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/trace/executors/TraceCallable.java @@ -24,10 +24,8 @@ package com.tencent.bk.job.common.trace.executors; -import brave.ScopedSpan; -import brave.Tracer; -import brave.Tracing; -import brave.propagation.TraceContext; +import org.springframework.cloud.sleuth.Span; +import org.springframework.cloud.sleuth.Tracer; import java.util.concurrent.Callable; @@ -39,25 +37,30 @@ public class TraceCallable implements Callable { /** * 调用链父上下文 */ - private final TraceContext parent; + private final Span parent; private Callable delegate; - public TraceCallable(Callable callable, Tracing tracing) { + public TraceCallable(Callable callable, Tracer tracer) { this.delegate = callable; - this.tracer = tracing.tracer(); - this.parent = tracing.currentTraceContext().get(); + this.tracer = tracer; + this.parent = tracer.currentSpan(); } @Override public V call() throws Exception { - ScopedSpan span = this.tracer.startScopedSpanWithParent("async", parent); + Span span = null; try { + span = tracer.nextSpan(parent).name("async"); return delegate.call(); } catch (Throwable e) { - span.error(e); + if (span != null) { + span.error(e); + } throw e; } finally { - span.finish(); + if (span != null) { + span.end(); + } } } } diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/trace/executors/TraceRunnable.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/trace/executors/TraceRunnable.java index 22e50970aa..f0a87977e0 100644 --- a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/trace/executors/TraceRunnable.java +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/trace/executors/TraceRunnable.java @@ -24,10 +24,8 @@ package com.tencent.bk.job.common.trace.executors; -import brave.ScopedSpan; -import brave.Tracer; -import brave.Tracing; -import brave.propagation.TraceContext; +import org.springframework.cloud.sleuth.Span; +import org.springframework.cloud.sleuth.Tracer; public class TraceRunnable implements Runnable { /** @@ -37,25 +35,30 @@ public class TraceRunnable implements Runnable { /** * 调用链父上下文 */ - private final TraceContext parent; + private final Span parent; private Runnable delegate; - public TraceRunnable(Runnable runnable, Tracing tracing) { + public TraceRunnable(Runnable runnable, Tracer tracer) { this.delegate = runnable; - this.tracer = tracing.tracer(); - this.parent = tracing.currentTraceContext().get(); + this.tracer = tracer; + this.parent = tracer.currentSpan(); } @Override public void run() { - ScopedSpan span = this.tracer.startScopedSpanWithParent("async", parent); + Span span = null; try { + span = tracer.nextSpan(parent).name("async"); delegate.run(); } catch (Throwable e) { - span.error(e); + if (span != null) { + span.error(e); + } throw e; } finally { - span.finish(); + if (span != null) { + span.end(); + } } } diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/trace/executors/TraceableExecutorService.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/trace/executors/TraceableExecutorService.java index 9e770fc86f..487c95a810 100644 --- a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/trace/executors/TraceableExecutorService.java +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/trace/executors/TraceableExecutorService.java @@ -24,7 +24,7 @@ package com.tencent.bk.job.common.trace.executors; -import brave.Tracing; +import org.springframework.cloud.sleuth.Tracer; import java.util.Collection; import java.util.List; @@ -38,11 +38,11 @@ public class TraceableExecutorService implements ExecutorService { protected ExecutorService delegateExecutorService; - protected Tracing tracing; + protected Tracer tracer; - public TraceableExecutorService(ExecutorService executorService, Tracing tracing) { + public TraceableExecutorService(ExecutorService executorService, Tracer tracer) { this.delegateExecutorService = executorService; - this.tracing = tracing; + this.tracer = tracer; } public ExecutorService getDelegateExecutorService() { @@ -76,17 +76,17 @@ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedE @Override public Future submit(Callable task) { - return delegateExecutorService.submit(new TraceCallable(task, tracing)); + return delegateExecutorService.submit(new TraceCallable(task, tracer)); } @Override public Future submit(Runnable task, T result) { - return delegateExecutorService.submit(new TraceRunnable(task, tracing), result); + return delegateExecutorService.submit(new TraceRunnable(task, tracer), result); } @Override public Future submit(Runnable task) { - return delegateExecutorService.submit(new TraceRunnable(task, tracing)); + return delegateExecutorService.submit(new TraceRunnable(task, tracer)); } @Override @@ -113,6 +113,6 @@ public T invokeAny(Collection> tasks, long timeout, @Override public void execute(Runnable command) { - delegateExecutorService.execute(new TraceRunnable(command, tracing)); + delegateExecutorService.execute(new TraceRunnable(command, tracer)); } } diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/trace/executors/TraceableScheduledExecutorService.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/trace/executors/TraceableScheduledExecutorService.java index 473edd7a33..03c487646f 100644 --- a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/trace/executors/TraceableScheduledExecutorService.java +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/trace/executors/TraceableScheduledExecutorService.java @@ -25,7 +25,7 @@ package com.tencent.bk.job.common.trace.executors; -import brave.Tracing; +import org.springframework.cloud.sleuth.Tracer; import java.util.concurrent.Callable; import java.util.concurrent.ScheduledExecutorService; @@ -33,8 +33,8 @@ import java.util.concurrent.TimeUnit; public class TraceableScheduledExecutorService extends TraceableExecutorService implements ScheduledExecutorService { - public TraceableScheduledExecutorService(final ScheduledExecutorService delegate, final Tracing tracing) { - super(delegate, tracing); + public TraceableScheduledExecutorService(final ScheduledExecutorService delegate, final Tracer tracer) { + super(delegate, tracer); } private ScheduledExecutorService getScheduledExecutorService() { @@ -43,26 +43,26 @@ private ScheduledExecutorService getScheduledExecutorService() { @Override public ScheduledFuture schedule(Runnable command, long delay, TimeUnit unit) { - Runnable r = new TraceRunnable(command, tracing); + Runnable r = new TraceRunnable(command, tracer); return getScheduledExecutorService().schedule(r, delay, unit); } @Override public ScheduledFuture schedule(Callable callable, long delay, TimeUnit unit) { - Callable c = new TraceCallable<>(callable, tracing); + Callable c = new TraceCallable<>(callable, tracer); return getScheduledExecutorService().schedule(c, delay, unit); } @Override public ScheduledFuture scheduleAtFixedRate(Runnable command, long initialDelay, long period, TimeUnit unit) { - Runnable r = new TraceRunnable(command, tracing); + Runnable r = new TraceRunnable(command, tracer); return getScheduledExecutorService().scheduleAtFixedRate(r, initialDelay, period, unit); } @Override public ScheduledFuture scheduleWithFixedDelay(Runnable command, long initialDelay, long delay, TimeUnit unit) { - Runnable r = new TraceRunnable(command, tracing); + Runnable r = new TraceRunnable(command, tracer); return getScheduledExecutorService().scheduleWithFixedDelay(r, initialDelay, delay, unit); } } diff --git a/src/backend/commons/common-service/src/main/resources/logback-spring.xml b/src/backend/commons/common-service/src/main/resources/logback-spring.xml index b521e63936..b283f4080a 100644 --- a/src/backend/commons/common-service/src/main/resources/logback-spring.xml +++ b/src/backend/commons/common-service/src/main/resources/logback-spring.xml @@ -5,8 +5,8 @@ - - + + diff --git a/src/backend/commons/common-web/build.gradle b/src/backend/commons/common-web/build.gradle index 1517be6d44..adadaef2d6 100644 --- a/src/backend/commons/common-web/build.gradle +++ b/src/backend/commons/common-web/build.gradle @@ -28,5 +28,6 @@ dependencies { api project(':commons:common-iam') api 'org.springframework.boot:spring-boot-starter-web' api 'org.springframework.boot:spring-boot-starter-security' + implementation 'com.google.guava:guava' testImplementation 'org.springframework.boot:spring-boot-starter-test' } diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/interceptor/JobCommonInterceptor.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/interceptor/JobCommonInterceptor.java index 6361f94281..d0f23cca3c 100644 --- a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/interceptor/JobCommonInterceptor.java +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/interceptor/JobCommonInterceptor.java @@ -24,7 +24,6 @@ package com.tencent.bk.job.common.web.interceptor; -import brave.Tracer; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.tencent.bk.job.common.constant.HttpRequestSourceEnum; @@ -42,11 +41,14 @@ import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpStatus; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cloud.sleuth.Span; +import org.springframework.cloud.sleuth.Tracer; import org.springframework.context.annotation.Lazy; import org.springframework.http.HttpMethod; +import org.springframework.lang.NonNull; import org.springframework.stereotype.Component; +import org.springframework.web.servlet.AsyncHandlerInterceptor; import org.springframework.web.servlet.ModelAndView; -import org.springframework.web.servlet.handler.HandlerInterceptorAdapter; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -63,10 +65,11 @@ */ @Slf4j @Component -public class JobCommonInterceptor extends HandlerInterceptorAdapter { +public class JobCommonInterceptor implements AsyncHandlerInterceptor { private static final Pattern SCOPE_PATTERN = Pattern.compile("/scope/(\\w+)/(\\d+)"); private final Tracer tracer; + private Tracer.SpanInScope spanInScope = null; private AppScopeMappingService appScopeMappingService; @@ -87,12 +90,14 @@ public JobCommonInterceptor(Tracer tracer) { } @Override - public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) { + public boolean preHandle(@NonNull HttpServletRequest request, + @NonNull HttpServletResponse response, + @NonNull Object handler) { JobContextUtil.setStartTime(); JobContextUtil.setRequest(request); JobContextUtil.setResponse(response); - addRequestId(); + initSpanAndAddRequestId(); if (!shouldFilter(request)) { return true; @@ -111,8 +116,14 @@ private boolean shouldFilter(HttpServletRequest request) { return uri.startsWith("/web/") || uri.startsWith("/service/") || uri.startsWith("/esb/"); } - private void addRequestId() { - String traceId = tracer.currentSpan().context().traceIdString(); + private void initSpanAndAddRequestId() { + Span currentSpan = tracer.currentSpan(); + if (currentSpan == null) { + currentSpan = tracer.nextSpan().start(); + } + spanInScope = tracer.withSpan(currentSpan); + log.debug("currentSpan={}", currentSpan); + String traceId = currentSpan.context().traceId(); JobContextUtil.setRequestId(traceId); } @@ -325,7 +336,9 @@ private String parseValueFromQueryStringOrBody(HttpServletRequest request, Strin } @Override - public void postHandle(HttpServletRequest request, HttpServletResponse response, Object handler, + public void postHandle(@NonNull HttpServletRequest request, + @NonNull HttpServletResponse response, + @NonNull Object handler, ModelAndView modelAndView) { if (log.isDebugEnabled()) { log.debug("Post handler|{}|{}|{}|{}|{}", JobContextUtil.getRequestId(), @@ -336,18 +349,19 @@ public void postHandle(HttpServletRequest request, HttpServletResponse response, } @Override - public void afterCompletion(HttpServletRequest request, HttpServletResponse response, - Object handler, + public void afterCompletion(@NonNull HttpServletRequest request, + @NonNull HttpServletResponse response, + @NonNull Object handler, Exception ex) { try { if (isClientOrServerError(response)) { log.warn("status {} given by {}", response.getStatus(), handler); } if (ex != null) { - log.error("After completion|{}|{}|{}|{}|{}|{}", JobContextUtil.getRequestId(), response.getStatus(), + log.error("After completion|{}|{}|{}|{}|{}|{}|{}", JobContextUtil.getRequestId(), response.getStatus(), JobContextUtil.getAppResourceScope(), JobContextUtil.getUsername(), System.currentTimeMillis() - JobContextUtil.getStartTime(), - request.getRequestURI(), ex); + request.getRequestURI(), ex.getMessage()); } else { log.debug("After completion|{}|{}|{}|{}|{}|{}", JobContextUtil.getRequestId(), response.getStatus(), JobContextUtil.getAppResourceScope(), @@ -355,6 +369,9 @@ public void afterCompletion(HttpServletRequest request, HttpServletResponse resp request.getRequestURI()); } } finally { + if (spanInScope != null) { + spanInScope.close(); + } JobContextUtil.unsetContext(); } } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/CompatibleImplementation.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/CompatibleImplementation.java index d47104fa78..d66af9c77a 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/CompatibleImplementation.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/CompatibleImplementation.java @@ -44,6 +44,11 @@ @Retention(RetentionPolicy.RUNTIME) @Target(value = {CONSTRUCTOR, FIELD, LOCAL_VARIABLE, METHOD, PACKAGE, PARAMETER, TYPE}) public @interface CompatibleImplementation { + /** + * 兼容实现的名称 + */ + String name() default ""; + /** * 说明 */ diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/PersistenceObject.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/PersistenceObject.java new file mode 100644 index 0000000000..14cd6fec0f --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/PersistenceObject.java @@ -0,0 +1,42 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB 蓝鲸作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB 蓝鲸作业平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.tencent.bk.job.common.annotation; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * 用于表示某个持久对象，该对象会被持久化到DB中,所以对于该对象的改动需要谨慎，避免破坏性的修改(比如修改字段名) + */ +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@Documented +public @interface PersistenceObject { +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/Bool.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/Bool.java index c1f8e83562..46e43bf79c 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/Bool.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/Bool.java @@ -25,15 +25,15 @@ package com.tencent.bk.job.common.constant; public enum Bool { - FALSE(0), TRUE(1); + FALSE((byte) 0), TRUE((byte) 1); - private final int value; + private final byte value; - Bool(int value) { + Bool(byte value) { this.value = value; } - public static Bool from(Integer value) { + public static Bool from(Byte value) { if (value == null) { return FALSE; } @@ -49,14 +49,14 @@ public static Bool from(boolean value) { return value ? TRUE : FALSE; } - public static boolean isTrue(Integer value) { + public static boolean isTrue(Byte value) { if (value == null) { return false; } return value.equals(Bool.TRUE.getValue()); } - public int getValue() { + public byte getValue() { return this.value; } } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ErrorCode.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ErrorCode.java index a3d5b04734..1537e9dff4 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ErrorCode.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ErrorCode.java @@ -188,8 +188,8 @@ public class ErrorCode { // 作业执行 start // 主机为空 public static final int SERVER_EMPTY = 1244001; - // 主机未注册:{} - public static final int SERVER_UNREGISTERED = 1244002; + // 主机[{0}]无效，请检查源或目标主机是否存在于配置平台；另外，主机跨业务需要联系作业平台管理员先将其添加到IP白名单。 + public static final int HOST_INVALID = 1244002; // 账号不存在 public static final int ACCOUNT_NOT_EXIST = 1244003; // 没有该账号的权限 @@ -234,6 +234,10 @@ public class ErrorCode { public static final int FAIL_TO_SAVE_FILE_TO_LOCAL = 1244024; // 任务被丢弃 public static final int TASK_ABANDONED = 1244025; + // 非法的滚动策略 + public static final int INVALID_ROLLING_EXPR = 1244026; + // 滚动批次不能大于{} + public static final int EXCEED_MAX_ALLOWED_BATCH_SIZE = 1244027; // 作业执行 end // 定时作业 start @@ -299,6 +303,12 @@ public class ErrorCode { public static final int FILE_SOURCE_SERVICE_INVALID = 1260013; // 通过[{0}]找不到file-worker public static final int FILE_WORKER_NOT_FOUND = 1260014; + // 文件源标识已存在：{0} + public static final int FILE_SOURCE_CODE_ALREADY_EXISTS = 1260015; + // 文件源ID与标识至少指定一个 + public static final int ID_AND_CODE_AT_LEAST_ONE = 1260016; + // 文件源[id={0}]不在业务/业务集下 + public static final int FILE_SOURCE_ID_NOT_IN_BIZ = 1260017; // 文件网关 end // 文件代理 start @@ -388,12 +398,16 @@ public class ErrorCode { public static final int FAIL_TO_FIND_DYNAMIC_GROUP_BY_BIZ = 1211004; // PaaS异常 - // PAAS服务不可达 - 地址配置错误或者地址无法正确解析 - public static final int PAAS_UNREACHABLE_SERVER = 1213001; - // paas接口返回数据结构异常-一般是被网关防火墙重定向返回统一登录页面 - public static final int PAAS_API_DATA_ERROR = 1213002; - // paas通知渠道接口数据获取异常 - public static final int PAAS_MSG_CHANNEL_DATA_ERROR = 1213003; + // CMSI接口访问异常 + public static final int CMSI_API_ACCESS_ERROR = 1213001; + // 用户管理接口访问异常 + public static final int USER_MANAGE_API_ACCESS_ERROR = 1213002; + // 调用CMSI接口获取通知渠道数据异常 + public static final int CMSI_MSG_CHANNEL_DATA_ERROR = 1213003; + // 调用CMSI接口发送通知失败，错误码：{0}，错误信息：{1} + public static final int CMSI_FAIL_TO_SEND_MSG = 1213004; + + // 制品库异常 // Artifactory接口返回数据结构异常 public static final int ARTIFACTORY_API_DATA_ERROR = 1214001; // 制品库中找不到节点:{0}，请到制品库核实 diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/AppTypeEnum.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/RollingModeEnum.java similarity index 73% rename from src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/AppTypeEnum.java rename to src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/RollingModeEnum.java index c016e36cdf..9e666898bd 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/AppTypeEnum.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/RollingModeEnum.java @@ -28,39 +28,43 @@ import com.fasterxml.jackson.annotation.JsonValue; /** - * 业务类型 + * 滚动模式 */ -public enum AppTypeEnum { +public enum RollingModeEnum { /** - * 普通业务 + * 执行失败则暂停 */ - NORMAL(1), + PAUSE_IF_FAIL(1), /** - * 业务集 + * 忽略失败，自动滚动下一批 */ - APP_SET(2), + IGNORE_ERROR(2), /** - * 全业务 + * 不自动，每批次都人工确认 + */ + MANUAL(3); + + /** + * 滚动模式 */ - ALL_APP(3); @JsonValue - private int type; + private final int mode; - AppTypeEnum(int type) { - this.type = type; + RollingModeEnum(int mode) { + this.mode = mode; } - @JsonCreator(mode = JsonCreator.Mode.DELEGATING) - public static AppTypeEnum valueOf(int type) { - for (AppTypeEnum appType : values()) { - if (appType.type == type) { - return appType; + @JsonCreator + public static RollingModeEnum valOf(int mode) { + for (RollingModeEnum modeEnum : values()) { + if (modeEnum.mode == mode) { + return modeEnum; } } - return null; + throw new IllegalArgumentException("No RollingModeEnum constant: " + mode); } public int getValue() { - return type; + return mode; } } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/ApplicationDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/ApplicationDTO.java index c6923df651..b9e7bdc59a 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/ApplicationDTO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/ApplicationDTO.java @@ -26,13 +26,12 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; -import com.tencent.bk.job.common.annotation.CompatibleImplementation; -import com.tencent.bk.job.common.constant.AppTypeEnum; import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; import lombok.Data; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; +import java.util.Collections; import java.util.List; /** @@ -85,30 +84,6 @@ public class ApplicationDTO { */ private ApplicationAttrsDO attrs; - /** - * 初始运维部门Id - */ - @CompatibleImplementation(explain = "兼容字段，等业务集全部迁移到cmdb之后可以删除", version = "3.6.x") - private Long operateDeptId; - - /** - * cmdb业务/业务集运维 - */ - @CompatibleImplementation(explain = "兼容字段，等业务集全部迁移到cmdb之后可以删除", version = "3.6.x") - private String maintainers; - - /** - * 业务集子业务ID - */ - @CompatibleImplementation(explain = "兼容字段，等业务集全部迁移到cmdb之后可以删除", version = "3.6.x") - private List subBizIds; - - /** - * 业务类型 - */ - @CompatibleImplementation(explain = "兼容字段，等业务集全部迁移到cmdb之后可以删除", version = "3.6.x") - private AppTypeEnum appType; - @JsonIgnore public boolean isBiz() { return scope != null && scope.getType() == ResourceScopeTypeEnum.BIZ; @@ -144,13 +119,10 @@ public Long getBizIdIfBizApp() { } public List getSubBizIds() { - // 业务集已迁移到cmdb if (attrs != null) { - return attrs.getSubBizIds(); + return attrs.getSubBizIds() == null ? Collections.emptyList() : attrs.getSubBizIds(); } else { - // 业务集还未迁移到cmdb，继续使用原来Job业务集配置的子业务;兼容方法，理论上不应该发生调用，此处日志仅用于排查，后续删除 - log.info("Get sub biz ids using field subBizIds"); - return subBizIds; + return Collections.emptyList(); } } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/ApplicationHostDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/ApplicationHostDTO.java index 52c61e12fa..3db525dd7f 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/ApplicationHostDTO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/ApplicationHostDTO.java @@ -111,6 +111,11 @@ public class ApplicationHostDTO { */ private List ipList; + /** + * 主机AgentId + */ + private String agentId; + public String getCloudIp() { if (StringUtils.isNotBlank(cloudIp)) { return cloudIp; diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/HostDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/HostDTO.java index 112c213951..cd32985f29 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/HostDTO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/HostDTO.java @@ -24,43 +24,124 @@ package com.tencent.bk.job.common.model.dto; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.annotation.PersistenceObject; import com.tencent.bk.job.common.model.vo.CloudAreaInfoVO; import com.tencent.bk.job.common.model.vo.HostInfoVO; -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; +import com.tencent.bk.job.common.util.ip.IpUtils; +import lombok.AllArgsConstructor; +import lombok.Getter; import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import org.apache.commons.lang3.StringUtils; + +import java.util.Objects; /** - * 服务器信息 + * 主机通用表示-内部服务使用 */ -@Data +@Setter +@Getter +@AllArgsConstructor @NoArgsConstructor -@ApiModel("服务器信息") -public class HostDTO { +@JsonInclude(JsonInclude.Include.NON_NULL) +@ToString +@PersistenceObject +public class HostDTO implements Cloneable { + /** + * 主机ID + */ + @JsonProperty("hostId") + private Long hostId; + + /** + * 主机 Agent ID + */ + @JsonProperty("agentId") + private String agentId; + /** * 云区域ID */ - @ApiModelProperty(value = "云区域ID", required = true) - private Long cloudAreaId; + @JsonProperty("cloudAreaId") + private Long bkCloudId; - @ApiModelProperty("云区域名称") - private String cloudAreaName; + /** + * 云区域名称 + */ + @JsonProperty("bkCloudName") + private String bkCloudName; /** - * 服务器IP + * 主机IP - ipv4 */ - @ApiModelProperty(value = "服务器IP", required = true) + @JsonProperty("ip") private String ip; - @ApiModelProperty(value = "agent状态，0-异常，1-正常") + /** + * 主机显示IP + */ + @JsonProperty("displayIp") + private String displayIp; + + /** + * 主机IP - ipv6 + */ + @JsonProperty("ipv6") + private String ipv6; + + /** + * agent状态，0-异常，1-正常 + */ + @JsonInclude(JsonInclude.Include.NON_NULL) private Integer alive; - public HostDTO(Long cloudAreaId, String ip) { - this.cloudAreaId = cloudAreaId; + public HostDTO(Long bkCloudId, String ip) { + this.bkCloudId = bkCloudId; this.ip = ip; } + public static HostDTO fromHostId(Long hostId) { + HostDTO hostDTO = new HostDTO(); + hostDTO.setHostId(hostId); + return hostDTO; + } + + public static HostDTO fromHostIdAndCloudIp(Long hostId, String cloudIp) { + HostDTO hostDTO = new HostDTO(); + hostDTO.setHostId(hostId); + if (StringUtils.isNotBlank(cloudIp)) { + String[] ipProps = cloudIp.split(IpUtils.COLON); + hostDTO.setBkCloudId(Long.valueOf(ipProps[0])); + hostDTO.setIp(ipProps[1]); + } + + return hostDTO; + } + + public static HostDTO fromCloudIp(String cloudIp) { + if (!IpUtils.checkCloudIp(cloudIp)) { + throw new IllegalArgumentException("Invalid cloudIp : " + cloudIp); + } + String[] ipProps = cloudIp.split(IpUtils.COLON); + return new HostDTO(Long.valueOf(ipProps[0]), ipProps[1]); + } + + public String toCloudIp() { + return bkCloudId + ":" + ip; + } + + public String getDisplayIp() { + if (StringUtils.isNotEmpty(displayIp)) { + return displayIp; + } else { + return ip; + } + } + public static HostInfoVO toVO(HostDTO host) { if (host == null) { return null; @@ -69,8 +150,8 @@ public static HostInfoVO toVO(HostDTO host) { hostInfo.setIp(host.getIp()); hostInfo.setAlive(host.getAlive()); CloudAreaInfoVO cloudAreaInfo = new CloudAreaInfoVO(); - cloudAreaInfo.setId(host.getCloudAreaId()); - cloudAreaInfo.setName(host.getCloudAreaName()); + cloudAreaInfo.setId(host.getBkCloudId()); + cloudAreaInfo.setName(host.getBkCloudName()); hostInfo.setCloudAreaInfo(cloudAreaInfo); return hostInfo; } @@ -81,9 +162,73 @@ public static HostDTO fromVO(HostInfoVO hostInfo) { } HostDTO host = new HostDTO(); host.setIp(hostInfo.getIp()); - host.setCloudAreaId(hostInfo.getCloudAreaInfo().getId()); - host.setCloudAreaName(hostInfo.getCloudAreaInfo().getName()); + host.setBkCloudId(hostInfo.getCloudAreaInfo().getId()); + host.setBkCloudName(hostInfo.getCloudAreaInfo().getName()); host.setAlive(hostInfo.getAlive()); return host; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + HostDTO hostDTO = (HostDTO) o; + if (hostId != null) { + return hostId.equals(hostDTO.getHostId()); + } else { + // 兼容没有hostId的场景 + return bkCloudId.equals(hostDTO.bkCloudId) && + ip.equals(hostDTO.ip); + } + } + + @Override + public int hashCode() { + if (hostId != null) { + return Objects.hashCode(hostId); + } else { + // 兼容没有hostId的场景 + return Objects.hash(bkCloudId, ip); + } + } + + public HostDTO clone() { + HostDTO clone = new HostDTO(); + clone.setHostId(hostId); + clone.setAgentId(agentId); + clone.setBkCloudId(bkCloudId); + clone.setBkCloudName(bkCloudName); + clone.setIp(ip); + clone.setIpv6(ipv6); + clone.setAlive(alive); + return clone; + } + + /** + * 获取最终的agentId，若agentId不存在，则使用cloudIp作为agentId + * + * @return 最终的agentId + */ + @JsonIgnore + public String getFinalAgentId() { + if (StringUtils.isNotBlank(agentId)) { + return agentId; + } else { + return toCloudIp(); + } + } + + /** + * 获取主机的唯一KEY，用于去重等操作 + * + * @return 主机KEY + */ + @JsonIgnore + public String getUniqueKey() { + if (hostId != null) { + return "HOST_ID:" + hostId; + } else { + return "HOST_IP:" + toCloudIp(); + } + } } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/IpDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/IpDTO.java deleted file mode 100644 index e00296b43d..0000000000 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/IpDTO.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.model.dto; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.tencent.bk.job.common.util.ip.IpUtils; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.ToString; - -import java.util.Objects; - -/** - * 主机IP通用表示-内部服务使用 - */ -@Setter -@Getter -@AllArgsConstructor -@NoArgsConstructor -@JsonInclude(JsonInclude.Include.NON_NULL) -@ToString -public class IpDTO implements Cloneable { - @JsonProperty("cloudAreaId") - private Long cloudAreaId; - - @JsonProperty("ip") - private String ip; - - /** - * agent状态，0-异常，1-正常 - */ - @JsonInclude(JsonInclude.Include.NON_NULL) - private Integer alive; - - public IpDTO(Long cloudAreaId, String ip) { - this.cloudAreaId = cloudAreaId; - this.ip = ip; - } - - public static IpDTO fromCloudAreaIdAndIpStr(String cloudAreaIdAndIpStr) { - if (!IpUtils.checkCloudAreaIdAndIpStr(cloudAreaIdAndIpStr)) { - return null; - } - String[] ipProps = cloudAreaIdAndIpStr.split(IpUtils.COLON); - return new IpDTO(Long.valueOf(ipProps[0]), ipProps[1]); - } - - public static HostDTO toHost(IpDTO ip) { - if (ip == null) { - return null; - } - HostDTO host = new HostDTO(); - host.setCloudAreaId(ip.getCloudAreaId()); - host.setIp(ip.getIp()); - return host; - } - - public String convertToStrIp() { - return cloudAreaId + ":" + ip; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - IpDTO ipDTO = (IpDTO) o; - return cloudAreaId.equals(ipDTO.cloudAreaId) && - ip.equals(ipDTO.ip); - } - - @Override - public int hashCode() { - return Objects.hash(cloudAreaId, ip); - } - - public IpDTO clone() { - IpDTO clone = new IpDTO(this.cloudAreaId, this.ip); - clone.setAlive(alive); - return clone; - } -} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/ResourceScope.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/ResourceScope.java index c94d460109..82fa4f5860 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/ResourceScope.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/ResourceScope.java @@ -45,7 +45,7 @@ public class ResourceScope { /** * 资源范围类型 */ - @ApiModelProperty(value = "枚举，资源范围类型(取值为biz/biz_set)", required = true) + @ApiModelProperty(value = "枚举，资源范围类型", required = true) private ResourceScopeTypeEnum type; /** * 资源范围ID,比如cmdb业务ID、cmdb业务集ID diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/ip/IpUtils.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/ip/IpUtils.java index 542f332509..93b75bd997 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/ip/IpUtils.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/ip/IpUtils.java @@ -24,18 +24,18 @@ package com.tencent.bk.job.common.util.ip; -import com.google.common.collect.Lists; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; -import org.springframework.util.CollectionUtils; import java.net.Inet4Address; import java.net.InetAddress; import java.net.NetworkInterface; import java.net.UnknownHostException; -import java.util.*; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -45,35 +45,36 @@ @Slf4j public class IpUtils { public static final String COLON = ":"; - public static final String SEMICOLON = ","; - /* + + /** * 直连云区域 */ public static final long DEFAULT_CLOUD_ID = 0; private static final Pattern IP_PATTERN = Pattern.compile( - "\\b((?!\\d\\d\\d)\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\.((?!\\d\\d\\d)\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\.((?!\\d\\d\\d)\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\.((?!\\d\\d\\d)\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\b"); + "\\b((?!\\d\\d\\d)\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\.((?!\\d\\d\\d)\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\.(" + + "(?!\\d\\d\\d)\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\.((?!\\d\\d\\d)\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\b"); private static final Pattern pattern = Pattern.compile( - "\\b((?!\\d\\d\\d)\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\.((?!\\d\\d\\d)\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\.((?!\\d\\d\\d)\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\.((?!\\d\\d\\d)\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\b"); + "\\b((?!\\d\\d\\d)\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\.((?!\\d\\d\\d)\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\.(" + + "(?!\\d\\d\\d)\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\.((?!\\d\\d\\d)\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\b"); /** * 校验云区域:服务器IP * - * @param cloudAreaIdAndIp 云区域ID:服务器IP - * @return + * @param cloudIp 云区域ID:服务器IP */ - public static boolean checkCloudAreaIdAndIpStr(String cloudAreaIdAndIp) { - if (StringUtils.isEmpty(cloudAreaIdAndIp)) { + public static boolean checkCloudIp(String cloudIp) { + if (StringUtils.isEmpty(cloudIp)) { return false; } - String[] ipProps = cloudAreaIdAndIp.split(":"); + String[] ipProps = cloudIp.split(":"); if (ipProps.length != 2) { - log.warn("Both cloudAreaId and ip is required, ip={}", cloudAreaIdAndIp); + log.warn("Both cloudAreaId and ip is required, ip={}", cloudIp); return false; } try { Integer.parseInt(ipProps[0]); } catch (NumberFormatException e) { - log.warn("CloudAreaId is illegal for ip:{}", cloudAreaIdAndIp); + log.warn("CloudAreaId is illegal for ip:{}", cloudIp); return false; } String ip = ipProps[1]; @@ -81,26 +82,10 @@ public static boolean checkCloudAreaIdAndIpStr(String cloudAreaIdAndIp) { return matcher.matches(); } - /** - * 校验ip - * - * @param ipDTO IP - * @return - */ - public static boolean checkIpDTO(IpDTO ipDTO) { - if (ipDTO == null || ipDTO.getCloudAreaId() == null || StringUtils.isEmpty(ipDTO.getIp().trim())) { - log.warn("Both cloudAreaId and ip is required, ip={}", ipDTO); - return false; - } - Matcher matcher = IP_PATTERN.matcher(ipDTO.getIp().trim()); - return matcher.matches(); - } - /** * 验证ip格式（不包含云区域ID) * * @param ipStr ip - * @return */ public static boolean checkIp(String ipStr) { if (StringUtils.isEmpty(ipStr.trim())) { @@ -110,58 +95,25 @@ public static boolean checkIp(String ipStr) { return matcher.matches(); } - /** - * 转换ip列表(1:127.0.0.1,1:127.0.0.2)到 IpDTO 列表 - * - * @param ipListStr - * @return - */ - public static Collection convertCloudAndIpListStrToIpDTOList(String ipListStr) { - if (StringUtils.isEmpty(ipListStr)) { - return Lists.newArrayList(); - } - return convertCloudAndIpStrListToIpDTOList(Arrays.asList(ipListStr.split(SEMICOLON))); - } - - /** - * 转换ip列表到IpDTO 列表 - * - * @param cloudAreaAndIpStrList ip列表 - * @return IpDTO列表 - */ - public static Collection convertCloudAndIpStrListToIpDTOList(Collection cloudAreaAndIpStrList) { - List ipDTOList = new ArrayList<>(); - if (!CollectionUtils.isEmpty(cloudAreaAndIpStrList)) { - for (String cloudAreaAndIp : cloudAreaAndIpStrList) { - IpDTO ipDto = transform(cloudAreaAndIp); - if (ipDto != null) { - ipDTOList.add(ipDto); - } - } - } - return ipDTOList; - } - /** * 转换到IpDTO * - * @param cloudAreaAndIp - * @return + * @param cloudIp 云区域+IP */ - public static IpDTO transform(String cloudAreaAndIp) { - IpDTO ipDTO = null; - if (cloudAreaAndIp != null) { - String[] split = cloudAreaAndIp.split(COLON); + public static HostDTO transform(String cloudIp) { + HostDTO hostDTO = null; + if (cloudIp != null) { + String[] split = cloudIp.split(COLON); if (split.length == 2) { long cloudAreaId = optLong(split[0].trim(), DEFAULT_CLOUD_ID); if (cloudAreaId == 1) cloudAreaId = DEFAULT_CLOUD_ID; - ipDTO = new IpDTO(cloudAreaId, split[1].trim()); + hostDTO = new HostDTO(cloudAreaId, split[1].trim()); } else { - ipDTO = new IpDTO(DEFAULT_CLOUD_ID, cloudAreaAndIp.trim()); + hostDTO = new HostDTO(DEFAULT_CLOUD_ID, cloudIp.trim()); } } - return ipDTO; + return hostDTO; } private static long optLong(String str, long defaultValue) { diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/EsbIpDTO.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/EsbIpDTO.java index b4644cfdd0..8ed00513a3 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/EsbIpDTO.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/EsbIpDTO.java @@ -46,7 +46,7 @@ public class EsbIpDTO { @JsonProperty("bk_cloud_id") @NotNull(message = "{validation.constraints.InvalidBkCloudId.message}") @Min(value = 0L, message = "{validation.constraints.InvalidBkCloudId.message}") - private Long cloudAreaId; + private Long bkCloudId; @JsonProperty("ip") @Pattern(regexp = "\\b((?!\\d\\d\\d)\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\.((?!\\d\\d\\d)" + @@ -60,7 +60,7 @@ public static EsbIpDTO fromApplicationHostInfo(ApplicationHostDTO applicationHos return null; } EsbIpDTO esbIp = new EsbIpDTO(); - esbIp.setCloudAreaId(applicationHostInfo.getCloudAreaId()); + esbIp.setBkCloudId(applicationHostInfo.getCloudAreaId()); esbIp.setIp(applicationHostInfo.getIp()); return esbIp; } @@ -70,13 +70,13 @@ public static EsbIpDTO fromHost(HostDTO host) { return null; } EsbIpDTO esbIp = new EsbIpDTO(); - esbIp.setCloudAreaId(host.getCloudAreaId()); + esbIp.setBkCloudId(host.getBkCloudId()); esbIp.setIp(host.getIp()); return esbIp; } public static EsbIpDTO fromCloudIp(String cloudIp) { - if (!IpUtils.checkCloudAreaIdAndIpStr(cloudIp)) { + if (!IpUtils.checkCloudIp(cloudIp)) { return null; } String[] ipProps = cloudIp.split(IpUtils.COLON); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/Utils.java b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/exception/PaasException.java similarity index 73% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/Utils.java rename to src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/exception/PaasException.java index 688d5ba232..e8f343c943 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/Utils.java +++ b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/exception/PaasException.java @@ -22,22 +22,19 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.execute.engine.util; +package com.tencent.bk.job.common.paas.exception; -import com.tencent.bk.job.execute.engine.consts.Consts; +import com.tencent.bk.job.common.exception.ServiceException; +import com.tencent.bk.job.common.model.error.ErrorType; -public class Utils { - /** - * 根据gse ErrorCode返回 Status - */ - public static int getStatusByGseErrorCode(int gseErrorCode) { - Integer status = Consts.GSE_ERROR_CODE_2_STATUS_MAP.get(gseErrorCode); - if (status == null) { - if (gseErrorCode > 0) - status = -1; - else - status = 9; - } - return status; +public class PaasException extends ServiceException { + + public PaasException(ErrorType errorType, Integer errorCode, Object[] errorParams) { + super(errorType, errorCode, errorParams); + } + + public PaasException(Throwable cause, ErrorType errorType, Integer errorCode, Object[] errorParams) { + super(cause, errorType, errorCode, errorParams); } + } diff --git a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/user/EEPaasClient.java b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/user/EEPaasClient.java index e575664f0c..b848c0f122 100644 --- a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/user/EEPaasClient.java +++ b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/user/EEPaasClient.java @@ -33,6 +33,8 @@ import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.dto.BkUserDTO; +import com.tencent.bk.job.common.model.error.ErrorType; +import com.tencent.bk.job.common.paas.exception.PaasException; import com.tencent.bk.job.common.paas.metrics.PaaSMetricTags; import com.tencent.bk.job.common.paas.model.EsbListUsersResult; import com.tencent.bk.job.common.paas.model.EsbNotifyChannelDTO; @@ -113,7 +115,7 @@ public List getUserList(String fields, } catch (Exception e) { String errorMsg = "Get " + API_GET_USER_LIST + " error"; log.error(errorMsg, e); - throw new InternalException(errorMsg, e, ErrorCode.PAAS_API_DATA_ERROR); + throw new InternalException(errorMsg, e, ErrorCode.USER_MANAGE_API_ACCESS_ERROR); } finally { HttpMetricUtil.clearHttpMetric(); } @@ -170,7 +172,7 @@ public List getNotifyChannelList(String uin) { } @Override - public boolean sendMsg( + public void sendMsg( String msgType, String sender, Set receivers, @@ -194,14 +196,21 @@ public boolean sendMsg( if (esbResp.getResult() == null || !esbResp.getResult() || esbResp.getCode() != 0) { status = checkRespAndGetStatus(uri, esbResp); - return false; + throw new PaasException( + ErrorType.INTERNAL, + ErrorCode.CMSI_FAIL_TO_SEND_MSG, + new Object[]{ + esbResp.getCode().toString(), + esbResp.getMessage() + }); } status = EsbMetricTags.VALUE_STATUS_SUCCESS; - return true; + } catch (PaasException e) { + throw e; } catch (Exception e) { log.error("Fail to request {}", uri, e); status = EsbMetricTags.VALUE_STATUS_ERROR; - return false; + throw new PaasException(e, ErrorType.INTERNAL, ErrorCode.CMSI_API_ACCESS_ERROR, new Object[]{}); } finally { HttpMetricUtil.clearHttpMetric(); recordMetrics(start, status, msgType); diff --git a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/user/IPaasClient.java b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/user/IPaasClient.java index 0f6509343c..b662396711 100644 --- a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/user/IPaasClient.java +++ b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/user/IPaasClient.java @@ -54,6 +54,5 @@ public interface IPaasClient { /** * ESB发通知信息接口 */ - boolean sendMsg(String msgType, String sender, Set receivers, String title, String content) - throws Exception; + void sendMsg(String msgType, String sender, Set receivers, String title, String content); } diff --git a/src/backend/job-analysis/boot-job-analysis/src/main/resources/application.yml b/src/backend/job-analysis/boot-job-analysis/src/main/resources/application.yml index 8453591469..c11c61c74c 100644 --- a/src/backend/job-analysis/boot-job-analysis/src/main/resources/application.yml +++ b/src/backend/job-analysis/boot-job-analysis/src/main/resources/application.yml @@ -1,6 +1,4 @@ spring: - application: - name: job-analysis profiles: active: prod diff --git a/src/backend/job-analysis/boot-job-analysis/src/main/resources/bootstrap.yml b/src/backend/job-analysis/boot-job-analysis/src/main/resources/bootstrap.yml index 77bd21fa13..687925871c 100644 --- a/src/backend/job-analysis/boot-job-analysis/src/main/resources/bootstrap.yml +++ b/src/backend/job-analysis/boot-job-analysis/src/main/resources/bootstrap.yml @@ -1,4 +1,6 @@ spring: + application: + name: job-analysis cloud: kubernetes: config: diff --git a/src/backend/job-analysis/boot-job-analysis/src/main/resources/logback/logback-app-props.xml b/src/backend/job-analysis/boot-job-analysis/src/main/resources/logback/logback-app-props.xml index 4a0cb3fca0..2ec806f590 100644 --- a/src/backend/job-analysis/boot-job-analysis/src/main/resources/logback/logback-app-props.xml +++ b/src/backend/job-analysis/boot-job-analysis/src/main/resources/logback/logback-app-props.xml @@ -1,4 +1,5 @@ + diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/impl/ForbiddenScriptFinder.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/impl/ForbiddenScriptFinder.java index 0a7885695b..a457883c2b 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/impl/ForbiddenScriptFinder.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/impl/ForbiddenScriptFinder.java @@ -102,7 +102,7 @@ private void findForbiddenScriptFromTaskTemplateStep( List badTplPlanInfoList ) { Long appId = taskTemplateInfoDTO.getAppId(); - if (taskStepDTO.getType() == TaskStepTypeEnum.SCRIPT.getType()) { + if (taskStepDTO.getType() == TaskStepTypeEnum.SCRIPT.getValue()) { if (taskStepDTO.getScriptStepInfo().getScriptSource() == TaskScriptSourceEnum.CITING.getType()) { ServiceScriptDTO scriptVersion = null; @@ -155,7 +155,7 @@ private void findForbiddenScriptFromTaskPlanStep( List badTplPlanInfoList ) { Long appId = taskTemplateInfoDTO.getAppId(); - if (taskStepDTO.getType() == TaskStepTypeEnum.SCRIPT.getType()) { + if (taskStepDTO.getType() == TaskStepTypeEnum.SCRIPT.getValue()) { if (taskStepDTO.getScriptStepInfo().getScriptSource() == TaskScriptSourceEnum.CITING.getType()) { ServiceScriptDTO scriptVersion = null; diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/impl/TaskPlanTargetChecker.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/impl/TaskPlanTargetChecker.java index a9d1a399d7..d7a0cda730 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/impl/TaskPlanTargetChecker.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/impl/TaskPlanTargetChecker.java @@ -178,10 +178,10 @@ public void run() { //检查步骤 for (ServiceTaskStepDTO taskStepDTO : taskPlanInfoDTO.getStepList()) { ServiceTaskHostNodeDTO taskHostNodeDTO = null; - if (taskStepDTO.getType() == TaskStepTypeEnum.SCRIPT.getType()) { + if (taskStepDTO.getType() == TaskStepTypeEnum.SCRIPT.getValue()) { taskHostNodeDTO = taskStepDTO.getScriptStepInfo().getExecuteTarget().getTargetServer(); - } else if (taskStepDTO.getType() == TaskStepTypeEnum.FILE.getType()) { + } else if (taskStepDTO.getType() == TaskStepTypeEnum.FILE.getValue()) { taskHostNodeDTO = taskStepDTO .getFileStepInfo() .getExecuteTarget() diff --git a/src/backend/job-backup/boot-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchivistAutoConfig.java b/src/backend/job-backup/boot-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchivistAutoConfig.java index 241c89fd25..d3657bbf80 100644 --- a/src/backend/job-backup/boot-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchivistAutoConfig.java +++ b/src/backend/job-backup/boot-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchivistAutoConfig.java @@ -26,9 +26,23 @@ import com.tencent.bk.job.backup.archive.JobExecuteArchiveManage; import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.JobExecuteDAO; import com.tencent.bk.job.backup.dao.impl.ExecuteArchiveDAOImpl; -import com.tencent.bk.job.backup.dao.impl.JobExecuteDAOImpl; +import com.tencent.bk.job.backup.dao.impl.FileSourceTaskRecordDAO; +import com.tencent.bk.job.backup.dao.impl.GseFileAgentTaskRecordDAO; +import com.tencent.bk.job.backup.dao.impl.GseScriptAgentTaskRecordDAO; +import com.tencent.bk.job.backup.dao.impl.GseTaskIpLogRecordDAO; +import com.tencent.bk.job.backup.dao.impl.GseTaskLogRecordDAO; +import com.tencent.bk.job.backup.dao.impl.GseTaskRecordDAO; +import com.tencent.bk.job.backup.dao.impl.OperationLogRecordDAO; +import com.tencent.bk.job.backup.dao.impl.RollingConfigRecordDAO; +import com.tencent.bk.job.backup.dao.impl.StepInstanceConfirmRecordDAO; +import com.tencent.bk.job.backup.dao.impl.StepInstanceFileRecordDAO; +import com.tencent.bk.job.backup.dao.impl.StepInstanceRecordDAO; +import com.tencent.bk.job.backup.dao.impl.StepInstanceRollingTaskRecordDAO; +import com.tencent.bk.job.backup.dao.impl.StepInstanceScriptRecordDAO; +import com.tencent.bk.job.backup.dao.impl.StepInstanceVariableRecordDAO; +import com.tencent.bk.job.backup.dao.impl.TaskInstanceRecordDAO; +import com.tencent.bk.job.backup.dao.impl.TaskInstanceVariableRecordDAO; import com.tencent.bk.job.backup.service.ArchiveProgressService; import lombok.extern.slf4j.Slf4j; import org.jooq.DSLContext; @@ -44,12 +58,135 @@ @Slf4j public class ArchivistAutoConfig { - @Bean(name = "execute-read-dao") @ConditionalOnExpression("${job.execute.archive.enabled:false} || ${job.execute.archive.delete.enabled:false}") - public JobExecuteDAO executeReadDAO(@Qualifier("job-execute-dsl-context") DSLContext context, - ArchiveConfig archiveConfig) { - log.info("Init JobExecuteDAO"); - return new JobExecuteDAOImpl(context, archiveConfig); + public static class ExecuteDaoAutoConfig { + + @Bean(name = "taskInstanceRecordDAO") + public TaskInstanceRecordDAO taskInstanceRecordDAO(@Qualifier("job-execute-dsl-context") DSLContext context, + ArchiveConfig archiveConfig) { + log.info("Init TaskInstanceRecordDAO"); + return new TaskInstanceRecordDAO(context, archiveConfig); + } + + @Bean(name = "stepInstanceRecordDAO") + public StepInstanceRecordDAO stepInstanceRecordDAO(@Qualifier("job-execute-dsl-context") DSLContext context, + ArchiveConfig archiveConfig) { + log.info("Init StepInstanceRecordDAO"); + return new StepInstanceRecordDAO(context, archiveConfig); + } + + @Bean(name = "stepInstanceScriptRecordDAO") + public StepInstanceScriptRecordDAO stepInstanceScriptRecordDAO( + @Qualifier("job-execute-dsl-context") DSLContext context, + ArchiveConfig archiveConfig) { + log.info("Init StepInstanceScriptRecordDAO"); + return new StepInstanceScriptRecordDAO(context, archiveConfig); + } + + @Bean(name = "stepInstanceFileRecordDAO") + public StepInstanceFileRecordDAO stepInstanceFileRecordDAO( + @Qualifier("job-execute-dsl-context") DSLContext context, + ArchiveConfig archiveConfig) { + log.info("Init StepInstanceFileRecordDAO"); + return new StepInstanceFileRecordDAO(context, archiveConfig); + } + + @Bean(name = "stepInstanceConfirmRecordDAO") + public StepInstanceConfirmRecordDAO stepInstanceConfirmRecordDAO( + @Qualifier("job-execute-dsl-context") DSLContext context, + ArchiveConfig archiveConfig) { + log.info("Init StepInstanceConfirmRecordDAO"); + return new StepInstanceConfirmRecordDAO(context, archiveConfig); + } + + @Bean(name = "stepInstanceVariableRecordDAO") + public StepInstanceVariableRecordDAO stepInstanceVariableRecordDAO( + @Qualifier("job-execute-dsl-context") DSLContext context, + ArchiveConfig archiveConfig) { + log.info("Init StepInstanceVariableRecordDAO"); + return new StepInstanceVariableRecordDAO(context, archiveConfig); + } + + @Bean(name = "taskInstanceVariableRecordDAO") + public TaskInstanceVariableRecordDAO taskInstanceVariableRecordDAO( + @Qualifier("job-execute-dsl-context") DSLContext context, + ArchiveConfig archiveConfig) { + log.info("Init TaskInstanceVariableRecordDAO"); + return new TaskInstanceVariableRecordDAO(context, archiveConfig); + } + + @Bean(name = "operationLogRecordDAO") + public OperationLogRecordDAO operationLogRecordDAO( + @Qualifier("job-execute-dsl-context") DSLContext context, + ArchiveConfig archiveConfig) { + log.info("Init OperationLogRecordDAO"); + return new OperationLogRecordDAO(context, archiveConfig); + } + + @Bean(name = "gseTaskLogRecordDAO") + public GseTaskLogRecordDAO gseTaskLogRecordDAO( + @Qualifier("job-execute-dsl-context") DSLContext context, + ArchiveConfig archiveConfig) { + log.info("Init GseTaskLogRecordDAO"); + return new GseTaskLogRecordDAO(context, archiveConfig); + } + + @Bean(name = "gseTaskIpLogRecordDAO") + public GseTaskIpLogRecordDAO gseTaskIpLogRecordDAO( + @Qualifier("job-execute-dsl-context") DSLContext context, + ArchiveConfig archiveConfig) { + log.info("Init GseTaskIpLogRecordDAO"); + return new GseTaskIpLogRecordDAO(context, archiveConfig); + } + + @Bean(name = "fileSourceTaskRecordDAO") + public FileSourceTaskRecordDAO fileSourceTaskRecordDAO( + @Qualifier("job-execute-dsl-context") DSLContext context, + ArchiveConfig archiveConfig) { + log.info("Init FileSourceTaskRecordDAO"); + return new FileSourceTaskRecordDAO(context, archiveConfig); + } + + @Bean(name = "gseTaskRecordDAO") + public GseTaskRecordDAO gseTaskRecordDAO( + @Qualifier("job-execute-dsl-context") DSLContext context, + ArchiveConfig archiveConfig) { + log.info("Init GseTaskRecordDAO"); + return new GseTaskRecordDAO(context, archiveConfig); + } + + @Bean(name = "gseScriptAgentTaskRecordDAO") + public GseScriptAgentTaskRecordDAO gseScriptAgentTaskRecordDAO( + @Qualifier("job-execute-dsl-context") DSLContext context, + ArchiveConfig archiveConfig) { + log.info("Init GseScriptAgentTaskRecordDAO"); + return new GseScriptAgentTaskRecordDAO(context, archiveConfig); + } + + @Bean(name = "gseFileAgentTaskRecordDAO") + public GseFileAgentTaskRecordDAO gseFileAgentTaskRecordDAO( + @Qualifier("job-execute-dsl-context") DSLContext context, + ArchiveConfig archiveConfig) { + log.info("Init GseFileAgentTaskRecordDAO"); + return new GseFileAgentTaskRecordDAO(context, archiveConfig); + } + + @Bean(name = "stepInstanceRollingTaskRecordDAO") + public StepInstanceRollingTaskRecordDAO stepInstanceRollingTaskRecordDAO( + @Qualifier("job-execute-dsl-context") DSLContext context, + ArchiveConfig archiveConfig) { + log.info("Init StepInstanceRollingTaskRecordDAO"); + return new StepInstanceRollingTaskRecordDAO(context, archiveConfig); + } + + @Bean(name = "rollingConfigRecordDAO") + public RollingConfigRecordDAO rollingConfigRecordDAO( + @Qualifier("job-execute-dsl-context") DSLContext context, + ArchiveConfig archiveConfig) { + log.info("Init RollingConfigRecordDAO"); + return new RollingConfigRecordDAO(context, archiveConfig); + } + } @Bean(name = "execute-archive-dao") @@ -61,11 +198,48 @@ public ExecuteArchiveDAO executeArchiveDAO(@Qualifier("job-execute-archive-dsl-c @Bean @ConditionalOnExpression("${job.execute.archive.enabled:false} || ${job.execute.archive.delete.enabled:false}") - public JobExecuteArchiveManage archiveExecuteLogExecutor(@Autowired(required = false) JobExecuteDAO jobExecuteDAO, - @Autowired(required = false) ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService, - ArchiveConfig archiveConfig) { + public JobExecuteArchiveManage jobExecuteArchiveManage( + @Autowired(required = false) TaskInstanceRecordDAO taskInstanceRecordDAO, + @Autowired(required = false) StepInstanceRecordDAO stepInstanceRecordDAO, + @Autowired(required = false) StepInstanceScriptRecordDAO stepInstanceScriptRecordDAO, + @Autowired(required = false) StepInstanceFileRecordDAO stepInstanceFileRecordDAO, + @Autowired(required = false) StepInstanceConfirmRecordDAO stepInstanceConfirmRecordDAO, + @Autowired(required = false) StepInstanceVariableRecordDAO stepInstanceVariableRecordDAO, + @Autowired(required = false) TaskInstanceVariableRecordDAO taskInstanceVariableRecordDAO, + @Autowired(required = false) OperationLogRecordDAO operationLogRecordDAO, + @Autowired(required = false) GseTaskLogRecordDAO gseTaskLogRecordDAO, + @Autowired(required = false) GseTaskIpLogRecordDAO gseTaskIpLogRecordDAO, + @Autowired(required = false) FileSourceTaskRecordDAO fileSourceTaskRecordDAO, + @Autowired(required = false) GseTaskRecordDAO gseTaskRecordDAO, + @Autowired(required = false) GseScriptAgentTaskRecordDAO gseScriptAgentTaskRecordDAO, + @Autowired(required = false) GseFileAgentTaskRecordDAO gseFileAgentTaskRecordDAO, + @Autowired(required = false) StepInstanceRollingTaskRecordDAO stepInstanceRollingTaskRecordDAO, + @Autowired(required = false) RollingConfigRecordDAO rollingConfigRecordDAO, + @Autowired(required = false) ExecuteArchiveDAO executeArchiveDAO, + ArchiveProgressService archiveProgressService, + ArchiveConfig archiveConfig) { + log.info("Init JobExecuteArchiveManage"); - return new JobExecuteArchiveManage(jobExecuteDAO, executeArchiveDAO, archiveProgressService, archiveConfig); + return new JobExecuteArchiveManage( + taskInstanceRecordDAO, + stepInstanceRecordDAO, + stepInstanceScriptRecordDAO, + stepInstanceFileRecordDAO, + stepInstanceConfirmRecordDAO, + stepInstanceVariableRecordDAO, + taskInstanceVariableRecordDAO, + operationLogRecordDAO, + gseTaskLogRecordDAO, + gseTaskIpLogRecordDAO, + fileSourceTaskRecordDAO, + gseTaskRecordDAO, + gseScriptAgentTaskRecordDAO, + gseFileAgentTaskRecordDAO, + stepInstanceRollingTaskRecordDAO, + rollingConfigRecordDAO, + executeArchiveDAO, + archiveProgressService, + archiveConfig + ); } } diff --git a/src/backend/job-backup/boot-job-backup/src/main/resources/application.yml b/src/backend/job-backup/boot-job-backup/src/main/resources/application.yml index f187ed6c87..c678b72aad 100644 --- a/src/backend/job-backup/boot-job-backup/src/main/resources/application.yml +++ b/src/backend/job-backup/boot-job-backup/src/main/resources/application.yml @@ -1,6 +1,4 @@ spring: - application: - name: job-backup profiles: active: prod ribbon: diff --git a/src/backend/job-backup/boot-job-backup/src/main/resources/bootstrap.yml b/src/backend/job-backup/boot-job-backup/src/main/resources/bootstrap.yml index 77bd21fa13..b134b23ecd 100644 --- a/src/backend/job-backup/boot-job-backup/src/main/resources/bootstrap.yml +++ b/src/backend/job-backup/boot-job-backup/src/main/resources/bootstrap.yml @@ -1,4 +1,6 @@ spring: + application: + name: job-backup cloud: kubernetes: config: diff --git a/src/backend/job-backup/boot-job-backup/src/main/resources/logback/logback-app-props.xml b/src/backend/job-backup/boot-job-backup/src/main/resources/logback/logback-app-props.xml index d8dab11a05..9791b58f20 100644 --- a/src/backend/job-backup/boot-job-backup/src/main/resources/logback/logback-app-props.xml +++ b/src/backend/job-backup/boot-job-backup/src/main/resources/logback/logback-app-props.xml @@ -1,4 +1,5 @@ + diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/api/inner/impl/ServiceArchiveResourceImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/api/inner/impl/ServiceArchiveResourceImpl.java index 2f4392179c..2b78021fe0 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/api/inner/impl/ServiceArchiveResourceImpl.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/api/inner/impl/ServiceArchiveResourceImpl.java @@ -37,7 +37,7 @@ @Slf4j public class ServiceArchiveResourceImpl implements ServiceArchiveResource { - private JobExecuteArchiveManage jobExecuteArchiveManage; + private final JobExecuteArchiveManage jobExecuteArchiveManage; public ServiceArchiveResourceImpl(@Autowired(required = false) JobExecuteArchiveManage jobExecuteArchiveManage) { this.jobExecuteArchiveManage = jobExecuteArchiveManage; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractArchivist.java index 89f586d1b5..64c99339c6 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractArchivist.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractArchivist.java @@ -26,7 +26,7 @@ import com.tencent.bk.job.backup.config.ArchiveConfig; import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.JobExecuteDAO; +import com.tencent.bk.job.backup.dao.ExecuteRecordDAO; import com.tencent.bk.job.backup.model.dto.ArchiveProgressDTO; import com.tencent.bk.job.backup.model.dto.ArchiveSummary; import com.tencent.bk.job.backup.service.ArchiveProgressService; @@ -40,10 +40,15 @@ import java.util.List; import java.util.concurrent.CountDownLatch; +/** + * 表归档基础实现 + * + * @param 表记录 + */ @Data @Slf4j public abstract class AbstractArchivist> { - protected JobExecuteDAO jobExecuteDAO; + protected ExecuteRecordDAO executeRecordDAO; protected ExecuteArchiveDAO executeArchiveDAO; protected ArchiveProgressService archiveProgressService; /** @@ -61,19 +66,14 @@ public abstract class AbstractArchivist> { protected String tableName; private ArchiveSummary archiveSummary; - /** - * 查询(start,stop]内的数据 - * @param start 不含start本身 - * @param stop 含stop在内 - * @return - */ - protected abstract List listRecord(Long start, Long stop); - - protected abstract int batchInsert(List recordList) throws IOException; - - protected abstract int deleteRecord(Long start, Long stop); - - protected abstract long getFirstInstanceId(); + public AbstractArchivist(ExecuteRecordDAO executeRecordDAO, + ExecuteArchiveDAO executeArchiveDAO, + ArchiveProgressService archiveProgressService) { + this.executeRecordDAO = executeRecordDAO; + this.executeArchiveDAO = executeArchiveDAO; + this.archiveProgressService = archiveProgressService; + this.tableName = executeRecordDAO.getTable().getName().toLowerCase(); + } public void archive(ArchiveConfig archiveConfig, Long maxNeedArchiveId, CountDownLatch countDownLatch) { boolean isAcquireLock = false; @@ -205,7 +205,6 @@ private boolean archive(Long maxNeedArchiveId) { /** * 获取上一次已归档完成的最后一个数据Id，后续用大于该Id选取范围数据 - * @return */ private long getLastArchivedId() { ArchiveProgressDTO archiveProgress = archiveProgressService.queryArchiveProgress(tableName); @@ -245,7 +244,7 @@ private void updateDeleteProgress(long lastDeletedId) { archiveProgressService.saveDeleteProgress(archiveProgress); } - private boolean delete(Long maxNeedArchiveId, ArchiveConfig archiveConfig) { + private void delete(Long maxNeedArchiveId, ArchiveConfig archiveConfig) { long startTime = System.currentTimeMillis(); Long maxNeedDeleteId; @@ -265,7 +264,7 @@ private boolean delete(Long maxNeedArchiveId, ArchiveConfig archiveConfig) { if (maxNeedDeleteId <= lastDeletedId) { log.info("LastDeletedId {} is greater than or equal to maxNeedDeleteId {}, skip delete {}!", lastDeletedId, maxNeedDeleteId, tableName); - return true; + return; } long start = minNeedDeleteId; @@ -287,10 +286,8 @@ private boolean delete(Long maxNeedArchiveId, ArchiveConfig archiveConfig) { "{}, cost: {}ms", tableName, minNeedDeleteId, maxNeedDeleteId, lastDeletedId, deletedRows, System.currentTimeMillis() - startTime); - return true; } catch (Throwable e) { log.error("Error while deleting {}", tableName, e); - return false; } finally { archiveSummary.setDeleteCost(System.currentTimeMillis() - startTime); archiveSummary.setDeleteIdStart(minNeedDeleteId); @@ -320,4 +317,20 @@ private int insertAndReset(long lastArchivedInstanceId, List recordList) thro private void storeArchiveSummary() { ArchiveSummaryHolder.getInstance().addArchiveSummary(this.archiveSummary); } + + private List listRecord(Long start, Long stop) { + return executeRecordDAO.listRecords(start, stop); + } + + private int batchInsert(List recordList) throws IOException { + return executeArchiveDAO.batchInsert(recordList, 1000); + } + + private int deleteRecord(Long start, Long stop) { + return executeRecordDAO.deleteRecords(start, stop); + } + + private long getFirstInstanceId() { + return executeRecordDAO.getFirstArchiveId(); + } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobExecuteArchiveManage.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobExecuteArchiveManage.java index 601a410750..5331e92279 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobExecuteArchiveManage.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobExecuteArchiveManage.java @@ -26,19 +26,39 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.tencent.bk.job.backup.archive.impl.FileSourceTaskLogArchivist; +import com.tencent.bk.job.backup.archive.impl.GseFileAgentTaskArchivist; +import com.tencent.bk.job.backup.archive.impl.GseScriptAgentTaskArchivist; +import com.tencent.bk.job.backup.archive.impl.GseTaskArchivist; import com.tencent.bk.job.backup.archive.impl.GseTaskIpLogArchivist; import com.tencent.bk.job.backup.archive.impl.GseTaskLogArchivist; import com.tencent.bk.job.backup.archive.impl.OperationLogArchivist; +import com.tencent.bk.job.backup.archive.impl.RollingConfigArchivist; import com.tencent.bk.job.backup.archive.impl.StepInstanceArchivist; import com.tencent.bk.job.backup.archive.impl.StepInstanceConfirmArchivist; import com.tencent.bk.job.backup.archive.impl.StepInstanceFileArchivist; +import com.tencent.bk.job.backup.archive.impl.StepInstanceRollingTaskArchivist; import com.tencent.bk.job.backup.archive.impl.StepInstanceScriptArchivist; import com.tencent.bk.job.backup.archive.impl.StepInstanceVariableArchivist; import com.tencent.bk.job.backup.archive.impl.TaskInstanceArchivist; import com.tencent.bk.job.backup.archive.impl.TaskInstanceVariableArchivist; import com.tencent.bk.job.backup.config.ArchiveConfig; import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.JobExecuteDAO; +import com.tencent.bk.job.backup.dao.impl.FileSourceTaskRecordDAO; +import com.tencent.bk.job.backup.dao.impl.GseFileAgentTaskRecordDAO; +import com.tencent.bk.job.backup.dao.impl.GseScriptAgentTaskRecordDAO; +import com.tencent.bk.job.backup.dao.impl.GseTaskIpLogRecordDAO; +import com.tencent.bk.job.backup.dao.impl.GseTaskLogRecordDAO; +import com.tencent.bk.job.backup.dao.impl.GseTaskRecordDAO; +import com.tencent.bk.job.backup.dao.impl.OperationLogRecordDAO; +import com.tencent.bk.job.backup.dao.impl.RollingConfigRecordDAO; +import com.tencent.bk.job.backup.dao.impl.StepInstanceConfirmRecordDAO; +import com.tencent.bk.job.backup.dao.impl.StepInstanceFileRecordDAO; +import com.tencent.bk.job.backup.dao.impl.StepInstanceRecordDAO; +import com.tencent.bk.job.backup.dao.impl.StepInstanceRollingTaskRecordDAO; +import com.tencent.bk.job.backup.dao.impl.StepInstanceScriptRecordDAO; +import com.tencent.bk.job.backup.dao.impl.StepInstanceVariableRecordDAO; +import com.tencent.bk.job.backup.dao.impl.TaskInstanceRecordDAO; +import com.tencent.bk.job.backup.dao.impl.TaskInstanceVariableRecordDAO; import com.tencent.bk.job.backup.service.ArchiveProgressService; import lombok.extern.slf4j.Slf4j; import org.joda.time.DateTime; @@ -54,7 +74,6 @@ @Slf4j public class JobExecuteArchiveManage implements SmartLifecycle { - private final JobExecuteDAO jobExecuteDAO; private final ArchiveConfig archiveConfig; private static final ExecutorService ARCHIVE_THREAD_POOL_EXECUTOR = new ThreadPoolExecutor(20, 20, @@ -73,6 +92,11 @@ public class JobExecuteArchiveManage implements SmartLifecycle { private final TaskInstanceArchivist taskInstanceArchivist; private final TaskInstanceVariableArchivist taskInstanceVariableArchivist; private final OperationLogArchivist operationLogArchivist; + private final GseTaskArchivist gseTaskArchivist; + private final GseScriptAgentTaskArchivist gseScriptAgentTaskArchivist; + private final GseFileAgentTaskArchivist gseFileAgentTaskArchivist; + private final StepInstanceRollingTaskArchivist stepInstanceRollingTaskArchivist; + private final RollingConfigArchivist rollingConfigArchivist; /** @@ -80,33 +104,57 @@ public class JobExecuteArchiveManage implements SmartLifecycle { */ private volatile boolean running = false; - public JobExecuteArchiveManage(JobExecuteDAO jobExecuteDAO, + public JobExecuteArchiveManage(TaskInstanceRecordDAO taskInstanceRecordDAO, + StepInstanceRecordDAO stepInstanceRecordDAO, + StepInstanceScriptRecordDAO stepInstanceScriptRecordDAO, + StepInstanceFileRecordDAO stepInstanceFileRecordDAO, + StepInstanceConfirmRecordDAO stepInstanceConfirmRecordDAO, + StepInstanceVariableRecordDAO stepInstanceVariableRecordDAO, + TaskInstanceVariableRecordDAO taskInstanceVariableRecordDAO, + OperationLogRecordDAO operationLogRecordDAO, + GseTaskLogRecordDAO gseTaskLogRecordDAO, + GseTaskIpLogRecordDAO gseTaskIpLogRecordDAO, + FileSourceTaskRecordDAO fileSourceTaskRecordDAO, + GseTaskRecordDAO gseTaskRecordDAO, + GseScriptAgentTaskRecordDAO gseScriptAgentTaskRecordDAO, + GseFileAgentTaskRecordDAO gseFileAgentTaskRecordDAO, + StepInstanceRollingTaskRecordDAO stepInstanceRollingTaskRecordDAO, + RollingConfigRecordDAO rollingConfigRecordDAO, ExecuteArchiveDAO executeArchiveDAO, ArchiveProgressService archiveProgressService, ArchiveConfig archiveConfig) { log.info("Init JobExecuteArchiveManage! archiveConfig: {}", archiveConfig); - this.jobExecuteDAO = jobExecuteDAO; this.archiveConfig = archiveConfig; - this.fileSourceTaskLogArchivist = new FileSourceTaskLogArchivist(jobExecuteDAO, executeArchiveDAO, + this.fileSourceTaskLogArchivist = new FileSourceTaskLogArchivist(fileSourceTaskRecordDAO, executeArchiveDAO, archiveProgressService); - this.stepInstanceArchivist = new StepInstanceArchivist(jobExecuteDAO, executeArchiveDAO, + this.stepInstanceArchivist = new StepInstanceArchivist(stepInstanceRecordDAO, executeArchiveDAO, archiveProgressService); - this.stepInstanceConfirmArchivist = new StepInstanceConfirmArchivist(jobExecuteDAO, executeArchiveDAO, + this.stepInstanceConfirmArchivist = new StepInstanceConfirmArchivist(stepInstanceConfirmRecordDAO, + executeArchiveDAO, archiveProgressService); + this.stepInstanceFileArchivist = new StepInstanceFileArchivist(stepInstanceFileRecordDAO, executeArchiveDAO, archiveProgressService); - this.stepInstanceFileArchivist = new StepInstanceFileArchivist(jobExecuteDAO, executeArchiveDAO, + this.stepInstanceScriptArchivist = new StepInstanceScriptArchivist(stepInstanceScriptRecordDAO, + executeArchiveDAO, archiveProgressService); + this.stepInstanceVariableArchivist = new StepInstanceVariableArchivist(stepInstanceVariableRecordDAO, + executeArchiveDAO, archiveProgressService); + this.gseTaskLogArchivist = new GseTaskLogArchivist(gseTaskLogRecordDAO, executeArchiveDAO, archiveProgressService); - this.stepInstanceScriptArchivist = new StepInstanceScriptArchivist(jobExecuteDAO, executeArchiveDAO, + this.gseTaskIpLogArchivist = new GseTaskIpLogArchivist(gseTaskIpLogRecordDAO, executeArchiveDAO, archiveProgressService); - this.stepInstanceVariableArchivist = new StepInstanceVariableArchivist(jobExecuteDAO, executeArchiveDAO, + this.taskInstanceArchivist = new TaskInstanceArchivist(taskInstanceRecordDAO, executeArchiveDAO, archiveProgressService); - this.gseTaskLogArchivist = new GseTaskLogArchivist(jobExecuteDAO, executeArchiveDAO, archiveProgressService); - this.gseTaskIpLogArchivist = new GseTaskIpLogArchivist(jobExecuteDAO, executeArchiveDAO, + this.taskInstanceVariableArchivist = new TaskInstanceVariableArchivist(taskInstanceVariableRecordDAO, + executeArchiveDAO, archiveProgressService); + this.operationLogArchivist = new OperationLogArchivist(operationLogRecordDAO, executeArchiveDAO, archiveProgressService); - this.taskInstanceArchivist = new TaskInstanceArchivist(jobExecuteDAO, executeArchiveDAO, - archiveProgressService); - this.taskInstanceVariableArchivist = new TaskInstanceVariableArchivist(jobExecuteDAO, executeArchiveDAO, - archiveProgressService); - this.operationLogArchivist = new OperationLogArchivist(jobExecuteDAO, executeArchiveDAO, + this.gseTaskArchivist = new GseTaskArchivist(gseTaskRecordDAO, executeArchiveDAO, archiveProgressService); + this.gseScriptAgentTaskArchivist = new GseScriptAgentTaskArchivist(gseScriptAgentTaskRecordDAO, + executeArchiveDAO, archiveProgressService); + this.gseFileAgentTaskArchivist = new GseFileAgentTaskArchivist(gseFileAgentTaskRecordDAO, + executeArchiveDAO, archiveProgressService); + this.stepInstanceRollingTaskArchivist = new StepInstanceRollingTaskArchivist(stepInstanceRollingTaskRecordDAO, + executeArchiveDAO, archiveProgressService); + this.rollingConfigArchivist = new RollingConfigArchivist(rollingConfigRecordDAO, executeArchiveDAO, archiveProgressService); } @@ -143,7 +191,7 @@ public boolean isRunning() { class ArchiveThread extends Thread { - private ArchiveConfig archiveConfig; + private final ArchiveConfig archiveConfig; ArchiveThread(ArchiveConfig archiveConfig) { @@ -182,9 +230,9 @@ private void doArchive(Long endTime) throws InterruptedException { try { log.info("Start job execute archive before {} at {}", endTime, System.currentTimeMillis()); - long maxNeedArchiveTaskInstanceId = jobExecuteDAO.getMaxNeedArchiveTaskInstanceId(endTime); + long maxNeedArchiveTaskInstanceId = taskInstanceArchivist.getMaxNeedArchiveTaskInstanceId(endTime); long maxNeedArchiveStepInstanceId = - jobExecuteDAO.getMaxNeedArchiveStepInstanceId(maxNeedArchiveTaskInstanceId); + stepInstanceArchivist.getMaxNeedArchiveStepInstanceId(maxNeedArchiveTaskInstanceId); log.info("Compute archive instance id range, maxNeedArchiveTaskInstanceId: {}, " + "maxNeedArchiveStepInstanceId: {}", maxNeedArchiveTaskInstanceId, maxNeedArchiveStepInstanceId); @@ -203,37 +251,57 @@ private void doArchive(Long endTime) throws InterruptedException { private void archive(long maxNeedArchiveTaskInstanceId, long maxNeedArchiveStepInstanceId) throws InterruptedException { - CountDownLatch countDownLatch = new CountDownLatch(8); + CountDownLatch countDownLatch = new CountDownLatch(15); log.info("Submitting archive task..."); + // task_instance ARCHIVE_THREAD_POOL_EXECUTOR.execute(() -> taskInstanceArchivist.archive(archiveConfig, maxNeedArchiveTaskInstanceId, countDownLatch)); - - + // step_instance ARCHIVE_THREAD_POOL_EXECUTOR.execute(() -> stepInstanceArchivist.archive(archiveConfig, maxNeedArchiveStepInstanceId, countDownLatch)); + // step_instance_confirm ARCHIVE_THREAD_POOL_EXECUTOR.execute(() -> stepInstanceConfirmArchivist.archive(archiveConfig, maxNeedArchiveStepInstanceId, countDownLatch)); + // step_instance_file ARCHIVE_THREAD_POOL_EXECUTOR.execute(() -> stepInstanceFileArchivist.archive(archiveConfig, maxNeedArchiveStepInstanceId, countDownLatch)); + // step_instance_script ARCHIVE_THREAD_POOL_EXECUTOR.execute(() -> stepInstanceScriptArchivist.archive(archiveConfig, maxNeedArchiveStepInstanceId, countDownLatch)); - - -// ARCHIVE_THREAD_POOL_EXECUTOR.execute(() -> gseTaskLogArchivist.archive(deleteAfterArchive, -// maxNeedArchiveStepInstanceId, countDownLatch)); -// ARCHIVE_THREAD_POOL_EXECUTOR.execute(() -> fileSourceTaskLogArchivist.archive(deleteAfterArchive, -// maxNeedArchiveStepInstanceId, countDownLatch)); + // gse_task_log + ARCHIVE_THREAD_POOL_EXECUTOR.execute(() -> gseTaskLogArchivist.archive(archiveConfig, + maxNeedArchiveStepInstanceId, countDownLatch)); + // file_source_task 非正式发布功能，暂时不开启 +// ARCHIVE_THREAD_POOL_EXECUTOR.execute(() -> fileSourceTaskLogArchivist.archive(archiveConfig, +// maxNeedArchiveStepInstanceId, countDownLatch)); + // gse_task_ip_log ARCHIVE_THREAD_POOL_EXECUTOR.execute(() -> gseTaskIpLogArchivist.archive(archiveConfig, maxNeedArchiveStepInstanceId, countDownLatch)); - + // task_instance_variable ARCHIVE_THREAD_POOL_EXECUTOR.execute(() -> taskInstanceVariableArchivist.archive(archiveConfig, maxNeedArchiveTaskInstanceId, countDownLatch)); -// ARCHIVE_THREAD_POOL_EXECUTOR.execute(() -> stepInstanceVariableArchivist.archive(deleteAfterArchive, -// maxNeedArchiveStepInstanceId, countDownLatch)); - + // step_instance_variable + ARCHIVE_THREAD_POOL_EXECUTOR.execute(() -> stepInstanceVariableArchivist.archive(archiveConfig, + maxNeedArchiveStepInstanceId, countDownLatch)); + // operation_log ARCHIVE_THREAD_POOL_EXECUTOR.execute(() -> operationLogArchivist.archive(archiveConfig, maxNeedArchiveTaskInstanceId, countDownLatch)); + // gse_task + ARCHIVE_THREAD_POOL_EXECUTOR.execute(() -> gseTaskArchivist.archive(archiveConfig, + maxNeedArchiveStepInstanceId, countDownLatch)); + // gse_script_agent_task + ARCHIVE_THREAD_POOL_EXECUTOR.execute(() -> gseScriptAgentTaskArchivist.archive(archiveConfig, + maxNeedArchiveStepInstanceId, countDownLatch)); + // gse_file_agent_task + ARCHIVE_THREAD_POOL_EXECUTOR.execute(() -> gseFileAgentTaskArchivist.archive(archiveConfig, + maxNeedArchiveStepInstanceId, countDownLatch)); + // step_instance_rolling_task + ARCHIVE_THREAD_POOL_EXECUTOR.execute(() -> stepInstanceRollingTaskArchivist.archive(archiveConfig, + maxNeedArchiveStepInstanceId, countDownLatch)); + // rolling_config + ARCHIVE_THREAD_POOL_EXECUTOR.execute(() -> rollingConfigArchivist.archive(archiveConfig, + maxNeedArchiveTaskInstanceId, countDownLatch)); log.info("Archive task submitted. Waiting for complete..."); countDownLatch.await(); diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/FileSourceTaskLogArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/FileSourceTaskLogArchivist.java index 96eaa8daea..d4a468bb2d 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/FileSourceTaskLogArchivist.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/FileSourceTaskLogArchivist.java @@ -26,47 +26,19 @@ import com.tencent.bk.job.backup.archive.AbstractArchivist; import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.JobExecuteDAO; +import com.tencent.bk.job.backup.dao.impl.FileSourceTaskRecordDAO; import com.tencent.bk.job.backup.service.ArchiveProgressService; -import org.jooq.generated.tables.FileSourceTaskLog; import org.jooq.generated.tables.records.FileSourceTaskLogRecord; -import java.io.IOException; -import java.util.List; - /** - * @since 18/3/2021 20:13 + * file_source_task_log 表归档 */ public class FileSourceTaskLogArchivist extends AbstractArchivist { - public FileSourceTaskLogArchivist(JobExecuteDAO jobExecuteDAO, + public FileSourceTaskLogArchivist(FileSourceTaskRecordDAO executeRecordDAO, ExecuteArchiveDAO executeArchiveDAO, ArchiveProgressService archiveProgressService) { - this.jobExecuteDAO = jobExecuteDAO; - this.executeArchiveDAO = executeArchiveDAO; - this.archiveProgressService = archiveProgressService; + super(executeRecordDAO, executeArchiveDAO, archiveProgressService); this.deleteIdStepSize = 100_000; - this.setTableName("file_source_task_log"); - } - - @Override - public List listRecord(Long start, Long stop) { - return jobExecuteDAO.listFileSourceTaskLog(start, stop); - } - - @Override - protected int batchInsert(List recordList) throws IOException { - return executeArchiveDAO.batchInsert(jobExecuteDAO.getFileSourceTaskLogFields(), recordList, 1000); - } - - @Override - protected int deleteRecord(Long start, Long stop) { - return jobExecuteDAO.deleteFileSourceTaskLog(start, stop); - } - - @Override - protected long getFirstInstanceId() { - return jobExecuteDAO.getFirstInstanceId(FileSourceTaskLog.FILE_SOURCE_TASK_LOG, - FileSourceTaskLog.FILE_SOURCE_TASK_LOG.STEP_INSTANCE_ID); } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseFileAgentTaskArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseFileAgentTaskArchivist.java new file mode 100644 index 0000000000..117abf3eba --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseFileAgentTaskArchivist.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.AbstractArchivist; +import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; +import com.tencent.bk.job.backup.dao.impl.GseFileAgentTaskRecordDAO; +import com.tencent.bk.job.backup.service.ArchiveProgressService; +import org.jooq.generated.tables.records.GseFileAgentTaskRecord; + +/** + * gse_file_agent_task 表归档 + */ +public class GseFileAgentTaskArchivist extends AbstractArchivist { + + public GseFileAgentTaskArchivist(GseFileAgentTaskRecordDAO executeRecordDAO, + ExecuteArchiveDAO executeArchiveDAO, + ArchiveProgressService archiveProgressService) { + super(executeRecordDAO, executeArchiveDAO, archiveProgressService); + this.deleteIdStepSize = 1_000; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseScriptAgentTaskArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseScriptAgentTaskArchivist.java new file mode 100644 index 0000000000..4580b37ba1 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseScriptAgentTaskArchivist.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.AbstractArchivist; +import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; +import com.tencent.bk.job.backup.dao.impl.GseScriptAgentTaskRecordDAO; +import com.tencent.bk.job.backup.service.ArchiveProgressService; +import org.jooq.generated.tables.records.GseScriptAgentTaskRecord; + +/** + * gse_script_agent_task 表归档 + */ +public class GseScriptAgentTaskArchivist extends AbstractArchivist { + + public GseScriptAgentTaskArchivist(GseScriptAgentTaskRecordDAO executeRecordDAO, + ExecuteArchiveDAO executeArchiveDAO, + ArchiveProgressService archiveProgressService) { + super(executeRecordDAO, executeArchiveDAO, archiveProgressService); + this.deleteIdStepSize = 1_000; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseTaskArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseTaskArchivist.java new file mode 100644 index 0000000000..ece9ec48a5 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseTaskArchivist.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.AbstractArchivist; +import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; +import com.tencent.bk.job.backup.dao.impl.GseTaskRecordDAO; +import com.tencent.bk.job.backup.service.ArchiveProgressService; +import org.jooq.generated.tables.records.GseTaskRecord; + +/** + * gse_task 表归档 + */ +public class GseTaskArchivist extends AbstractArchivist { + + public GseTaskArchivist(GseTaskRecordDAO executeRecordDAO, + ExecuteArchiveDAO executeArchiveDAO, + ArchiveProgressService archiveProgressService) { + super(executeRecordDAO, executeArchiveDAO, archiveProgressService); + this.deleteIdStepSize = 10_000; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseTaskIpLogArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseTaskIpLogArchivist.java index f99aaa4288..f53428c3c7 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseTaskIpLogArchivist.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseTaskIpLogArchivist.java @@ -26,47 +26,19 @@ import com.tencent.bk.job.backup.archive.AbstractArchivist; import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.JobExecuteDAO; +import com.tencent.bk.job.backup.dao.impl.GseTaskIpLogRecordDAO; import com.tencent.bk.job.backup.service.ArchiveProgressService; -import org.jooq.generated.tables.GseTaskIpLog; import org.jooq.generated.tables.records.GseTaskIpLogRecord; -import java.io.IOException; -import java.util.List; - /** - * @since 18/3/2021 20:13 + * gse_task_ip_log 表归档 */ public class GseTaskIpLogArchivist extends AbstractArchivist { - public GseTaskIpLogArchivist(JobExecuteDAO jobExecuteDAO, + public GseTaskIpLogArchivist(GseTaskIpLogRecordDAO executeRecordDAO, ExecuteArchiveDAO executeArchiveDAO, ArchiveProgressService archiveProgressService) { - this.jobExecuteDAO = jobExecuteDAO; - this.executeArchiveDAO = executeArchiveDAO; - this.archiveProgressService = archiveProgressService; + super(executeRecordDAO, executeArchiveDAO, archiveProgressService); this.deleteIdStepSize = 1_000; - this.setTableName("gse_task_ip_log"); - } - - @Override - public List listRecord(Long start, Long stop) { - return jobExecuteDAO.listGseTaskIpLog(start, stop); - } - - @Override - protected int batchInsert(List recordList) throws IOException { - return executeArchiveDAO.batchInsert(jobExecuteDAO.getGseTaskIpLogFields(), recordList, 1000); - } - - @Override - protected int deleteRecord(Long start, Long stop) { - return jobExecuteDAO.deleteGseTaskIpLog(start, stop); - } - - @Override - protected long getFirstInstanceId() { - return jobExecuteDAO.getFirstInstanceId(GseTaskIpLog.GSE_TASK_IP_LOG, - GseTaskIpLog.GSE_TASK_IP_LOG.STEP_INSTANCE_ID); } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseTaskLogArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseTaskLogArchivist.java index 0ea7a462c1..85bfc6fd86 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseTaskLogArchivist.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseTaskLogArchivist.java @@ -26,46 +26,19 @@ import com.tencent.bk.job.backup.archive.AbstractArchivist; import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.JobExecuteDAO; +import com.tencent.bk.job.backup.dao.impl.GseTaskLogRecordDAO; import com.tencent.bk.job.backup.service.ArchiveProgressService; -import org.jooq.generated.tables.GseTaskLog; import org.jooq.generated.tables.records.GseTaskLogRecord; -import java.io.IOException; -import java.util.List; - /** - * @since 18/3/2021 20:13 + * gse_task_log 表归档 */ public class GseTaskLogArchivist extends AbstractArchivist { - public GseTaskLogArchivist(JobExecuteDAO jobExecuteDAO, + public GseTaskLogArchivist(GseTaskLogRecordDAO executeRecordDAO, ExecuteArchiveDAO executeArchiveDAO, ArchiveProgressService archiveProgressService) { - this.jobExecuteDAO = jobExecuteDAO; - this.executeArchiveDAO = executeArchiveDAO; - this.archiveProgressService = archiveProgressService; + super(executeRecordDAO, executeArchiveDAO, archiveProgressService); this.deleteIdStepSize = 10_000; - this.setTableName("gse_task_log"); - } - - @Override - public List listRecord(Long start, Long stop) { - return jobExecuteDAO.listGseTaskLog(start, stop); - } - - @Override - protected int batchInsert(List recordList) throws IOException { - return executeArchiveDAO.batchInsert(jobExecuteDAO.getGseTaskLogFields(), recordList, 1000); - } - - @Override - protected int deleteRecord(Long start, Long stop) { - return jobExecuteDAO.deleteGseTaskLog(start, stop); - } - - @Override - protected long getFirstInstanceId() { - return jobExecuteDAO.getFirstInstanceId(GseTaskLog.GSE_TASK_LOG, GseTaskLog.GSE_TASK_LOG.STEP_INSTANCE_ID); } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/OperationLogArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/OperationLogArchivist.java index fdfead0ca0..2619917fe6 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/OperationLogArchivist.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/OperationLogArchivist.java @@ -26,47 +26,19 @@ import com.tencent.bk.job.backup.archive.AbstractArchivist; import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.JobExecuteDAO; +import com.tencent.bk.job.backup.dao.impl.OperationLogRecordDAO; import com.tencent.bk.job.backup.service.ArchiveProgressService; -import org.jooq.generated.tables.OperationLog; import org.jooq.generated.tables.records.OperationLogRecord; -import java.io.IOException; -import java.util.List; - /** - * @since 18/3/2021 20:13 + * operation_log 表归档 */ public class OperationLogArchivist extends AbstractArchivist { - public OperationLogArchivist(JobExecuteDAO jobExecuteDAO, + public OperationLogArchivist(OperationLogRecordDAO executeRecordDAO, ExecuteArchiveDAO executeArchiveDAO, ArchiveProgressService archiveProgressService) { - this.jobExecuteDAO = jobExecuteDAO; - this.executeArchiveDAO = executeArchiveDAO; - this.archiveProgressService = archiveProgressService; + super(executeRecordDAO, executeArchiveDAO, archiveProgressService); this.deleteIdStepSize = 10_000; - this.setTableName("operation_log"); - } - - @Override - public List listRecord(Long start, Long stop) { - return jobExecuteDAO.listOperationLog(start, stop); - } - - @Override - protected int batchInsert(List recordList) throws IOException { - return executeArchiveDAO.batchInsert(jobExecuteDAO.getOperationLogFields(), recordList, 1000); - } - - @Override - protected int deleteRecord(Long start, Long stop) { - return jobExecuteDAO.deleteOperationLog(start, stop); - } - - @Override - protected long getFirstInstanceId() { - return jobExecuteDAO.getFirstInstanceId(OperationLog.OPERATION_LOG, - OperationLog.OPERATION_LOG.TASK_INSTANCE_ID); } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/RollingConfigArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/RollingConfigArchivist.java new file mode 100644 index 0000000000..c0f94ee9be --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/RollingConfigArchivist.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.AbstractArchivist; +import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; +import com.tencent.bk.job.backup.dao.impl.RollingConfigRecordDAO; +import com.tencent.bk.job.backup.service.ArchiveProgressService; +import org.jooq.generated.tables.records.RollingConfigRecord; + +/** + * rolling_config 表归档 + */ +public class RollingConfigArchivist extends AbstractArchivist { + + public RollingConfigArchivist(RollingConfigRecordDAO executeRecordDAO, + ExecuteArchiveDAO executeArchiveDAO, + ArchiveProgressService archiveProgressService) { + super(executeRecordDAO, executeArchiveDAO, archiveProgressService); + this.deleteIdStepSize = 100_000; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceArchivist.java index f781cac832..7d9f5eda65 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceArchivist.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceArchivist.java @@ -26,46 +26,24 @@ import com.tencent.bk.job.backup.archive.AbstractArchivist; import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.JobExecuteDAO; +import com.tencent.bk.job.backup.dao.impl.StepInstanceRecordDAO; import com.tencent.bk.job.backup.service.ArchiveProgressService; -import org.jooq.generated.tables.StepInstance; import org.jooq.generated.tables.records.StepInstanceRecord; -import java.io.IOException; -import java.util.List; - /** - * @since 18/3/2021 20:13 + * step_instance 表归档 */ public class StepInstanceArchivist extends AbstractArchivist { - public StepInstanceArchivist(JobExecuteDAO jobExecuteDAO, + public StepInstanceArchivist(StepInstanceRecordDAO executeRecordDAO, ExecuteArchiveDAO executeArchiveDAO, ArchiveProgressService archiveProgressService) { - this.jobExecuteDAO = jobExecuteDAO; - this.executeArchiveDAO = executeArchiveDAO; - this.archiveProgressService = archiveProgressService; + super(executeRecordDAO, executeArchiveDAO, archiveProgressService); this.deleteIdStepSize = 10_000; - this.setTableName("step_instance"); - } - - @Override - public List listRecord(Long start, Long stop) { - return jobExecuteDAO.listStepInstance(start, stop); - } - - @Override - protected int batchInsert(List recordList) throws IOException { - return executeArchiveDAO.batchInsert(jobExecuteDAO.getStepInstanceFields(), recordList, 200); - } - - @Override - protected int deleteRecord(Long start, Long stop) { - return jobExecuteDAO.deleteStepInstance(start, stop); } - @Override - protected long getFirstInstanceId() { - return jobExecuteDAO.getFirstInstanceId(StepInstance.STEP_INSTANCE, StepInstance.STEP_INSTANCE.ID); + public Long getMaxNeedArchiveStepInstanceId(Long taskInstanceId) { + StepInstanceRecordDAO stepInstanceRecordDAO = (StepInstanceRecordDAO) executeRecordDAO; + return stepInstanceRecordDAO.getMaxNeedArchiveStepInstanceId(taskInstanceId); } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceConfirmArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceConfirmArchivist.java index d60b5178b7..e144c72d45 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceConfirmArchivist.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceConfirmArchivist.java @@ -26,47 +26,19 @@ import com.tencent.bk.job.backup.archive.AbstractArchivist; import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.JobExecuteDAO; +import com.tencent.bk.job.backup.dao.impl.StepInstanceConfirmRecordDAO; import com.tencent.bk.job.backup.service.ArchiveProgressService; -import org.jooq.generated.tables.StepInstanceConfirm; import org.jooq.generated.tables.records.StepInstanceConfirmRecord; -import java.io.IOException; -import java.util.List; - /** - * @since 18/3/2021 20:13 + * step_instance_confirm 表归档 */ public class StepInstanceConfirmArchivist extends AbstractArchivist { - public StepInstanceConfirmArchivist(JobExecuteDAO jobExecuteDAO, + public StepInstanceConfirmArchivist(StepInstanceConfirmRecordDAO executeRecordDAO, ExecuteArchiveDAO executeArchiveDAO, ArchiveProgressService archiveProgressService) { - this.jobExecuteDAO = jobExecuteDAO; - this.executeArchiveDAO = executeArchiveDAO; - this.archiveProgressService = archiveProgressService; + super(executeRecordDAO, executeArchiveDAO, archiveProgressService); this.deleteIdStepSize = 100_000; - this.setTableName("step_instance_confirm"); - } - - @Override - public List listRecord(Long start, Long stop) { - return jobExecuteDAO.listStepInstanceConfirm(start, stop); - } - - @Override - protected int batchInsert(List recordList) throws IOException { - return executeArchiveDAO.batchInsert(jobExecuteDAO.getStepInstanceConfirmFields(), recordList, 1000); - } - - @Override - protected int deleteRecord(Long start, Long stop) { - return jobExecuteDAO.deleteStepInstanceConfirm(start, stop); - } - - @Override - protected long getFirstInstanceId() { - return jobExecuteDAO.getFirstInstanceId(StepInstanceConfirm.STEP_INSTANCE_CONFIRM, - StepInstanceConfirm.STEP_INSTANCE_CONFIRM.STEP_INSTANCE_ID); } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceFileArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceFileArchivist.java index 51b2da851c..e98f51bda1 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceFileArchivist.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceFileArchivist.java @@ -26,47 +26,19 @@ import com.tencent.bk.job.backup.archive.AbstractArchivist; import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.JobExecuteDAO; +import com.tencent.bk.job.backup.dao.impl.StepInstanceFileRecordDAO; import com.tencent.bk.job.backup.service.ArchiveProgressService; -import org.jooq.generated.tables.StepInstanceFile; import org.jooq.generated.tables.records.StepInstanceFileRecord; -import java.io.IOException; -import java.util.List; - /** - * @since 18/3/2021 20:13 + * step_instance_file 表归档 */ public class StepInstanceFileArchivist extends AbstractArchivist { - public StepInstanceFileArchivist(JobExecuteDAO jobExecuteDAO, + public StepInstanceFileArchivist(StepInstanceFileRecordDAO executeRecordDAO, ExecuteArchiveDAO executeArchiveDAO, ArchiveProgressService archiveProgressService) { - this.jobExecuteDAO = jobExecuteDAO; - this.executeArchiveDAO = executeArchiveDAO; - this.archiveProgressService = archiveProgressService; + super(executeRecordDAO, executeArchiveDAO, archiveProgressService); this.deleteIdStepSize = 10_000; - this.setTableName("step_instance_file"); - } - - @Override - public List listRecord(Long start, Long stop) { - return jobExecuteDAO.listStepInstanceFile(start, stop); - } - - @Override - protected int batchInsert(List recordList) throws IOException { - return executeArchiveDAO.batchInsert(jobExecuteDAO.getStepInstanceFileFields(), recordList, 500); - } - - @Override - protected int deleteRecord(Long start, Long stop) { - return jobExecuteDAO.deleteStepInstanceFile(start, stop); - } - - @Override - protected long getFirstInstanceId() { - return jobExecuteDAO.getFirstInstanceId(StepInstanceFile.STEP_INSTANCE_FILE, - StepInstanceFile.STEP_INSTANCE_FILE.STEP_INSTANCE_ID); } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceRollingTaskArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceRollingTaskArchivist.java new file mode 100644 index 0000000000..8b19a8941a --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceRollingTaskArchivist.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.AbstractArchivist; +import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; +import com.tencent.bk.job.backup.dao.impl.StepInstanceRollingTaskRecordDAO; +import com.tencent.bk.job.backup.service.ArchiveProgressService; +import org.jooq.generated.tables.records.StepInstanceRollingTaskRecord; + +/** + * step_instance_rolling_task 表归档 + */ +public class StepInstanceRollingTaskArchivist extends AbstractArchivist { + + public StepInstanceRollingTaskArchivist(StepInstanceRollingTaskRecordDAO executeRecordDAO, + ExecuteArchiveDAO executeArchiveDAO, + ArchiveProgressService archiveProgressService) { + super(executeRecordDAO, executeArchiveDAO, archiveProgressService); + this.deleteIdStepSize = 100_000; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceScriptArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceScriptArchivist.java index 27e057a8a8..c9374ae363 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceScriptArchivist.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceScriptArchivist.java @@ -26,47 +26,19 @@ import com.tencent.bk.job.backup.archive.AbstractArchivist; import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.JobExecuteDAO; +import com.tencent.bk.job.backup.dao.impl.StepInstanceScriptRecordDAO; import com.tencent.bk.job.backup.service.ArchiveProgressService; -import org.jooq.generated.tables.StepInstanceScript; import org.jooq.generated.tables.records.StepInstanceScriptRecord; -import java.io.IOException; -import java.util.List; - /** - * @since 18/3/2021 20:13 + * step_instance_script 表归档 */ public class StepInstanceScriptArchivist extends AbstractArchivist { - public StepInstanceScriptArchivist(JobExecuteDAO jobExecuteDAO, + public StepInstanceScriptArchivist(StepInstanceScriptRecordDAO executeRecordDAO, ExecuteArchiveDAO executeArchiveDAO, ArchiveProgressService archiveProgressService) { - this.jobExecuteDAO = jobExecuteDAO; - this.executeArchiveDAO = executeArchiveDAO; - this.archiveProgressService = archiveProgressService; + super(executeRecordDAO, executeArchiveDAO, archiveProgressService); this.deleteIdStepSize = 10_000; - this.setTableName("step_instance_script"); - } - - @Override - public List listRecord(Long start, Long stop) { - return jobExecuteDAO.listStepInstanceScript(start, stop); - } - - @Override - protected int batchInsert(List recordList) throws IOException { - return executeArchiveDAO.batchInsert(jobExecuteDAO.getStepInstanceScriptFields(), recordList, 200); - } - - @Override - protected int deleteRecord(Long start, Long stop) { - return jobExecuteDAO.deleteStepInstanceScript(start, stop); - } - - @Override - protected long getFirstInstanceId() { - return jobExecuteDAO.getFirstInstanceId(StepInstanceScript.STEP_INSTANCE_SCRIPT, - StepInstanceScript.STEP_INSTANCE_SCRIPT.STEP_INSTANCE_ID); } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceVariableArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceVariableArchivist.java index dc61df7a34..892adb58ef 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceVariableArchivist.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceVariableArchivist.java @@ -26,47 +26,19 @@ import com.tencent.bk.job.backup.archive.AbstractArchivist; import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.JobExecuteDAO; +import com.tencent.bk.job.backup.dao.impl.StepInstanceVariableRecordDAO; import com.tencent.bk.job.backup.service.ArchiveProgressService; -import org.jooq.generated.tables.StepInstanceVariable; import org.jooq.generated.tables.records.StepInstanceVariableRecord; -import java.io.IOException; -import java.util.List; - /** - * @since 18/3/2021 20:13 + * step_instance_variable 表归档 */ public class StepInstanceVariableArchivist extends AbstractArchivist { - public StepInstanceVariableArchivist(JobExecuteDAO jobExecuteDAO, + public StepInstanceVariableArchivist(StepInstanceVariableRecordDAO executeRecordDAO, ExecuteArchiveDAO executeArchiveDAO, ArchiveProgressService archiveProgressService) { - this.jobExecuteDAO = jobExecuteDAO; - this.executeArchiveDAO = executeArchiveDAO; - this.archiveProgressService = archiveProgressService; + super(executeRecordDAO, executeArchiveDAO, archiveProgressService); this.deleteIdStepSize = 100_000; - this.setTableName("step_instance_variable"); - } - - @Override - public List listRecord(Long start, Long stop) { - return jobExecuteDAO.listStepInstanceVariable(start, stop); - } - - @Override - protected int batchInsert(List recordList) throws IOException { - return executeArchiveDAO.batchInsert(jobExecuteDAO.getStepInstanceVariableFields(), recordList, 200); - } - - @Override - protected int deleteRecord(Long start, Long stop) { - return jobExecuteDAO.deleteStepInstanceVariable(start, stop); - } - - @Override - protected long getFirstInstanceId() { - return jobExecuteDAO.getFirstInstanceId(StepInstanceVariable.STEP_INSTANCE_VARIABLE, - StepInstanceVariable.STEP_INSTANCE_VARIABLE.STEP_INSTANCE_ID); } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceArchivist.java index 84ce5756ae..9c6cfcfbc4 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceArchivist.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceArchivist.java @@ -26,43 +26,24 @@ import com.tencent.bk.job.backup.archive.AbstractArchivist; import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.JobExecuteDAO; +import com.tencent.bk.job.backup.dao.impl.TaskInstanceRecordDAO; import com.tencent.bk.job.backup.service.ArchiveProgressService; -import org.jooq.generated.tables.TaskInstance; import org.jooq.generated.tables.records.TaskInstanceRecord; -import java.io.IOException; -import java.util.List; - +/** + * task_instance 表归档 + */ public class TaskInstanceArchivist extends AbstractArchivist { - public TaskInstanceArchivist(JobExecuteDAO jobExecuteDAO, + public TaskInstanceArchivist(TaskInstanceRecordDAO executeRecordDAO, ExecuteArchiveDAO executeArchiveDAO, ArchiveProgressService archiveProgressService) { - this.jobExecuteDAO = jobExecuteDAO; - this.executeArchiveDAO = executeArchiveDAO; - this.archiveProgressService = archiveProgressService; + super(executeRecordDAO, executeArchiveDAO, archiveProgressService); this.deleteIdStepSize = 10_000; - this.setTableName("task_instance"); - } - - @Override - public List listRecord(Long start, Long stop) { - return jobExecuteDAO.listTaskInstance(start, stop); - } - - @Override - protected int batchInsert(List recordList) throws IOException { - return executeArchiveDAO.batchInsert(jobExecuteDAO.getTaskInstanceFields(), recordList, 500); - } - - @Override - protected int deleteRecord(Long start, Long stop) { - return jobExecuteDAO.deleteTaskInstance(start, stop); } - @Override - protected long getFirstInstanceId() { - return jobExecuteDAO.getFirstInstanceId(TaskInstance.TASK_INSTANCE, TaskInstance.TASK_INSTANCE.ID); + public Long getMaxNeedArchiveTaskInstanceId(Long endTime) { + TaskInstanceRecordDAO taskInstanceRecordDAO = (TaskInstanceRecordDAO) executeRecordDAO; + return taskInstanceRecordDAO.getMaxNeedArchiveTaskInstanceId(endTime); } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceVariableArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceVariableArchivist.java index e65faf14d2..76ea71bec2 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceVariableArchivist.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceVariableArchivist.java @@ -26,47 +26,19 @@ import com.tencent.bk.job.backup.archive.AbstractArchivist; import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.JobExecuteDAO; +import com.tencent.bk.job.backup.dao.impl.TaskInstanceVariableRecordDAO; import com.tencent.bk.job.backup.service.ArchiveProgressService; -import org.jooq.generated.tables.TaskInstanceVariable; import org.jooq.generated.tables.records.TaskInstanceVariableRecord; -import java.io.IOException; -import java.util.List; - /** - * @since 18/3/2021 20:13 + * task_instance_variable 表归档 */ public class TaskInstanceVariableArchivist extends AbstractArchivist { - public TaskInstanceVariableArchivist(JobExecuteDAO jobExecuteDAO, + public TaskInstanceVariableArchivist(TaskInstanceVariableRecordDAO executeRecordDAO, ExecuteArchiveDAO executeArchiveDAO, ArchiveProgressService archiveProgressService) { - this.jobExecuteDAO = jobExecuteDAO; - this.executeArchiveDAO = executeArchiveDAO; - this.archiveProgressService = archiveProgressService; + super(executeRecordDAO, executeArchiveDAO, archiveProgressService); this.deleteIdStepSize = 100_000; - this.setTableName("task_instance_variable"); - } - - @Override - public List listRecord(Long start, Long stop) { - return jobExecuteDAO.listTaskInstanceVariable(start, stop); - } - - @Override - protected int batchInsert(List recordList) throws IOException { - return executeArchiveDAO.batchInsert(jobExecuteDAO.getTaskInstanceVariableFields(), recordList, 200); - } - - @Override - protected int deleteRecord(Long start, Long stop) { - return jobExecuteDAO.deleteTaskInstanceVariable(start, stop); - } - - @Override - protected long getFirstInstanceId() { - return jobExecuteDAO.getFirstInstanceId(TaskInstanceVariable.TASK_INSTANCE_VARIABLE, - TaskInstanceVariable.TASK_INSTANCE_VARIABLE.TASK_INSTANCE_ID); } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/ExecuteArchiveDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/ExecuteArchiveDAO.java index e03ce9c431..60fcd5cf09 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/ExecuteArchiveDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/ExecuteArchiveDAO.java @@ -24,13 +24,12 @@ package com.tencent.bk.job.backup.dao; -import org.jooq.Field; import org.jooq.TableRecord; import java.io.IOException; import java.util.List; public interface ExecuteArchiveDAO { - Integer batchInsert(List> fieldList, List> recordList, int bulkSize) + Integer batchInsert(List> recordList, int bulkSize) throws IOException; } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/ExecuteRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/ExecuteRecordDAO.java new file mode 100644 index 0000000000..2b13ac02c8 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/ExecuteRecordDAO.java @@ -0,0 +1,54 @@ +package com.tencent.bk.job.backup.dao; + +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.TableField; + +import java.util.List; + +/** + * job-execute 微服务的表数据DAO + * + * @param 表记录 + */ +public interface ExecuteRecordDAO { + + /** + * 获取表 + * + * @return 表 + */ + Table getTable(); + + /** + * 获取用于查询归档记录的ID字段 + * + * @return ID字段 + */ + TableField getArchiveIdField(); + + /** + * 根据起始/结束ID获取表记录 + * + * @param start 起始ID + * @param end 结束ID + * @return 表记录 + */ + List listRecords(Long start, Long end); + + /** + * 根据起始/结束ID删除表记录 + * + * @param start 起始ID + * @param end 结束ID + * @return 删除的记录数量 + */ + int deleteRecords(Long start, Long end); + + /** + * 获取表中第一条数据的ID(按照ID升序) + * + * @return id值 + */ + Long getFirstArchiveId(); +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/JobExecuteDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/JobExecuteDAO.java deleted file mode 100644 index cbb96c04f0..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/JobExecuteDAO.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.dao; - -import org.jooq.Field; -import org.jooq.Record; -import org.jooq.Table; -import org.jooq.TableField; -import org.jooq.generated.tables.records.*; - -import java.util.List; - -public interface JobExecuteDAO { - Long getMaxNeedArchiveTaskInstanceId(Long endTime); - - Long getMaxNeedArchiveStepInstanceId(Long taskInstanceId); - - Long getFirstInstanceId(Table table, TableField field); - - List> getFileSourceTaskLogFields(); - - List listFileSourceTaskLog(Long start, Long stop); - - List> getGseTaskIpLogFields(); - - List listGseTaskIpLog(Long start, Long stop); - - List> getGseTaskLogFields(); - - List listGseTaskLog(Long start, Long stop); - - List> getStepInstanceFields(); - - List listStepInstance(Long start, Long stop); - - List> getStepInstanceConfirmFields(); - - List listStepInstanceConfirm(Long start, Long stop); - - List> getStepInstanceFileFields(); - - List listStepInstanceFile(Long start, Long stop); - - List> getStepInstanceScriptFields(); - - List listStepInstanceScript(Long start, Long stop); - - List> getStepInstanceVariableFields(); - - List listStepInstanceVariable(Long start, Long stop); - - List> getTaskInstanceFields(); - - List listTaskInstance(Long start, Long stop); - - List> getTaskInstanceVariableFields(); - - List> getOperationLogFields(); - - List listTaskInstanceVariable(Long start, Long stop); - - List listOperationLog(Long lastInstanceId, Long stop); - - int deleteFileSourceTaskLog(Long start, Long stop); - - int deleteGseTaskIpLog(Long start, Long stop); - - int deleteGseTaskLog(Long start, Long stop); - - int deleteOperationLog(Long start, Long stop); - - int deleteStepInstance(Long start, Long stop); - - int deleteStepInstanceConfirm(Long start, Long stop); - - int deleteStepInstanceFile(Long start, Long stop); - - int deleteStepInstanceScript(Long start, Long stop); - - int deleteStepInstanceVariable(Long start, Long stop); - - int deleteTaskInstance(Long start, Long stop); - - int deleteTaskInstanceVariable(Long start, Long stop); -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/AbstractExecuteRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/AbstractExecuteRecordDAO.java new file mode 100644 index 0000000000..e4bf2ec29d --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/AbstractExecuteRecordDAO.java @@ -0,0 +1,75 @@ +package com.tencent.bk.job.backup.dao.impl; + +import com.tencent.bk.job.backup.config.ArchiveConfig; +import com.tencent.bk.job.backup.dao.ExecuteRecordDAO; +import org.jooq.Condition; +import org.jooq.DSLContext; +import org.jooq.Record; +import org.jooq.Record1; +import org.jooq.Result; +import org.jooq.Table; + +import java.util.ArrayList; +import java.util.List; + +import static org.jooq.impl.DSL.min; + +public abstract class AbstractExecuteRecordDAO implements ExecuteRecordDAO { + + protected final DSLContext context; + protected final ArchiveConfig archiveConfig; + + public AbstractExecuteRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { + this.context = context; + this.archiveConfig = archiveConfig; + } + + @Override + public List listRecords(Long start, Long end) { + Result result = query(getTable(), buildConditions(start, end)); + return result.into(getTable()); + } + + @Override + public int deleteRecords(Long start, Long end) { + return deleteWithLimit(getTable(), buildConditions(start, end)); + } + + private List buildConditions(Long start, Long end) { + List conditions = new ArrayList<>(); + conditions.add(getArchiveIdField().greaterThan(start)); + conditions.add(getArchiveIdField().lessOrEqual(end)); + return conditions; + } + + private int deleteWithLimit(Table table, List conditions) { + int totalDeleteRows = 0; + int maxLimitedDeleteRows = archiveConfig.getDeleteLimitRowCount(); + while (true) { + int deletedRows = context.delete(table).where(conditions).limit(maxLimitedDeleteRows).execute(); + totalDeleteRows += deletedRows; + if (deletedRows < maxLimitedDeleteRows) { + break; + } + } + return totalDeleteRows; + } + + private Result query(Table table, List conditions) { + return context.select() + .from(table) + .where(conditions) + .fetch(); + } + + @Override + public Long getFirstArchiveId() { + Record1 firstArchiveIdRecord = context.select(min(getArchiveIdField())).from(getTable()).fetchOne(); + if (firstArchiveIdRecord != null && firstArchiveIdRecord.get(0) != null) { + return (Long) firstArchiveIdRecord.get(0); + } + return 0L; + } + + public abstract Table getTable(); +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/ExecuteArchiveDAOImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/ExecuteArchiveDAOImpl.java index 2a16345972..ce350a0ed8 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/ExecuteArchiveDAOImpl.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/ExecuteArchiveDAOImpl.java @@ -28,7 +28,6 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.jooq.DSLContext; -import org.jooq.Field; import org.jooq.Loader; import org.jooq.LoaderError; import org.jooq.TableRecord; @@ -38,9 +37,7 @@ import java.io.IOException; import java.util.List; -/** - * @since 17/3/2021 15:46 - */ + @Slf4j public class ExecuteArchiveDAOImpl implements ExecuteArchiveDAO { @@ -54,8 +51,7 @@ public ExecuteArchiveDAOImpl(DSLContext context) { } @Override - public Integer batchInsert(List> fieldList, List> recordList, int bulkSize) - throws IOException { + public Integer batchInsert(List> recordList, int bulkSize) throws IOException { long start = System.currentTimeMillis(); int executeResult = 0; String table = recordList.get(0).getTable().getName(); @@ -66,7 +62,7 @@ public Integer batchInsert(List> fieldList, List { + + private static final FileSourceTaskLog TABLE = FileSourceTaskLog.FILE_SOURCE_TASK_LOG; + + public FileSourceTaskRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { + super(context, archiveConfig); + } + + @Override + public Table getTable() { + return TABLE; + } + + @Override + public TableField getArchiveIdField() { + return TABLE.STEP_INSTANCE_ID; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseFileAgentTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseFileAgentTaskRecordDAO.java new file mode 100644 index 0000000000..b41d333963 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseFileAgentTaskRecordDAO.java @@ -0,0 +1,30 @@ +package com.tencent.bk.job.backup.dao.impl; + +import com.tencent.bk.job.backup.config.ArchiveConfig; +import org.jooq.DSLContext; +import org.jooq.Table; +import org.jooq.TableField; +import org.jooq.generated.tables.GseFileAgentTask; +import org.jooq.generated.tables.records.GseFileAgentTaskRecord; + +/** + * gse_file_agent_task DAO + */ +public class GseFileAgentTaskRecordDAO extends AbstractExecuteRecordDAO { + + private static final GseFileAgentTask TABLE = GseFileAgentTask.GSE_FILE_AGENT_TASK; + + public GseFileAgentTaskRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { + super(context, archiveConfig); + } + + @Override + public Table getTable() { + return TABLE; + } + + @Override + public TableField getArchiveIdField() { + return TABLE.STEP_INSTANCE_ID; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseScriptAgentTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseScriptAgentTaskRecordDAO.java new file mode 100644 index 0000000000..bcbb40850b --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseScriptAgentTaskRecordDAO.java @@ -0,0 +1,30 @@ +package com.tencent.bk.job.backup.dao.impl; + +import com.tencent.bk.job.backup.config.ArchiveConfig; +import org.jooq.DSLContext; +import org.jooq.Table; +import org.jooq.TableField; +import org.jooq.generated.tables.GseScriptAgentTask; +import org.jooq.generated.tables.records.GseScriptAgentTaskRecord; + +/** + * gse_script_agent_task DAO + */ +public class GseScriptAgentTaskRecordDAO extends AbstractExecuteRecordDAO { + + private static final GseScriptAgentTask TABLE = GseScriptAgentTask.GSE_SCRIPT_AGENT_TASK; + + public GseScriptAgentTaskRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { + super(context, archiveConfig); + } + + @Override + public Table getTable() { + return TABLE; + } + + @Override + public TableField getArchiveIdField() { + return TABLE.STEP_INSTANCE_ID; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseTaskIpLogRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseTaskIpLogRecordDAO.java new file mode 100644 index 0000000000..eccbe21a04 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseTaskIpLogRecordDAO.java @@ -0,0 +1,30 @@ +package com.tencent.bk.job.backup.dao.impl; + +import com.tencent.bk.job.backup.config.ArchiveConfig; +import org.jooq.DSLContext; +import org.jooq.Table; +import org.jooq.TableField; +import org.jooq.generated.tables.GseTaskIpLog; +import org.jooq.generated.tables.records.GseTaskIpLogRecord; + +/** + * gse_task_ip_log DAO + */ +public class GseTaskIpLogRecordDAO extends AbstractExecuteRecordDAO { + + private static final GseTaskIpLog TABLE = GseTaskIpLog.GSE_TASK_IP_LOG; + + public GseTaskIpLogRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { + super(context, archiveConfig); + } + + @Override + public Table getTable() { + return TABLE; + } + + @Override + public TableField getArchiveIdField() { + return TABLE.STEP_INSTANCE_ID; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseTaskLogRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseTaskLogRecordDAO.java new file mode 100644 index 0000000000..348c9531b6 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseTaskLogRecordDAO.java @@ -0,0 +1,30 @@ +package com.tencent.bk.job.backup.dao.impl; + +import com.tencent.bk.job.backup.config.ArchiveConfig; +import org.jooq.DSLContext; +import org.jooq.Table; +import org.jooq.TableField; +import org.jooq.generated.tables.GseTaskLog; +import org.jooq.generated.tables.records.GseTaskLogRecord; + +/** + * gse_task_log DAO + */ +public class GseTaskLogRecordDAO extends AbstractExecuteRecordDAO { + + private static final GseTaskLog TABLE = GseTaskLog.GSE_TASK_LOG; + + public GseTaskLogRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { + super(context, archiveConfig); + } + + @Override + public Table getTable() { + return TABLE; + } + + @Override + public TableField getArchiveIdField() { + return TABLE.STEP_INSTANCE_ID; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseTaskRecordDAO.java new file mode 100644 index 0000000000..854921c598 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseTaskRecordDAO.java @@ -0,0 +1,27 @@ +package com.tencent.bk.job.backup.dao.impl; + +import com.tencent.bk.job.backup.config.ArchiveConfig; +import org.jooq.DSLContext; +import org.jooq.Table; +import org.jooq.TableField; +import org.jooq.generated.tables.GseTask; +import org.jooq.generated.tables.records.GseTaskRecord; + +public class GseTaskRecordDAO extends AbstractExecuteRecordDAO { + + private static final GseTask TABLE = GseTask.GSE_TASK; + + public GseTaskRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { + super(context, archiveConfig); + } + + @Override + public Table getTable() { + return TABLE; + } + + @Override + public TableField getArchiveIdField() { + return TABLE.STEP_INSTANCE_ID; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/JobExecuteDAOImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/JobExecuteDAOImpl.java deleted file mode 100644 index 19f9e47fb6..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/JobExecuteDAOImpl.java +++ /dev/null @@ -1,541 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.backup.config.ArchiveConfig; -import com.tencent.bk.job.backup.dao.JobExecuteDAO; -import lombok.extern.slf4j.Slf4j; -import org.jooq.Condition; -import org.jooq.DSLContext; -import org.jooq.Field; -import org.jooq.Record; -import org.jooq.Record1; -import org.jooq.Result; -import org.jooq.Table; -import org.jooq.TableField; -import org.jooq.generated.tables.FileSourceTaskLog; -import org.jooq.generated.tables.GseTaskIpLog; -import org.jooq.generated.tables.GseTaskLog; -import org.jooq.generated.tables.OperationLog; -import org.jooq.generated.tables.StepInstance; -import org.jooq.generated.tables.StepInstanceConfirm; -import org.jooq.generated.tables.StepInstanceFile; -import org.jooq.generated.tables.StepInstanceScript; -import org.jooq.generated.tables.StepInstanceVariable; -import org.jooq.generated.tables.TaskInstance; -import org.jooq.generated.tables.TaskInstanceVariable; -import org.jooq.generated.tables.records.FileSourceTaskLogRecord; -import org.jooq.generated.tables.records.GseTaskIpLogRecord; -import org.jooq.generated.tables.records.GseTaskLogRecord; -import org.jooq.generated.tables.records.OperationLogRecord; -import org.jooq.generated.tables.records.StepInstanceConfirmRecord; -import org.jooq.generated.tables.records.StepInstanceFileRecord; -import org.jooq.generated.tables.records.StepInstanceRecord; -import org.jooq.generated.tables.records.StepInstanceScriptRecord; -import org.jooq.generated.tables.records.StepInstanceVariableRecord; -import org.jooq.generated.tables.records.TaskInstanceRecord; -import org.jooq.generated.tables.records.TaskInstanceVariableRecord; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import static org.jooq.impl.DSL.max; -import static org.jooq.impl.DSL.min; - -/** - * @since 17/3/2021 15:46 - */ -@Slf4j -public class JobExecuteDAOImpl implements JobExecuteDAO { - - private static final FileSourceTaskLog FILE_SOURCE_TASK_LOG_TABLE = FileSourceTaskLog.FILE_SOURCE_TASK_LOG; - private static final List> FILE_SOURCE_TASK_LOG_FIELDS = - Arrays.asList(FILE_SOURCE_TASK_LOG_TABLE.STEP_INSTANCE_ID, FILE_SOURCE_TASK_LOG_TABLE.EXECUTE_COUNT, - FILE_SOURCE_TASK_LOG_TABLE.START_TIME, FILE_SOURCE_TASK_LOG_TABLE.END_TIME, - FILE_SOURCE_TASK_LOG_TABLE.TOTAL_TIME, FILE_SOURCE_TASK_LOG_TABLE.STATUS, - FILE_SOURCE_TASK_LOG_TABLE.FILE_SOURCE_BATCH_TASK_ID, FILE_SOURCE_TASK_LOG_TABLE.ROW_CREATE_TIME, - FILE_SOURCE_TASK_LOG_TABLE.ROW_UPDATE_TIME); - - private static final GseTaskIpLog GSE_TASK_IP_LOG_TABLE = GseTaskIpLog.GSE_TASK_IP_LOG; - private static final List> GSE_TASK_IP_LOG_FIELDS = Arrays.asList(GSE_TASK_IP_LOG_TABLE.STEP_INSTANCE_ID, - GSE_TASK_IP_LOG_TABLE.EXECUTE_COUNT, - GSE_TASK_IP_LOG_TABLE.IP, GSE_TASK_IP_LOG_TABLE.STATUS, GSE_TASK_IP_LOG_TABLE.START_TIME, - GSE_TASK_IP_LOG_TABLE.END_TIME, GSE_TASK_IP_LOG_TABLE.TOTAL_TIME, GSE_TASK_IP_LOG_TABLE.ERROR_CODE, - GSE_TASK_IP_LOG_TABLE.EXIT_CODE, GSE_TASK_IP_LOG_TABLE.TAG, GSE_TASK_IP_LOG_TABLE.LOG_OFFSET, - GSE_TASK_IP_LOG_TABLE.DISPLAY_IP, GSE_TASK_IP_LOG_TABLE.IS_TARGET, - GSE_TASK_IP_LOG_TABLE.ROW_CREATE_TIME, GSE_TASK_IP_LOG_TABLE.ROW_UPDATE_TIME, - GSE_TASK_IP_LOG_TABLE.IS_SOURCE); - - private static final GseTaskLog GSE_TASK_LOG_TABLE = GseTaskLog.GSE_TASK_LOG; - private static final List> GSE_TASK_LOG_FIELDS = Arrays.asList(GSE_TASK_LOG_TABLE.STEP_INSTANCE_ID, - GSE_TASK_LOG_TABLE.EXECUTE_COUNT, - GSE_TASK_LOG_TABLE.START_TIME, GSE_TASK_LOG_TABLE.END_TIME, GSE_TASK_LOG_TABLE.TOTAL_TIME, - GSE_TASK_LOG_TABLE.STATUS, GSE_TASK_LOG_TABLE.GSE_TASK_ID, GSE_TASK_LOG_TABLE.ROW_CREATE_TIME, - GSE_TASK_LOG_TABLE.ROW_UPDATE_TIME); - - private static final OperationLog OPERATION_LOG_TABLE = OperationLog.OPERATION_LOG; - private static final List> OPERATION_LOG_FIELDS = Arrays.asList(OPERATION_LOG_TABLE.ID, - OPERATION_LOG_TABLE.TASK_INSTANCE_ID, OPERATION_LOG_TABLE.OP_CODE, OPERATION_LOG_TABLE.OPERATOR, - OPERATION_LOG_TABLE.DETAIL, OPERATION_LOG_TABLE.CREATE_TIME, OPERATION_LOG_TABLE.ROW_CREATE_TIME, - OPERATION_LOG_TABLE.ROW_UPDATE_TIME); - - private static final StepInstance STEP_INSTANCE_TABLE = StepInstance.STEP_INSTANCE; - private static final List> STEP_INSTANCE_FIELDS = Arrays.asList(STEP_INSTANCE_TABLE.ID, - STEP_INSTANCE_TABLE.STEP_ID, - STEP_INSTANCE_TABLE.TASK_INSTANCE_ID, STEP_INSTANCE_TABLE.APP_ID, STEP_INSTANCE_TABLE.NAME, - STEP_INSTANCE_TABLE.TYPE, STEP_INSTANCE_TABLE.OPERATOR, STEP_INSTANCE_TABLE.STATUS, - STEP_INSTANCE_TABLE.EXECUTE_COUNT, STEP_INSTANCE_TABLE.TARGET_SERVERS, - STEP_INSTANCE_TABLE.ABNORMAL_AGENT_IP_LIST, STEP_INSTANCE_TABLE.START_TIME, - STEP_INSTANCE_TABLE.END_TIME, STEP_INSTANCE_TABLE.TOTAL_TIME, STEP_INSTANCE_TABLE.TOTAL_IP_NUM, - STEP_INSTANCE_TABLE.ABNORMAL_AGENT_NUM, STEP_INSTANCE_TABLE.RUN_IP_NUM, STEP_INSTANCE_TABLE.FAIL_IP_NUM, - STEP_INSTANCE_TABLE.SUCCESS_IP_NUM, STEP_INSTANCE_TABLE.CREATE_TIME, STEP_INSTANCE_TABLE.IGNORE_ERROR, - STEP_INSTANCE_TABLE.ROW_CREATE_TIME, STEP_INSTANCE_TABLE.ROW_UPDATE_TIME, STEP_INSTANCE_TABLE.STEP_NUM, - STEP_INSTANCE_TABLE.STEP_ORDER); - - private static final StepInstanceConfirm STEP_INSTANCE_CONFIRM_TABLE = StepInstanceConfirm.STEP_INSTANCE_CONFIRM; - private static final List> STEP_INSTANCE_CONFIRM_FIELDS = - Arrays.asList(STEP_INSTANCE_CONFIRM_TABLE.STEP_INSTANCE_ID, STEP_INSTANCE_CONFIRM_TABLE.CONFIRM_MESSAGE, - STEP_INSTANCE_CONFIRM_TABLE.CONFIRM_USERS, STEP_INSTANCE_CONFIRM_TABLE.CONFIRM_ROLES, - STEP_INSTANCE_CONFIRM_TABLE.NOTIFY_CHANNELS, STEP_INSTANCE_CONFIRM_TABLE.ROW_CREATE_TIME, - STEP_INSTANCE_CONFIRM_TABLE.ROW_UPDATE_TIME, STEP_INSTANCE_CONFIRM_TABLE.CONFIRM_REASON); - - private static final StepInstanceFile STEP_INSTANCE_FILE_TABLE = StepInstanceFile.STEP_INSTANCE_FILE; - private static final List> STEP_INSTANCE_FILE_FIELDS = - Arrays.asList(STEP_INSTANCE_FILE_TABLE.STEP_INSTANCE_ID, STEP_INSTANCE_FILE_TABLE.FILE_SOURCE, - STEP_INSTANCE_FILE_TABLE.RESOLVED_FILE_SOURCE, STEP_INSTANCE_FILE_TABLE.FILE_TARGET_PATH, - STEP_INSTANCE_FILE_TABLE.RESOLVED_FILE_TARGET_PATH, STEP_INSTANCE_FILE_TABLE.FILE_UPLOAD_SPEED_LIMIT, - STEP_INSTANCE_FILE_TABLE.FILE_DOWNLOAD_SPEED_LIMIT, STEP_INSTANCE_FILE_TABLE.FILE_DUPLICATE_HANDLE, - STEP_INSTANCE_FILE_TABLE.NOT_EXIST_PATH_HANDLER, STEP_INSTANCE_FILE_TABLE.EXECUTION_TIMEOUT, - STEP_INSTANCE_FILE_TABLE.SYSTEM_ACCOUNT_ID, STEP_INSTANCE_FILE_TABLE.SYSTEM_ACCOUNT, - STEP_INSTANCE_FILE_TABLE.ROW_CREATE_TIME, STEP_INSTANCE_FILE_TABLE.ROW_UPDATE_TIME); - - private static final StepInstanceScript STEP_INSTANCE_SCRIPT_TABLE = StepInstanceScript.STEP_INSTANCE_SCRIPT; - private static final List> STEP_INSTANCE_SCRIPT_FIELDS = - Arrays.asList(STEP_INSTANCE_SCRIPT_TABLE.STEP_INSTANCE_ID, STEP_INSTANCE_SCRIPT_TABLE.SCRIPT_CONTENT, - STEP_INSTANCE_SCRIPT_TABLE.SCRIPT_TYPE, STEP_INSTANCE_SCRIPT_TABLE.SCRIPT_PARAM, - STEP_INSTANCE_SCRIPT_TABLE.RESOLVED_SCRIPT_PARAM, STEP_INSTANCE_SCRIPT_TABLE.EXECUTION_TIMEOUT, - STEP_INSTANCE_SCRIPT_TABLE.SYSTEM_ACCOUNT_ID, STEP_INSTANCE_SCRIPT_TABLE.SYSTEM_ACCOUNT, - STEP_INSTANCE_SCRIPT_TABLE.DB_ACCOUNT_ID, STEP_INSTANCE_SCRIPT_TABLE.DB_TYPE, - STEP_INSTANCE_SCRIPT_TABLE.DB_ACCOUNT, STEP_INSTANCE_SCRIPT_TABLE.DB_PASSWORD, - STEP_INSTANCE_SCRIPT_TABLE.DB_PORT, STEP_INSTANCE_SCRIPT_TABLE.ROW_CREATE_TIME, - STEP_INSTANCE_SCRIPT_TABLE.ROW_UPDATE_TIME, STEP_INSTANCE_SCRIPT_TABLE.SCRIPT_SOURCE, - STEP_INSTANCE_SCRIPT_TABLE.SCRIPT_ID, STEP_INSTANCE_SCRIPT_TABLE.SCRIPT_VERSION_ID, - STEP_INSTANCE_SCRIPT_TABLE.IS_SECURE_PARAM); - - private static final StepInstanceVariable STEP_INSTANCE_VARIABLE_TABLE = - StepInstanceVariable.STEP_INSTANCE_VARIABLE; - private static final List> STEP_INSTANCE_VARIABLE_FIELDS = - Arrays.asList(STEP_INSTANCE_VARIABLE_TABLE.TASK_INSTANCE_ID, STEP_INSTANCE_VARIABLE_TABLE.STEP_INSTANCE_ID, - STEP_INSTANCE_VARIABLE_TABLE.TYPE, STEP_INSTANCE_VARIABLE_TABLE.PARAM_VALUES, - STEP_INSTANCE_VARIABLE_TABLE.ROW_CREATE_TIME, STEP_INSTANCE_VARIABLE_TABLE.ROW_UPDATE_TIME); - private static final TaskInstance TASK_INSTANCE_TABLE = TaskInstance.TASK_INSTANCE; - private static final List> TASK_INSTANCE_FIELDS = Arrays.asList(TASK_INSTANCE_TABLE.ID, - TASK_INSTANCE_TABLE.APP_ID, TASK_INSTANCE_TABLE.TASK_ID, - TASK_INSTANCE_TABLE.TASK_TEMPLATE_ID, TASK_INSTANCE_TABLE.NAME, TASK_INSTANCE_TABLE.TYPE, - TASK_INSTANCE_TABLE.OPERATOR, TASK_INSTANCE_TABLE.STATUS, TASK_INSTANCE_TABLE.CURRENT_STEP_ID, - TASK_INSTANCE_TABLE.STARTUP_MODE, TASK_INSTANCE_TABLE.TOTAL_TIME, TASK_INSTANCE_TABLE.CALLBACK_URL, - TASK_INSTANCE_TABLE.IS_DEBUG_TASK, TASK_INSTANCE_TABLE.CRON_TASK_ID, TASK_INSTANCE_TABLE.CREATE_TIME, - TASK_INSTANCE_TABLE.START_TIME, TASK_INSTANCE_TABLE.END_TIME, TASK_INSTANCE_TABLE.APP_CODE, - TASK_INSTANCE_TABLE.ROW_CREATE_TIME, TASK_INSTANCE_TABLE.ROW_UPDATE_TIME); - - private static final TaskInstanceVariable TASK_INSTANCE_VARIABLE_TABLE = - TaskInstanceVariable.TASK_INSTANCE_VARIABLE; - private static final List> TASK_INSTANCE_VARIABLE_FIELDS = Arrays.asList(TASK_INSTANCE_VARIABLE_TABLE.ID, - TASK_INSTANCE_VARIABLE_TABLE.TASK_INSTANCE_ID, - TASK_INSTANCE_VARIABLE_TABLE.NAME, TASK_INSTANCE_VARIABLE_TABLE.TYPE, - TASK_INSTANCE_VARIABLE_TABLE.VALUE, TASK_INSTANCE_VARIABLE_TABLE.IS_CHANGEABLE, - TASK_INSTANCE_VARIABLE_TABLE.ROW_CREATE_TIME, TASK_INSTANCE_VARIABLE_TABLE.ROW_UPDATE_TIME); - - private final DSLContext context; - private final ArchiveConfig archiveConfig; - - public JobExecuteDAOImpl(DSLContext context, ArchiveConfig archiveConfig) { - log.info("Init ExecuteReadDAO."); - this.context = context; - this.archiveConfig = archiveConfig; - } - - @Override - public Long getMaxNeedArchiveTaskInstanceId(Long endTime) { - Record1 maxNeedTaskInstanceIdRecord = - context.select(max(TASK_INSTANCE_TABLE.ID)) - .from(TASK_INSTANCE_TABLE) - .where(TASK_INSTANCE_TABLE.CREATE_TIME.lessOrEqual(endTime)) - .fetchOne(); - if (maxNeedTaskInstanceIdRecord != null) { - Long maxNeedTaskInstanceId = (Long) maxNeedTaskInstanceIdRecord.get(0); - if (maxNeedTaskInstanceId != null) { - return maxNeedTaskInstanceId; - } - } - return 0L; - } - - @Override - public Long getMaxNeedArchiveStepInstanceId(Long taskInstanceId) { - Record1 maxNeedStepInstanceIdRecord = - context.select(max(STEP_INSTANCE_TABLE.ID)) - .from(STEP_INSTANCE_TABLE) - .where(STEP_INSTANCE_TABLE.TASK_INSTANCE_ID.lessOrEqual(taskInstanceId)) - .fetchOne(); - if (maxNeedStepInstanceIdRecord != null) { - Long maxNeedStepInstanceId = (Long) maxNeedStepInstanceIdRecord.get(0); - if (maxNeedStepInstanceId != null) { - return maxNeedStepInstanceId; - } - } - return 0L; - } - - public Long getFirstInstanceId(Table table, TableField field) { - Record1 firstInstanceIdRecord = context.select(min(field)).from(table).fetchOne(); - if (firstInstanceIdRecord != null && firstInstanceIdRecord.get(0) != null) { - return (Long) firstInstanceIdRecord.get(0); - } - return 0L; - } - - @Override - public List> getFileSourceTaskLogFields() { - return FILE_SOURCE_TASK_LOG_FIELDS; - } - - @Override - public List> getGseTaskIpLogFields() { - return GSE_TASK_IP_LOG_FIELDS; - } - - @Override - public List> getGseTaskLogFields() { - return GSE_TASK_LOG_FIELDS; - } - - @Override - public List> getStepInstanceFields() { - return STEP_INSTANCE_FIELDS; - } - - @Override - public List> getStepInstanceConfirmFields() { - return STEP_INSTANCE_CONFIRM_FIELDS; - } - - @Override - public List> getStepInstanceFileFields() { - return STEP_INSTANCE_FILE_FIELDS; - } - - @Override - public List> getStepInstanceScriptFields() { - return STEP_INSTANCE_SCRIPT_FIELDS; - } - - @Override - public List> getStepInstanceVariableFields() { - return STEP_INSTANCE_VARIABLE_FIELDS; - } - - @Override - public List> getTaskInstanceFields() { - return TASK_INSTANCE_FIELDS; - } - - @Override - public List> getTaskInstanceVariableFields() { - return TASK_INSTANCE_VARIABLE_FIELDS; - } - - @Override - public List> getOperationLogFields() { - return OPERATION_LOG_FIELDS; - } - - @Override - public List listFileSourceTaskLog(Long start, Long stop) { - List conditions = new ArrayList<>(); - conditions.add(FILE_SOURCE_TASK_LOG_TABLE.STEP_INSTANCE_ID.greaterThan(start)); - conditions.add(FILE_SOURCE_TASK_LOG_TABLE.STEP_INSTANCE_ID.lessOrEqual(stop)); - - Result record = - context.select(FILE_SOURCE_TASK_LOG_FIELDS) - .from(FILE_SOURCE_TASK_LOG_TABLE) - .where(conditions) - .fetch(); - - return record.into(FILE_SOURCE_TASK_LOG_TABLE); - } - - @Override - public List listGseTaskIpLog(Long start, Long stop) { - List conditions = new ArrayList<>(); - conditions.add(GSE_TASK_IP_LOG_TABLE.STEP_INSTANCE_ID.greaterThan(start)); - conditions.add(GSE_TASK_IP_LOG_TABLE.STEP_INSTANCE_ID.lessOrEqual(stop)); - - Result record = - context.select(GSE_TASK_IP_LOG_FIELDS) - .from(GSE_TASK_IP_LOG_TABLE) - .where(conditions) - .fetch(); - - return record.into(GSE_TASK_IP_LOG_TABLE); - } - - @Override - public List listGseTaskLog(Long start, Long stop) { - List conditions = new ArrayList<>(); - conditions.add(GSE_TASK_LOG_TABLE.STEP_INSTANCE_ID.greaterThan(start)); - conditions.add(GSE_TASK_LOG_TABLE.STEP_INSTANCE_ID.lessOrEqual(stop)); - - Result record = - context.select(GSE_TASK_LOG_FIELDS) - .from(GSE_TASK_LOG_TABLE) - .where(conditions) - .fetch(); - - return record.into(GSE_TASK_LOG_TABLE); - } - - @Override - public List listStepInstance(Long start, Long stop) { - List conditions = new ArrayList<>(); - conditions.add(STEP_INSTANCE_TABLE.ID.greaterThan(start)); - conditions.add(STEP_INSTANCE_TABLE.ID.lessOrEqual(stop)); - - Result record = - context.select(STEP_INSTANCE_FIELDS) - .from(STEP_INSTANCE_TABLE) - .where(conditions) - .fetch(); - - return record.into(STEP_INSTANCE_TABLE); - } - - @Override - public List listStepInstanceConfirm(Long start, Long stop) { - List conditions = new ArrayList<>(); - conditions.add(STEP_INSTANCE_CONFIRM_TABLE.STEP_INSTANCE_ID.greaterThan(start)); - conditions.add(STEP_INSTANCE_CONFIRM_TABLE.STEP_INSTANCE_ID.lessOrEqual(stop)); - - Result record = - context.select(STEP_INSTANCE_CONFIRM_FIELDS) - .from(STEP_INSTANCE_CONFIRM_TABLE) - .where(conditions) - .fetch(); - - return record.into(STEP_INSTANCE_CONFIRM_TABLE); - } - - @Override - public List listStepInstanceFile(Long start, Long stop) { - List conditions = new ArrayList<>(); - conditions.add(STEP_INSTANCE_FILE_TABLE.STEP_INSTANCE_ID.greaterThan(start)); - conditions.add(STEP_INSTANCE_FILE_TABLE.STEP_INSTANCE_ID.lessOrEqual(stop)); - - Result record = - context.select(STEP_INSTANCE_FILE_FIELDS) - .from(STEP_INSTANCE_FILE_TABLE) - .where(conditions) - .fetch(); - - return record.into(STEP_INSTANCE_FILE_TABLE); - } - - @Override - public List listStepInstanceScript(Long start, Long stop) { - List conditions = new ArrayList<>(); - conditions.add(STEP_INSTANCE_SCRIPT_TABLE.STEP_INSTANCE_ID.greaterThan(start)); - conditions.add(STEP_INSTANCE_SCRIPT_TABLE.STEP_INSTANCE_ID.lessOrEqual(stop)); - - Result record = - context.select(STEP_INSTANCE_SCRIPT_FIELDS) - .from(STEP_INSTANCE_SCRIPT_TABLE) - .where(conditions) - .fetch(); - - return record.into(STEP_INSTANCE_SCRIPT_TABLE); - } - - @Override - public List listStepInstanceVariable(Long start, Long stop) { - List conditions = new ArrayList<>(); - conditions.add(STEP_INSTANCE_VARIABLE_TABLE.STEP_INSTANCE_ID.greaterThan(start)); - conditions.add(STEP_INSTANCE_VARIABLE_TABLE.STEP_INSTANCE_ID.lessOrEqual(stop)); - - Result record = - context.select(STEP_INSTANCE_VARIABLE_FIELDS) - .from(STEP_INSTANCE_VARIABLE_TABLE) - .where(conditions) - .fetch(); - - return record.into(STEP_INSTANCE_VARIABLE_TABLE); - } - - @Override - public List listTaskInstance(Long start, Long stop) { - List conditions = new ArrayList<>(); - conditions.add(TASK_INSTANCE_TABLE.ID.greaterThan(start)); - conditions.add(TASK_INSTANCE_TABLE.ID.lessOrEqual(stop)); - - Result record = - context.select(TASK_INSTANCE_FIELDS) - .from(TASK_INSTANCE_TABLE) - .where(conditions) - .fetch(); - - return record.into(TASK_INSTANCE_TABLE); - } - - @Override - public List listTaskInstanceVariable(Long start, Long stop) { - List conditions = new ArrayList<>(); - conditions.add(TASK_INSTANCE_VARIABLE_TABLE.TASK_INSTANCE_ID.greaterThan(start)); - conditions.add(TASK_INSTANCE_VARIABLE_TABLE.TASK_INSTANCE_ID.lessOrEqual(stop)); - - Result record = - context.select(TASK_INSTANCE_VARIABLE_FIELDS) - .from(TASK_INSTANCE_VARIABLE_TABLE) - .where(conditions) - .fetch(); - - return record.into(TASK_INSTANCE_VARIABLE_TABLE); - } - - @Override - public List listOperationLog(Long start, Long stop) { - List conditions = new ArrayList<>(); - conditions.add(OPERATION_LOG_TABLE.TASK_INSTANCE_ID.greaterThan(start)); - conditions.add(OPERATION_LOG_TABLE.TASK_INSTANCE_ID.lessOrEqual(stop)); - - Result record = - context.select(OPERATION_LOG_FIELDS) - .from(OPERATION_LOG_TABLE) - .where(conditions) - .fetch(); - - return record.into(OPERATION_LOG_TABLE); - } - - @Override - public int deleteFileSourceTaskLog(Long start, Long stop) { - List conditions = new ArrayList<>(); - conditions.add(FILE_SOURCE_TASK_LOG_TABLE.STEP_INSTANCE_ID.greaterThan(start)); - conditions.add(FILE_SOURCE_TASK_LOG_TABLE.STEP_INSTANCE_ID.lessOrEqual(stop)); - return deleteWithLimit(FILE_SOURCE_TASK_LOG_TABLE, conditions); - } - - @Override - public int deleteGseTaskIpLog(Long start, Long stop) { - List conditions = new ArrayList<>(); - conditions.add(GSE_TASK_IP_LOG_TABLE.STEP_INSTANCE_ID.greaterThan(start)); - conditions.add(GSE_TASK_IP_LOG_TABLE.STEP_INSTANCE_ID.lessOrEqual(stop)); - return deleteWithLimit(GSE_TASK_IP_LOG_TABLE, conditions); - } - - private int deleteWithLimit(Table table, List conditions) { - int totalDeleteRows = 0; - int maxLimitedDeleteRows = archiveConfig.getDeleteLimitRowCount(); - while (true) { - int deletedRows = context.delete(table).where(conditions).limit(maxLimitedDeleteRows).execute(); - totalDeleteRows += deletedRows; - if (deletedRows < maxLimitedDeleteRows) { - break; - } - } - return totalDeleteRows; - } - - @Override - public int deleteGseTaskLog(Long start, Long stop) { - List conditions = new ArrayList<>(); - conditions.add(GSE_TASK_LOG_TABLE.STEP_INSTANCE_ID.greaterThan(start)); - conditions.add(GSE_TASK_LOG_TABLE.STEP_INSTANCE_ID.lessOrEqual(stop)); - return deleteWithLimit(GSE_TASK_LOG_TABLE, conditions); - } - - @Override - public int deleteOperationLog(Long start, Long stop) { - List conditions = new ArrayList<>(); - conditions.add(OPERATION_LOG_TABLE.TASK_INSTANCE_ID.greaterThan(start)); - conditions.add(OPERATION_LOG_TABLE.TASK_INSTANCE_ID.lessOrEqual(stop)); - return deleteWithLimit(OPERATION_LOG_TABLE, conditions); - } - - @Override - public int deleteStepInstance(Long start, Long stop) { - List conditions = new ArrayList<>(); - conditions.add(STEP_INSTANCE_TABLE.ID.greaterThan(start)); - conditions.add(STEP_INSTANCE_TABLE.ID.lessOrEqual(stop)); - return deleteWithLimit(STEP_INSTANCE_TABLE, conditions); - } - - @Override - public int deleteStepInstanceConfirm(Long start, Long stop) { - List conditions = new ArrayList<>(); - conditions.add(STEP_INSTANCE_CONFIRM_TABLE.STEP_INSTANCE_ID.greaterThan(start)); - conditions.add(STEP_INSTANCE_CONFIRM_TABLE.STEP_INSTANCE_ID.lessOrEqual(stop)); - return deleteWithLimit(STEP_INSTANCE_CONFIRM_TABLE, conditions); - } - - @Override - public int deleteStepInstanceFile(Long start, Long stop) { - List conditions = new ArrayList<>(); - conditions.add(STEP_INSTANCE_FILE_TABLE.STEP_INSTANCE_ID.greaterThan(start)); - conditions.add(STEP_INSTANCE_FILE_TABLE.STEP_INSTANCE_ID.lessOrEqual(stop)); - return deleteWithLimit(STEP_INSTANCE_FILE_TABLE, conditions); - } - - @Override - public int deleteStepInstanceScript(Long start, Long stop) { - List conditions = new ArrayList<>(); - conditions.add(STEP_INSTANCE_SCRIPT_TABLE.STEP_INSTANCE_ID.greaterThan(start)); - conditions.add(STEP_INSTANCE_SCRIPT_TABLE.STEP_INSTANCE_ID.lessOrEqual(stop)); - return deleteWithLimit(STEP_INSTANCE_SCRIPT_TABLE, conditions); - } - - @Override - public int deleteStepInstanceVariable(Long start, Long stop) { - List conditions = new ArrayList<>(); - conditions.add(STEP_INSTANCE_VARIABLE_TABLE.STEP_INSTANCE_ID.greaterThan(start)); - conditions.add(STEP_INSTANCE_VARIABLE_TABLE.STEP_INSTANCE_ID.lessOrEqual(stop)); - return deleteWithLimit(STEP_INSTANCE_VARIABLE_TABLE, conditions); - } - - @Override - public int deleteTaskInstance(Long start, Long stop) { - List conditions = new ArrayList<>(); - conditions.add(TASK_INSTANCE_TABLE.ID.greaterThan(start)); - conditions.add(TASK_INSTANCE_TABLE.ID.lessOrEqual(stop)); - return deleteWithLimit(TASK_INSTANCE_TABLE, conditions); - } - - @Override - public int deleteTaskInstanceVariable(Long start, Long stop) { - List conditions = new ArrayList<>(); - conditions.add(TASK_INSTANCE_VARIABLE_TABLE.TASK_INSTANCE_ID.greaterThan(start)); - conditions.add(TASK_INSTANCE_VARIABLE_TABLE.TASK_INSTANCE_ID.lessOrEqual(stop)); - return deleteWithLimit(TASK_INSTANCE_VARIABLE_TABLE, conditions); - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/OperationLogRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/OperationLogRecordDAO.java new file mode 100644 index 0000000000..0fa7ea537e --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/OperationLogRecordDAO.java @@ -0,0 +1,30 @@ +package com.tencent.bk.job.backup.dao.impl; + +import com.tencent.bk.job.backup.config.ArchiveConfig; +import org.jooq.DSLContext; +import org.jooq.Table; +import org.jooq.TableField; +import org.jooq.generated.tables.OperationLog; +import org.jooq.generated.tables.records.OperationLogRecord; + +/** + * operation_log DAO + */ +public class OperationLogRecordDAO extends AbstractExecuteRecordDAO { + + private static final OperationLog TABLE = OperationLog.OPERATION_LOG; + + public OperationLogRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { + super(context, archiveConfig); + } + + @Override + public Table getTable() { + return TABLE; + } + + @Override + public TableField getArchiveIdField() { + return TABLE.TASK_INSTANCE_ID; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/RollingConfigRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/RollingConfigRecordDAO.java new file mode 100644 index 0000000000..9f2243e41e --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/RollingConfigRecordDAO.java @@ -0,0 +1,30 @@ +package com.tencent.bk.job.backup.dao.impl; + +import com.tencent.bk.job.backup.config.ArchiveConfig; +import org.jooq.DSLContext; +import org.jooq.Table; +import org.jooq.TableField; +import org.jooq.generated.tables.RollingConfig; +import org.jooq.generated.tables.records.RollingConfigRecord; + +/** + * rolling_config DAO + */ +public class RollingConfigRecordDAO extends AbstractExecuteRecordDAO { + + private static final RollingConfig TABLE = RollingConfig.ROLLING_CONFIG; + + public RollingConfigRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { + super(context, archiveConfig); + } + + @Override + public Table getTable() { + return TABLE; + } + + @Override + public TableField getArchiveIdField() { + return TABLE.TASK_INSTANCE_ID; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceConfirmRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceConfirmRecordDAO.java new file mode 100644 index 0000000000..4789097fc6 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceConfirmRecordDAO.java @@ -0,0 +1,31 @@ +package com.tencent.bk.job.backup.dao.impl; + +import com.tencent.bk.job.backup.config.ArchiveConfig; +import org.jooq.DSLContext; +import org.jooq.Table; +import org.jooq.TableField; +import org.jooq.generated.tables.StepInstanceConfirm; +import org.jooq.generated.tables.records.StepInstanceConfirmRecord; + +/** + * step_instance_confirm DAO + */ +public class StepInstanceConfirmRecordDAO extends AbstractExecuteRecordDAO { + + + private static final StepInstanceConfirm TABLE = StepInstanceConfirm.STEP_INSTANCE_CONFIRM; + + public StepInstanceConfirmRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { + super(context, archiveConfig); + } + + @Override + public Table getTable() { + return TABLE; + } + + @Override + public TableField getArchiveIdField() { + return TABLE.STEP_INSTANCE_ID; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceFileRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceFileRecordDAO.java new file mode 100644 index 0000000000..2693aafe2c --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceFileRecordDAO.java @@ -0,0 +1,30 @@ +package com.tencent.bk.job.backup.dao.impl; + +import com.tencent.bk.job.backup.config.ArchiveConfig; +import org.jooq.DSLContext; +import org.jooq.Table; +import org.jooq.TableField; +import org.jooq.generated.tables.StepInstanceFile; +import org.jooq.generated.tables.records.StepInstanceFileRecord; + +/** + * step_instance_file DAO + */ +public class StepInstanceFileRecordDAO extends AbstractExecuteRecordDAO { + + private static final StepInstanceFile TABLE = StepInstanceFile.STEP_INSTANCE_FILE; + + public StepInstanceFileRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { + super(context, archiveConfig); + } + + @Override + public Table getTable() { + return TABLE; + } + + @Override + public TableField getArchiveIdField() { + return TABLE.STEP_INSTANCE_ID; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceRecordDAO.java new file mode 100644 index 0000000000..d3854e7a43 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceRecordDAO.java @@ -0,0 +1,48 @@ +package com.tencent.bk.job.backup.dao.impl; + +import com.tencent.bk.job.backup.config.ArchiveConfig; +import org.jooq.DSLContext; +import org.jooq.Record1; +import org.jooq.Table; +import org.jooq.TableField; +import org.jooq.generated.tables.StepInstance; +import org.jooq.generated.tables.records.StepInstanceRecord; + +import static org.jooq.impl.DSL.max; + +/** + * step_instance DAO + */ +public class StepInstanceRecordDAO extends AbstractExecuteRecordDAO { + + private static final StepInstance TABLE = StepInstance.STEP_INSTANCE; + + public StepInstanceRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { + super(context, archiveConfig); + } + + @Override + public Table getTable() { + return TABLE; + } + + public Long getMaxNeedArchiveStepInstanceId(Long taskInstanceId) { + Record1 maxNeedStepInstanceIdRecord = + context.select(max(TABLE.ID)) + .from(TABLE) + .where(TABLE.TASK_INSTANCE_ID.lessOrEqual(taskInstanceId)) + .fetchOne(); + if (maxNeedStepInstanceIdRecord != null) { + Long maxNeedStepInstanceId = (Long) maxNeedStepInstanceIdRecord.get(0); + if (maxNeedStepInstanceId != null) { + return maxNeedStepInstanceId; + } + } + return 0L; + } + + @Override + public TableField getArchiveIdField() { + return TABLE.ID; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceRollingTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceRollingTaskRecordDAO.java new file mode 100644 index 0000000000..9a3e3b1406 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceRollingTaskRecordDAO.java @@ -0,0 +1,30 @@ +package com.tencent.bk.job.backup.dao.impl; + +import com.tencent.bk.job.backup.config.ArchiveConfig; +import org.jooq.DSLContext; +import org.jooq.Table; +import org.jooq.TableField; +import org.jooq.generated.tables.StepInstanceRollingTask; +import org.jooq.generated.tables.records.StepInstanceRollingTaskRecord; + +/** + * step_instance_rolling_task DAO + */ +public class StepInstanceRollingTaskRecordDAO extends AbstractExecuteRecordDAO { + + private static final StepInstanceRollingTask TABLE = StepInstanceRollingTask.STEP_INSTANCE_ROLLING_TASK; + + public StepInstanceRollingTaskRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { + super(context, archiveConfig); + } + + @Override + public Table getTable() { + return TABLE; + } + + @Override + public TableField getArchiveIdField() { + return TABLE.STEP_INSTANCE_ID; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceScriptRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceScriptRecordDAO.java new file mode 100644 index 0000000000..27d73c4e97 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceScriptRecordDAO.java @@ -0,0 +1,30 @@ +package com.tencent.bk.job.backup.dao.impl; + +import com.tencent.bk.job.backup.config.ArchiveConfig; +import org.jooq.DSLContext; +import org.jooq.Table; +import org.jooq.TableField; +import org.jooq.generated.tables.StepInstanceScript; +import org.jooq.generated.tables.records.StepInstanceScriptRecord; + +/** + * step_instance_script DAO + */ +public class StepInstanceScriptRecordDAO extends AbstractExecuteRecordDAO { + + private static final StepInstanceScript TABLE = StepInstanceScript.STEP_INSTANCE_SCRIPT; + + public StepInstanceScriptRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { + super(context, archiveConfig); + } + + @Override + public Table getTable() { + return TABLE; + } + + @Override + public TableField getArchiveIdField() { + return TABLE.STEP_INSTANCE_ID; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceVariableRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceVariableRecordDAO.java new file mode 100644 index 0000000000..d863edf502 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceVariableRecordDAO.java @@ -0,0 +1,30 @@ +package com.tencent.bk.job.backup.dao.impl; + +import com.tencent.bk.job.backup.config.ArchiveConfig; +import org.jooq.DSLContext; +import org.jooq.Table; +import org.jooq.TableField; +import org.jooq.generated.tables.StepInstanceVariable; +import org.jooq.generated.tables.records.StepInstanceVariableRecord; + +/** + * step_instance_variable DAO + */ +public class StepInstanceVariableRecordDAO extends AbstractExecuteRecordDAO { + + private static final StepInstanceVariable TABLE = StepInstanceVariable.STEP_INSTANCE_VARIABLE; + + public StepInstanceVariableRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { + super(context, archiveConfig); + } + + @Override + public Table getTable() { + return TABLE; + } + + @Override + public TableField getArchiveIdField() { + return TABLE.STEP_INSTANCE_ID; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/TaskInstanceRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/TaskInstanceRecordDAO.java new file mode 100644 index 0000000000..b06c50b688 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/TaskInstanceRecordDAO.java @@ -0,0 +1,50 @@ +package com.tencent.bk.job.backup.dao.impl; + +import com.tencent.bk.job.backup.config.ArchiveConfig; +import org.jooq.DSLContext; +import org.jooq.Record1; +import org.jooq.Table; +import org.jooq.TableField; +import org.jooq.generated.tables.TaskInstance; +import org.jooq.generated.tables.records.TaskInstanceRecord; + +import static org.jooq.impl.DSL.max; + +/** + * task_instance DAO + */ +public class TaskInstanceRecordDAO extends AbstractExecuteRecordDAO { + + private static final TaskInstance TABLE = TaskInstance.TASK_INSTANCE; + + public TaskInstanceRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { + super(context, archiveConfig); + } + + @Override + public Table getTable() { + return TABLE; + } + + @Override + public TableField getArchiveIdField() { + return TABLE.ID; + } + + public Long getMaxNeedArchiveTaskInstanceId(Long endTime) { + Record1 maxNeedTaskInstanceIdRecord = + context.select(max(TABLE.ID)) + .from(TABLE) + .where(TABLE.CREATE_TIME.lessOrEqual(endTime)) + .fetchOne(); + if (maxNeedTaskInstanceIdRecord != null) { + Long maxNeedTaskInstanceId = (Long) maxNeedTaskInstanceIdRecord.get(0); + if (maxNeedTaskInstanceId != null) { + return maxNeedTaskInstanceId; + } + } + return 0L; + } + + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/TaskInstanceVariableRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/TaskInstanceVariableRecordDAO.java new file mode 100644 index 0000000000..491a610254 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/TaskInstanceVariableRecordDAO.java @@ -0,0 +1,30 @@ +package com.tencent.bk.job.backup.dao.impl; + +import com.tencent.bk.job.backup.config.ArchiveConfig; +import org.jooq.DSLContext; +import org.jooq.Table; +import org.jooq.TableField; +import org.jooq.generated.tables.TaskInstanceVariable; +import org.jooq.generated.tables.records.TaskInstanceVariableRecord; + +/** + * task_instance_variable DAO + */ +public class TaskInstanceVariableRecordDAO extends AbstractExecuteRecordDAO { + + private static final TaskInstanceVariable TABLE = TaskInstanceVariable.TASK_INSTANCE_VARIABLE; + + public TaskInstanceVariableRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { + super(context, archiveConfig); + } + + @Override + public Table getTable() { + return TABLE; + } + + @Override + public TableField getArchiveIdField() { + return TABLE.TASK_INSTANCE_ID; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/executor/ExportJobExecutor.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/executor/ExportJobExecutor.java index 12285d5b7f..8dc8db40f0 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/executor/ExportJobExecutor.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/executor/ExportJobExecutor.java @@ -445,7 +445,7 @@ private void processLinkScript(ExportJobInfoDTO exportJobInfo, JobBackupInfoDTO private void extractScriptInfo(List stepList, Map> linkScriptContentMap) { for (TaskStepVO taskStep : stepList) { - if (TaskStepTypeEnum.SCRIPT.getType() == taskStep.getType()) { + if (TaskStepTypeEnum.SCRIPT.getValue() == taskStep.getType()) { TaskScriptStepVO scriptStepInfo = taskStep.getScriptStepInfo(); if (TaskScriptSourceEnum.CITING.getType() == scriptStepInfo.getScriptSource() || TaskScriptSourceEnum.PUBLIC.getType() == scriptStepInfo.getScriptSource()) { @@ -549,7 +549,7 @@ private void processLocalFile(ExportJobInfoDTO exportInfo, JobBackupInfoDTO jobB private List extractLocalFileList(List stepList, File uploadFileDirectory) { List localFileList = new ArrayList<>(); for (TaskStepVO taskStep : stepList) { - if (TaskStepTypeEnum.FILE.getType() == taskStep.getType()) { + if (TaskStepTypeEnum.FILE.getValue() == taskStep.getType()) { TaskFileStepVO fileStepInfo = taskStep.getFileStepInfo(); for (TaskFileSourceInfoVO taskFileInfo : fileStepInfo.getFileSourceList()) { if (TaskFileTypeEnum.LOCAL.getType() == taskFileInfo.getFileType()) { diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/executor/ImportJobExecutor.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/executor/ImportJobExecutor.java index 1081f9fa9e..373889f4cc 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/executor/ImportJobExecutor.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/executor/ImportJobExecutor.java @@ -652,7 +652,7 @@ private void processFile(ImportJobInfoDTO importJob, JobBackupInfoDTO jobBackupI private void fixScript(ImportJobInfoDTO importJob, Map> linkScriptContentMap, List stepList) { for (TaskStepVO taskStep : stepList) { - if (TaskStepTypeEnum.SCRIPT.getType() == taskStep.getType()) { + if (TaskStepTypeEnum.SCRIPT.getValue() == taskStep.getType()) { TaskScriptStepVO scriptStepInfo = taskStep.getScriptStepInfo(); if (TaskScriptSourceEnum.CITING.getType() == scriptStepInfo.getScriptSource() || TaskScriptSourceEnum.PUBLIC.getType() == scriptStepInfo.getScriptSource()) { diff --git a/src/backend/job-config/src/main/resources/logback/logback-app-props.xml b/src/backend/job-config/src/main/resources/logback/logback-app-props.xml index ee3a5b9dbd..69bd3ec4b5 100644 --- a/src/backend/job-config/src/main/resources/logback/logback-app-props.xml +++ b/src/backend/job-config/src/main/resources/logback/logback-app-props.xml @@ -1,4 +1,5 @@ + diff --git a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/inner/ServerDTO.java b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/inner/ServerDTO.java index b9bb934c44..532e91ad39 100644 --- a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/inner/ServerDTO.java +++ b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/inner/ServerDTO.java @@ -134,7 +134,7 @@ public static ServerDTO fromEsbServerV3(EsbServerV3DTO server) { } if (CollectionUtils.isNotEmpty(server.getIps())) { List staticIpList = new ArrayList<>(); - server.getIps().forEach(ip -> staticIpList.add(new HostDTO(ip.getCloudAreaId(), ip.getIp()))); + server.getIps().forEach(ip -> staticIpList.add(new HostDTO(ip.getBkCloudId(), ip.getIp()))); serverDTO.setIps(staticIpList); } return serverDTO; diff --git a/src/backend/job-crontab/boot-job-crontab/src/main/resources/application.yml b/src/backend/job-crontab/boot-job-crontab/src/main/resources/application.yml index 841fecd843..033cf5770c 100644 --- a/src/backend/job-crontab/boot-job-crontab/src/main/resources/application.yml +++ b/src/backend/job-crontab/boot-job-crontab/src/main/resources/application.yml @@ -1,6 +1,4 @@ spring: - application: - name: job-crontab profiles: active: prod diff --git a/src/backend/job-crontab/boot-job-crontab/src/main/resources/bootstrap.yml b/src/backend/job-crontab/boot-job-crontab/src/main/resources/bootstrap.yml index 77bd21fa13..50878cc860 100644 --- a/src/backend/job-crontab/boot-job-crontab/src/main/resources/bootstrap.yml +++ b/src/backend/job-crontab/boot-job-crontab/src/main/resources/bootstrap.yml @@ -1,4 +1,6 @@ spring: + application: + name: job-crontab cloud: kubernetes: config: diff --git a/src/backend/job-crontab/boot-job-crontab/src/main/resources/logback/logback-app-props.xml b/src/backend/job-crontab/boot-job-crontab/src/main/resources/logback/logback-app-props.xml index 6d17e3dd33..61489b7fef 100644 --- a/src/backend/job-crontab/boot-job-crontab/src/main/resources/logback/logback-app-props.xml +++ b/src/backend/job-crontab/boot-job-crontab/src/main/resources/logback/logback-app-props.xml @@ -1,4 +1,5 @@ + diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/web/impl/WebCronJobResourceImpl.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/web/impl/WebCronJobResourceImpl.java index 23b68a56f9..3e89544fba 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/web/impl/WebCronJobResourceImpl.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/web/impl/WebCronJobResourceImpl.java @@ -273,7 +273,7 @@ private void fillStatisticInfo(CronJobVO resultCronJob, // 正在执行 case RUNNING: // 等待用户 - case WAITING: + case WAITING_USER: default: break; } diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/AbstractQuartzJobBean.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/AbstractQuartzJobBean.java index b2b2cd9ed3..5af9abce60 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/AbstractQuartzJobBean.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/AbstractQuartzJobBean.java @@ -24,7 +24,6 @@ package com.tencent.bk.job.crontab.timer; -import brave.Tracer; import com.tencent.bk.job.common.redis.util.LockUtils; import com.tencent.bk.job.common.util.JobContextUtil; import com.tencent.bk.job.crontab.metrics.ScheduleMeasureService; @@ -32,6 +31,7 @@ import org.quartz.JobExecutionContext; import org.quartz.JobExecutionException; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cloud.sleuth.Tracer; import org.springframework.scheduling.quartz.QuartzJobBean; import java.time.Instant; @@ -67,7 +67,7 @@ protected static String getLockKey(JobExecutionContext context) { @Override protected void executeInternal(JobExecutionContext context) { scheduleMeasureService.recordCronScheduleDelay(name(), context); - JobContextUtil.setRequestId(tracer.currentSpan().context().traceIdString()); + JobContextUtil.setRequestId(tracer.currentSpan().context().traceId()); String executeId = JobContextUtil.getRequestId(); try { if (log.isDebugEnabled()) { diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java index cd4af5fcbc..35948b606c 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java @@ -121,7 +121,6 @@ Response> getTaskHistoryList( String ip ); - @ApiOperation(value = "获取作业执行信息", produces = "application/json") @GetMapping(value = {"/task-execution-result/{taskInstanceId}"}) Response getTaskExecutionResult( @ApiParam("用户名，网关自动传入") @@ -163,8 +162,9 @@ Response getStepExecutionResult( @PathVariable("stepInstanceId") Long stepInstanceId, @ApiParam(value = "执行次数，首次传0", name = "executeCount", required = true) - @PathVariable(value = "executeCount", required = false) - Integer executeCount, + @PathVariable(value = "executeCount", required = false) Integer executeCount, + @ApiParam(value = "滚动执行批次。如果不传表示返回最新批次，传入0表示返回全部批次", name = "batch") + @RequestParam(value = "batch", required = false) Integer batch, @ApiParam(value = "任务执行结果", name = "resultType") @RequestParam(value = "resultType", required = false) Integer resultType, @@ -204,8 +204,9 @@ Response getFastTaskStepExecutionResult( @PathVariable(value = "scopeId") String scopeId, @ApiParam(value = "任务实例ID", name = "taskInstanceId", required = true) - @PathVariable("taskInstanceId") - Long taskInstanceId, + @PathVariable("taskInstanceId") Long taskInstanceId, + @ApiParam(value = "滚动执行批次", name = "batch") + @RequestParam(value = "batch", required = false) Integer batch, @ApiParam(value = "任务执行结果", name = "resultType") @RequestParam(value = "resultType", required = false) Integer resultType, @@ -245,7 +246,9 @@ Response getScriptLogContentByIp( @PathVariable("executeCount") Integer executeCount, @ApiParam(value = "IP，格式为云区域ID:IP,比如1:10.10.10.10", name = "ip", required = true) - @PathVariable("ip") String ip + @PathVariable("ip") String ip, + @ApiParam(value = "滚动批次，非滚动步骤不需要传入", name = "batch") + @RequestParam(value = "batch", required = false) Integer batch ); @ApiOperation(value = "获取文件分发步骤IP对应的日志", produces = "application/json") @@ -274,7 +277,10 @@ Response getFileLogContentByIp( String ip, @ApiParam(value = "文件任务上传下载标识,upload-上传,download-下载", name = "mode", required = true) @RequestParam(value = "mode") - String mode + String mode, + @ApiParam(value = "滚动批次，非滚动步骤不需要传入", name = "batch") + @RequestParam(value = "batch", required = false) + Integer batch ); @ApiOperation(value = "获取文件分发步骤文件任务ID对应的执行日志", produces = "application/json") @@ -298,6 +304,9 @@ Response> getFileLogContentByFileTaskIds( @ApiParam(value = "执行次数，首次传0", name = "executeCount", required = true) @PathVariable("executeCount") Integer executeCount, + @ApiParam(value = "滚动批次，非滚动步骤不需要传入", name = "batch") + @RequestParam(value = "batch", required = false) + Integer batch, @ApiParam(value = "文件任务ID列表", name = "taskIds", required = true) @RequestBody List taskIds @@ -345,8 +354,9 @@ Response> getHostsByResultType( @PathVariable("stepInstanceId") Long stepInstanceId, @ApiParam(value = "执行次数，首次传0", name = "executeCount", required = true) - @PathVariable("executeCount") - Integer executeCount, + @PathVariable("executeCount") Integer executeCount, + @ApiParam(value = "滚动执行批次，该步骤为滚动步骤时并且用户指定了批次的场景下需要传入该参数", name = "batch") + @RequestParam(value = "batch", required = false) Integer batch, @ApiParam(value = "任务执行结果", name = "resultType", required = true) @RequestParam(value = "resultType") Integer resultType, @@ -375,7 +385,10 @@ Response> listStepExecutionHistory( String scopeId, @ApiParam(value = "步骤实例ID", name = "stepInstanceId", required = true) @PathVariable("stepInstanceId") - Long stepInstanceId + Long stepInstanceId, + @ApiParam(value = "滚动批次，非滚动步骤不需要传入", name = "batch") + @RequestParam(value = "batch", required = false) + Integer batch ); } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskLogResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskLogResource.java index 0f95f229c8..6645b97289 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskLogResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskLogResource.java @@ -71,9 +71,12 @@ Response requestDownloadLogFile( @ApiParam(value = "步骤实例 ID", name = "stepInstanceId", required = true) @PathVariable("stepInstanceId") Long stepInstanceId, + @ApiParam(value = "hostId", name = "hostId") + @RequestParam(value = "hostId", required = false) + Long hostId, @ApiParam(value = "ip", name = "ip") @RequestParam(value = "ip", required = false) - String ip, + String cloudIp, @ApiParam(value = "重新打包", name = "repackage") @RequestParam(value = "repackage", required = false) Boolean repackage @@ -98,8 +101,11 @@ ResponseEntity downloadLogFile( @ApiParam(value = "步骤实例 ID", name = "stepInstanceId", required = true) @PathVariable("stepInstanceId") Long stepInstanceId, + @ApiParam(value = "hostId", name = "hostId") + @RequestParam(value = "hostId", required = false) + Long hostId, @ApiParam(value = "ip", name = "ip") @RequestParam(value = "ip", required = false) - String ip + String cloudIp ); } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/RunStatusEnum.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/RunStatusEnum.java index e02c13243f..954aa2ee5e 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/RunStatusEnum.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/RunStatusEnum.java @@ -58,7 +58,7 @@ public enum RunStatusEnum { /** * 等待用户 */ - WAITING(7), + WAITING_USER(7), /** * 手动结束 */ @@ -82,7 +82,11 @@ public enum RunStatusEnum { /** * 被丢弃 */ - ABANDONED(14); + ABANDONED(14), + /** + * 滚动等待 + */ + ROLLING_WAITING(15); private final Integer value; private static final Set FINISHED_STATUS_SET = new HashSet<>(); @@ -112,8 +116,12 @@ public static RunStatusEnum valueOf(int status) { throw new IllegalArgumentException("Invalid run status[" + status + "]"); } + public static RunStatusEnum valueOf(byte status) { + return valueOf((int) status); + } + /** - * 判断是否是终止态的状态 + * 是否终止态 * * @param status 状态 */ @@ -125,6 +133,9 @@ public Integer getValue() { return value; } + /** + * 获取国际化Key + **/ public String getI18nKey() { return "task.run.status." + this.name().toLowerCase(); } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/StepExecuteTypeEnum.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/StepExecuteTypeEnum.java index d5ffce3463..5b7992bf85 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/StepExecuteTypeEnum.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/StepExecuteTypeEnum.java @@ -28,8 +28,10 @@ * 步骤执行类型 */ public enum StepExecuteTypeEnum { - DEFAULT(0, "未定义"), EXECUTE_SCRIPT(1, "执行脚本"), SEND_FILE(2, "分发文件"), MANUAL_CONFIRM(3, "人工确认"), EXECUTE_SQL(4, - "执行SQL脚本"); + EXECUTE_SCRIPT(1, "执行脚本"), + SEND_FILE(2, "分发文件"), + MANUAL_CONFIRM(3, "人工确认"), + EXECUTE_SQL(4, "执行SQL脚本"); private final Integer value; private final String name; @@ -45,7 +47,7 @@ public static StepExecuteTypeEnum valueOf(int type) { return stepType; } } - return DEFAULT; + throw new IllegalArgumentException("StepExecuteTypeEnum:" + type); } public Integer getValue() { diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/StepRunModeEnum.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/StepRunModeEnum.java new file mode 100644 index 0000000000..582beb124d --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/StepRunModeEnum.java @@ -0,0 +1,62 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.common.constants; + +/** + * 步骤执行模式 + */ +public enum StepRunModeEnum { + /** + * 全量执行 + */ + RUN_ALL(1), + /** + * 滚动全量执行 + */ + ROLLING_ALL(2), + /** + * 滚动分批执行 + */ + ROLLING_IN_BATCH(3); + + private final int value; + + StepRunModeEnum(int value) { + this.value = value; + } + + public static StepRunModeEnum valueOf(int status) { + for (StepRunModeEnum runStatusEnum : values()) { + if (runStatusEnum.getValue() == status) { + return runStatusEnum; + } + } + return null; + } + + public int getValue() { + return value; + } +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/EsbStepInstanceResultAndLog.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/EsbStepInstanceResultAndLog.java index b342908b21..66d1490d5c 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/EsbStepInstanceResultAndLog.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/EsbStepInstanceResultAndLog.java @@ -82,11 +82,11 @@ public static class StepInstResultDTO { * ip 日志内容 */ @JsonProperty("ip_logs") - private List ipLogs; + private List ipLogs; } @Data - public static class IpLogDTO { + public static class EsbGseAgentTaskDTO { /** * 开始执行时间，YYYY-MM-DD HH:mm:ss */ diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/GseTaskLogDTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbRollingConfigDTO.java similarity index 79% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/GseTaskLogDTO.java rename to src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbRollingConfigDTO.java index 23cfc41033..3894a9b396 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/GseTaskLogDTO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbRollingConfigDTO.java @@ -22,29 +22,25 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.execute.model; +package com.tencent.bk.job.execute.model.esb.v3; import lombok.Data; /** - * GSE任务执行结果 + * 滚动执行配置 */ @Data -public class GseTaskLogDTO { - private Long stepInstanceId; - - private Integer executeCount; - - private Long startTime; - - private Long endTime; +public class EsbRollingConfigDTO { /** - * 耗时，毫秒 + * 滚动分批策略表达式 */ - private Long totalTime; + private String expression; - private Integer status = 1; - - private String gseTaskId; + /** + * 滚动机制,1-执行失败则暂停；2-忽略失败，自动滚动下一批；3-人工确认 + * + * @see com.tencent.bk.job.common.constant.RollingModeEnum + */ + private Integer mode; } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbFastExecuteScriptV3Request.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbFastExecuteScriptV3Request.java index 729ce4ad52..bb51a1cbb9 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbFastExecuteScriptV3Request.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbFastExecuteScriptV3Request.java @@ -29,6 +29,7 @@ import com.tencent.bk.job.common.esb.model.EsbAppScopeReq; import com.tencent.bk.job.common.esb.model.job.EsbIpDTO; import com.tencent.bk.job.common.esb.model.job.v3.EsbServerV3DTO; +import com.tencent.bk.job.execute.model.esb.v3.EsbRollingConfigDTO; import lombok.Getter; import lombok.Setter; import org.hibernate.validator.constraints.Range; @@ -115,6 +116,12 @@ public class EsbFastExecuteScriptV3Request extends EsbAppScopeReq { @JsonProperty("callback_url") private String callbackUrl; + /** + * 滚动配置 + */ + @JsonProperty("rolling_config") + private EsbRollingConfigDTO rollingConfig; + public void trimIps() { if (this.targetServer != null) { trimIps(this.targetServer.getIps()); diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbFastTransferFileV3Request.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbFastTransferFileV3Request.java index 88a5d99940..fe942c48fa 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbFastTransferFileV3Request.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbFastTransferFileV3Request.java @@ -30,6 +30,7 @@ import com.tencent.bk.job.common.esb.model.job.EsbIpDTO; import com.tencent.bk.job.common.esb.model.job.v3.EsbFileSourceV3DTO; import com.tencent.bk.job.common.esb.model.job.v3.EsbServerV3DTO; +import com.tencent.bk.job.execute.model.esb.v3.EsbRollingConfigDTO; import lombok.Getter; import lombok.Setter; import org.hibernate.validator.constraints.Range; @@ -112,6 +113,12 @@ public class EsbFastTransferFileV3Request extends EsbAppScopeReq { */ private Integer transferMode; + /** + * 滚动配置 + */ + @JsonProperty("rolling_config") + private EsbRollingConfigDTO rollingConfig; + public void trimIps() { if (this.targetServer != null) { trimIps(this.targetServer.getIps()); diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/WebFastExecuteScriptRequest.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/WebFastExecuteScriptRequest.java index f2d2189671..e0309d2dbc 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/WebFastExecuteScriptRequest.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/WebFastExecuteScriptRequest.java @@ -27,6 +27,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.tencent.bk.job.common.constant.JobConstants; import com.tencent.bk.job.execute.model.web.vo.ExecuteTargetVO; +import com.tencent.bk.job.execute.model.web.vo.RollingConfigVO; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; @@ -108,4 +109,10 @@ public class WebFastExecuteScriptRequest { @ApiModelProperty(value = "任务实例ID,重做的时候需要传入") private Long taskInstanceId; + @ApiModelProperty(value = "滚动配置, 滚动执行需要传入") + private RollingConfigVO rollingConfig; + + @ApiModelProperty(value = "是否启用滚动执行") + private boolean rollingEnabled; + } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/WebFastPushFileRequest.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/WebFastPushFileRequest.java index f4fc8f0e58..5a7d8f5a44 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/WebFastPushFileRequest.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/WebFastPushFileRequest.java @@ -30,6 +30,7 @@ import com.tencent.bk.job.execute.common.constants.FileTransferModeEnum; import com.tencent.bk.job.execute.model.web.vo.ExecuteFileDestinationInfoVO; import com.tencent.bk.job.execute.model.web.vo.ExecuteFileSourceInfoVO; +import com.tencent.bk.job.execute.model.web.vo.RollingConfigVO; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; @@ -81,6 +82,12 @@ public class WebFastPushFileRequest { message = "{validation.constraints.InvalidJobTimeout_outOfRange.message}") private Integer timeout; + @ApiModelProperty(value = "是否启用滚动执行") + private boolean rollingEnabled; + + @ApiModelProperty(value = "滚动配置, 滚动执行需要传入") + private RollingConfigVO rollingConfig; + /** * 传输模式 */ diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/WebStepOperation.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/WebStepOperation.java index 721c091a51..a4bb87a0c7 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/WebStepOperation.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/WebStepOperation.java @@ -34,7 +34,7 @@ @Data @ApiModel("作业步骤操作") public class WebStepOperation { - @ApiModelProperty("步骤实例操作，2-失败IP重做，3-忽略错误，6-确认继续,8-全部重试，9-终止流程（人工确认），10-重新发起确认, 11-进入下一步, 12-强制跳过") + @ApiModelProperty("步骤实例操作，2-失败IP重做，3-忽略错误，6-确认继续,8-全部重试，9-终止流程（人工确认），10-重新发起确认, 11-进入下一步, 12-强制跳过, 13 - 继续滚动") private Integer operationCode; @ApiModelProperty("确认/驳回原因") diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/AgentTaskExecutionVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/AgentTaskExecutionVO.java index b736f95c67..638a3eeaee 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/AgentTaskExecutionVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/AgentTaskExecutionVO.java @@ -36,6 +36,10 @@ public class AgentTaskExecutionVO { @ApiModelProperty("执行次数") private Integer retryCount; + @ApiModelProperty("滚动批次") + private Integer batch; + @ApiModelProperty("主机ID") + private Long hostId; @ApiModelProperty("Agent绑定的ip，包含云区域") private String ip; @ApiModelProperty("Agent ip显示名称，展示给用户使用该ip") diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/DangerousRecordVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/DangerousRecordVO.java index 6e2e9193c9..7103d55a3b 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/DangerousRecordVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/DangerousRecordVO.java @@ -64,15 +64,8 @@ public class DangerousRecordVO { /** * 业务名称 */ - @Deprecated @ApiModelProperty("业务名称") private String appName; - - /** - * 资源范围名称 - */ - @ApiModelProperty("资源范围名称") - private String scopeName; /** * 执行人 */ diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteStepVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteStepVO.java index 2bcd647c05..f5b973ef4a 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteStepVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteStepVO.java @@ -49,4 +49,11 @@ public class ExecuteStepVO { @ApiModelProperty("审批步骤信息") private ExecuteApprovalStepVO approvalStepInfo; + + @ApiModelProperty(value = "是否启用滚动执行") + private boolean rollingEnabled; + + @ApiModelProperty(value = "滚动配置") + private RollingConfigVO rollingConfig; + } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/tmp/TmpUpdateAppSetRequest.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/RollingBatchTaskVO.java similarity index 71% rename from src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/tmp/TmpUpdateAppSetRequest.java rename to src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/RollingBatchTaskVO.java index 0cf3983cd4..ac286cd5ce 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/tmp/TmpUpdateAppSetRequest.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/RollingBatchTaskVO.java @@ -22,43 +22,24 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.manage.model.tmp; +package com.tencent.bk.job.execute.model.web.vo; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; -/** - * 更新业务集请求 - */ +@ApiModel("分批滚动任务") @Data -@ApiModel("更新业务集请求") -public class TmpUpdateAppSetRequest { - - @ApiModelProperty("业务集ID") - private Long appId; - - /** - * 新增运维 - */ - @ApiModelProperty("新增运维") - private String addMaintainers; - - /** - * 删除运维 - */ - @ApiModelProperty("删除运维") - private String delMaintainers; - - /** - * 新增子业务 - */ - @ApiModelProperty("新增子业务") - private String addSubBizIds; +public class RollingBatchTaskVO { + @ApiModelProperty("滚动批次") + private Integer batch; /** - * 删除子业务 + * 任务状态 + * + * @see com.tencent.bk.job.execute.common.constants.RunStatusEnum */ - @ApiModelProperty("删除子业务") - private String delSubBizIds; + @ApiModelProperty("任务状态,值同步骤状态一致。1-等待执行，2-正在执行，3-执行成功，4-执行失败，5-跳过，6-忽略错误，7-等待用户" + + "，8-手动结束，9-状态异常，10-强制终止中，11-强制终止成功，12-强制终止失败") + private Integer status; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/StepControlMessage.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/RollingConfigVO.java similarity index 73% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/StepControlMessage.java rename to src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/RollingConfigVO.java index 31835cb84b..fd688f95cc 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/StepControlMessage.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/RollingConfigVO.java @@ -22,26 +22,25 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.execute.engine.model; +package com.tencent.bk.job.execute.model.web.vo; -import com.fasterxml.jackson.annotation.JsonInclude; -import lombok.AllArgsConstructor; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; import lombok.Data; -import lombok.NoArgsConstructor; - -import java.time.LocalDateTime; +@ApiModel("滚动执行配置") @Data -@NoArgsConstructor -@AllArgsConstructor -@JsonInclude(JsonInclude.Include.NON_NULL) -public class StepControlMessage { - private int action; - private long stepInstanceId; - private Integer executeCount; - private LocalDateTime time; +public class RollingConfigVO { /** - * 请求ID,防止重复下发任务 + * 滚动配置名称 */ - private String requestId; + @ApiModelProperty(value = "滚动配置名称") + private String name; + + @ApiModelProperty(value = "滚动分批策略表达式") + private String expr; + + @ApiModelProperty(value = "滚动机制,1-执行失败则暂停；2-忽略失败，自动滚动下一批；3-人工确认") + private Integer mode; + } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/RollingStepBatchTaskVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/RollingStepBatchTaskVO.java new file mode 100644 index 0000000000..3eb5e44730 --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/RollingStepBatchTaskVO.java @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.web.vo; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +@ApiModel("步骤滚动任务") +@Data +public class RollingStepBatchTaskVO { + @ApiModelProperty("滚动批次") + private Integer batch; + + /** + * 任务状态 + * + * @see com.tencent.bk.job.execute.common.constants.RunStatusEnum + */ + @ApiModelProperty("任务状态,值同步骤状态一致。1-等待执行，2-正在执行，3-执行成功，4-执行失败，5-跳过，6-忽略错误，7-等待用户" + + "，8-手动结束，9-状态异常，10-强制终止中，11-强制终止成功，12-强制终止失败，13-确认终止，14-被丢弃，15-滚动等待") + private Integer status; + + @ApiModelProperty(name = "isLatestBatch", value = "是否最新批次的滚动任务") + private boolean latestBatch; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/tmp/TmpAddAppSetRequest.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/RollingTaskVO.java similarity index 69% rename from src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/tmp/TmpAddAppSetRequest.java rename to src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/RollingTaskVO.java index b187f43847..c6265d2c7d 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/tmp/TmpAddAppSetRequest.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/RollingTaskVO.java @@ -22,50 +22,29 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.manage.model.tmp; +package com.tencent.bk.job.execute.model.web.vo; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; +import java.util.List; + /** - * 新增业务集请求 + * 滚动任务 */ +@ApiModel("滚动任务") @Data -@ApiModel("新增业务集请求") -public class TmpAddAppSetRequest { - - @ApiModelProperty("业务ID") - private Long id; - - /** - * 业务名称 - */ - @ApiModelProperty("业务名称") +public class RollingTaskVO { + @ApiModelProperty("滚动配置(任务)名称") private String name; + @ApiModelProperty("总批次") + private Integer totalBatch; + @ApiModelProperty("滚动区间包含的步骤实例ID, 按步骤先后排序") + private List includeStepInstanceIdList; + @ApiModelProperty("当前执行批次") + private Integer currentBatch; + @ApiModelProperty("分批滚动任务") + private List rollingBatchTaskList; - /** - * 运维 - */ - private String maintainers; - - /** - * 子业务 - */ - @ApiModelProperty("子业务ID") - private String subAppIds; - - @ApiModelProperty("是否是动态业务集") - private boolean dynamicAppSet = false; - - /** - * 时区 - */ - private String timeZone; - - /** - * 组织架构ID - */ - @ApiModelProperty("组织架构ID") - private Long deptId; } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepExecutionDetailVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepExecutionDetailVO.java index 2dc5e99a03..c4cb22a13c 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepExecutionDetailVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepExecutionDetailVO.java @@ -53,16 +53,24 @@ public class StepExecutionDetailVO { @ApiModelProperty("总耗时") @JsonSerialize(using = DecimalFormatJsonSerializer.class) private Long totalTime; - @ApiModelProperty("步骤状态,1-等待执行，2-正在执行，3-执行成功，4-执行失败，5-跳过，6-忽略错误，7-等待用户，8-手动结束，9-状态异常，10-强制终止中，11-强制终止成功，12-强制终止失败") + @ApiModelProperty("步骤状态,1-等待执行，2-正在执行，3-执行成功，4-执行失败，5-跳过，6-忽略错误，7-等待用户，8-手动结束，9-状态异常" + + "，10-强制终止中，11-强制终止成功，12-强制终止失败，13-确认终止，14-被丢弃，15-滚动等待") private Integer status; @ApiModelProperty("步骤状态描述") private String statusDesc; - @ApiModelProperty("gseTaskId") - private String gseTaskId; @ApiModelProperty("Agent作业执行结果分组") private List resultGroups; @ApiModelProperty("是否是作业中最后一个步骤") private Boolean isLastStep; @ApiModelProperty("步骤类型，1-脚本，2-文件，3-人工确认") private Integer type; + /** + * 步骤执行模式 + * + * @see com.tencent.bk.job.execute.common.constants.StepRunModeEnum + */ + @ApiModelProperty("步骤执行模式。1-单次全量执行(非滚动步骤)；2-滚动全量执行(滚动步骤)；3-滚动分批执行(滚动步骤)") + private Integer runMode; + @ApiModelProperty("步骤包含的滚动任务;如果非滚动步骤，那么该值为空") + private List rollingTasks; } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepExecutionVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepExecutionVO.java index 5ad146b32f..23aa3c9283 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepExecutionVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepExecutionVO.java @@ -75,4 +75,10 @@ public class StepExecutionVO { private boolean currentStepRunning; @ApiModelProperty("是否最后一个步骤") private Boolean isLastStep; + @ApiModelProperty("滚动执行批次总数;如果非滚动执行，那么该值为空") + private Integer totalBatch; + @ApiModelProperty("滚动执行当前执行批次;如果非滚动执行，那么该值为空") + private Integer currentBatch; + @ApiModelProperty("滚动配置名称") + private String rollingConfigName; } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskExecuteResultVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskExecuteResultVO.java index ec53071763..897f85c511 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskExecuteResultVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskExecuteResultVO.java @@ -45,4 +45,7 @@ public class TaskExecuteResultVO { @ApiModelProperty("步骤执行情况") private List stepExecution; + + @ApiModelProperty("滚动任务列表") + private List rollingTaskList; } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskOperationLogVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskOperationLogVO.java index 1e21128e72..513f54392f 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskOperationLogVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskOperationLogVO.java @@ -47,6 +47,8 @@ public class TaskOperationLogVO { private Long stepInstanceId; @ApiModelProperty("步骤执行次数") private Integer retry; + @ApiModelProperty("滚动执行批次") + private Integer batch; @ApiModelProperty("步骤名称") private String stepName; @ApiModelProperty("操作时间") diff --git a/src/backend/job-execute/boot-job-execute/src/main/java/com/tencent/bk/job/execute/JobExecuteBootApplication.java b/src/backend/job-execute/boot-job-execute/src/main/java/com/tencent/bk/job/execute/JobExecuteBootApplication.java index dd0db6df50..065e9f81f5 100644 --- a/src/backend/job-execute/boot-job-execute/src/main/java/com/tencent/bk/job/execute/JobExecuteBootApplication.java +++ b/src/backend/job-execute/boot-job-execute/src/main/java/com/tencent/bk/job/execute/JobExecuteBootApplication.java @@ -32,9 +32,7 @@ import org.springframework.cloud.client.discovery.EnableDiscoveryClient; import org.springframework.cloud.openfeign.EnableFeignClients; -/** - * @date 2019/09/20 - */ + @SpringBootApplication(scanBasePackages = "com.tencent.bk.job", exclude = {RedisAutoConfiguration.class}) @EnableDiscoveryClient @EnableFeignClients diff --git a/src/backend/job-execute/boot-job-execute/src/main/resources/application.yml b/src/backend/job-execute/boot-job-execute/src/main/resources/application.yml index fa0c68b99f..2a2849f7ab 100644 --- a/src/backend/job-execute/boot-job-execute/src/main/resources/application.yml +++ b/src/backend/job-execute/boot-job-execute/src/main/resources/application.yml @@ -1,6 +1,4 @@ spring: - application: - name: job-execute profiles: active: prod diff --git a/src/backend/job-execute/boot-job-execute/src/main/resources/bootstrap.yml b/src/backend/job-execute/boot-job-execute/src/main/resources/bootstrap.yml index 77bd21fa13..58da1565a1 100644 --- a/src/backend/job-execute/boot-job-execute/src/main/resources/bootstrap.yml +++ b/src/backend/job-execute/boot-job-execute/src/main/resources/bootstrap.yml @@ -1,4 +1,6 @@ spring: + application: + name: job-execute cloud: kubernetes: config: diff --git a/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message.properties b/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message.properties index 63990e9f70..e9c0fbcef3 100644 --- a/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message.properties +++ b/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message.properties @@ -36,13 +36,14 @@ task.run.status.success=执行成功 task.run.status.fail=执行失败 task.run.status.skipped=跳过 task.run.status.ignore_error=忽略错误 -task.run.status.waiting=等待确认 +task.run.status.waiting_user=等待确认 task.run.status.terminated=手动结束 task.run.status.abnormal_state=状态异常 task.run.status.stopping=强制终止中 task.run.status.stop_success=强制终止成功 task.run.status.confirm_terminated=确认终止 task.run.status.abandoned=被丢弃 +task.run.status.rolling_waiting=滚动等待 agent.task.status.unknown_error=未知错误 agent.task.status.agent_error=Agent异常 diff --git a/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message_en.properties b/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message_en.properties index c7de1c353d..c640e2fc6c 100644 --- a/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message_en.properties +++ b/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message_en.properties @@ -1,11 +1,11 @@ # -# Tencent is pleased to support the open source community by making BK-JOBҵƽ̨ available. +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # # Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. # -# BK-JOBҵƽ̨ is licensed under the MIT License. +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # -# License for BK-JOBҵƽ̨: +# License for BK-JOB蓝鲸智云作业平台: # -------------------------------------------------------------------- # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated # documentation files (the "Software"), to deal in the Software without restriction, including without limitation @@ -36,13 +36,14 @@ task.run.status.success=Success task.run.status.fail=Failed task.run.status.skipped=Skipped task.run.status.ignore_error=Ignore Error -task.run.status.waiting=Waiting +task.run.status.waiting_user=Waiting task.run.status.terminated=Terminated task.run.status.abnormal_state=Abnormal task.run.status.stopping=Terminating task.run.status.stop_success=Terminate Success task.run.status.confirm_terminated=Termination Confirmed task.run.status.abandoned=Abandoned +task.run.status.rolling_waiting=Rolling Waiting agent.task.status.unknown_error=Unknown Error agent.task.status.agent_error=Abnormal Agent diff --git a/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message_en_US.properties b/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message_en_US.properties index e15f6d9ac7..f82a53a08b 100644 --- a/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message_en_US.properties +++ b/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message_en_US.properties @@ -1,11 +1,11 @@ # -# Tencent is pleased to support the open source community by making BK-JOBҵƽ̨ available. +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # # Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. # -# BK-JOBҵƽ̨ is licensed under the MIT License. +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # -# License for BK-JOBҵƽ̨: +# License for BK-JOB蓝鲸智云作业平台: # -------------------------------------------------------------------- # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated # documentation files (the "Software"), to deal in the Software without restriction, including without limitation @@ -36,13 +36,14 @@ task.run.status.success=Success task.run.status.fail=Failed task.run.status.skipped=Skipped task.run.status.ignore_error=Ignore Error -task.run.status.waiting=Waiting +task.run.status.waiting_user=Waiting task.run.status.terminated=Terminated task.run.status.abnormal_state=Abnormal task.run.status.stopping=Terminating task.run.status.stop_success=Terminate Success task.run.status.confirm_terminated=Termination Confirmed task.run.status.abandoned=Abandoned +task.run.status.rolling_waiting=Rolling Waiting agent.task.status.unknown_error=Unknown Error agent.task.status.agent_error=Abnormal Agent diff --git a/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message_zh.properties b/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message_zh.properties index 63990e9f70..e9c0fbcef3 100644 --- a/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message_zh.properties +++ b/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message_zh.properties @@ -36,13 +36,14 @@ task.run.status.success=执行成功 task.run.status.fail=执行失败 task.run.status.skipped=跳过 task.run.status.ignore_error=忽略错误 -task.run.status.waiting=等待确认 +task.run.status.waiting_user=等待确认 task.run.status.terminated=手动结束 task.run.status.abnormal_state=状态异常 task.run.status.stopping=强制终止中 task.run.status.stop_success=强制终止成功 task.run.status.confirm_terminated=确认终止 task.run.status.abandoned=被丢弃 +task.run.status.rolling_waiting=滚动等待 agent.task.status.unknown_error=未知错误 agent.task.status.agent_error=Agent异常 diff --git a/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message_zh_CN.properties b/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message_zh_CN.properties index 63990e9f70..e9c0fbcef3 100644 --- a/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message_zh_CN.properties +++ b/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message_zh_CN.properties @@ -36,13 +36,14 @@ task.run.status.success=执行成功 task.run.status.fail=执行失败 task.run.status.skipped=跳过 task.run.status.ignore_error=忽略错误 -task.run.status.waiting=等待确认 +task.run.status.waiting_user=等待确认 task.run.status.terminated=手动结束 task.run.status.abnormal_state=状态异常 task.run.status.stopping=强制终止中 task.run.status.stop_success=强制终止成功 task.run.status.confirm_terminated=确认终止 task.run.status.abandoned=被丢弃 +task.run.status.rolling_waiting=滚动等待 agent.task.status.unknown_error=未知错误 agent.task.status.agent_error=Agent异常 diff --git a/src/backend/job-execute/boot-job-execute/src/main/resources/logback/logback-app-props.xml b/src/backend/job-execute/boot-job-execute/src/main/resources/logback/logback-app-props.xml index 24dafcd349..88a771523e 100644 --- a/src/backend/job-execute/boot-job-execute/src/main/resources/logback/logback-app-props.xml +++ b/src/backend/job-execute/boot-job-execute/src/main/resources/logback/logback-app-props.xml @@ -1,4 +1,5 @@ + diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImplIntegrationTest.java new file mode 100644 index 0000000000..c9fd43c502 --- /dev/null +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImplIntegrationTest.java @@ -0,0 +1,262 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao.impl; + +import com.tencent.bk.job.execute.dao.FileAgentTaskDAO; +import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; +import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.model.AgentTaskResultGroupBaseDTO; +import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.jdbc.Sql; +import org.springframework.test.context.jdbc.SqlConfig; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import java.util.ArrayList; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +@ExtendWith(SpringExtension.class) +@SpringBootTest +@ActiveProfiles("test") +@TestPropertySource(locations = "classpath:test.properties") +@SqlConfig(encoding = "utf-8") +@Sql({"/init_file_agent_task_data.sql"}) +public class FileAgentTaskDAOImplIntegrationTest { + @Autowired + private FileAgentTaskDAO fileAgentTaskDAO; + + @Test + @DisplayName("根据主机ID获取Agent任务") + public void testGetAgentTaskByHostId() { + long hostId = 101L; + long stepInstanceId = 1L; + int executeCount = 0; + int batch = 1; + FileTaskModeEnum mode = FileTaskModeEnum.UPLOAD; + AgentTaskDTO agentTask = fileAgentTaskDAO.getAgentTaskByHostId(stepInstanceId, executeCount, batch, mode, hostId); + + assertThat(agentTask.getStepInstanceId()).isEqualTo(stepInstanceId); + assertThat(agentTask.getExecuteCount()).isEqualTo(executeCount); + assertThat(agentTask.getBatch()).isEqualTo(batch); + assertThat(agentTask.getFileTaskMode()).isEqualTo(FileTaskModeEnum.UPLOAD); + assertThat(agentTask.getHostId()).isEqualTo(hostId); + assertThat(agentTask.getAgentId()).isEqualTo("0:127.0.0.1"); + assertThat(agentTask.getGseTaskId()).isEqualTo(1L); + assertThat(agentTask.getStatus()).isEqualTo(AgentTaskStatusEnum.SUCCESS); + assertThat(agentTask.getStartTime()).isEqualTo(1565767148000L); + assertThat(agentTask.getEndTime()).isEqualTo(1565767149000L); + assertThat(agentTask.getTotalTime()).isEqualTo(1000L); + assertThat(agentTask.getErrorCode()).isEqualTo(0); + } + + @Test + @DisplayName("批量新增Agent任务") + public void testBatchSaveAgentTasks() { + List agentTaskList = new ArrayList<>(); + AgentTaskDTO agentTask1 = new AgentTaskDTO(); + agentTask1.setStepInstanceId(100L); + agentTask1.setExecuteCount(1); + agentTask1.setActualExecuteCount(1); + agentTask1.setBatch(1); + agentTask1.setFileTaskMode(FileTaskModeEnum.UPLOAD); + agentTask1.setHostId(101L); + agentTask1.setAgentId("0:127.0.0.1"); + agentTask1.setGseTaskId(1000L); + agentTask1.setStartTime(1572858334000L); + agentTask1.setEndTime(1572858335000L); + agentTask1.setTotalTime(1000L); + agentTask1.setErrorCode(99); + agentTask1.setStatus(AgentTaskStatusEnum.AGENT_ERROR); + agentTaskList.add(agentTask1); + + AgentTaskDTO agentTask2 = new AgentTaskDTO(); + agentTask2.setStepInstanceId(100L); + agentTask2.setExecuteCount(1); + agentTask2.setActualExecuteCount(1); + agentTask2.setBatch(1); + agentTask2.setFileTaskMode(FileTaskModeEnum.DOWNLOAD); + agentTask2.setHostId(102L); + agentTask2.setAgentId("0:127.0.0.2"); + agentTask2.setGseTaskId(1001L); + agentTask2.setErrorCode(88); + agentTask2.setStartTime(1572858330000L); + agentTask2.setEndTime(1572858331000L); + agentTask2.setTotalTime(1000L); + agentTask2.setErrorCode(88); + agentTask2.setStatus(AgentTaskStatusEnum.HOST_NOT_EXIST); + agentTaskList.add(agentTask2); + + fileAgentTaskDAO.batchSaveAgentTasks(agentTaskList); + + AgentTaskDTO agentTask1Return = fileAgentTaskDAO.getAgentTaskByHostId(100L, 1, 1, + FileTaskModeEnum.UPLOAD, 101L); + assertThat(agentTask1Return.getStepInstanceId()).isEqualTo(100L); + assertThat(agentTask1Return.getExecuteCount()).isEqualTo(1L); + assertThat(agentTask1Return.getActualExecuteCount()).isEqualTo(1L); + assertThat(agentTask1Return.getBatch()).isEqualTo(1); + assertThat(agentTask1Return.getFileTaskMode()).isEqualTo(FileTaskModeEnum.UPLOAD); + assertThat(agentTask1Return.getHostId()).isEqualTo(101L); + assertThat(agentTask1Return.getAgentId()).isEqualTo("0:127.0.0.1"); + assertThat(agentTask1Return.getGseTaskId()).isEqualTo(1000L); + assertThat(agentTask1Return.getStartTime()).isEqualTo(1572858334000L); + assertThat(agentTask1Return.getEndTime()).isEqualTo(1572858335000L); + assertThat(agentTask1Return.getTotalTime()).isEqualTo(1000L); + assertThat(agentTask1Return.getErrorCode()).isEqualTo(99); + assertThat(agentTask1Return.getStatus()).isEqualTo(AgentTaskStatusEnum.AGENT_ERROR); + + + AgentTaskDTO agentTask2Return = fileAgentTaskDAO.getAgentTaskByHostId(100L, 1, 1, + FileTaskModeEnum.DOWNLOAD, 102L); + assertThat(agentTask2Return.getStepInstanceId()).isEqualTo(100L); + assertThat(agentTask2Return.getExecuteCount()).isEqualTo(1L); + assertThat(agentTask2Return.getActualExecuteCount()).isEqualTo(1L); + assertThat(agentTask2Return.getBatch()).isEqualTo(1); + assertThat(agentTask2Return.getFileTaskMode()).isEqualTo(FileTaskModeEnum.DOWNLOAD); + assertThat(agentTask2Return.getHostId()).isEqualTo(102L); + assertThat(agentTask2Return.getAgentId()).isEqualTo("0:127.0.0.2"); + assertThat(agentTask2Return.getGseTaskId()).isEqualTo(1001L); + assertThat(agentTask2Return.getStartTime()).isEqualTo(1572858330000L); + assertThat(agentTask2Return.getEndTime()).isEqualTo(1572858331000L); + assertThat(agentTask2Return.getTotalTime()).isEqualTo(1000L); + assertThat(agentTask2Return.getErrorCode()).isEqualTo(88); + assertThat(agentTask2Return.getStatus()).isEqualTo(AgentTaskStatusEnum.HOST_NOT_EXIST); + } + + @Test + @DisplayName("批量更新Agent任务") + public void testBatchUpdateAgentTasks() { + List agentTaskList = new ArrayList<>(); + AgentTaskDTO agentTask1 = new AgentTaskDTO(); + agentTask1.setStepInstanceId(1L); + agentTask1.setExecuteCount(0); + agentTask1.setBatch(2); + agentTask1.setFileTaskMode(FileTaskModeEnum.UPLOAD); + agentTask1.setHostId(101L); + agentTask1.setAgentId("0:127.0.0.1"); + agentTask1.setGseTaskId(1000L); + agentTask1.setStartTime(1572858334000L); + agentTask1.setEndTime(1572858335000L); + agentTask1.setTotalTime(1000L); + agentTask1.setErrorCode(99); + agentTask1.setStatus(AgentTaskStatusEnum.AGENT_ERROR); + agentTaskList.add(agentTask1); + + AgentTaskDTO agentTask2 = new AgentTaskDTO(); + agentTask2.setStepInstanceId(1L); + agentTask2.setExecuteCount(0); + agentTask2.setBatch(2); + agentTask2.setFileTaskMode(FileTaskModeEnum.DOWNLOAD); + agentTask2.setHostId(103L); + agentTask2.setAgentId("0:127.0.0.3"); + agentTask2.setGseTaskId(1001L); + agentTask2.setErrorCode(88); + agentTask2.setStartTime(1572858330000L); + agentTask2.setEndTime(1572858331000L); + agentTask2.setTotalTime(1000L); + agentTask2.setErrorCode(88); + agentTask2.setStatus(AgentTaskStatusEnum.HOST_NOT_EXIST); + agentTaskList.add(agentTask2); + + fileAgentTaskDAO.batchUpdateAgentTasks(agentTaskList); + + AgentTaskDTO agentTask1Return = fileAgentTaskDAO.getAgentTaskByHostId(1L, 0, 2, + FileTaskModeEnum.UPLOAD, 101L); + assertThat(agentTask1Return.getStepInstanceId()).isEqualTo(1L); + assertThat(agentTask1Return.getExecuteCount()).isEqualTo(0L); + assertThat(agentTask1Return.getBatch()).isEqualTo(2); + assertThat(agentTask1Return.getAgentId()).isEqualTo("0:127.0.0.1"); + assertThat(agentTask1Return.getHostId()).isEqualTo(101L); + assertThat(agentTask1Return.getGseTaskId()).isEqualTo(1000L); + assertThat(agentTask1Return.getStartTime()).isEqualTo(1572858334000L); + assertThat(agentTask1Return.getEndTime()).isEqualTo(1572858335000L); + assertThat(agentTask1Return.getTotalTime()).isEqualTo(1000L); + assertThat(agentTask1Return.getErrorCode()).isEqualTo(99); + assertThat(agentTask1Return.getStatus()).isEqualTo(AgentTaskStatusEnum.AGENT_ERROR); + + + AgentTaskDTO agentTask2Return = fileAgentTaskDAO.getAgentTaskByHostId(1L, 0, 2, + FileTaskModeEnum.DOWNLOAD, 103L); + assertThat(agentTask2Return.getStepInstanceId()).isEqualTo(1L); + assertThat(agentTask2Return.getExecuteCount()).isEqualTo(0L); + assertThat(agentTask2Return.getBatch()).isEqualTo(2); + assertThat(agentTask2Return.getGseTaskId()).isEqualTo(1001L); + assertThat(agentTask2Return.getHostId()).isEqualTo(103L); + assertThat(agentTask2Return.getAgentId()).isEqualTo("0:127.0.0.3"); + assertThat(agentTask2Return.getStartTime()).isEqualTo(1572858330000L); + assertThat(agentTask2Return.getEndTime()).isEqualTo(1572858331000L); + assertThat(agentTask2Return.getTotalTime()).isEqualTo(1000L); + assertThat(agentTask2Return.getErrorCode()).isEqualTo(88); + assertThat(agentTask2Return.getStatus()).isEqualTo(AgentTaskStatusEnum.HOST_NOT_EXIST); + } + + @Test + @DisplayName("Agent任务结果分组") + public void listResultGroups() { + List resultGroups = fileAgentTaskDAO.listResultGroups(1L, 0, null); + + assertThat(resultGroups.size()).isEqualTo(2); + assertThat(resultGroups).extracting("status").containsOnly(9, 11); + AgentTaskResultGroupBaseDTO resultGroup = resultGroups.get(0); + if (resultGroup.getStatus().equals(9)) { + assertThat(resultGroup.getTotalAgentTasks()).isEqualTo(2); + } + if (resultGroup.getStatus().equals(11)) { + assertThat(resultGroup.getTotalAgentTasks()).isEqualTo(1); + } + + // 根据滚动执行批次查询 + resultGroups = fileAgentTaskDAO.listResultGroups(1L, 0, 2); + + assertThat(resultGroups.size()).isEqualTo(2); + assertThat(resultGroups).extracting("status").containsOnly(9, 11); + resultGroup = resultGroups.get(0); + if (resultGroup.getStatus().equals(9)) { + assertThat(resultGroup.getTotalAgentTasks()).isEqualTo(1); + } + if (resultGroup.getStatus().equals(11)) { + assertThat(resultGroup.getTotalAgentTasks()).isEqualTo(1); + } + } + + @Test + public void testListAgentTaskByResultGroup() { + List agentTasks = fileAgentTaskDAO.listAgentTaskByResultGroup(1L, 0, 2, 9); + assertThat(agentTasks.size()).isEqualTo(1); + assertThat(agentTasks.get(0).getStepInstanceId()).isEqualTo(1L); + assertThat(agentTasks.get(0).getExecuteCount()).isEqualTo(0); + assertThat(agentTasks.get(0).getBatch()).isEqualTo(2); + assertThat(agentTasks.get(0).getStatus()).isEqualTo(AgentTaskStatusEnum.SUCCESS); + assertThat(agentTasks).extracting("hostId").containsOnly(103L); + } + +} diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/GseTaskDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/GseTaskDAOImplIntegrationTest.java new file mode 100644 index 0000000000..159bea4f45 --- /dev/null +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/GseTaskDAOImplIntegrationTest.java @@ -0,0 +1,111 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao.impl; + +import com.tencent.bk.job.execute.common.constants.RunStatusEnum; +import com.tencent.bk.job.execute.dao.GseTaskDAO; +import com.tencent.bk.job.execute.model.GseTaskDTO; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.jdbc.Sql; +import org.springframework.test.context.jdbc.SqlConfig; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import static org.assertj.core.api.Assertions.assertThat; + +@ExtendWith(SpringExtension.class) +@SpringBootTest +@ActiveProfiles("test") +@TestPropertySource(locations = "classpath:test.properties") +@SqlConfig(encoding = "utf-8") +@Sql({"/init_gse_task_data.sql"}) +public class GseTaskDAOImplIntegrationTest { + + @Autowired + private GseTaskDAO gseTaskDAO; + + @Test + @DisplayName("保存GseTask") + void saveGseTask() { + GseTaskDTO gseTask = new GseTaskDTO(); + gseTask.setStepInstanceId(10L); + gseTask.setExecuteCount(0); + gseTask.setBatch(1); + gseTask.setStatus(RunStatusEnum.SUCCESS.getValue()); + gseTask.setStartTime(1660639907757L); + gseTask.setEndTime(1660639914013L); + gseTask.setTotalTime(6256L); + gseTask.setGseTaskId("GSE_TASK_10"); + + long id = gseTaskDAO.saveGseTask(gseTask); + assertThat(id).isGreaterThan(0); + + GseTaskDTO savedGseTask = gseTaskDAO.getGseTask(id); + + assertThat(savedGseTask.getStepInstanceId()).isEqualTo(10L); + assertThat(savedGseTask.getExecuteCount()).isEqualTo(0); + assertThat(savedGseTask.getBatch()).isEqualTo(1); + assertThat(savedGseTask.getStatus()).isEqualTo(RunStatusEnum.SUCCESS.getValue()); + assertThat(savedGseTask.getStartTime()).isEqualTo(1660639907757L); + assertThat(savedGseTask.getEndTime()).isEqualTo(1660639914013L); + assertThat(savedGseTask.getTotalTime()).isEqualTo(6256L); + assertThat(savedGseTask.getGseTaskId()).isEqualTo("GSE_TASK_10"); + } + + @Test + @DisplayName("更新GseTask") + void updateGseTask() { + GseTaskDTO gseTask = new GseTaskDTO(); + gseTask.setId(1L); + gseTask.setStepInstanceId(1L); + gseTask.setExecuteCount(0); + gseTask.setBatch(0); + gseTask.setStatus(RunStatusEnum.SUCCESS.getValue()); + gseTask.setStartTime(1660639907757L); + gseTask.setEndTime(1660639914013L); + gseTask.setTotalTime(6256L); + gseTask.setGseTaskId("GSE_TASK_2"); + + boolean result = gseTaskDAO.updateGseTask(gseTask); + assertThat(result).isTrue(); + + GseTaskDTO savedGseTask = gseTaskDAO.getGseTask(1L); + + assertThat(savedGseTask.getStepInstanceId()).isEqualTo(1L); + assertThat(savedGseTask.getExecuteCount()).isEqualTo(0); + assertThat(savedGseTask.getBatch()).isEqualTo(0); + assertThat(savedGseTask.getStatus()).isEqualTo(RunStatusEnum.SUCCESS.getValue()); + assertThat(savedGseTask.getStartTime()).isEqualTo(1660639907757L); + assertThat(savedGseTask.getEndTime()).isEqualTo(1660639914013L); + assertThat(savedGseTask.getTotalTime()).isEqualTo(6256L); + assertThat(savedGseTask.getGseTaskId()).isEqualTo("GSE_TASK_2"); + } +} + diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/GseTaskIpLogDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/GseTaskIpLogDAOImplIntegrationTest.java index 9ce2c78400..ba4f7bccfa 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/GseTaskIpLogDAOImplIntegrationTest.java +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/GseTaskIpLogDAOImplIntegrationTest.java @@ -25,8 +25,8 @@ package com.tencent.bk.job.execute.dao.impl; import com.tencent.bk.job.execute.dao.GseTaskIpLogDAO; -import com.tencent.bk.job.execute.model.GseTaskIpLogDTO; -import com.tencent.bk.job.execute.model.ResultGroupBaseDTO; +import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; +import com.tencent.bk.job.execute.model.AgentTaskDTO; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; @@ -37,7 +37,6 @@ import org.springframework.test.context.jdbc.SqlConfig; import org.springframework.test.context.junit.jupiter.SpringExtension; -import java.util.ArrayList; import java.util.List; import static org.assertj.core.api.Assertions.assertThat; @@ -53,142 +52,39 @@ public class GseTaskIpLogDAOImplIntegrationTest { private GseTaskIpLogDAO gseTaskIpLogDAO; @Test - public void testGetIpLogByIp() { + public void testGetAgentTaskByIp() { String ip = "0:127.0.0.1"; long stepInstanceId = 1L; int executeCount = 0; - GseTaskIpLogDTO gseTaskIpLog = gseTaskIpLogDAO.getIpLogByIp(stepInstanceId, executeCount, ip); + AgentTaskDTO agentTask = gseTaskIpLogDAO.getAgentTaskByIp(stepInstanceId, executeCount, ip); - assertThat(gseTaskIpLog.getStepInstanceId()).isEqualTo(stepInstanceId); - assertThat(gseTaskIpLog.getExecuteCount()).isEqualTo(executeCount); - assertThat(gseTaskIpLog.getCloudAreaAndIp()).isEqualTo(ip); - assertThat(gseTaskIpLog.getStatus()).isEqualTo(9); + assertThat(agentTask.getStepInstanceId()).isEqualTo(stepInstanceId); + assertThat(agentTask.getExecuteCount()).isEqualTo(executeCount); + assertThat(agentTask.getCloudIp()).isEqualTo(ip); + assertThat(agentTask.getStatus()).isEqualTo(AgentTaskStatusEnum.SUCCESS); Long expectStartTime = 1565767148000L; Long expectEndTime = 1565767149000L; - assertThat(gseTaskIpLog.getStartTime()).isEqualTo(expectStartTime); - assertThat(gseTaskIpLog.getEndTime()).isEqualTo(expectEndTime); - assertThat(gseTaskIpLog.getTotalTime()).isEqualTo(1316L); - assertThat(gseTaskIpLog.getErrCode()).isEqualTo(0); - assertThat(gseTaskIpLog.getExitCode()).isEqualTo(0); - assertThat(gseTaskIpLog.getTag()).isEqualTo("succ"); - assertThat(gseTaskIpLog.getOffset()).isEqualTo(0); - assertThat(gseTaskIpLog.getCloudAreaId()).isEqualTo(0); - assertThat(gseTaskIpLog.getDisplayIp()).isEqualTo("127.0.0.1"); - assertThat(gseTaskIpLog.isTargetServer()).isEqualTo(true); - assertThat(gseTaskIpLog.isSourceServer()).isEqualTo(false); + assertThat(agentTask.getStartTime()).isEqualTo(expectStartTime); + assertThat(agentTask.getEndTime()).isEqualTo(expectEndTime); + assertThat(agentTask.getTotalTime()).isEqualTo(1316L); + assertThat(agentTask.getErrorCode()).isEqualTo(0); + assertThat(agentTask.getExitCode()).isEqualTo(0); + assertThat(agentTask.getTag()).isEqualTo("succ"); + assertThat(agentTask.getScriptLogOffset()).isEqualTo(0); } @Test - public void testBatchSaveIpLog() { - List ipLogList = new ArrayList<>(); - GseTaskIpLogDTO ipLog1 = new GseTaskIpLogDTO(); - ipLog1.setStepInstanceId(1L); - ipLog1.setExecuteCount(0); - ipLog1.setCloudAreaAndIp("0:127.0.0.1"); - ipLog1.setDisplayIp("127.0.0.1"); - ipLog1.setErrCode(99); - ipLog1.setStatus(1); - ipLog1.setExitCode(1); - ipLogList.add(ipLog1); - - GseTaskIpLogDTO ipLog2 = new GseTaskIpLogDTO(); - ipLog2.setStepInstanceId(3L); - ipLog2.setExecuteCount(0); - ipLog2.setCloudAreaAndIp("0:127.0.0.1"); - ipLog2.setErrCode(88); - ipLog2.setExitCode(1); - ipLog2.setDisplayIp("127.0.0.1"); - long startTime = 1572858330000L; - ipLog2.setStartTime(startTime); - long endTime = 1572858331000L; - ipLog2.setEndTime(endTime); - ipLog2.setStatus(2); - ipLogList.add(ipLog2); - - gseTaskIpLogDAO.batchSaveIpLog(ipLogList); - - GseTaskIpLogDTO ipLog1Return = gseTaskIpLogDAO.getIpLogByIp(1L, 0, "0:127.0.0.1"); - assertThat(ipLog1Return.getStepInstanceId()).isEqualTo(1L); - assertThat(ipLog1Return.getExecuteCount()).isEqualTo(0L); - assertThat(ipLog1Return.getCloudAreaAndIp()).isEqualTo("0:127.0.0.1"); - assertThat(ipLog1Return.getErrCode()).isEqualTo(99); - assertThat(ipLog1Return.getStatus()).isEqualTo(1); - assertThat(ipLog1Return.getExitCode()).isEqualTo(1); - - - GseTaskIpLogDTO ipLog2Return = gseTaskIpLogDAO.getIpLogByIp(3L, 0, "0:127.0.0.1"); - assertThat(ipLog2Return.getStepInstanceId()).isEqualTo(3L); - assertThat(ipLog2Return.getExecuteCount()).isEqualTo(0L); - assertThat(ipLog2Return.getCloudAreaAndIp()).isEqualTo("0:127.0.0.1"); - assertThat(ipLog2Return.getStartTime()).isEqualTo(startTime); - assertThat(ipLog2Return.getEndTime()).isEqualTo(endTime); - assertThat(ipLog2Return.getErrCode()).isEqualTo(88); - assertThat(ipLog2Return.getStatus()).isEqualTo(2); - assertThat(ipLog2Return.getExitCode()).isEqualTo(1); - } - - @Test - public void testGetSuccessIpCount() { - Integer count = gseTaskIpLogDAO.getSuccessIpCount(1L, 0); + public void testGetSuccessAgentTaskCount() { + Integer count = gseTaskIpLogDAO.getSuccessAgentTaskCount(1L, 0); assertThat(count).isEqualTo(2); } @Test - public void testGetSuccessIpList() { - List gseTaskIpLogList = gseTaskIpLogDAO.getSuccessGseTaskIp(1L, 0); - assertThat(gseTaskIpLogList).extracting("cloudAreaAndIp").containsOnly("0:127.0.0.1", "0:127.0.0.2"); - } - - @Test - public void testGetIpErrorStatList() { - List resultGroups = gseTaskIpLogDAO.getResultGroups(1L, 0); - - assertThat(resultGroups.size()).isEqualTo(1); - assertThat(resultGroups.get(0).getTag()).isEqualTo("succ"); - assertThat(resultGroups.get(0).getResultType()).isEqualTo(9); - assertThat(resultGroups.get(0).getAgentTaskCount()).isEqualTo(2); - } - - @Test - public void testGetIpLogByResultType() { - List ipLogs = gseTaskIpLogDAO.getIpLogByResultType(1L, 0, 9, "succ"); - assertThat(ipLogs.size()).isEqualTo(2); - assertThat(ipLogs).extracting("tag").containsOnly("succ", "succ"); - assertThat(ipLogs).extracting("stepInstanceId").containsOnly(1L, 1L); + public void testListAgentTaskByResultGroup() { + List agentTasks = gseTaskIpLogDAO.listAgentTaskByResultGroup(1L, 0, 9, "succ"); + assertThat(agentTasks.size()).isEqualTo(2); + assertThat(agentTasks).extracting("tag").containsOnly("succ", "succ"); + assertThat(agentTasks).extracting("stepInstanceId").containsOnly(1L, 1L); } - @Test - public void testGetIpLogByIps() { - String[] ipArray = {"0:127.0.0.1", "0:127.0.0.2"}; - List ipLogs = gseTaskIpLogDAO.getIpLogByIps(1L, 0, ipArray); - - assertThat(ipLogs.size()).isEqualTo(2); - assertThat(ipLogs).extracting("cloudAreaAndIp").containsOnly("0:127.0.0.1", "0:127.0.0.2"); - assertThat(ipLogs).extracting("stepInstanceId").containsOnly(1L, 1L); - assertThat(ipLogs).extracting("executeCount").containsOnly(0, 0); - } - - @Test - public void testDeleteAllIpLog() { - gseTaskIpLogDAO.deleteAllIpLog(1L, 0); - - List ipLogs = gseTaskIpLogDAO.getIpLog(1L, 0, false); - assertThat(ipLogs.size()).isEqualTo(0); - } - - @Test - public void testGetTaskFileSourceIps() { - List fileSourceIps = gseTaskIpLogDAO.getTaskFileSourceIps(1L, 0); - assertThat(fileSourceIps.size()).isEqualTo(1); - assertThat(fileSourceIps.get(0)).isEqualTo("0:127.0.0.3"); - } - - @Test - public void testFuzzySearchTargetIpsByIp() { - List matchIps = gseTaskIpLogDAO.fuzzySearchTargetIpsByIp(1L, 0, "0.0.2"); - assertThat(matchIps.size()).isEqualTo(1); - assertThat(matchIps.get(0)).isEqualTo("0:127.0.0.2"); - } - - } diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/RollingConfigDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/RollingConfigDAOImplIntegrationTest.java new file mode 100644 index 0000000000..388c6f2c1a --- /dev/null +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/RollingConfigDAOImplIntegrationTest.java @@ -0,0 +1,154 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao.impl; + +import com.tencent.bk.job.common.constant.RollingModeEnum; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.dao.RollingConfigDAO; +import com.tencent.bk.job.execute.model.RollingConfigDTO; +import com.tencent.bk.job.execute.model.db.RollingConfigDetailDO; +import com.tencent.bk.job.execute.model.db.RollingHostsBatchDO; +import com.tencent.bk.job.execute.model.db.StepRollingConfigDO; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.jdbc.Sql; +import org.springframework.test.context.jdbc.SqlConfig; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +@ExtendWith(SpringExtension.class) +@SpringBootTest +@ActiveProfiles("test") +@TestPropertySource(locations = "classpath:test.properties") +@SqlConfig(encoding = "utf-8") +@Sql({"/init_rolling_config_data.sql"}) +public class RollingConfigDAOImplIntegrationTest { + + @Autowired + private RollingConfigDAO rollingConfigDAO; + + @Test + @DisplayName("根据ID获取滚动配置") + void queryRollingConfig() { + RollingConfigDTO savedTaskInstanceRollingConfig = + rollingConfigDAO.queryRollingConfigById(1L); + + assertThat(savedTaskInstanceRollingConfig.getId()).isEqualTo(1L); + assertThat(savedTaskInstanceRollingConfig.getConfigName()).isEqualTo("config1"); + assertThat(savedTaskInstanceRollingConfig.getTaskInstanceId()).isEqualTo(1L); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail()).isNotNull(); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getExpr()).isEqualTo("1 10% 100%"); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getMode()).isEqualTo(RollingModeEnum.PAUSE_IF_FAIL.getValue()); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getName()).isEqualTo("config1"); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getStepRollingConfigs()).hasSize(4); + Map stepRollingConfigs = savedTaskInstanceRollingConfig.getConfigDetail().getStepRollingConfigs(); + assertThat(stepRollingConfigs.get(100L).isBatch()).isEqualTo(true); + assertThat(stepRollingConfigs.get(101L).isBatch()).isEqualTo(false); + assertThat(stepRollingConfigs.get(102L).isBatch()).isEqualTo(true); + assertThat(stepRollingConfigs.get(103L).isBatch()).isEqualTo(true); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList()).hasSize(3); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList().get(0).getBatch()).isEqualTo(1); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList().get(0).getHosts()).hasSize(1); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList().get(0).getHosts().get(0).getBkCloudId()).isEqualTo(0L); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList().get(0).getHosts().get(0).getIp()).isEqualTo("127.0.0.1"); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList().get(1).getBatch()).isEqualTo(2); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList().get(1).getHosts()).hasSize(1); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList().get(1).getHosts().get(0).getBkCloudId()).isEqualTo(0L); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList().get(1).getHosts().get(0).getIp()).isEqualTo("127.0.0.2"); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList().get(2).getBatch()).isEqualTo(3); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList().get(2).getHosts()).hasSize(2); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList().get(2).getHosts().get(0).getBkCloudId()).isEqualTo(0L); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList().get(2).getHosts().get(0).getIp()).isEqualTo("127.0.0.3"); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList().get(2).getHosts().get(1).getBkCloudId()).isEqualTo(0L); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList().get(2).getHosts().get(1).getIp()).isEqualTo("127.0.0.4"); + } + + @Test + @DisplayName("保存作业实例滚动配置") + void saveRollingConfig() { + RollingConfigDTO taskInstanceRollingConfig = new RollingConfigDTO(); + taskInstanceRollingConfig.setTaskInstanceId(10L); + taskInstanceRollingConfig.setConfigName("default"); + RollingConfigDetailDO rollingConfig = new RollingConfigDetailDO(); + rollingConfig.setName("default"); + rollingConfig.setExpr("10%"); + rollingConfig.setMode(RollingModeEnum.PAUSE_IF_FAIL.getValue()); + List includeStepInstanceIdList = new ArrayList<>(); + includeStepInstanceIdList.add(1000L); + includeStepInstanceIdList.add(1001L); + includeStepInstanceIdList.add(1002L); + includeStepInstanceIdList.add(1003L); + rollingConfig.setIncludeStepInstanceIdList(includeStepInstanceIdList); + Map stepRollingConfigs = new HashMap<>(); + stepRollingConfigs.put(1000L, new StepRollingConfigDO(true)); + stepRollingConfigs.put(1001L, new StepRollingConfigDO(false)); + stepRollingConfigs.put(1002L, new StepRollingConfigDO(true)); + stepRollingConfigs.put(1003L, new StepRollingConfigDO(true)); + rollingConfig.setStepRollingConfigs(stepRollingConfigs); + List hostsBatchList = new ArrayList<>(); + List servers = new ArrayList<>(); + servers.add(new HostDTO(0L, "127.0.0.1")); + RollingHostsBatchDO hostBatch1 = new RollingHostsBatchDO(1, servers); + hostsBatchList.add(hostBatch1); + rollingConfig.setHostsBatchList(hostsBatchList); + taskInstanceRollingConfig.setConfigDetail(rollingConfig); + + long rollingConfigId = rollingConfigDAO.saveRollingConfig(taskInstanceRollingConfig); + assertThat(rollingConfigId).isGreaterThan(0); + + RollingConfigDTO savedTaskInstanceRollingConfig = + rollingConfigDAO.queryRollingConfigById(rollingConfigId); + + assertThat(savedTaskInstanceRollingConfig.getId()).isEqualTo(rollingConfigId); + assertThat(savedTaskInstanceRollingConfig.getConfigName()).isEqualTo("default"); + assertThat(savedTaskInstanceRollingConfig.getTaskInstanceId()).isEqualTo(10L); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail()).isNotNull(); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getExpr()).isEqualTo("10%"); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getMode()).isEqualTo(RollingModeEnum.PAUSE_IF_FAIL.getValue()); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getName()).isEqualTo("default"); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getStepRollingConfigs()).hasSize(4); + Map savedStepRollingConfigs = savedTaskInstanceRollingConfig.getConfigDetail().getStepRollingConfigs(); + assertThat(savedStepRollingConfigs.get(1000L).isBatch()).isEqualTo(true); + assertThat(savedStepRollingConfigs.get(1001L).isBatch()).isEqualTo(false); + assertThat(savedStepRollingConfigs.get(1002L).isBatch()).isEqualTo(true); + assertThat(savedStepRollingConfigs.get(1003L).isBatch()).isEqualTo(true); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList()).hasSize(1); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList().get(0).getBatch()).isEqualTo(1); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList().get(0).getHosts().get(0).getBkCloudId()).isEqualTo(0L); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList().get(0).getHosts().get(0).getIp()).isEqualTo("127.0.0.1"); + } +} + diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImplIntegrationTest.java new file mode 100644 index 0000000000..957fe52d1f --- /dev/null +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImplIntegrationTest.java @@ -0,0 +1,280 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao.impl; + +import com.tencent.bk.job.execute.dao.ScriptAgentTaskDAO; +import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; +import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.model.AgentTaskResultGroupBaseDTO; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.jdbc.Sql; +import org.springframework.test.context.jdbc.SqlConfig; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import java.util.ArrayList; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +@ExtendWith(SpringExtension.class) +@SpringBootTest +@ActiveProfiles("test") +@TestPropertySource(locations = "classpath:test.properties") +@SqlConfig(encoding = "utf-8") +@Sql({"/init_script_agent_task_data.sql"}) +public class ScriptAgentTaskDAOImplIntegrationTest { + @Autowired + private ScriptAgentTaskDAO scriptAgentTaskDAO; + + @Test + @DisplayName("根据主机获取Agent任务") + public void testGetAgentTaskByHostId() { + long hostId = 101L; + long stepInstanceId = 1L; + int executeCount = 0; + int batch = 1; + AgentTaskDTO agentTask = scriptAgentTaskDAO.getAgentTaskByHostId(stepInstanceId, executeCount, batch, hostId); + + assertThat(agentTask.getStepInstanceId()).isEqualTo(stepInstanceId); + assertThat(agentTask.getExecuteCount()).isEqualTo(executeCount); + assertThat(agentTask.getBatch()).isEqualTo(batch); + assertThat(agentTask.getHostId()).isEqualTo(hostId); + assertThat(agentTask.getAgentId()).isEqualTo("0:127.0.0.1"); + assertThat(agentTask.getGseTaskId()).isEqualTo(1L); + assertThat(agentTask.getStatus()).isEqualTo(AgentTaskStatusEnum.SUCCESS); + Long expectStartTime = 1565767148000L; + Long expectEndTime = 1565767149000L; + assertThat(agentTask.getStartTime()).isEqualTo(expectStartTime); + assertThat(agentTask.getEndTime()).isEqualTo(expectEndTime); + assertThat(agentTask.getTotalTime()).isEqualTo(1316L); + assertThat(agentTask.getErrorCode()).isEqualTo(0); + assertThat(agentTask.getExitCode()).isEqualTo(0); + assertThat(agentTask.getTag()).isEqualTo("succ"); + assertThat(agentTask.getScriptLogOffset()).isEqualTo(0); + } + + @Test + @DisplayName("批量新增Agent任务") + public void testBatchSaveAgentTasks() { + List agentTaskList = new ArrayList<>(); + AgentTaskDTO agentTask1 = new AgentTaskDTO(); + agentTask1.setStepInstanceId(100L); + agentTask1.setExecuteCount(1); + agentTask1.setActualExecuteCount(1); + agentTask1.setBatch(1); + agentTask1.setHostId(101L); + agentTask1.setAgentId("0:127.0.0.1"); + agentTask1.setGseTaskId(1000L); + agentTask1.setStartTime(1572858334000L); + agentTask1.setEndTime(1572858335000L); + agentTask1.setTotalTime(1000L); + agentTask1.setErrorCode(99); + agentTask1.setStatus(AgentTaskStatusEnum.AGENT_ERROR); + agentTask1.setTag("aa"); + agentTask1.setExitCode(1); + agentTaskList.add(agentTask1); + + AgentTaskDTO agentTask2 = new AgentTaskDTO(); + agentTask2.setStepInstanceId(100L); + agentTask2.setExecuteCount(1); + agentTask2.setActualExecuteCount(1); + agentTask2.setBatch(1); + agentTask2.setHostId(102L); + agentTask2.setAgentId("0:127.0.0.2"); + agentTask2.setGseTaskId(1001L); + agentTask2.setErrorCode(88); + agentTask2.setExitCode(1); + agentTask2.setStartTime(1572858330000L); + agentTask2.setEndTime(1572858331000L); + agentTask2.setTotalTime(1000L); + agentTask2.setErrorCode(88); + agentTask2.setStatus(AgentTaskStatusEnum.HOST_NOT_EXIST); + agentTask2.setTag("bb"); + agentTask2.setExitCode(2); + agentTaskList.add(agentTask2); + + scriptAgentTaskDAO.batchSaveAgentTasks(agentTaskList); + + AgentTaskDTO agentTask1Return = scriptAgentTaskDAO.getAgentTaskByHostId(100L, 1, 1, 101L); + assertThat(agentTask1Return.getStepInstanceId()).isEqualTo(100L); + assertThat(agentTask1Return.getExecuteCount()).isEqualTo(1L); + assertThat(agentTask1Return.getActualExecuteCount()).isEqualTo(1L); + assertThat(agentTask1Return.getBatch()).isEqualTo(1); + assertThat(agentTask1Return.getHostId()).isEqualTo(101L); + assertThat(agentTask1Return.getAgentId()).isEqualTo("0:127.0.0.1"); + assertThat(agentTask1Return.getGseTaskId()).isEqualTo(1000L); + assertThat(agentTask1Return.getStartTime()).isEqualTo(1572858334000L); + assertThat(agentTask1Return.getEndTime()).isEqualTo(1572858335000L); + assertThat(agentTask1Return.getTotalTime()).isEqualTo(1000L); + assertThat(agentTask1Return.getErrorCode()).isEqualTo(99); + assertThat(agentTask1Return.getStatus()).isEqualTo(AgentTaskStatusEnum.AGENT_ERROR); + assertThat(agentTask1Return.getTag()).isEqualTo("aa"); + assertThat(agentTask1Return.getExitCode()).isEqualTo(1); + + + AgentTaskDTO agentTask2Return = scriptAgentTaskDAO.getAgentTaskByHostId(100L, 1, 1, 102L); + assertThat(agentTask2Return.getStepInstanceId()).isEqualTo(100L); + assertThat(agentTask2Return.getExecuteCount()).isEqualTo(1L); + assertThat(agentTask2Return.getActualExecuteCount()).isEqualTo(1L); + assertThat(agentTask2Return.getBatch()).isEqualTo(1); + assertThat(agentTask2Return.getHostId()).isEqualTo(102L); + assertThat(agentTask2Return.getGseTaskId()).isEqualTo(1001L); + assertThat(agentTask2Return.getAgentId()).isEqualTo("0:127.0.0.2"); + assertThat(agentTask2Return.getStartTime()).isEqualTo(1572858330000L); + assertThat(agentTask2Return.getEndTime()).isEqualTo(1572858331000L); + assertThat(agentTask2Return.getTotalTime()).isEqualTo(1000L); + assertThat(agentTask2Return.getErrorCode()).isEqualTo(88); + assertThat(agentTask2Return.getStatus()).isEqualTo(AgentTaskStatusEnum.HOST_NOT_EXIST); + assertThat(agentTask2Return.getTag()).isEqualTo("bb"); + assertThat(agentTask2Return.getExitCode()).isEqualTo(2); + } + + @Test + @DisplayName("批量更新Agent任务") + public void testBatchUpdateAgentTasks() { + List agentTaskList = new ArrayList<>(); + AgentTaskDTO agentTask1 = new AgentTaskDTO(); + agentTask1.setStepInstanceId(1L); + agentTask1.setExecuteCount(0); + agentTask1.setBatch(3); + agentTask1.setHostId(103L); + agentTask1.setAgentId("0:127.0.0.3"); + agentTask1.setGseTaskId(1000L); + agentTask1.setStartTime(1572858334000L); + agentTask1.setEndTime(1572858335000L); + agentTask1.setTotalTime(1000L); + agentTask1.setErrorCode(99); + agentTask1.setStatus(AgentTaskStatusEnum.AGENT_ERROR); + agentTask1.setTag("aa"); + agentTask1.setExitCode(1); + agentTaskList.add(agentTask1); + + AgentTaskDTO agentTask2 = new AgentTaskDTO(); + agentTask2.setStepInstanceId(1L); + agentTask2.setExecuteCount(0); + agentTask2.setBatch(3); + agentTask2.setHostId(104L); + agentTask2.setAgentId("0:127.0.0.4"); + agentTask2.setGseTaskId(1001L); + agentTask2.setErrorCode(88); + agentTask2.setExitCode(1); + agentTask2.setStartTime(1572858330000L); + agentTask2.setEndTime(1572858331000L); + agentTask2.setTotalTime(1000L); + agentTask2.setErrorCode(88); + agentTask2.setStatus(AgentTaskStatusEnum.HOST_NOT_EXIST); + agentTask2.setTag("bb"); + agentTask2.setExitCode(2); + agentTaskList.add(agentTask2); + + scriptAgentTaskDAO.batchUpdateAgentTasks(agentTaskList); + + AgentTaskDTO agentTask1Return = scriptAgentTaskDAO.getAgentTaskByHostId(1L, 0, 3, 103L); + assertThat(agentTask1Return.getStepInstanceId()).isEqualTo(1L); + assertThat(agentTask1Return.getExecuteCount()).isEqualTo(0L); + assertThat(agentTask1Return.getBatch()).isEqualTo(3); + assertThat(agentTask1Return.getHostId()).isEqualTo(103L); + assertThat(agentTask1Return.getAgentId()).isEqualTo("0:127.0.0.3"); + assertThat(agentTask1Return.getGseTaskId()).isEqualTo(1000L); + assertThat(agentTask1Return.getStartTime()).isEqualTo(1572858334000L); + assertThat(agentTask1Return.getEndTime()).isEqualTo(1572858335000L); + assertThat(agentTask1Return.getTotalTime()).isEqualTo(1000L); + assertThat(agentTask1Return.getErrorCode()).isEqualTo(99); + assertThat(agentTask1Return.getStatus()).isEqualTo(AgentTaskStatusEnum.AGENT_ERROR); + assertThat(agentTask1Return.getTag()).isEqualTo("aa"); + assertThat(agentTask1Return.getExitCode()).isEqualTo(1); + + + AgentTaskDTO agentTask2Return = scriptAgentTaskDAO.getAgentTaskByHostId(1L, 0, 3, 104L); + assertThat(agentTask2Return.getStepInstanceId()).isEqualTo(1L); + assertThat(agentTask2Return.getExecuteCount()).isEqualTo(0L); + assertThat(agentTask2Return.getBatch()).isEqualTo(3); + assertThat(agentTask2Return.getHostId()).isEqualTo(104L); + assertThat(agentTask2Return.getGseTaskId()).isEqualTo(1001L); + assertThat(agentTask2Return.getAgentId()).isEqualTo("0:127.0.0.4"); + assertThat(agentTask2Return.getStartTime()).isEqualTo(1572858330000L); + assertThat(agentTask2Return.getEndTime()).isEqualTo(1572858331000L); + assertThat(agentTask2Return.getTotalTime()).isEqualTo(1000L); + assertThat(agentTask2Return.getErrorCode()).isEqualTo(88); + assertThat(agentTask2Return.getStatus()).isEqualTo(AgentTaskStatusEnum.HOST_NOT_EXIST); + assertThat(agentTask2Return.getTag()).isEqualTo("bb"); + assertThat(agentTask2Return.getExitCode()).isEqualTo(2); + } + + @Test + public void testGetSuccessIpCount() { + Integer count = scriptAgentTaskDAO.getSuccessAgentTaskCount(1L, 0); + assertThat(count).isEqualTo(4); + } + + @Test + @DisplayName("Agent任务结果分组") + public void listResultGroups() { + List resultGroups = scriptAgentTaskDAO.listResultGroups(1L, 0, null); + + assertThat(resultGroups.size()).isEqualTo(2); + assertThat(resultGroups).extracting("status").containsOnly(9, 11); + assertThat(resultGroups).extracting("tag").containsOnly("succ", "fail"); + AgentTaskResultGroupBaseDTO resultGroup = resultGroups.get(0); + if (resultGroup.getStatus().equals(9)) { + assertThat(resultGroup.getTotalAgentTasks()).isEqualTo(4); + } + if (resultGroup.getStatus().equals(11)) { + assertThat(resultGroup.getTotalAgentTasks()).isEqualTo(1); + } + + // 根据滚动执行批次查询 + resultGroups = scriptAgentTaskDAO.listResultGroups(1L, 0, 3); + + assertThat(resultGroups.size()).isEqualTo(2); + assertThat(resultGroups).extracting("status").containsOnly(9, 11); + assertThat(resultGroups).extracting("tag").containsOnly("succ", "fail"); + resultGroup = resultGroups.get(0); + if (resultGroup.getStatus().equals(9)) { + assertThat(resultGroup.getTotalAgentTasks()).isEqualTo(2); + } + if (resultGroup.getStatus().equals(11)) { + assertThat(resultGroup.getTotalAgentTasks()).isEqualTo(1); + } + } + + @Test + public void testListAgentTaskByResultGroup() { + List agentTasks = scriptAgentTaskDAO.listAgentTaskByResultGroup(1L, 0, 1, 9, "succ"); + assertThat(agentTasks.size()).isEqualTo(1); + assertThat(agentTasks.get(0).getStepInstanceId()).isEqualTo(1L); + assertThat(agentTasks.get(0).getExecuteCount()).isEqualTo(0); + assertThat(agentTasks.get(0).getBatch()).isEqualTo(1); + assertThat(agentTasks.get(0).getStatus()).isEqualTo(AgentTaskStatusEnum.SUCCESS); + assertThat(agentTasks.get(0).getTag()).isEqualTo("succ"); + } + +} diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImplIntegrationTest.java index ef761dd6ed..785ba7156c 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImplIntegrationTest.java +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImplIntegrationTest.java @@ -24,7 +24,7 @@ package com.tencent.bk.job.execute.dao.impl; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; import com.tencent.bk.job.execute.dao.StepInstanceDAO; @@ -80,16 +80,17 @@ public void testGetStepInstanceBase() { assertThat(stepInstance.getExecuteType()).isEqualTo(1); assertThat(stepInstance.getIpList()).isEqualTo("0:127.0.0.1"); assertThat(stepInstance.getOperator()).isEqualTo("admin"); - assertThat(stepInstance.getStatus()).isEqualTo(3); + assertThat(stepInstance.getStatus()).isEqualTo(RunStatusEnum.SUCCESS); assertThat(stepInstance.getExecuteCount()).isEqualTo(0); assertThat(stepInstance.getStartTime()).isEqualTo(1572868800000L); assertThat(stepInstance.getEndTime()).isEqualTo(1572868801000L); assertThat(stepInstance.getTotalTime()).isEqualTo(1111L); assertThat(stepInstance.getCreateTime()).isEqualTo(1572868800000L); assertThat(stepInstance.getTargetServers()).isNotNull(); - List expectedServer = new ArrayList<>(); - expectedServer.add(new IpDTO(0L, "127.0.0.1")); + List expectedServer = new ArrayList<>(); + expectedServer.add(new HostDTO(0L, "127.0.0.1")); assertThat(stepInstance.getTargetServers().getIpList()).containsAll(expectedServer); + assertThat(stepInstance.getBatch()).isEqualTo(0); } @Test @@ -101,12 +102,12 @@ public void testAddStepInstanceBase() { stepInstanceDTO.setStepId(1L); stepInstanceDTO.setExecuteType(StepExecuteTypeEnum.EXECUTE_SCRIPT.getValue()); ServersDTO servers = new ServersDTO(); - List ipList = new ArrayList<>(); - ipList.add(new IpDTO(0L, "127.0.0.1")); + List ipList = new ArrayList<>(); + ipList.add(new HostDTO(0L, "127.0.0.1")); servers.setIpList(ipList); stepInstanceDTO.setTargetServers(servers); stepInstanceDTO.setOperator("admin"); - stepInstanceDTO.setStatus(RunStatusEnum.SUCCESS.getValue()); + stepInstanceDTO.setStatus(RunStatusEnum.SUCCESS); stepInstanceDTO.setExecuteCount(0); stepInstanceDTO.setStartTime(1572868800000L); stepInstanceDTO.setEndTime(1572868801000L); @@ -127,11 +128,11 @@ public void testAddStepInstanceBase() { assertThat(returnStepInstance.getExecuteType()).isEqualTo(StepExecuteTypeEnum.EXECUTE_SCRIPT.getValue()); assertThat(returnStepInstance.getIpList()).isEqualTo("0:127.0.0.1"); assertThat(returnStepInstance.getTargetServers().getIpList()).hasSize(1); - List expectedServer = new ArrayList<>(); - expectedServer.add(new IpDTO(0L, "127.0.0.1")); + List expectedServer = new ArrayList<>(); + expectedServer.add(new HostDTO(0L, "127.0.0.1")); assertThat(returnStepInstance.getTargetServers().getIpList()).containsAll(expectedServer); assertThat(returnStepInstance.getOperator()).isEqualTo("admin"); - assertThat(returnStepInstance.getStatus()).isEqualTo(RunStatusEnum.SUCCESS.getValue()); + assertThat(returnStepInstance.getStatus()).isEqualTo(RunStatusEnum.SUCCESS); assertThat(returnStepInstance.getExecuteCount()).isEqualTo(0); assertThat(returnStepInstance.getStartTime()).isEqualTo(1572868800000L); assertThat(returnStepInstance.getEndTime()).isEqualTo(1572868801000L); @@ -139,6 +140,7 @@ public void testAddStepInstanceBase() { assertThat(returnStepInstance.getCreateTime()).isEqualTo(1572868800000L); assertThat(returnStepInstance.getStepNum()).isEqualTo(3); assertThat(returnStepInstance.getStepOrder()).isEqualTo(1); + assertThat(returnStepInstance.getBatch()).isEqualTo(0); } @Test @@ -163,9 +165,6 @@ public void testResetStepStatus() { assertThat(returnStepInstance.getStartTime()).isNull(); assertThat(returnStepInstance.getEndTime()).isNull(); assertThat(returnStepInstance.getTotalTime()).isNull(); - assertThat(returnStepInstance.getSuccessIPNum()).isEqualTo(0); - assertThat(returnStepInstance.getFailIPNum()).isEqualTo(0); - assertThat(returnStepInstance.getRunIPNum()).isEqualTo(0); } @Test @@ -183,14 +182,12 @@ public void testAddStepRetryCount() { @Test public void testUpdateStepStatus() { long stepInstanceId = 1L; - int status = RunStatusEnum.RUNNING.getValue(); - - stepInstanceDAO.updateStepStatus(stepInstanceId, status); + stepInstanceDAO.updateStepStatus(stepInstanceId, RunStatusEnum.RUNNING.getValue()); StepInstanceBaseDTO returnStepInstance = stepInstanceDAO.getStepInstanceBase(stepInstanceId); assertThat(returnStepInstance.getId()).isEqualTo(stepInstanceId); - assertThat(returnStepInstance.getStatus()).isEqualTo(status); + assertThat(returnStepInstance.getStatus()).isEqualTo(RunStatusEnum.RUNNING); } @Test @@ -249,51 +246,6 @@ public void testUpdateStepTotalTime() { assertThat(returnStepInstance.getTotalTime()).isEqualTo(totalTime); } - @Test - public void testUpdateStepStatInfo() { - long stepInstanceId = 1L; - int successIPNum = 1; - int failIPNum = 2; - int runIPNum = 3; - - stepInstanceDAO.updateStepStatInfo(stepInstanceId, runIPNum, successIPNum, failIPNum); - - StepInstanceBaseDTO returnStepInstance = stepInstanceDAO.getStepInstanceBase(stepInstanceId); - - assertThat(returnStepInstance.getId()).isEqualTo(stepInstanceId); - assertThat(returnStepInstance.getSuccessIPNum()).isEqualTo(successIPNum); - assertThat(returnStepInstance.getFailIPNum()).isEqualTo(failIPNum); - assertThat(returnStepInstance.getRunIPNum()).isEqualTo(runIPNum); - - } - - @Test - public void testGetFirstStepStartTime() { - long taskInstanceId = 1L; - - Long startTime = stepInstanceDAO.getFirstStepStartTime(taskInstanceId); - - assertThat(startTime).isEqualTo(1572868800000L); - } - - @Test - public void testGetLastStepEndTime() { - long taskInstanceId = 1L; - - Long endTime = stepInstanceDAO.getLastStepEndTime(taskInstanceId); - - assertThat(endTime).isEqualTo(1572868802000L); - } - - @Test - public void testGetAllStepTotalTime() { - long taskInstanceId = 1L; - - float totalTime = stepInstanceDAO.getAllStepTotalTime(taskInstanceId); - - assertThat(totalTime).isEqualTo(2223L); - } - @Test public void testResetStepExecuteInfoForRetry() { long stepInstanceId = 1L; @@ -303,11 +255,8 @@ public void testResetStepExecuteInfoForRetry() { assertThat(returnStepInstance).isNotNull(); assertThat(returnStepInstance.getStartTime()).isNotNull(); assertThat(returnStepInstance.getEndTime()).isNull(); - assertThat(returnStepInstance.getStatus()).isEqualTo(RunStatusEnum.RUNNING.getValue()); + assertThat(returnStepInstance.getStatus()).isEqualTo(RunStatusEnum.RUNNING); assertThat(returnStepInstance.getTotalTime()).isNull(); - assertThat(returnStepInstance.getSuccessIPNum()).isEqualTo(0); - assertThat(returnStepInstance.getFailIPNum()).isEqualTo(0); - assertThat(returnStepInstance.getRunIPNum()).isEqualTo(0); } @Test @@ -387,7 +336,7 @@ public void testAddFileStepInstance() { fileSource.setLocalUpload(false); fileSource.setFileType(TaskFileTypeEnum.SERVER.getType()); ServersDTO fileSourceServers = new ServersDTO(); - fileSourceServers.setIpList(Lists.newArrayList(new IpDTO(1L, "10.10.10.10"))); + fileSourceServers.setIpList(Lists.newArrayList(new HostDTO(1L, "10.10.10.10"))); fileSource.setServers(fileSourceServers); FileDetailDTO fileDetail = new FileDetailDTO(); fileDetail.setFilePath("/tmp/1.log"); @@ -417,7 +366,7 @@ public void testAddFileStepInstance() { assertThat(savedStepInstance.getFileSourceList().get(0).getServers()).isNotNull(); assertThat(savedStepInstance.getFileSourceList().get(0).getServers().getIpList()).isNotEmpty(); assertThat(savedStepInstance.getFileSourceList().get(0).getServers().getIpList()) - .containsOnly(new IpDTO(1L, + .containsOnly(new HostDTO(1L, "10.10.10.10")); assertThat(savedStepInstance.getFileSourceList().get(0).getFiles()).isNotEmpty(); assertThat(savedStepInstance.getFileSourceList().get(0).getFiles().get(0).getFilePath()) @@ -506,8 +455,8 @@ void testUpdateResolvedFileSource() { fileSourceDTO.setAccountId(1L); fileSourceDTO.setAccount("root"); ServersDTO servers = new ServersDTO(); - List ips = new ArrayList<>(); - ips.add(new IpDTO(1L, "10.10.10.10")); + List ips = new ArrayList<>(); + ips.add(new HostDTO(1L, "10.10.10.10")); servers.setIpList(ips); fileSourceDTO.setServers(servers); fileSources.add(fileSourceDTO); @@ -524,7 +473,7 @@ void testUpdateResolvedFileSource() { "/data/logs/1.log"); assertThat(updatedStepInstance.getFileSourceList().get(0).getFiles().get(0).getResolvedFilePath()).isEqualTo( "/data/logs/1.log"); - assertThat(updatedStepInstance.getFileSourceList().get(0).getServers().getIpList()).contains(new IpDTO(1L, + assertThat(updatedStepInstance.getFileSourceList().get(0).getServers().getIpList()).contains(new HostDTO(1L, "10.10.10.10")); } @@ -562,5 +511,21 @@ void testGetPreExecutableStepInstance() { assertThat(preStepInstance.getId()).isEqualTo(1L); } + @Test + void updateStepCurrentBatch() { + stepInstanceDAO.updateStepCurrentBatch(1L, 1); + + StepInstanceBaseDTO stepInstance = stepInstanceDAO.getStepInstanceBase(1L); + assertThat(stepInstance.getBatch()).isEqualTo(1); + } + + @Test + void updateStepRollingConfigId() { + stepInstanceDAO.updateStepRollingConfigId(1L, 1000L); + + StepInstanceBaseDTO stepInstance = stepInstanceDAO.getStepInstanceBase(1L); + assertThat(stepInstance.getRollingConfigId()).isEqualTo(1000L); + } + } diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImplIntegrationTest.java new file mode 100644 index 0000000000..31378cb33d --- /dev/null +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImplIntegrationTest.java @@ -0,0 +1,138 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao.impl; + +import com.tencent.bk.job.execute.common.constants.RunStatusEnum; +import com.tencent.bk.job.execute.dao.StepInstanceRollingTaskDAO; +import com.tencent.bk.job.execute.model.StepInstanceRollingTaskDTO; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.jdbc.Sql; +import org.springframework.test.context.jdbc.SqlConfig; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +@ExtendWith(SpringExtension.class) +@SpringBootTest +@ActiveProfiles("test") +@TestPropertySource(locations = "classpath:test.properties") +@SqlConfig(encoding = "utf-8") +@Sql({"/init_step_instance_rolling_task_data.sql"}) +public class StepInstanceRollingTaskDAOImplIntegrationTest { + + @Autowired + private StepInstanceRollingTaskDAO stepInstanceRollingTaskDAO; + + @Test + @DisplayName("根据步骤实例ID、执行次数、滚动批次查询步骤滚动任务") + void queryRollingTask() { + StepInstanceRollingTaskDTO rollingTask = stepInstanceRollingTaskDAO.queryRollingTask(1, 0, 1); + assertThat(rollingTask).isNotNull(); + assertThat(rollingTask.getStepInstanceId()).isEqualTo(1L); + assertThat(rollingTask.getExecuteCount()).isEqualTo(0); + assertThat(rollingTask.getBatch()).isEqualTo(1); + assertThat(rollingTask.getStatus()).isEqualTo(RunStatusEnum.BLANK); + assertThat(rollingTask.getStartTime()).isEqualTo(1642247802000L); + assertThat(rollingTask.getEndTime()).isEqualTo(1642247803000L); + assertThat(rollingTask.getTotalTime()).isEqualTo(1000L); + } + + @Test + @DisplayName("根据步骤实例ID查询步骤滚动任务") + void listRollingTasks() { + List rollingTasks = stepInstanceRollingTaskDAO.listRollingTasks(1L, null, null); + assertThat(rollingTasks).hasSize(2); + + StepInstanceRollingTaskDTO rollingTask1 = rollingTasks.get(0); + assertThat(rollingTask1).isNotNull(); + assertThat(rollingTask1.getStepInstanceId()).isEqualTo(1L); + assertThat(rollingTask1.getExecuteCount()).isEqualTo(0); + assertThat(rollingTask1.getBatch()).isEqualTo(1); + assertThat(rollingTask1.getStatus()).isEqualTo(RunStatusEnum.BLANK); + assertThat(rollingTask1.getStartTime()).isEqualTo(1642247802000L); + assertThat(rollingTask1.getEndTime()).isEqualTo(1642247803000L); + assertThat(rollingTask1.getTotalTime()).isEqualTo(1000L); + + StepInstanceRollingTaskDTO rollingTask2 = rollingTasks.get(1); + assertThat(rollingTask2).isNotNull(); + assertThat(rollingTask2.getStepInstanceId()).isEqualTo(1L); + assertThat(rollingTask2.getExecuteCount()).isEqualTo(0); + assertThat(rollingTask2.getBatch()).isEqualTo(2); + assertThat(rollingTask2.getStatus()).isEqualTo(RunStatusEnum.BLANK); + assertThat(rollingTask2.getStartTime()).isEqualTo(1642247804000L); + assertThat(rollingTask2.getEndTime()).isEqualTo(1642247805000L); + assertThat(rollingTask2.getTotalTime()).isEqualTo(1000L); + } + + @Test + @DisplayName("保存步骤滚动任务") + void saveRollingTask() { + StepInstanceRollingTaskDTO rollingTask = new StepInstanceRollingTaskDTO(); + rollingTask.setStepInstanceId(100L); + rollingTask.setExecuteCount(0); + rollingTask.setBatch(1); + rollingTask.setStatus(RunStatusEnum.ROLLING_WAITING); + rollingTask.setStartTime(1642247802000L); + stepInstanceRollingTaskDAO.saveRollingTask(rollingTask); + + StepInstanceRollingTaskDTO savedRollingTask = stepInstanceRollingTaskDAO.queryRollingTask(100L, 0, 1); + assertThat(savedRollingTask).isNotNull(); + assertThat(savedRollingTask.getStepInstanceId()).isEqualTo(100L); + assertThat(savedRollingTask.getExecuteCount()).isEqualTo(0); + assertThat(savedRollingTask.getBatch()).isEqualTo(1); + assertThat(savedRollingTask.getStatus()).isEqualTo(RunStatusEnum.ROLLING_WAITING); + assertThat(savedRollingTask.getStartTime()).isEqualTo(1642247802000L); + assertThat(savedRollingTask.getEndTime()).isNull(); + assertThat(savedRollingTask.getTotalTime()).isNull(); + } + + @Test + void updateRollingTask() { + long startTime = System.currentTimeMillis(); + Long endTime = startTime + 1000L; + Long totalTime = 1000L; + stepInstanceRollingTaskDAO.updateRollingTask(1L, 0, 1, RunStatusEnum.SUCCESS, startTime, endTime, totalTime); + + StepInstanceRollingTaskDTO savedRollingTask = stepInstanceRollingTaskDAO.queryRollingTask(1L, 0, 1); + + assertThat(savedRollingTask).isNotNull(); + assertThat(savedRollingTask.getStepInstanceId()).isEqualTo(1L); + assertThat(savedRollingTask.getExecuteCount()).isEqualTo(0); + assertThat(savedRollingTask.getBatch()).isEqualTo(1); + assertThat(savedRollingTask.getStatus()).isEqualTo(RunStatusEnum.SUCCESS); + assertThat(savedRollingTask.getStartTime()).isEqualTo(startTime); + assertThat(savedRollingTask.getEndTime()).isEqualTo(endTime); + assertThat(savedRollingTask.getTotalTime()).isEqualTo(totalTime); + + } +} diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceDAOImplIntegrationTest.java index 274ae3689b..9c36844bb0 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceDAOImplIntegrationTest.java +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceDAOImplIntegrationTest.java @@ -73,7 +73,7 @@ void testGetTaskInstance() { assertThat(taskInstance.getCreateTime()).isEqualTo(1572868800000L); assertThat(taskInstance.getStartTime()).isEqualTo(1572868800000L); assertThat(taskInstance.getEndTime()).isEqualTo(1572868801000L); - assertThat(taskInstance.getStatus()).isEqualTo(3); + assertThat(taskInstance.getStatus()).isEqualTo(RunStatusEnum.SUCCESS); assertThat(taskInstance.getTotalTime()).isEqualTo(1111L); assertThat(taskInstance.getStartupMode()).isEqualTo(1); assertThat(taskInstance.getCallbackUrl()).isEqualTo("http://bkjob.com"); @@ -90,7 +90,7 @@ void testAddTaskInstance() { taskInstance.setOperator("user1"); taskInstance.setType(TaskTypeEnum.NORMAL.getValue()); taskInstance.setCreateTime(1572955200000L); - taskInstance.setStatus(RunStatusEnum.BLANK.getValue()); + taskInstance.setStatus(RunStatusEnum.BLANK); taskInstance.setStartupMode(TaskStartupModeEnum.API.getValue()); taskInstance.setCallbackUrl("http://bkjob.com"); taskInstance.setAppCode("bk_monitor"); @@ -109,7 +109,7 @@ void testAddTaskInstance() { assertThat(returnTaskInstance.getCreateTime()).isEqualTo(1572955200000L); assertThat(returnTaskInstance.getStartTime()).isNull(); assertThat(returnTaskInstance.getEndTime()).isNull(); - assertThat(returnTaskInstance.getStatus()).isEqualTo(RunStatusEnum.BLANK.getValue()); + assertThat(returnTaskInstance.getStatus()).isEqualTo(RunStatusEnum.BLANK); assertThat(returnTaskInstance.getStartupMode()).isEqualTo(TaskStartupModeEnum.API.getValue()); assertThat(returnTaskInstance.getCallbackUrl()).isEqualTo("http://bkjob.com"); assertThat(taskInstance.getAppCode()).isEqualTo("bk_monitor"); @@ -128,13 +128,11 @@ void testGetTaskInstanceByTaskId() { @Test void testUpdateTaskStatus() { long taskInstanceId = 2L; - int status = RunStatusEnum.ABNORMAL_STATE.getValue(); - - taskInstanceDAO.updateTaskStatus(taskInstanceId, status); + taskInstanceDAO.updateTaskStatus(taskInstanceId, RunStatusEnum.ABNORMAL_STATE.getValue()); TaskInstanceDTO taskInstanceDTO = taskInstanceDAO.getTaskInstance(taskInstanceId); assertThat(taskInstanceDTO.getId()).isEqualTo(taskInstanceId); - assertThat(taskInstanceDTO.getStatus()).isEqualTo(status); + assertThat(taskInstanceDTO.getStatus()).isEqualTo(RunStatusEnum.ABNORMAL_STATE); } @Test @@ -170,7 +168,7 @@ void testUpdateTaskCurrentStepId() { TaskInstanceDTO taskInstanceDTO = taskInstanceDAO.getTaskInstance(taskInstanceId); assertThat(taskInstanceDTO.getId()).isEqualTo(taskInstanceId); - assertThat(taskInstanceDTO.getCurrentStepId()).isEqualTo(stepInstanceId); + assertThat(taskInstanceDTO.getCurrentStepInstanceId()).isEqualTo(stepInstanceId); } @@ -185,7 +183,7 @@ void testResetTaskStatus() { assertThat(taskInstanceDTO.getStartTime()).isNull(); assertThat(taskInstanceDTO.getEndTime()).isNull(); assertThat(taskInstanceDTO.getTotalTime()).isNull(); - assertThat(taskInstanceDTO.getCurrentStepId()).isEqualTo(0L); + assertThat(taskInstanceDTO.getCurrentStepInstanceId()).isEqualTo(0L); } @Test @@ -306,20 +304,20 @@ void testUpdateTaskExecutionInfo() { assertThat(taskInstanceDTO.getStartTime()).isEqualTo(startTime); assertThat(taskInstanceDTO.getEndTime()).isEqualTo(endTime); assertThat(taskInstanceDTO.getTotalTime()).isEqualTo(totalTime); - assertThat(taskInstanceDTO.getStatus()).isEqualTo(RunStatusEnum.SUCCESS.getValue()); + assertThat(taskInstanceDTO.getStatus()).isEqualTo(RunStatusEnum.SUCCESS); } @Test void testResetTaskExecuteInfoForResume() { long taskInstanceId = 2L; - taskInstanceDAO.resetTaskExecuteInfoForResume(taskInstanceId); + taskInstanceDAO.resetTaskExecuteInfoForRetry(taskInstanceId); TaskInstanceDTO taskInstanceDTO = taskInstanceDAO.getTaskInstance(taskInstanceId); assertThat(taskInstanceDTO.getId()).isEqualTo(taskInstanceId); assertThat(taskInstanceDTO.getEndTime()).isNull(); assertThat(taskInstanceDTO.getTotalTime()).isNull(); - assertThat(taskInstanceDTO.getStatus()).isEqualTo(RunStatusEnum.RUNNING.getValue()); + assertThat(taskInstanceDTO.getStatus()).isEqualTo(RunStatusEnum.RUNNING); } } diff --git a/src/backend/job-execute/boot-job-execute/src/test/resources/init_file_agent_task_data.sql b/src/backend/job-execute/boot-job-execute/src/test/resources/init_file_agent_task_data.sql new file mode 100644 index 0000000000..ab4d5a8a5d --- /dev/null +++ b/src/backend/job-execute/boot-job-execute/src/test/resources/init_file_agent_task_data.sql @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +TRUNCATE TABLE job_execute.gse_file_agent_task; + +INSERT INTO job_execute.gse_file_agent_task + (step_instance_id, + execute_count, + batch, + mode, + host_id, + agent_id, + gse_task_id, + status, + start_time, + end_time, + total_time, + error_code) +VALUES + (1,0,1,0,101,'0:127.0.0.1',1,9,1565767148000,1565767149000,1000,0), + (1,0,1,1,102,'0:127.0.0.2',2,9,1565767148000,1565767149000,1000,0), + (1,0,2,0,101,'0:127.0.0.1',3,9,1565767150000,1565767151000,1000,0), + (1,0,2,1,103,'0:127.0.0.3',3,9,1565767150000,1565767151000,1000,0), + (1,0,2,1,104,'0:127.0.0.4',3,11,1565767150000,1565767151000,1000,99), + (2,0,0,0,101,'0:127.0.0.1',4,9,1565767148000,1565767149000,1000,0), + (2,0,0,1,102,'0:127.0.0.2',4,9,1565767148000,1565767149000,1000,0); diff --git a/src/backend/job-execute/boot-job-execute/src/test/resources/init_gse_task_data.sql b/src/backend/job-execute/boot-job-execute/src/test/resources/init_gse_task_data.sql new file mode 100644 index 0000000000..69c531371a --- /dev/null +++ b/src/backend/job-execute/boot-job-execute/src/test/resources/init_gse_task_data.sql @@ -0,0 +1,31 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +TRUNCATE TABLE job_execute.gse_task; + +INSERT INTO job_execute.gse_task (id,step_instance_id,execute_count,batch,status,start_time,end_time,total_time,gse_task_id) +VALUES +(1,1,0,0,4,1660639901000,1660639902000,1000,'GSE_TASK_1'), +(2,2,0,1,3,1660639903000,1660639904000,1000,'GSE_TASK_2'), +(3,2,0,2,4,1660639905000,1660639906000,1000,'GSE_TASK_3'); diff --git a/src/backend/job-execute/boot-job-execute/src/test/resources/init_rolling_config_data.sql b/src/backend/job-execute/boot-job-execute/src/test/resources/init_rolling_config_data.sql new file mode 100644 index 0000000000..049b763a5d --- /dev/null +++ b/src/backend/job-execute/boot-job-execute/src/test/resources/init_rolling_config_data.sql @@ -0,0 +1,28 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +truncate table rolling_config; +insert into job_execute.rolling_config (id,task_instance_id,config_name,config) values +(1,1,'config1', '{"name":"config1","includeStepInstanceIdList":[100,101,102,103],"stepRollingConfigs": {"100": {"batch": true},"101": {"batch": false},"102": {"batch": true},"103": {"batch": true}},"mode":1,"expr":"1 10% 100%","hostsBatchList":[{"batch":1,"hosts":[{"cloudAreaId":0,"ip":"127.0.0.1"}]},{"batch":2,"hosts":[{"cloudAreaId":0,"ip":"127.0.0.2"}]},{"batch":3,"hosts":[{"cloudAreaId":0,"ip":"127.0.0.3"},{"cloudAreaId":0,"ip":"127.0.0.4"}]}]}'); + diff --git a/src/backend/job-execute/boot-job-execute/src/test/resources/init_schema.sql b/src/backend/job-execute/boot-job-execute/src/test/resources/init_schema.sql index 12c2e4ca3d..a659b2740e 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/resources/init_schema.sql +++ b/src/backend/job-execute/boot-job-execute/src/test/resources/init_schema.sql @@ -60,31 +60,27 @@ CREATE TABLE IF NOT EXISTS `task_instance` CREATE TABLE IF NOT EXISTS `step_instance` ( - `id` bigint(20) NOT NULL AUTO_INCREMENT, - `step_id` bigint(20) NOT NULL, - `task_instance_id` bigint(20) NOT NULL, - `app_id` bigint(20) NOT NULL, - `name` varchar(512) DEFAULT NULL, - `type` tinyint(4) NOT NULL, - `operator` varchar(128) DEFAULT NULL, - `status` tinyint(4) NOT NULL DEFAULT '1', - `execute_count` int(11) NOT NULL DEFAULT '0', - `target_servers` longtext, - `abnormal_agent_ip_list` longtext, - `start_time` bigint(20) DEFAULT NULL, - `end_time` bigint(20) DEFAULT NULL, - `total_time` bigint(20) DEFAULT NULL, - `total_ip_num` int(11) DEFAULT '0', - `abnormal_agent_num` int(11) DEFAULT '0', - `run_ip_num` int(11) DEFAULT '0', - `fail_ip_num` int(11) DEFAULT '0', - `success_ip_num` int(11) DEFAULT '0', - `create_time` bigint(20) DEFAULT NULL, - `ignore_error` tinyint(4) NOT NULL DEFAULT 0, - `step_num` int(11) NOT NULL DEFAULT 0, - `step_order` int(11) NOT NULL DEFAULT 0, - `row_create_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `row_update_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `step_id` bigint(20) NOT NULL, + `task_instance_id` bigint(20) NOT NULL, + `app_id` bigint(20) NOT NULL, + `name` varchar(512) DEFAULT NULL, + `type` tinyint(4) NOT NULL, + `operator` varchar(128) DEFAULT NULL, + `status` tinyint(4) NOT NULL DEFAULT '1', + `execute_count` int(11) NOT NULL DEFAULT '0', + `batch` smallint(6) NOT NULL DEFAULT '0', + `target_servers` longtext, + `start_time` bigint(20) DEFAULT NULL, + `end_time` bigint(20) DEFAULT NULL, + `total_time` bigint(20) DEFAULT NULL, + `create_time` bigint(20) DEFAULT NULL, + `ignore_error` tinyint(4) NOT NULL DEFAULT 0, + `step_num` int(11) NOT NULL DEFAULT 0, + `step_order` int(11) NOT NULL DEFAULT 0, + `rolling_config_id` bigint(20) DEFAULT NULL, + `row_create_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + `row_update_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`id`), KEY (`task_instance_id`) ) ENGINE = InnoDB @@ -170,6 +166,7 @@ CREATE TABLE IF NOT EXISTS `gse_task_ip_log` ( `step_instance_id` bigint(20) NOT NULL, `execute_count` int(11) NOT NULL DEFAULT '0', + `gse_task_id` bigint(20) NOT NULL DEFAULT '0', `ip` varchar(30) NOT NULL, `status` int(11) DEFAULT '1', `start_time` bigint(20) DEFAULT NULL, @@ -185,10 +182,78 @@ CREATE TABLE IF NOT EXISTS `gse_task_ip_log` `row_create_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, `row_update_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`step_instance_id`, `execute_count`, `ip`), + KEY (`gse_task_id`), KEY (`display_ip`) ) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; +CREATE TABLE IF NOT EXISTS `gse_task` +( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `step_instance_id` bigint(20) NOT NULL DEFAULT '0', + `execute_count` smallint(6) NOT NULL DEFAULT '0', + `batch` smallint(6) NOT NULL DEFAULT '0', + `start_time` bigint(20) DEFAULT NULL, + `end_time` bigint(20) DEFAULT NULL, + `total_time` bigint(11) DEFAULT NULL, + `status` tinyint(4) DEFAULT '1', + `gse_task_id` varchar(64) DEFAULT NULL, + `row_create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `row_update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY (`step_instance_id`, `execute_count`, `batch`), + KEY (`gse_task_id`) +) ENGINE = InnoDB + DEFAULT CHARSET = utf8mb4; + +CREATE TABLE IF NOT EXISTS `gse_script_agent_task` +( + `id` bigint(20) NOT NULL AUTO_INCREMENT PRIMARY KEY, + `step_instance_id` bigint(20) NOT NULL, + `execute_count` smallint(6) NOT NULL DEFAULT '0', + `actual_execute_count` smallint(6) DEFAULT NULL, + `batch` smallint(6) NOT NULL DEFAULT '0', + `host_id` bigint(20) NOT NULL DEFAULT '0', + `agent_id` varchar(64) NOT NULL DEFAULT '', + `gse_task_id` bigint(20) NOT NULL DEFAULT '0', + `status` int(11) DEFAULT '1', + `start_time` bigint(20) DEFAULT NULL, + `end_time` bigint(20) DEFAULT NULL, + `total_time` bigint(20) DEFAULT NULL, + `error_code` int(11) DEFAULT '0', + `exit_code` int(11) DEFAULT NULL, + `tag` varchar(256) DEFAULT '', + `log_offset` int(11) NOT NULL DEFAULT '0', + `row_create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `row_update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + UNIQUE KEY (`step_instance_id`, `execute_count`, `batch`, `host_id`), + KEY (`step_instance_id`, `host_id`) +) ENGINE = InnoDB + DEFAULT CHARSET = utf8mb4; + +CREATE TABLE IF NOT EXISTS `gse_file_agent_task` +( + `id` bigint(20) NOT NULL AUTO_INCREMENT PRIMARY KEY, + `step_instance_id` bigint(20) NOT NULL, + `execute_count` smallint(6) NOT NULL DEFAULT '0', + `actual_execute_count` smallint(6) DEFAULT NULL, + `batch` smallint(6) NOT NULL DEFAULT '0', + `host_id` bigint(20) NOT NULL DEFAULT '0', + `agent_id` varchar(64) NOT NULL DEFAULT '', + `mode` tinyint(1) NOT NULL, + `gse_task_id` bigint(20) NOT NULL DEFAULT '0', + `status` int(11) DEFAULT '1', + `start_time` bigint(20) DEFAULT NULL, + `end_time` bigint(20) DEFAULT NULL, + `total_time` bigint(20) DEFAULT NULL, + `error_code` int(11) DEFAULT '0', + `row_create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `row_update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + UNIQUE KEY (`step_instance_id`, `execute_count`, `batch`, `mode`, `host_id`), + KEY (`step_instance_id`, `host_id`) +) ENGINE = InnoDB + DEFAULT CHARSET = utf8mb4; + CREATE TABLE IF NOT EXISTS `operation_log` ( `id` bigint(20) NOT NULL AUTO_INCREMENT, @@ -221,31 +286,32 @@ CREATE TABLE IF NOT EXISTS `task_instance_variable` CREATE TABLE IF NOT EXISTS `dangerous_record` ( - `row_create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, - `row_update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - `id` bigint(20) NOT NULL AUTO_INCREMENT, - `rule_id` bigint(20) NOT NULL, - `rule_expression` varchar(255) NOT NULL, - `app_id` bigint(20) NOT NULL, + `row_create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `row_update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `rule_id` bigint(20) NOT NULL, + `rule_expression` varchar(255) NOT NULL, + `app_id` bigint(20) NOT NULL, `app_name` varchar(1024) NOT NULL, - `operator` varchar(128) NOT NULL, - `script_language` tinyint(4) NOT NULL, - `script_content` longtext NOT NULL, - `create_time` bigint(20) NOT NULL, - `startup_mode` tinyint(4) NOT NULL, - `client` varchar(128) NOT NULL, - `action` tinyint(4) NOT NULL, - `check_result` text NOT NULL, - `ext_data` text, + `operator` varchar(128) NOT NULL, + `script_language` tinyint(4) NOT NULL, + `script_content` longtext NOT NULL, + `create_time` bigint(20) NOT NULL, + `startup_mode` tinyint(4) NOT NULL, + `client` varchar(128) NOT NULL, + `action` tinyint(4) NOT NULL, + `check_result` text NOT NULL, + `ext_data` text, PRIMARY KEY (`id`), - KEY `idx_create_time_rule_id` (`create_time`,`rule_id`), - KEY `idx_create_time_rule_expression` (`create_time`,`rule_expression`), - KEY `idx_create_time_app_id` (`create_time`,`app_id`), - KEY `idx_create_time_operator` (`create_time`,`operator`), - KEY `idx_create_time_startup_mode` (`create_time`,`startup_mode`), - KEY `idx_create_time_client` (`create_time`,`client`), - KEY `idx_create_time_mode` (`create_time`,`action`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + KEY `idx_create_time_rule_id` (`create_time`, `rule_id`), + KEY `idx_create_time_rule_expression` (`create_time`, `rule_expression`), + KEY `idx_create_time_app_id` (`create_time`, `app_id`), + KEY `idx_create_time_operator` (`create_time`, `operator`), + KEY `idx_create_time_startup_mode` (`create_time`, `startup_mode`), + KEY `idx_create_time_client` (`create_time`, `client`), + KEY `idx_create_time_mode` (`create_time`, `action`) +) ENGINE = InnoDB + DEFAULT CHARSET = utf8mb4; CREATE TABLE IF NOT EXISTS `step_instance_variable` ( @@ -262,3 +328,34 @@ CREATE TABLE IF NOT EXISTS `step_instance_variable` ) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; + +CREATE TABLE IF NOT EXISTS `rolling_config` +( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `task_instance_id` bigint(20) NOT NULL DEFAULT '0', + `config_name` varchar(128) NOT NULL, + `config` longtext NOT NULL, + `row_create_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + `row_update_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY (`task_instance_id`, `config_name`) +) ENGINE = InnoDB + DEFAULT CHARSET = utf8mb4; + +CREATE TABLE IF NOT EXISTS `step_instance_rolling_task` +( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `step_instance_id` bigint(20) NOT NULL DEFAULT '0', + `batch` smallint(6) NOT NULL DEFAULT '0', + `execute_count` smallint(6) NOT NULL DEFAULT '0', + `start_time` bigint(20) DEFAULT NULL, + `end_time` bigint(20) DEFAULT NULL, + `total_time` bigint(11) DEFAULT NULL, + `status` tinyint(4) NOT NULL DEFAULT '1', + `row_create_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + `row_update_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY (`step_instance_id`, `batch`, `execute_count`) +) ENGINE = InnoDB + DEFAULT CHARSET = utf8mb4; + diff --git a/src/backend/job-execute/boot-job-execute/src/test/resources/init_script_agent_task_data.sql b/src/backend/job-execute/boot-job-execute/src/test/resources/init_script_agent_task_data.sql new file mode 100644 index 0000000000..f8d2fc25d3 --- /dev/null +++ b/src/backend/job-execute/boot-job-execute/src/test/resources/init_script_agent_task_data.sql @@ -0,0 +1,51 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +TRUNCATE TABLE job_execute.gse_script_agent_task; + +INSERT INTO job_execute.gse_script_agent_task + (step_instance_id, + execute_count, + batch, + host_id, + agent_id, + gse_task_id, + status, + start_time, + end_time, + total_time, + error_code, + exit_code, + tag, + log_offset) +VALUES + (1,0,1,101,'0:127.0.0.1',1,9,1565767148000,1565767149000,1316,0,0,'succ',0), + (1,0,2,102,'0:127.0.0.2',2,9,1565767148000,1565767149000,1211,0,0,'succ',0), + (1,0,3,103,'0:127.0.0.3',3,9,1565767148000,1565767149000,1211,0,0,'succ',0), + (1,0,3,104,'0:127.0.0.4',3,9,1565767148000,1565767149000,1211,0,0,'succ',0), + (1,0,3,105,'0:127.0.0.5',3,11,1565767148000,1565767149000,1211,0,0,'fail',0), + (2,0,0,101,'0:127.0.0.1',4,9,1565767148000,1565767209000,1211,0,0,'succ',0), + (2,1,0,101,'0:127.0.0.1',4,9,1565766610000,1565767211000,1215,0,0,'succ',0), + (3,0,0,101,'0:127.0.0.1',5,9,1565766610000,1565767211000,1215,0,0,'succ',0), + (3,0,0,102,'0:127.0.0.2',5,9,1565766610000,1565767211000,1215,0,0,'succ',0); diff --git a/src/backend/job-execute/boot-job-execute/src/test/resources/init_step_instance_rolling_task_data.sql b/src/backend/job-execute/boot-job-execute/src/test/resources/init_step_instance_rolling_task_data.sql new file mode 100644 index 0000000000..780a6c7e77 --- /dev/null +++ b/src/backend/job-execute/boot-job-execute/src/test/resources/init_step_instance_rolling_task_data.sql @@ -0,0 +1,41 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +TRUNCATE TABLE step_instance_rolling_task; + +INSERT INTO job_execute.step_instance_rolling_task + (step_instance_id, + execute_count, + batch, + status, + start_time, + end_time, + total_time) +VALUES + (1,0,1,1,1642247802000,1642247803000,1000), + (1,0,2,1,1642247804000,1642247805000,1000), + (2,0,1,1,1642247804000,1642247805000,1000), + (2,0,2,1,1642247805000,1642247806000,1000), + (2,0,3,1,1642247806000,1642247807000,1000); + diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/impl/GseManageProcessResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/impl/GseManageProcessResourceImpl.java index 4ea4879a30..6b2973519d 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/impl/GseManageProcessResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/impl/GseManageProcessResourceImpl.java @@ -28,7 +28,7 @@ import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.i18n.service.MessageI18nService; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.api.esb.gse.GseManageProcessResource; import com.tencent.bk.job.execute.engine.model.GseTaskResponse; import com.tencent.bk.job.execute.gse.GseApiExecutor; @@ -84,7 +84,7 @@ private GseProcessInfoDTO buildGseProcessInfoDTO(EsbGseManageProcRequest.ManageP String procName = manageProcInfo.getProcName(); String contact = manageProcInfo.getProcOwner(); gseProcessInfo.setIpList(manageProcInfo.getIpList().stream() - .map(ipDTO -> new IpDTO(ipDTO.getCloudAreaId(), ipDTO.getIp())).collect(Collectors.toList())); + .map(ipDTO -> new HostDTO(ipDTO.getBkCloudId(), ipDTO.getIp())).collect(Collectors.toList())); gseProcessInfo.setProcName(procName); gseProcessInfo.setContact(contact); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/impl/GseOperateProcessResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/impl/GseOperateProcessResourceImpl.java index a638247078..e1c4f5a45f 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/impl/GseOperateProcessResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/impl/GseOperateProcessResourceImpl.java @@ -29,7 +29,7 @@ import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.i18n.service.MessageI18nService; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.api.esb.gse.GseOperateProcessResource; import com.tencent.bk.job.execute.engine.model.GseTaskResponse; import com.tencent.bk.job.execute.gse.GseApiExecutor; @@ -67,7 +67,7 @@ public EsbResp gseOperateProcess(EsbGseOperateProcessReques List gseProcessInfos = new ArrayList<>(); for (EsbProcessInfoDTO processInfo : request.getProcessInfos()) { GseProcessInfoDTO gseProcessInfo = new GseProcessInfoDTO(); - gseProcessInfo.setIpList(processInfo.getIpList().stream().map(ipDTO -> new IpDTO(ipDTO.getCloudAreaId(), + gseProcessInfo.setIpList(processInfo.getIpList().stream().map(ipDTO -> new HostDTO(ipDTO.getBkCloudId(), ipDTO.getIp())).collect(Collectors.toList())); String setupPath = processInfo.getSetupPath(); String procName = processInfo.getProcName(); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/impl/GseOperateProcessV2ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/impl/GseOperateProcessV2ResourceImpl.java index 930fcf9339..540fefcdd2 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/impl/GseOperateProcessV2ResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/impl/GseOperateProcessV2ResourceImpl.java @@ -29,7 +29,7 @@ import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.i18n.service.MessageI18nService; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.api.esb.gse.GseOperateProcessV2Resource; import com.tencent.bk.job.execute.engine.model.GseTaskResponse; import com.tencent.bk.job.execute.gse.GseApiExecutor; @@ -68,7 +68,7 @@ public EsbResp gseOperateProcessV2(EsbGseOperateProcessRequ List gseProcessInfos = new ArrayList<>(); for (EsbProcessInfoDTO processInfo : request.getProcessInfos()) { GseProcessInfoDTO gseProcessInfo = new GseProcessInfoDTO(); - gseProcessInfo.setIpList(processInfo.getIpList().stream().map(ipDTO -> new IpDTO(ipDTO.getCloudAreaId(), + gseProcessInfo.setIpList(processInfo.getIpList().stream().map(ipDTO -> new HostDTO(ipDTO.getBkCloudId(), ipDTO.getIp())).collect(Collectors.toList())); String setupPath = processInfo.getSetupPath(); String procName = processInfo.getProcName(); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbExecuteTaskResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbExecuteTaskResourceImpl.java index f32098befa..53362cba60 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbExecuteTaskResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbExecuteTaskResourceImpl.java @@ -33,7 +33,7 @@ import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.ValidateResult; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.common.web.metrics.CustomTimed; @@ -107,7 +107,7 @@ public EsbResp executeJob(EsbExecuteJobRequest request) { executeVariableValues.add(taskVariableDTO); } } - TaskInstanceDTO taskInstanceDTO = taskExecuteService.createTaskInstanceForTask( + TaskInstanceDTO taskInstanceDTO = taskExecuteService.executeJobPlan( TaskExecuteParam .builder() .appId(request.getAppId()) @@ -118,7 +118,6 @@ public EsbResp executeJob(EsbExecuteJobRequest request) { .callbackUrl(request.getCallbackUrl()) .appCode(request.getAppCode()) .build()); - taskExecuteService.startTask(taskInstanceDTO.getId()); EsbJobExecuteDTO result = new EsbJobExecuteDTO(); result.setTaskInstanceId(taskInstanceDTO.getId()); @@ -152,8 +151,8 @@ private ServersDTO convertToServersDTO(EsbServerDTO servers, List ipLi ServersDTO serversDTO = new ServersDTO(); if (servers != null) { if (servers.getIps() != null) { - List staticIpList = new ArrayList<>(); - servers.getIps().forEach(ip -> staticIpList.add(new IpDTO(ip.getCloudAreaId(), ip.getIp()))); + List staticIpList = new ArrayList<>(); + servers.getIps().forEach(ip -> staticIpList.add(new HostDTO(ip.getBkCloudId(), ip.getIp()))); serversDTO.setStaticIpList(staticIpList); } if (servers.getDynamicGroupIds() != null) { @@ -170,8 +169,8 @@ private ServersDTO convertToServersDTO(EsbServerDTO servers, List ipLi } } else { if (ipList != null) { - List staticIpList = new ArrayList<>(); - ipList.forEach(ip -> staticIpList.add(new IpDTO(ip.getCloudAreaId(), ip.getIp()))); + List staticIpList = new ArrayList<>(); + ipList.forEach(ip -> staticIpList.add(new HostDTO(ip.getBkCloudId(), ip.getIp()))); serversDTO.setStaticIpList(staticIpList); } if (dynamicGroupIdList != null) { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastExecuteSQLResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastExecuteSQLResourceImpl.java index e292dac800..679adc3c88 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastExecuteSQLResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastExecuteSQLResourceImpl.java @@ -46,6 +46,7 @@ import com.tencent.bk.job.execute.common.constants.TaskTypeEnum; import com.tencent.bk.job.execute.metrics.ExecuteMetricsConstants; import com.tencent.bk.job.execute.model.AccountDTO; +import com.tencent.bk.job.execute.model.FastTaskDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.esb.v2.EsbJobExecuteDTO; @@ -112,8 +113,9 @@ public EsbResp fastExecuteSQL(EsbFastExecuteSQLRequest request scriptService); TaskInstanceDTO taskInstance = buildFastSQLTaskInstance(request); StepInstanceDTO stepInstance = buildFastSQLStepInstance(request, script); - long taskInstanceId = taskExecuteService.createTaskInstanceFast(taskInstance, stepInstance); - taskExecuteService.startTask(taskInstanceId); + long taskInstanceId = taskExecuteService.executeFastTask( + FastTaskDTO.builder().taskInstance(taskInstance).stepInstance(stepInstance).build() + ); EsbJobExecuteDTO jobExecuteInfo = new EsbJobExecuteDTO(); jobExecuteInfo.setTaskInstanceId(taskInstanceId); @@ -167,9 +169,9 @@ private TaskInstanceDTO buildFastSQLTaskInstance(EsbFastExecuteSQLRequest reques taskInstance.setTaskId(-1L); taskInstance.setTaskTemplateId(-1L); taskInstance.setDebugTask(false); - taskInstance.setStatus(RunStatusEnum.BLANK.getValue()); + taskInstance.setStatus(RunStatusEnum.BLANK); taskInstance.setStartupMode(TaskStartupModeEnum.API.getValue()); - taskInstance.setCurrentStepId(0L); + taskInstance.setCurrentStepInstanceId(0L); taskInstance.setCreateTime(DateUtils.currentTimeMillis()); taskInstance.setType(TaskTypeEnum.SCRIPT.getValue()); taskInstance.setAppCode(request.getAppCode()); @@ -199,7 +201,7 @@ private StepInstanceDTO buildFastSQLStepInstance(EsbFastExecuteSQLRequest reques stepInstance.setTimeout(request.getTimeout() == null ? JobConstants.DEFAULT_JOB_TIMEOUT_SECONDS : request.getTimeout()); stepInstance.setExecuteType(StepExecuteTypeEnum.EXECUTE_SQL.getValue()); - stepInstance.setStatus(RunStatusEnum.BLANK.getValue()); + stepInstance.setStatus(RunStatusEnum.BLANK); stepInstance.setTargetServers(convertToStandardServers(request.getTargetServer(), request.getIpList(), request.getDynamicGroupIdList())); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastExecuteScriptResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastExecuteScriptResourceImpl.java index 8904a95fd0..a3355747fd 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastExecuteScriptResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastExecuteScriptResourceImpl.java @@ -46,6 +46,7 @@ import com.tencent.bk.job.execute.common.constants.TaskTypeEnum; import com.tencent.bk.job.execute.metrics.ExecuteMetricsConstants; import com.tencent.bk.job.execute.model.AccountDTO; +import com.tencent.bk.job.execute.model.FastTaskDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.esb.v2.EsbJobExecuteDTO; @@ -114,8 +115,9 @@ public EsbResp fastExecuteScript(EsbFastExecuteScriptRequest r scriptService); TaskInstanceDTO taskInstance = buildFastScriptTaskInstance(request); StepInstanceDTO stepInstance = buildFastScriptStepInstance(request, script); - long taskInstanceId = taskExecuteService.createTaskInstanceFast(taskInstance, stepInstance); - taskExecuteService.startTask(taskInstanceId); + long taskInstanceId = taskExecuteService.executeFastTask( + FastTaskDTO.builder().taskInstance(taskInstance).stepInstance(stepInstance).build() + ); EsbJobExecuteDTO jobExecuteInfo = new EsbJobExecuteDTO(); jobExecuteInfo.setTaskInstanceId(taskInstanceId); @@ -174,11 +176,11 @@ private TaskInstanceDTO buildFastScriptTaskInstance(EsbFastExecuteScriptRequest taskInstance.setDebugTask(false); taskInstance.setAppId(request.getAppId()); taskInstance.setStartupMode(TaskStartupModeEnum.API.getValue()); - taskInstance.setStatus(RunStatusEnum.BLANK.getValue()); + taskInstance.setStatus(RunStatusEnum.BLANK); taskInstance.setOperator(request.getUserName()); taskInstance.setCreateTime(DateUtils.currentTimeMillis()); taskInstance.setType(TaskTypeEnum.SCRIPT.getValue()); - taskInstance.setCurrentStepId(0L); + taskInstance.setCurrentStepInstanceId(0L); taskInstance.setCallbackUrl(request.getCallbackUrl()); taskInstance.setAppCode(request.getAppCode()); return taskInstance; @@ -215,7 +217,7 @@ private StepInstanceDTO buildFastScriptStepInstance(EsbFastExecuteScriptRequest JobConstants.DEFAULT_JOB_TIMEOUT_SECONDS : request.getTimeout()); stepInstance.setExecuteType(StepExecuteTypeEnum.EXECUTE_SCRIPT.getValue()); - stepInstance.setStatus(RunStatusEnum.BLANK.getValue()); + stepInstance.setStatus(RunStatusEnum.BLANK); stepInstance.setTargetServers(convertToStandardServers(request.getTargetServer(), request.getIpList(), request.getDynamicGroupIdList())); AccountDTO account = accountService.getSystemAccountByAlias(request.getAccount(), request.getAppId()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastPushFileResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastPushFileResourceImpl.java index b5e77c8d2f..f3a82c8d2d 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastPushFileResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastPushFileResourceImpl.java @@ -47,6 +47,7 @@ import com.tencent.bk.job.execute.common.constants.TaskTypeEnum; import com.tencent.bk.job.execute.metrics.ExecuteMetricsConstants; import com.tencent.bk.job.execute.model.AccountDTO; +import com.tencent.bk.job.execute.model.FastTaskDTO; import com.tencent.bk.job.execute.model.FileDetailDTO; import com.tencent.bk.job.execute.model.FileSourceDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; @@ -114,8 +115,9 @@ public EsbResp fastPushFile(EsbFastPushFileRequest request) { TaskInstanceDTO taskInstance = buildFastFileTaskInstance(request); StepInstanceDTO stepInstance = buildFastFileStepInstance(request); - long taskInstanceId = taskExecuteService.createTaskInstanceFast(taskInstance, stepInstance); - taskExecuteService.startTask(taskInstanceId); + long taskInstanceId = taskExecuteService.executeFastTask( + FastTaskDTO.builder().taskInstance(taskInstance).stepInstance(stepInstance).build() + ); EsbJobExecuteDTO jobExecuteInfo = new EsbJobExecuteDTO(); jobExecuteInfo.setTaskInstanceId(taskInstanceId); @@ -217,11 +219,11 @@ private TaskInstanceDTO buildFastFileTaskInstance(EsbFastPushFileRequest request taskInstance.setCronTaskId(-1L); taskInstance.setTaskTemplateId(-1L); taskInstance.setAppId(request.getAppId()); - taskInstance.setStatus(RunStatusEnum.BLANK.getValue()); + taskInstance.setStatus(RunStatusEnum.BLANK); taskInstance.setStartupMode(TaskStartupModeEnum.API.getValue()); taskInstance.setOperator(request.getUserName()); taskInstance.setCreateTime(DateUtils.currentTimeMillis()); - taskInstance.setCurrentStepId(0L); + taskInstance.setCurrentStepInstanceId(0L); taskInstance.setDebugTask(false); taskInstance.setCallbackUrl(request.getCallbackUrl()); taskInstance.setAppCode(request.getAppCode()); @@ -247,7 +249,7 @@ private StepInstanceDTO buildFastFileStepInstance(EsbFastPushFileRequest request stepInstance.setTargetServers(convertToStandardServers(request.getTargetServer(), request.getIpList(), request.getDynamicGroupIdList())); stepInstance.setOperator(request.getUserName()); - stepInstance.setStatus(RunStatusEnum.BLANK.getValue()); + stepInstance.setStatus(RunStatusEnum.BLANK); stepInstance.setCreateTime(DateUtils.currentTimeMillis()); stepInstance.setTimeout(request.getTimeout() == null ? JobConstants.DEFAULT_JOB_TIMEOUT_SECONDS : request.getTimeout()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetJobInstanceLogResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetJobInstanceLogResourceImpl.java index 20bff54dec..58ba78f1bb 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetJobInstanceLogResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetJobInstanceLogResourceImpl.java @@ -34,19 +34,25 @@ import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.Utils; import com.tencent.bk.job.execute.api.esb.v2.EsbGetJobInstanceLogResource; +import com.tencent.bk.job.execute.common.constants.FileDistModeEnum; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; +import com.tencent.bk.job.execute.model.AgentTaskDetailDTO; import com.tencent.bk.job.execute.model.AgentTaskResultGroupDTO; -import com.tencent.bk.job.execute.model.GseTaskIpLogDTO; -import com.tencent.bk.job.execute.model.GseTaskLogDTO; +import com.tencent.bk.job.execute.model.FileIpLogContent; +import com.tencent.bk.job.execute.model.ScriptHostLogContent; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.esb.v2.EsbStepInstanceResultAndLog; import com.tencent.bk.job.execute.model.esb.v2.request.EsbGetJobInstanceLogRequest; -import com.tencent.bk.job.execute.service.GseTaskLogService; +import com.tencent.bk.job.execute.service.FileAgentTaskService; +import com.tencent.bk.job.execute.service.GseTaskService; +import com.tencent.bk.job.execute.service.LogService; +import com.tencent.bk.job.execute.service.ScriptAgentTaskService; import com.tencent.bk.job.execute.service.TaskInstanceService; import lombok.extern.slf4j.Slf4j; import org.springframework.web.bind.annotation.RestController; +import java.util.Collections; import java.util.List; @RestController @@ -54,15 +60,24 @@ public class EsbGetJobInstanceLogResourceImpl extends JobQueryCommonProcessor implements EsbGetJobInstanceLogResource { private final TaskInstanceService taskInstanceService; - private final GseTaskLogService gseTaskLogService; private final AppScopeMappingService appScopeMappingService; + private final ScriptAgentTaskService scriptAgentTaskService; + private final FileAgentTaskService fileAgentTaskService; + private final GseTaskService gseTaskService; + private final LogService logService; - public EsbGetJobInstanceLogResourceImpl(GseTaskLogService gseTaskLogService, - TaskInstanceService taskInstanceService, - AppScopeMappingService appScopeMappingService) { - this.gseTaskLogService = gseTaskLogService; + public EsbGetJobInstanceLogResourceImpl(TaskInstanceService taskInstanceService, + AppScopeMappingService appScopeMappingService, + ScriptAgentTaskService scriptAgentTaskService, + FileAgentTaskService fileAgentTaskService, + GseTaskService gseTaskService, + LogService logService) { this.taskInstanceService = taskInstanceService; this.appScopeMappingService = appScopeMappingService; + this.scriptAgentTaskService = scriptAgentTaskService; + this.fileAgentTaskService = fileAgentTaskService; + this.gseTaskService = gseTaskService; + this.logService = logService; } @Override @@ -84,61 +99,60 @@ public EsbResp> getJobInstanceLogUsingPost(Esb taskInstanceService.listStepInstanceByTaskInstanceId(taskInstanceId); List stepInstResultAndLogList = Lists.newArrayList(); for (StepInstanceBaseDTO stepInstance : stepInstanceList) { - GseTaskLogDTO gseTaskLog = gseTaskLogService.getGseTaskLog(stepInstance.getId(), - stepInstance.getExecuteCount()); - if (null == gseTaskLog) { - EsbStepInstanceResultAndLog stepInstResultAndLog = new EsbStepInstanceResultAndLog(); - stepInstResultAndLog.setFinished(false); - stepInstResultAndLog.setStepInstanceId(stepInstance.getId()); - stepInstResultAndLog.setName(stepInstance.getName()); - stepInstResultAndLog.setStatus(stepInstance.getStatus()); - stepInstResultAndLog.setStepResults(Lists.newArrayList()); - stepInstResultAndLogList.add(stepInstResultAndLog); - continue; - } - EsbStepInstanceResultAndLog stepInstResultAndLog = new EsbStepInstanceResultAndLog(); - stepInstResultAndLog.setFinished(!gseTaskLog.getStatus().equals(RunStatusEnum.BLANK.getValue()) - && !gseTaskLog.getStatus().equals(RunStatusEnum.RUNNING.getValue())); - List resultGroups = gseTaskLogService.getIpLogStatInfo(stepInstance.getId(), - stepInstance.getExecuteCount()); - List stepInstResultList = - Lists.newArrayListWithCapacity(resultGroups.size()); - for (AgentTaskResultGroupDTO resultGroup : resultGroups) { - EsbStepInstanceResultAndLog.StepInstResultDTO stepInstResult = - new EsbStepInstanceResultAndLog.StepInstResultDTO(); - stepInstResult.setIpStatus(resultGroup.getResultType().getValue()); - stepInstResult.setTag(resultGroup.getTag()); - List ipLogList = gseTaskLogService.getIpLogContentByResultType(stepInstance.getId(), - stepInstance.getExecuteCount(), resultGroup.getResultType().getValue(), resultGroup.getTag()); - List resultIpLogList = - Lists.newArrayListWithCapacity(ipLogList.size()); - for (GseTaskIpLogDTO taskIpLog : ipLogList) { - EsbStepInstanceResultAndLog.IpLogDTO ipLogDTO = new EsbStepInstanceResultAndLog.IpLogDTO(); - ipLogDTO.setLogContent(Utils.htmlEncode(taskIpLog.getLogContent())); - ipLogDTO.setExecuteCount(taskIpLog.getExecuteCount()); - ipLogDTO.setEndTime(taskIpLog.getEndTime()); - ipLogDTO.setStartTime(taskIpLog.getStartTime()); - ipLogDTO.setErrCode(taskIpLog.getErrCode()); - ipLogDTO.setExitCode(taskIpLog.getExitCode()); - ipLogDTO.setTotalTime(taskIpLog.getTotalTime()); - ipLogDTO.setCloudAreaId(taskIpLog.getCloudAreaId()); - ipLogDTO.setIp(taskIpLog.getIp()); - resultIpLogList.add(ipLogDTO); - } - stepInstResult.setIpLogs(resultIpLogList); - stepInstResultList.add(stepInstResult); - } - + stepInstResultAndLog.setFinished(RunStatusEnum.isFinishedStatus(stepInstance.getStatus())); stepInstResultAndLog.setStepInstanceId(stepInstance.getId()); stepInstResultAndLog.setName(stepInstance.getName()); - stepInstResultAndLog.setStatus(stepInstance.getStatus()); - stepInstResultAndLog.setStepResults(stepInstResultList); + stepInstResultAndLog.setStatus(stepInstance.getStatus().getValue()); + stepInstResultAndLog.setStepResults(buildStepInstResult(stepInstance)); stepInstResultAndLogList.add(stepInstResultAndLog); } return EsbResp.buildSuccessResp(stepInstResultAndLogList); } + private List buildStepInstResult(StepInstanceBaseDTO stepInstance) { + List resultGroups = Collections.emptyList(); + if (stepInstance.isScriptStep()) { + resultGroups = scriptAgentTaskService.listAndGroupAgentTasks(stepInstance, + stepInstance.getExecuteCount(), null); + } else if (stepInstance.isFileStep()) { + resultGroups = fileAgentTaskService.listAndGroupAgentTasks(stepInstance, + stepInstance.getExecuteCount(), null); + } + + List stepInstResultList = + Lists.newArrayListWithCapacity(resultGroups.size()); + + for (AgentTaskResultGroupDTO resultGroup : resultGroups) { + EsbStepInstanceResultAndLog.StepInstResultDTO stepInstResult = + new EsbStepInstanceResultAndLog.StepInstResultDTO(); + stepInstResult.setIpStatus(resultGroup.getStatus()); + stepInstResult.setTag(resultGroup.getTag()); + List agentTasks = resultGroup.getAgentTasks(); + addLogContent(stepInstance, agentTasks); + List esbGseAgentTaskList = + Lists.newArrayListWithCapacity(agentTasks.size()); + for (AgentTaskDetailDTO agentTask : agentTasks) { + EsbStepInstanceResultAndLog.EsbGseAgentTaskDTO esbGseAgentTaskDTO = + new EsbStepInstanceResultAndLog.EsbGseAgentTaskDTO(); + esbGseAgentTaskDTO.setLogContent(Utils.htmlEncode(agentTask.getScriptLogContent())); + esbGseAgentTaskDTO.setExecuteCount(agentTask.getExecuteCount()); + esbGseAgentTaskDTO.setEndTime(agentTask.getEndTime()); + esbGseAgentTaskDTO.setStartTime(agentTask.getStartTime()); + esbGseAgentTaskDTO.setErrCode(agentTask.getErrorCode()); + esbGseAgentTaskDTO.setExitCode(agentTask.getExitCode()); + esbGseAgentTaskDTO.setTotalTime(agentTask.getTotalTime()); + esbGseAgentTaskDTO.setCloudAreaId(agentTask.getBkCloudId()); + esbGseAgentTaskDTO.setIp(agentTask.getIp()); + esbGseAgentTaskList.add(esbGseAgentTaskDTO); + } + stepInstResult.setIpLogs(esbGseAgentTaskList); + stepInstResultList.add(stepInstResult); + } + + return stepInstResultList; + } + private ValidateResult checkRequest(EsbGetJobInstanceLogRequest request) { if (request.getTaskInstanceId() == null || request.getTaskInstanceId() < 1) { log.warn("TaskInstanceId is empty or illegal, taskInstanceId={}", request.getTaskInstanceId()); @@ -147,6 +161,24 @@ private ValidateResult checkRequest(EsbGetJobInstanceLogRequest request) { return ValidateResult.pass(); } + private void addLogContent(StepInstanceBaseDTO stepInstance, List agentTasks) { + long stepInstanceId = stepInstance.getId(); + int executeCount = stepInstance.getExecuteCount(); + + for (AgentTaskDetailDTO agentTask : agentTasks) { + if (stepInstance.isScriptStep()) { + ScriptHostLogContent scriptHostLogContent = logService.getScriptHostLogContent(stepInstanceId, + executeCount, + null, agentTask.getHost()); + agentTask.setScriptLogContent(scriptHostLogContent == null ? "" : scriptHostLogContent.getContent()); + } else if (stepInstance.isFileStep()) { + FileIpLogContent fileIpLogContent = logService.getFileIpLogContent(stepInstanceId, executeCount, + null, agentTask.getHost(), FileDistModeEnum.DOWNLOAD.getValue()); + agentTask.setScriptLogContent(fileIpLogContent == null ? "" : fileIpLogContent.getContent()); + } + } + } + @Override public EsbResp> getJobInstanceLog(String appCode, String username, diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetJobInstanceStatusResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetJobInstanceStatusResourceImpl.java index cee3f54d1e..5499a61518 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetJobInstanceStatusResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetJobInstanceStatusResourceImpl.java @@ -36,21 +36,25 @@ import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.execute.api.esb.v2.EsbGetJobInstanceStatusResource; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; -import com.tencent.bk.job.execute.model.GseTaskIpLogDTO; +import com.tencent.bk.job.execute.model.AgentTaskDetailDTO; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.esb.v2.EsbIpStatusDTO; import com.tencent.bk.job.execute.model.esb.v2.EsbJobInstanceStatusDTO; import com.tencent.bk.job.execute.model.esb.v2.request.EsbGetJobInstanceStatusRequest; -import com.tencent.bk.job.execute.service.GseTaskLogService; +import com.tencent.bk.job.execute.service.FileAgentTaskService; +import com.tencent.bk.job.execute.service.ScriptAgentTaskService; import com.tencent.bk.job.execute.service.TaskInstanceService; +import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; import org.springframework.web.bind.annotation.RestController; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @RestController @Slf4j @@ -59,15 +63,18 @@ public class EsbGetJobInstanceStatusResourceImpl implements EsbGetJobInstanceStatusResource { private final TaskInstanceService taskInstanceService; - private final GseTaskLogService gseTaskLogService; private final AppScopeMappingService appScopeMappingService; + private final ScriptAgentTaskService scriptAgentTaskService; + private final FileAgentTaskService fileAgentTaskService; - public EsbGetJobInstanceStatusResourceImpl(GseTaskLogService gseTaskLogService, - TaskInstanceService taskInstanceService, - AppScopeMappingService appScopeMappingService) { - this.gseTaskLogService = gseTaskLogService; + public EsbGetJobInstanceStatusResourceImpl(TaskInstanceService taskInstanceService, + AppScopeMappingService appScopeMappingService, + ScriptAgentTaskService scriptAgentTaskService, + FileAgentTaskService fileAgentTaskService) { this.taskInstanceService = taskInstanceService; this.appScopeMappingService = appScopeMappingService; + this.scriptAgentTaskService = scriptAgentTaskService; + this.fileAgentTaskService = fileAgentTaskService; } @Override @@ -109,14 +116,31 @@ private ValidateResult checkRequest(EsbGetJobInstanceStatusRequest request) { private Map> getStepIpResult(List stepInstanceList) { Map> stepIpResult = new HashMap<>(); for (StepInstanceBaseDTO stepInstance : stepInstanceList) { - List ipLogList = gseTaskLogService.getIpLog(stepInstance.getId(), - stepInstance.getExecuteCount(), true); + List agentTasks = null; + if (stepInstance.isScriptStep()) { + agentTasks = scriptAgentTaskService.listAgentTaskDetail(stepInstance, + stepInstance.getExecuteCount(), null); + } else if (stepInstance.isFileStep()) { + agentTasks = fileAgentTaskService.listAgentTaskDetail(stepInstance, + stepInstance.getExecuteCount(), null); + if (CollectionUtils.isNotEmpty(agentTasks)) { + // 如果是文件分发任务，只返回目标Agent结果 + agentTasks = agentTasks.stream() + .filter(agentTask -> agentTask.getFileTaskMode() == FileTaskModeEnum.DOWNLOAD) + .collect(Collectors.toList()); + } + } List ipResultList = Lists.newArrayList(); - for (GseTaskIpLogDTO ipLog : ipLogList) { + if (CollectionUtils.isEmpty(agentTasks)) { + stepIpResult.put(stepInstance.getId(), ipResultList); + continue; + } + for (AgentTaskDetailDTO agentTask : agentTasks) { EsbIpStatusDTO ipStatus = new EsbIpStatusDTO(); - ipStatus.setIp(ipLog.getIp()); - ipStatus.setCloudAreaId(ipLog.getCloudAreaId()); - ipStatus.setStatus(ipLog.getStatus()); + ipStatus.setIp(agentTask.getIp()); + ipStatus.setCloudAreaId(agentTask.getBkCloudId()); + + ipStatus.setStatus(agentTask.getStatus().getValue()); ipResultList.add(ipStatus); } stepIpResult.put(stepInstance.getId(), ipResultList); @@ -129,11 +153,11 @@ private EsbJobInstanceStatusDTO buildEsbJobInstanceStatusDTO(TaskInstanceDTO tas Map> stepIpResultMap) { EsbJobInstanceStatusDTO jobInstanceStatus = new EsbJobInstanceStatusDTO(); jobInstanceStatus.setIsFinished( - RunStatusEnum.isFinishedStatus(RunStatusEnum.valueOf(taskInstance.getStatus()))); + RunStatusEnum.isFinishedStatus(taskInstance.getStatus())); EsbJobInstanceStatusDTO.JobInstance jobInstance = new EsbJobInstanceStatusDTO.JobInstance(); EsbDTOAppScopeMappingHelper.fillEsbAppScopeDTOByAppId(taskInstance.getAppId(), jobInstance); - jobInstance.setCurrentStepId(taskInstance.getCurrentStepId()); + jobInstance.setCurrentStepId(taskInstance.getCurrentStepInstanceId()); jobInstance.setId(taskInstance.getId()); jobInstance.setName(taskInstance.getName()); jobInstance.setOperator(taskInstance.getOperator()); @@ -141,7 +165,7 @@ private EsbJobInstanceStatusDTO buildEsbJobInstanceStatusDTO(TaskInstanceDTO tas jobInstance.setStartTime(taskInstance.getStartTime()); jobInstance.setEndTime(taskInstance.getEndTime()); jobInstance.setStartWay(taskInstance.getStartupMode()); - jobInstance.setStatus(taskInstance.getStatus()); + jobInstance.setStatus(taskInstance.getStatus().getValue()); jobInstance.setTaskId(taskInstance.getTaskId()); jobInstance.setTotalTime(taskInstance.getTotalTime()); jobInstanceStatus.setJobInstance(jobInstance); @@ -160,7 +184,7 @@ private EsbJobInstanceStatusDTO buildEsbJobInstanceStatusDTO(TaskInstanceDTO tas stepInst.setType(stepInstance.getExecuteType()); stepInst.setOperator(stepInstance.getOperator()); stepInst.setExecuteCount(stepInstance.getExecuteCount()); - stepInst.setStatus(stepInstance.getStatus()); + stepInst.setStatus(stepInstance.getStatus().getValue()); stepInst.setStepId(stepInstance.getStepId()); stepInst.setTotalTime(stepInstance.getTotalTime()); List stepIpResult = stepIpResultMap.get(stepInstance.getId()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetStepInstanceStatusResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetStepInstanceStatusResourceImpl.java index d921b9eba3..cf1f3e6c13 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetStepInstanceStatusResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetStepInstanceStatusResourceImpl.java @@ -34,20 +34,21 @@ import com.tencent.bk.job.common.i18n.service.MessageI18nService; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.ValidateResult; -import com.tencent.bk.job.common.model.dto.IpDTO; import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.execute.api.esb.v2.EsbGetStepInstanceStatusResource; -import com.tencent.bk.job.execute.common.constants.RunStatusEnum; +import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; +import com.tencent.bk.job.execute.model.AgentTaskDetailDTO; import com.tencent.bk.job.execute.model.AgentTaskResultGroupDTO; -import com.tencent.bk.job.execute.model.GseTaskLogDTO; +import com.tencent.bk.job.execute.model.StepExecutionDetailDTO; +import com.tencent.bk.job.execute.model.StepExecutionResultQuery; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.esb.v2.EsbStepInstanceStatusDTO; import com.tencent.bk.job.execute.model.esb.v2.request.EsbGetStepInstanceStatusRequest; -import com.tencent.bk.job.execute.service.GseTaskLogService; import com.tencent.bk.job.execute.service.TaskInstanceService; +import com.tencent.bk.job.execute.service.TaskResultService; import lombok.extern.slf4j.Slf4j; -import org.jooq.tools.StringUtils; +import org.apache.commons.collections4.CollectionUtils; import org.springframework.web.bind.annotation.RestController; import java.util.ArrayList; @@ -61,18 +62,18 @@ public class EsbGetStepInstanceStatusResourceImpl extends JobQueryCommonProcessor implements EsbGetStepInstanceStatusResource { private final TaskInstanceService taskInstanceService; - private final GseTaskLogService gseTaskLogService; + private final TaskResultService taskResultService; private final MessageI18nService i18nService; private final AppScopeMappingService appScopeMappingService; public EsbGetStepInstanceStatusResourceImpl(MessageI18nService i18nService, - GseTaskLogService gseTaskLogService, TaskInstanceService taskInstanceService, - AppScopeMappingService appScopeMappingService) { + AppScopeMappingService appScopeMappingService, + TaskResultService taskResultService) { this.i18nService = i18nService; - this.gseTaskLogService = gseTaskLogService; this.taskInstanceService = taskInstanceService; this.appScopeMappingService = appScopeMappingService; + this.taskResultService = taskResultService; } @Override @@ -86,57 +87,42 @@ public EsbResp getJobStepInstanceStatus(EsbGetStepInst throw new InvalidParamException(checkResult); } - long taskInstanceId = request.getTaskInstanceId(); - long stepInstanceId = request.getStepInstanceId(); + EsbStepInstanceStatusDTO resultData = new EsbStepInstanceStatusDTO(); + + StepExecutionResultQuery query = StepExecutionResultQuery.builder() + .stepInstanceId(request.getStepInstanceId()).build(); + StepExecutionDetailDTO stepExecutionDetail = taskResultService.getStepExecutionResult(request.getUserName(), + request.getAppId(), query); TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(request.getTaskInstanceId()); authViewTaskInstance(request.getUserName(), request.getAppResourceScope(), taskInstance); + resultData.setIsFinished(stepExecutionDetail.isFinished()); + resultData.setAyalyseResult(convertToStandardAnalyseResult(stepExecutionDetail)); - StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(stepInstanceId); + StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(request.getStepInstanceId()); if (stepInstance == null) { log.warn("Get step instance status by taskInstanceId:{}, stepInstanceId:{}, stepInstance is null!", - taskInstanceId, stepInstanceId); + request.getTaskInstanceId(), request.getStepInstanceId()); throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); } - - EsbStepInstanceStatusDTO resultData = new EsbStepInstanceStatusDTO(); - GseTaskLogDTO gseTaskLog = gseTaskLogService.getGseTaskLog(stepInstanceId, stepInstance.getExecuteCount()); - if (null == gseTaskLog) { - resultData.setIsFinished(false); - return EsbResp.buildSuccessResp(null); - } else { - resultData.setIsFinished(!gseTaskLog.getStatus().equals(RunStatusEnum.BLANK.getValue()) - && !gseTaskLog.getStatus().equals(RunStatusEnum.RUNNING.getValue())); - List analyseResult = gseTaskLogService.getLogStatInfoWithIp(stepInstance.getId() - , stepInstance.getExecuteCount()); - resultData.setAyalyseResult(convertToStandardAnalyseResult(analyseResult)); - - EsbStepInstanceStatusDTO.StepInstance stepDetail = convertStepInstance(stepInstance); - resultData.setStepInstance(stepDetail); - return EsbResp.buildSuccessResp(resultData); - } + EsbStepInstanceStatusDTO.StepInstance stepDetail = convertStepInstance(stepInstance); + resultData.setStepInstance(stepDetail); + return EsbResp.buildSuccessResp(resultData); } private EsbStepInstanceStatusDTO.StepInstance convertStepInstance(StepInstanceBaseDTO stepInstance) { EsbStepInstanceStatusDTO.StepInstance stepInst = new EsbStepInstanceStatusDTO.StepInstance(); EsbDTOAppScopeMappingHelper.fillEsbAppScopeDTOByAppId(stepInstance.getAppId(), stepInst); stepInst.setId(stepInstance.getId()); - stepInst.setBadIpList(stepInstance.getBadIpList()); - stepInst.setBadIPNum(StringUtils.isEmpty(stepInstance.getBadIpList()) ? 0 : - stepInstance.getBadIpList().split(",").length); stepInst.setEndTime(stepInstance.getEndTime()); stepInst.setStartTime(stepInstance.getStartTime()); - stepInst.setFailIPNum(stepInstance.getFailIPNum()); stepInst.setIpList(stepInstance.getIpList()); stepInst.setName(stepInstance.getName()); stepInst.setOperator(stepInstance.getOperator()); stepInst.setExecuteCount(stepInstance.getExecuteCount()); - stepInst.setRunIPNum(stepInstance.getRunIPNum()); - stepInst.setStatus(stepInstance.getStatus()); + stepInst.setStatus(stepInstance.getStatus().getValue()); stepInst.setStepId(stepInstance.getStepId()); - stepInst.setSuccessIPNum(stepInstance.getSuccessIPNum()); stepInst.setTaskInstanceId(stepInstance.getTaskInstanceId()); - stepInst.setTotalIPNum(stepInstance.getTotalIPNum()); stepInst.setTotalTime(stepInstance.getTotalTime()); stepInst.setType(stepInstance.getExecuteType()); @@ -156,25 +142,29 @@ private ValidateResult checkRequest(EsbGetStepInstanceStatusRequest request) { return ValidateResult.pass(); } - private List> convertToStandardAnalyseResult(List resultGroups) { + private List> convertToStandardAnalyseResult(StepExecutionDetailDTO stepExecutionDetail) { List> standardStepAnalyseResultList = new ArrayList<>(); + List resultGroups = stepExecutionDetail.getResultGroups(); if (resultGroups == null || resultGroups.isEmpty()) { return standardStepAnalyseResultList; } + for (AgentTaskResultGroupDTO resultGroup : resultGroups) { Map standardStepAnalyseResult = new HashMap<>(); - standardStepAnalyseResult.put("count", resultGroup.getCount()); - if (resultGroup.getIpList() != null) { - List ipDTOS = new ArrayList<>(); - for (IpDTO ipDTO : resultGroup.getIpList()) { - ipDTOS.add(new EsbIpDTO(ipDTO.getCloudAreaId(), ipDTO.getIp())); + List agentTasks = resultGroup.getAgentTasks(); + standardStepAnalyseResult.put("count", CollectionUtils.isEmpty(agentTasks) ? 0 : agentTasks.size()); + if (CollectionUtils.isNotEmpty(agentTasks)) { + List ips = new ArrayList<>(); + for (AgentTaskDetailDTO agentTask : agentTasks) { + ips.add(new EsbIpDTO(agentTask.getBkCloudId(), agentTask.getIp())); } - standardStepAnalyseResult.put("ip_list", ipDTOS); + standardStepAnalyseResult.put("ip_list", ips); + } - standardStepAnalyseResult.put("result_type", resultGroup.getResultType().getValue()); + standardStepAnalyseResult.put("result_type", resultGroup.getStatus()); standardStepAnalyseResult.put("result_type_text", - i18nService.getI18n(resultGroup.getResultType().getI18nKey())); + i18nService.getI18n(AgentTaskStatusEnum.valueOf(resultGroup.getStatus()).getI18nKey())); standardStepAnalyseResultList.add(standardStepAnalyseResult); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbPushConfigFileResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbPushConfigFileResourceImpl.java index 3e9bcdc228..7ba656406b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbPushConfigFileResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbPushConfigFileResourceImpl.java @@ -46,6 +46,7 @@ import com.tencent.bk.job.execute.config.StorageSystemConfig; import com.tencent.bk.job.execute.metrics.ExecuteMetricsConstants; import com.tencent.bk.job.execute.model.AccountDTO; +import com.tencent.bk.job.execute.model.FastTaskDTO; import com.tencent.bk.job.execute.model.FileDetailDTO; import com.tencent.bk.job.execute.model.FileSourceDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; @@ -108,8 +109,9 @@ public EsbResp pushConfigFile(EsbPushConfigFileRequest request TaskInstanceDTO taskInstance = buildFastFileTaskInstance(request); StepInstanceDTO stepInstance = buildFastFileStepInstance(request, request.getFileList()); - long taskInstanceId = taskExecuteService.createTaskInstanceFast(taskInstance, stepInstance); - taskExecuteService.startTask(taskInstanceId); + long taskInstanceId = taskExecuteService.executeFastTask( + FastTaskDTO.builder().taskInstance(taskInstance).stepInstance(stepInstance).build() + ); EsbJobExecuteDTO jobExecuteInfo = new EsbJobExecuteDTO(); jobExecuteInfo.setTaskInstanceId(taskInstanceId); @@ -129,11 +131,11 @@ private TaskInstanceDTO buildFastFileTaskInstance(EsbPushConfigFileRequest reque taskInstance.setCronTaskId(-1L); taskInstance.setTaskTemplateId(-1L); taskInstance.setAppId(request.getAppId()); - taskInstance.setStatus(RunStatusEnum.BLANK.getValue()); + taskInstance.setStatus(RunStatusEnum.BLANK); taskInstance.setStartupMode(TaskStartupModeEnum.API.getValue()); taskInstance.setOperator(request.getUserName()); taskInstance.setCreateTime(DateUtils.currentTimeMillis()); - taskInstance.setCurrentStepId(0L); + taskInstance.setCurrentStepInstanceId(0L); taskInstance.setDebugTask(false); taskInstance.setAppCode(request.getAppCode()); return taskInstance; @@ -157,7 +159,7 @@ private StepInstanceDTO buildFastFileStepInstance(EsbPushConfigFileRequest reque stepInstance.setAppId(request.getAppId()); stepInstance.setTargetServers(convertToStandardServers(null, request.getIpList(), null)); stepInstance.setOperator(request.getUserName()); - stepInstance.setStatus(RunStatusEnum.BLANK.getValue()); + stepInstance.setStatus(RunStatusEnum.BLANK); stepInstance.setCreateTime(DateUtils.currentTimeMillis()); stepInstance.setTimeout(JobConstants.DEFAULT_JOB_TIMEOUT_SECONDS); return stepInstance; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/JobExecuteCommonProcessor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/JobExecuteCommonProcessor.java index 210d6fdc5c..5a7a3b9bde 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/JobExecuteCommonProcessor.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/JobExecuteCommonProcessor.java @@ -29,7 +29,7 @@ import com.tencent.bk.job.common.esb.model.job.EsbServerDTO; import com.tencent.bk.job.common.exception.NotFoundException; import com.tencent.bk.job.common.exception.ServiceException; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.model.DynamicServerGroupDTO; import com.tencent.bk.job.execute.model.DynamicServerTopoNodeDTO; import com.tencent.bk.job.execute.model.ServersDTO; @@ -84,8 +84,8 @@ protected ServersDTO convertToStandardServers(EsbServerDTO requestTargetServers, serversDTO.setDynamicServerGroups(dynamicServerGroups); } if (requestTargetServers.getIps() != null) { - List staticIpList = new ArrayList<>(); - requestTargetServers.getIps().forEach(ip -> staticIpList.add(new IpDTO(ip.getCloudAreaId(), + List staticIpList = new ArrayList<>(); + requestTargetServers.getIps().forEach(ip -> staticIpList.add(new HostDTO(ip.getBkCloudId(), ip.getIp()))); serversDTO.setStaticIpList(staticIpList); } @@ -94,8 +94,8 @@ protected ServersDTO convertToStandardServers(EsbServerDTO requestTargetServers, // 兼容历史版本API ServersDTO serversDTO = new ServersDTO(); if (ipList != null) { - List staticIpList = new ArrayList<>(); - ipList.forEach(ip -> staticIpList.add(new IpDTO(ip.getCloudAreaId(), ip.getIp()))); + List staticIpList = new ArrayList<>(); + ipList.forEach(ip -> staticIpList.add(new HostDTO(ip.getBkCloudId(), ip.getIp()))); serversDTO.setStaticIpList(staticIpList); } if (dynamicGroupIdList != null) { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBatchGetJobInstanceIpLogV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBatchGetJobInstanceIpLogV3ResourceImpl.java index 3e7fd69838..d71e6f6ef4 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBatchGetJobInstanceIpLogV3ResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBatchGetJobInstanceIpLogV3ResourceImpl.java @@ -32,13 +32,13 @@ import com.tencent.bk.job.common.exception.NotFoundException; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.ValidateResult; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.common.util.ip.IpUtils; import com.tencent.bk.job.execute.api.esb.v2.impl.JobQueryCommonProcessor; import com.tencent.bk.job.execute.common.constants.FileDistModeEnum; -import com.tencent.bk.job.execute.model.ScriptIpLogContent; +import com.tencent.bk.job.execute.model.ScriptHostLogContent; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.esb.v3.EsbFileIpLogV3DTO; @@ -50,7 +50,7 @@ import com.tencent.bk.job.execute.service.TaskInstanceService; import com.tencent.bk.job.logsvr.consts.LogTypeEnum; import com.tencent.bk.job.logsvr.model.service.ServiceFileTaskLogDTO; -import com.tencent.bk.job.logsvr.model.service.ServiceIpLogsDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceHostLogsDTO; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -150,8 +150,8 @@ private ValidateResult checkIps(List cloudIpList) { return ValidateResult.fail(ErrorCode.MISSING_OR_ILLEGAL_PARAM_WITH_PARAM_NAME, "ip_list"); } for (EsbIpDTO cloudIp : cloudIpList) { - if (cloudIp.getCloudAreaId() == null || cloudIp.getCloudAreaId() < 0) { - log.warn("CloudAreaId is empty or illegal, cloudAreaId={}", cloudIp.getCloudAreaId()); + if (cloudIp.getBkCloudId() == null || cloudIp.getBkCloudId() < 0) { + log.warn("CloudAreaId is empty or illegal, cloudAreaId={}", cloudIp.getBkCloudId()); return ValidateResult.fail(ErrorCode.MISSING_OR_ILLEGAL_PARAM_WITH_PARAM_NAME, "bk_cloud_id"); } @@ -166,16 +166,15 @@ private ValidateResult checkIps(List cloudIpList) { } return ValidateResult.pass(); } - private void buildScriptLogs(EsbIpLogsV3DTO ipLogs, StepInstanceBaseDTO stepInstance, List ipList) { ipLogs.setLogType(LogTypeEnum.SCRIPT.getValue()); String jobCreateDate = DateUtils.formatUnixTimestamp(stepInstance.getCreateTime(), ChronoUnit.MILLIS, "yyyy_MM_dd", ZoneId.of("UTC")); - List ipLogContentList = logService.batchGetScriptIpLogContent(jobCreateDate, - stepInstance.getId(), stepInstance.getExecuteCount(), - ipList.stream().map(cloudIp -> new IpDTO(cloudIp.getCloudAreaId(), cloudIp.getIp())) + List ipLogContentList = logService.batchGetScriptHostLogContent(jobCreateDate, + stepInstance.getId(), stepInstance.getExecuteCount(), null, + ipList.stream().map(cloudIp -> new HostDTO(cloudIp.getBkCloudId(), cloudIp.getIp())) .collect(Collectors.toList())); if (CollectionUtils.isEmpty(ipLogContentList)) { @@ -184,8 +183,8 @@ private void buildScriptLogs(EsbIpLogsV3DTO ipLogs, StepInstanceBaseDTO stepInst List scriptTaskLogs = ipLogContentList.stream().map(ipLogContent -> { EsbScriptIpLogV3DTO scriptIpLog = new EsbScriptIpLogV3DTO(); - IpDTO cloudIp = IpUtils.transform(ipLogContent.getIp()); - scriptIpLog.setCloudAreaId(cloudIp.getCloudAreaId()); + HostDTO cloudIp = IpUtils.transform(ipLogContent.getIp()); + scriptIpLog.setCloudAreaId(cloudIp.getBkCloudId()); scriptIpLog.setIp(cloudIp.getIp()); scriptIpLog.setLogContent(ipLogContent.getContent()); return scriptIpLog; @@ -197,9 +196,9 @@ private void buildFileLogs(EsbIpLogsV3DTO esbIpLogs, StepInstanceBaseDTO stepIns List ipList) { esbIpLogs.setLogType(LogTypeEnum.FILE.getValue()); - ServiceIpLogsDTO ipLogs = logService.batchGetFileIpLogContent( - stepInstance.getId(), stepInstance.getExecuteCount(), - ipList.stream().map(cloudIp -> new IpDTO(cloudIp.getCloudAreaId(), cloudIp.getIp())) + ServiceHostLogsDTO ipLogs = logService.batchGetFileIpLogContent( + stepInstance.getId(), stepInstance.getExecuteCount(), null, + ipList.stream().map(cloudIp -> new HostDTO(cloudIp.getBkCloudId(), cloudIp.getIp())) .collect(Collectors.toList())); if (ipLogs == null || CollectionUtils.isEmpty(ipLogs.getIpLogs())) { @@ -210,11 +209,9 @@ private void buildFileLogs(EsbIpLogsV3DTO esbIpLogs, StepInstanceBaseDTO stepIns List ipFileLogs = ipLog.getFileTaskLogs(); EsbFileIpLogV3DTO esbFileIpLog = new EsbFileIpLogV3DTO(); if (CollectionUtils.isNotEmpty(ipFileLogs)) { - IpDTO cloudIp = IpDTO.fromCloudAreaIdAndIpStr(ipLog.getIp()); - if (cloudIp != null) { - esbFileIpLog.setCloudAreaId(cloudIp.getCloudAreaId()); - esbFileIpLog.setIp(cloudIp.getIp()); - } + HostDTO cloudIp = HostDTO.fromCloudIp(ipLog.getIp()); + esbFileIpLog.setCloudAreaId(cloudIp.getBkCloudId()); + esbFileIpLog.setIp(cloudIp.getIp()); List esbFileLogs = ipFileLogs.stream() .map(this::toEsbFileLogV3DTO).collect(Collectors.toList()); esbFileIpLog.setFileLogs(esbFileLogs); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbExecuteJobPlanV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbExecuteJobPlanV3ResourceImpl.java index c96cd4deae..57365ff922 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbExecuteJobPlanV3ResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbExecuteJobPlanV3ResourceImpl.java @@ -106,7 +106,7 @@ public EsbResp executeJobPlan(EsbExecuteJobV3Request request executeVariableValues.add(taskVariableDTO); } } - TaskInstanceDTO taskInstanceDTO = taskExecuteService.createTaskInstanceForTask( + TaskInstanceDTO taskInstanceDTO = taskExecuteService.executeJobPlan( TaskExecuteParam .builder() .appId(request.getAppId()) @@ -117,7 +117,6 @@ public EsbResp executeJobPlan(EsbExecuteJobV3Request request .callbackUrl(request.getCallbackUrl()) .appCode(request.getAppCode()) .build()); - taskExecuteService.startTask(taskInstanceDTO.getId()); EsbJobExecuteV3DTO result = new EsbJobExecuteV3DTO(); result.setTaskInstanceId(taskInstanceDTO.getId()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastExecuteSQLV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastExecuteSQLV3ResourceImpl.java index 88806f19af..1bae1a7ecc 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastExecuteSQLV3ResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastExecuteSQLV3ResourceImpl.java @@ -41,6 +41,7 @@ import com.tencent.bk.job.execute.common.constants.TaskStartupModeEnum; import com.tencent.bk.job.execute.common.constants.TaskTypeEnum; import com.tencent.bk.job.execute.metrics.ExecuteMetricsConstants; +import com.tencent.bk.job.execute.model.FastTaskDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.esb.v3.EsbJobExecuteV3DTO; @@ -93,8 +94,9 @@ public EsbResp fastExecuteSQL(EsbFastExecuteSQLV3Request req TaskInstanceDTO taskInstance = buildFastSQLTaskInstance(request); StepInstanceDTO stepInstance = buildFastSQLStepInstance(request); - long taskInstanceId = taskExecuteService.createTaskInstanceFast(taskInstance, stepInstance); - taskExecuteService.startTask(taskInstanceId); + long taskInstanceId = taskExecuteService.executeFastTask( + FastTaskDTO.builder().taskInstance(taskInstance).stepInstance(stepInstance).build() + ); EsbJobExecuteV3DTO jobExecuteInfo = new EsbJobExecuteV3DTO(); jobExecuteInfo.setTaskInstanceId(taskInstanceId); @@ -149,9 +151,9 @@ private TaskInstanceDTO buildFastSQLTaskInstance(EsbFastExecuteSQLV3Request requ taskInstance.setTaskId(-1L); taskInstance.setTaskTemplateId(-1L); taskInstance.setDebugTask(false); - taskInstance.setStatus(RunStatusEnum.BLANK.getValue()); + taskInstance.setStatus(RunStatusEnum.BLANK); taskInstance.setStartupMode(TaskStartupModeEnum.API.getValue()); - taskInstance.setCurrentStepId(0L); + taskInstance.setCurrentStepInstanceId(0L); taskInstance.setCreateTime(DateUtils.currentTimeMillis()); taskInstance.setType(TaskTypeEnum.SCRIPT.getValue()); taskInstance.setAppCode(request.getAppCode()); @@ -181,7 +183,7 @@ private StepInstanceDTO buildFastSQLStepInstance(EsbFastExecuteSQLV3Request requ stepInstance.setTimeout(request.getTimeout() == null ? JobConstants.DEFAULT_JOB_TIMEOUT_SECONDS : request.getTimeout()); stepInstance.setExecuteType(StepExecuteTypeEnum.EXECUTE_SQL.getValue()); - stepInstance.setStatus(RunStatusEnum.BLANK.getValue()); + stepInstance.setStatus(RunStatusEnum.BLANK); stepInstance.setTargetServers(convertToServersDTO(request.getTargetServer())); stepInstance.setDbAccountId(request.getDbAccountId()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastExecuteScriptV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastExecuteScriptV3ResourceImpl.java index 62f5c82c42..81ed235175 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastExecuteScriptV3ResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastExecuteScriptV3ResourceImpl.java @@ -43,7 +43,9 @@ import com.tencent.bk.job.execute.common.constants.TaskTypeEnum; import com.tencent.bk.job.execute.engine.evict.TaskEvictPolicyExecutor; import com.tencent.bk.job.execute.metrics.ExecuteMetricsConstants; +import com.tencent.bk.job.execute.model.FastTaskDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; +import com.tencent.bk.job.execute.model.StepRollingConfigDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.esb.v3.EsbJobExecuteV3DTO; import com.tencent.bk.job.execute.model.esb.v3.request.EsbFastExecuteScriptV3Request; @@ -99,8 +101,17 @@ public EsbResp fastExecuteScript(EsbFastExecuteScriptV3Reque return EsbResp.buildCommonFailResp(ErrorCode.TASK_ABANDONED); } StepInstanceDTO stepInstance = buildFastScriptStepInstance(request); - long taskInstanceId = taskExecuteService.createTaskInstanceFast(taskInstance, stepInstance); - taskExecuteService.startTask(taskInstanceId); + StepRollingConfigDTO rollingConfig = null; + if (request.getRollingConfig() != null) { + rollingConfig = StepRollingConfigDTO.fromEsbRollingConfig(request.getRollingConfig()); + } + long taskInstanceId = taskExecuteService.executeFastTask( + FastTaskDTO.builder() + .taskInstance(taskInstance) + .stepInstance(stepInstance) + .rollingConfig(rollingConfig) + .build() + ); EsbJobExecuteV3DTO jobExecuteInfo = new EsbJobExecuteV3DTO(); jobExecuteInfo.setTaskInstanceId(taskInstanceId); @@ -163,11 +174,11 @@ private TaskInstanceDTO buildFastScriptTaskInstance(EsbFastExecuteScriptV3Reques taskInstance.setDebugTask(false); taskInstance.setAppId(request.getAppId()); taskInstance.setStartupMode(TaskStartupModeEnum.API.getValue()); - taskInstance.setStatus(RunStatusEnum.BLANK.getValue()); + taskInstance.setStatus(RunStatusEnum.BLANK); taskInstance.setOperator(request.getUserName()); taskInstance.setCreateTime(DateUtils.currentTimeMillis()); taskInstance.setType(TaskTypeEnum.SCRIPT.getValue()); - taskInstance.setCurrentStepId(0L); + taskInstance.setCurrentStepInstanceId(0L); taskInstance.setCallbackUrl(request.getCallbackUrl()); taskInstance.setAppCode(request.getAppCode()); return taskInstance; @@ -204,12 +215,14 @@ private StepInstanceDTO buildFastScriptStepInstance(EsbFastExecuteScriptV3Reques request.getTimeout() == null ? JobConstants.DEFAULT_JOB_TIMEOUT_SECONDS : request.getTimeout()); stepInstance.setExecuteType(StepExecuteTypeEnum.EXECUTE_SCRIPT.getValue()); - stepInstance.setStatus(RunStatusEnum.BLANK.getValue()); + stepInstance.setStatus(RunStatusEnum.BLANK); stepInstance.setTargetServers(convertToServersDTO(request.getTargetServer())); stepInstance.setAccountId(request.getAccountId()); stepInstance.setAccountAlias(request.getAccountAlias()); stepInstance.setOperator(request.getUserName()); stepInstance.setCreateTime(DateUtils.currentTimeMillis()); + return stepInstance; } + } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastTransferFileV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastTransferFileV3ResourceImpl.java index 5b51378341..d472a7b898 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastTransferFileV3ResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastTransferFileV3ResourceImpl.java @@ -49,9 +49,11 @@ import com.tencent.bk.job.execute.common.constants.TaskStartupModeEnum; import com.tencent.bk.job.execute.common.constants.TaskTypeEnum; import com.tencent.bk.job.execute.metrics.ExecuteMetricsConstants; +import com.tencent.bk.job.execute.model.FastTaskDTO; import com.tencent.bk.job.execute.model.FileDetailDTO; import com.tencent.bk.job.execute.model.FileSourceDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; +import com.tencent.bk.job.execute.model.StepRollingConfigDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.esb.v3.EsbJobExecuteV3DTO; import com.tencent.bk.job.execute.model.esb.v3.request.EsbFastTransferFileV3Request; @@ -120,8 +122,17 @@ public EsbResp fastTransferFile(EsbFastTransferFileV3Request TaskInstanceDTO taskInstance = buildFastFileTaskInstance(request); StepInstanceDTO stepInstance = buildFastFileStepInstance(request); - long taskInstanceId = taskExecuteService.createTaskInstanceFast(taskInstance, stepInstance); - taskExecuteService.startTask(taskInstanceId); + StepRollingConfigDTO rollingConfig = null; + if (request.getRollingConfig() != null) { + rollingConfig = StepRollingConfigDTO.fromEsbRollingConfig(request.getRollingConfig()); + } + long taskInstanceId = taskExecuteService.executeFastTask( + FastTaskDTO.builder() + .taskInstance(taskInstance) + .stepInstance(stepInstance) + .rollingConfig(rollingConfig) + .build() + ); EsbJobExecuteV3DTO jobExecuteInfo = new EsbJobExecuteV3DTO(); jobExecuteInfo.setTaskInstanceId(taskInstanceId); @@ -133,8 +144,8 @@ public EsbResp fastTransferFile(EsbFastTransferFileV3Request private ValidateResult checkFileSource(EsbFileSourceV3DTO fileSource) { Integer fileType = fileSource.getFileType(); // fileType是后加的字段，为null则默认为服务器文件不校验 - if (fileType != null && TaskFileTypeEnum.valueOf(fileType) == null) { - return ValidateResult.fail(ErrorCode.MISSING_PARAM_WITH_PARAM_NAME, "file_source.file_type"); + if (fileType != null && !TaskFileTypeEnum.isValid(fileType)) { + return ValidateResult.fail(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME, "file_source.file_type"); } List files = fileSource.getFiles(); if (files == null || files.isEmpty()) { @@ -239,11 +250,11 @@ private TaskInstanceDTO buildFastFileTaskInstance(EsbFastTransferFileV3Request r taskInstance.setCronTaskId(-1L); taskInstance.setTaskTemplateId(-1L); taskInstance.setAppId(request.getAppId()); - taskInstance.setStatus(RunStatusEnum.BLANK.getValue()); + taskInstance.setStatus(RunStatusEnum.BLANK); taskInstance.setStartupMode(TaskStartupModeEnum.API.getValue()); taskInstance.setOperator(request.getUserName()); taskInstance.setCreateTime(DateUtils.currentTimeMillis()); - taskInstance.setCurrentStepId(0L); + taskInstance.setCurrentStepInstanceId(0L); taskInstance.setDebugTask(false); taskInstance.setCallbackUrl(request.getCallbackUrl()); taskInstance.setAppCode(request.getAppCode()); @@ -268,7 +279,7 @@ private StepInstanceDTO buildFastFileStepInstance(EsbFastTransferFileV3Request r stepInstance.setAppId(request.getAppId()); stepInstance.setTargetServers(convertToServersDTO(request.getTargetServer())); stepInstance.setOperator(request.getUserName()); - stepInstance.setStatus(RunStatusEnum.BLANK.getValue()); + stepInstance.setStatus(RunStatusEnum.BLANK); stepInstance.setCreateTime(DateUtils.currentTimeMillis()); stepInstance.setTimeout(request.getTimeout() == null ? JobConstants.DEFAULT_JOB_TIMEOUT_SECONDS : request.getTimeout()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceIpLogV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceIpLogV3ResourceImpl.java index 9e4108c0f8..be5e9973b7 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceIpLogV3ResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceIpLogV3ResourceImpl.java @@ -32,13 +32,13 @@ import com.tencent.bk.job.common.exception.NotFoundException; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.ValidateResult; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.ip.IpUtils; import com.tencent.bk.job.execute.api.esb.v2.impl.JobQueryCommonProcessor; import com.tencent.bk.job.execute.common.constants.FileDistModeEnum; import com.tencent.bk.job.execute.model.FileIpLogContent; -import com.tencent.bk.job.execute.model.ScriptIpLogContent; +import com.tencent.bk.job.execute.model.ScriptHostLogContent; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.esb.v3.EsbFileLogV3DTO; @@ -139,8 +139,8 @@ private ValidateResult checkRequest(EsbGetJobInstanceIpLogV3Request request) { private void buildScriptLog(EsbIpLogV3DTO ipLog, Long stepInstanceId, Integer executeCount, Long cloudAreaId, String ip) { ipLog.setLogType(LogTypeEnum.SCRIPT.getValue()); - ScriptIpLogContent logContent = logService.getScriptIpLogContent(stepInstanceId, - executeCount, new IpDTO(cloudAreaId, ip)); + ScriptHostLogContent logContent = logService.getScriptHostLogContent(stepInstanceId, + executeCount, null, new HostDTO(cloudAreaId, ip)); if (logContent != null && StringUtils.isNotBlank(logContent.getContent())) { ipLog.setScriptLogContent(logContent.getContent()); } @@ -149,10 +149,10 @@ private void buildScriptLog(EsbIpLogV3DTO ipLog, Long stepInstanceId, Integer ex private void buildFileLog(EsbIpLogV3DTO ipLog, Long stepInstanceId, Integer executeCount, Long cloudAreaId, String ip) { ipLog.setLogType(LogTypeEnum.FILE.getValue()); - FileIpLogContent downloadIpLog = logService.getFileIpLogContent(stepInstanceId, executeCount, - new IpDTO(cloudAreaId, ip), FileDistModeEnum.DOWNLOAD.getValue()); + FileIpLogContent downloadIpLog = logService.getFileIpLogContent(stepInstanceId, executeCount, null, + new HostDTO(cloudAreaId, ip), FileDistModeEnum.DOWNLOAD.getValue()); List uploadTaskLogs = logService.batchGetFileSourceIpLogContent( - stepInstanceId, executeCount); + stepInstanceId, executeCount, null); List fileLogs = new ArrayList<>(); fileLogs.addAll(buildDownloadFileLogs(downloadIpLog)); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceListV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceListV3ResourceImpl.java index afe1745626..6ecdb3c18c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceListV3ResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceListV3ResourceImpl.java @@ -126,7 +126,7 @@ private EsbPageDataV3 convertToEsbTaskInstancePageData(Pag taskInstance.setStartupMode(taskInstanceDTO.getStartupMode()); taskInstance.setTaskId(taskInstanceDTO.getTaskId()); taskInstance.setOperator(taskInstanceDTO.getOperator()); - taskInstance.setStatus(taskInstanceDTO.getStatus()); + taskInstance.setStatus(taskInstanceDTO.getStatus().getValue()); taskInstance.setTemplateId(taskInstanceDTO.getTaskTemplateId()); taskInstance.setTotalTime(taskInstanceDTO.getTotalTime()); taskInstance.setType(taskInstanceDTO.getType()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceStatusV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceStatusV3ResourceImpl.java index 7535197b5d..27da93865e 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceStatusV3ResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceStatusV3ResourceImpl.java @@ -35,19 +35,22 @@ import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.execute.api.esb.v2.impl.JobQueryCommonProcessor; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; -import com.tencent.bk.job.execute.model.GseTaskIpLogDTO; +import com.tencent.bk.job.execute.model.AgentTaskDetailDTO; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.esb.v3.EsbJobInstanceStatusV3DTO; import com.tencent.bk.job.execute.model.esb.v3.request.EsbGetJobInstanceStatusV3Request; -import com.tencent.bk.job.execute.service.GseTaskLogService; +import com.tencent.bk.job.execute.service.FileAgentTaskService; +import com.tencent.bk.job.execute.service.ScriptAgentTaskService; import com.tencent.bk.job.execute.service.TaskInstanceService; +import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.springframework.web.bind.annotation.RestController; import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; @RestController @Slf4j @@ -56,15 +59,18 @@ public class EsbGetJobInstanceStatusV3ResourceImpl implements EsbGetJobInstanceStatusV3Resource { private final TaskInstanceService taskInstanceService; - private final GseTaskLogService gseTaskLogService; private final AppScopeMappingService appScopeMappingService; + private final ScriptAgentTaskService scriptAgentTaskService; + private final FileAgentTaskService fileAgentTaskService; - public EsbGetJobInstanceStatusV3ResourceImpl(GseTaskLogService gseTaskLogService, - TaskInstanceService taskInstanceService, - AppScopeMappingService appScopeMappingService) { - this.gseTaskLogService = gseTaskLogService; + public EsbGetJobInstanceStatusV3ResourceImpl(TaskInstanceService taskInstanceService, + AppScopeMappingService appScopeMappingService, + ScriptAgentTaskService scriptAgentTaskService, + FileAgentTaskService fileAgentTaskService) { this.taskInstanceService = taskInstanceService; this.appScopeMappingService = appScopeMappingService; + this.scriptAgentTaskService = scriptAgentTaskService; + this.fileAgentTaskService = fileAgentTaskService; } @Override @@ -110,7 +116,7 @@ private EsbJobInstanceStatusV3DTO buildEsbJobInstanceStatusDTO(TaskInstanceDTO t List stepInstances, boolean isReturnIpResult) { EsbJobInstanceStatusV3DTO jobInstanceStatus = new EsbJobInstanceStatusV3DTO(); - jobInstanceStatus.setFinished(RunStatusEnum.isFinishedStatus(RunStatusEnum.valueOf(taskInstance.getStatus()))); + jobInstanceStatus.setFinished(RunStatusEnum.isFinishedStatus(taskInstance.getStatus())); EsbJobInstanceStatusV3DTO.JobInstance jobInstance = new EsbJobInstanceStatusV3DTO.JobInstance(); EsbDTOAppScopeMappingHelper.fillEsbAppScopeDTOByAppId(taskInstance.getAppId(), jobInstance); @@ -119,7 +125,7 @@ private EsbJobInstanceStatusV3DTO buildEsbJobInstanceStatusDTO(TaskInstanceDTO t jobInstance.setCreateTime(taskInstance.getCreateTime()); jobInstance.setStartTime(taskInstance.getStartTime()); jobInstance.setEndTime(taskInstance.getEndTime()); - jobInstance.setStatus(taskInstance.getStatus()); + jobInstance.setStatus(taskInstance.getStatus().getValue()); jobInstance.setTotalTime(taskInstance.getTotalTime()); jobInstanceStatus.setJobInstance(jobInstance); @@ -133,25 +139,37 @@ private EsbJobInstanceStatusV3DTO buildEsbJobInstanceStatusDTO(TaskInstanceDTO t stepInst.setStartTime(stepInstance.getStartTime()); stepInst.setType(stepInstance.getExecuteType()); stepInst.setExecuteCount(stepInstance.getExecuteCount()); - stepInst.setStatus(stepInstance.getStatus()); + stepInst.setStatus(stepInstance.getStatus().getValue()); stepInst.setTotalTime(stepInstance.getTotalTime()); if (isReturnIpResult) { List stepIpResults = new ArrayList<>(); - List ipLogList = gseTaskLogService.getIpLog(stepInstance.getId(), - stepInstance.getExecuteCount(), true); - if (CollectionUtils.isNotEmpty(ipLogList)) { - for (GseTaskIpLogDTO ipLog : ipLogList) { + List agentTaskList = null; + if (stepInstance.isScriptStep()) { + agentTaskList = scriptAgentTaskService.listAgentTaskDetail(stepInstance, + stepInstance.getExecuteCount(), null); + } else if (stepInstance.isFileStep()) { + agentTaskList = fileAgentTaskService.listAgentTaskDetail(stepInstance, + stepInstance.getExecuteCount(), null); + if (CollectionUtils.isNotEmpty(agentTaskList)) { + // 如果是文件分发任务，只返回目标Agent结果 + agentTaskList = agentTaskList.stream() + .filter(agentTask -> agentTask.getFileTaskMode() == FileTaskModeEnum.DOWNLOAD) + .collect(Collectors.toList()); + } + } + if (CollectionUtils.isNotEmpty(agentTaskList)) { + for (AgentTaskDetailDTO agentTask : agentTaskList) { EsbJobInstanceStatusV3DTO.IpResult stepIpResult = new EsbJobInstanceStatusV3DTO.IpResult(); - stepIpResult.setCloudAreaId(ipLog.getCloudAreaId()); - stepIpResult.setIp(ipLog.getIp()); - stepIpResult.setExitCode(ipLog.getExitCode()); - stepIpResult.setErrorCode(ipLog.getErrCode()); - stepIpResult.setStartTime(ipLog.getStartTime()); - stepIpResult.setEndTime(ipLog.getEndTime()); - stepIpResult.setTotalTime(ipLog.getTotalTime()); - stepIpResult.setTag(ipLog.getTag()); - stepIpResult.setStatus(ipLog.getStatus()); + stepIpResult.setCloudAreaId(agentTask.getBkCloudId()); + stepIpResult.setIp(agentTask.getIp()); + stepIpResult.setExitCode(agentTask.getExitCode()); + stepIpResult.setErrorCode(agentTask.getErrorCode()); + stepIpResult.setStartTime(agentTask.getStartTime()); + stepIpResult.setEndTime(agentTask.getEndTime()); + stepIpResult.setTotalTime(agentTask.getTotalTime()); + stepIpResult.setTag(agentTask.getTag()); + stepIpResult.setStatus(agentTask.getStatus().getValue()); stepIpResults.add(stepIpResult); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbPushConfigFileResourceV3Impl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbPushConfigFileResourceV3Impl.java index 2a0bf03abb..eccf28a855 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbPushConfigFileResourceV3Impl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbPushConfigFileResourceV3Impl.java @@ -42,6 +42,7 @@ import com.tencent.bk.job.execute.common.constants.TaskTypeEnum; import com.tencent.bk.job.execute.config.StorageSystemConfig; import com.tencent.bk.job.execute.metrics.ExecuteMetricsConstants; +import com.tencent.bk.job.execute.model.FastTaskDTO; import com.tencent.bk.job.execute.model.FileDetailDTO; import com.tencent.bk.job.execute.model.FileSourceDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; @@ -99,8 +100,8 @@ public EsbResp pushConfigFile(EsbPushConfigFileV3Request req TaskInstanceDTO taskInstance = buildFastFileTaskInstance(request); StepInstanceDTO stepInstance = buildFastFileStepInstance(request, request.getFileList()); - long taskInstanceId = taskExecuteService.createTaskInstanceFast(taskInstance, stepInstance); - taskExecuteService.startTask(taskInstanceId); + long taskInstanceId = taskExecuteService.executeFastTask( + FastTaskDTO.builder().taskInstance(taskInstance).stepInstance(stepInstance).build()); EsbJobExecuteV3DTO jobExecuteInfo = new EsbJobExecuteV3DTO(); jobExecuteInfo.setTaskInstanceId(taskInstanceId); @@ -120,11 +121,11 @@ private TaskInstanceDTO buildFastFileTaskInstance(EsbPushConfigFileV3Request req taskInstance.setCronTaskId(-1L); taskInstance.setTaskTemplateId(-1L); taskInstance.setAppId(request.getAppId()); - taskInstance.setStatus(RunStatusEnum.BLANK.getValue()); + taskInstance.setStatus(RunStatusEnum.BLANK); taskInstance.setStartupMode(TaskStartupModeEnum.API.getValue()); taskInstance.setOperator(request.getUserName()); taskInstance.setCreateTime(DateUtils.currentTimeMillis()); - taskInstance.setCurrentStepId(0L); + taskInstance.setCurrentStepInstanceId(0L); taskInstance.setDebugTask(false); taskInstance.setAppCode(request.getAppCode()); taskInstance.setCallbackUrl(request.getCallbackUrl()); @@ -150,7 +151,7 @@ private StepInstanceDTO buildFastFileStepInstance( stepInstance.setAppId(request.getAppId()); stepInstance.setTargetServers(convertToServersDTO(request.getTargetServer())); stepInstance.setOperator(request.getUserName()); - stepInstance.setStatus(RunStatusEnum.BLANK.getValue()); + stepInstance.setStatus(RunStatusEnum.BLANK); stepInstance.setCreateTime(DateUtils.currentTimeMillis()); stepInstance.setTimeout(JobConstants.DEFAULT_JOB_TIMEOUT_SECONDS); return stepInstance; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/JobExecuteCommonV3Processor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/JobExecuteCommonV3Processor.java index 7c23067e27..c5aba13938 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/JobExecuteCommonV3Processor.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/JobExecuteCommonV3Processor.java @@ -27,7 +27,7 @@ import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.esb.model.job.v3.EsbServerV3DTO; import com.tencent.bk.job.common.model.ValidateResult; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.model.DynamicServerGroupDTO; import com.tencent.bk.job.execute.model.DynamicServerTopoNodeDTO; import com.tencent.bk.job.execute.model.ServersDTO; @@ -73,8 +73,8 @@ protected ServersDTO convertToServersDTO(EsbServerV3DTO server) { serversDTO.setDynamicServerGroups(dynamicServerGroups); } if (CollectionUtils.isNotEmpty(server.getIps())) { - List staticIpList = new ArrayList<>(); - server.getIps().forEach(ip -> staticIpList.add(new IpDTO(ip.getCloudAreaId(), ip.getIp()))); + List staticIpList = new ArrayList<>(); + server.getIps().forEach(ip -> staticIpList.add(new HostDTO(ip.getBkCloudId(), ip.getIp()))); serversDTO.setStaticIpList(staticIpList); } return serversDTO; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceExecuteTaskResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceExecuteTaskResourceImpl.java index d914094aa1..3dc71c002b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceExecuteTaskResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceExecuteTaskResourceImpl.java @@ -31,7 +31,6 @@ import com.tencent.bk.job.common.iam.model.AuthResult; import com.tencent.bk.job.common.iam.service.WebAuthService; import com.tencent.bk.job.common.model.InternalResponse; -import com.tencent.bk.job.common.model.dto.IpDTO; import com.tencent.bk.job.common.model.iam.AuthResultDTO; import com.tencent.bk.job.common.web.metrics.CustomTimed; import com.tencent.bk.job.execute.common.constants.TaskStartupModeEnum; @@ -82,8 +81,7 @@ public InternalResponse executeTask(ServiceTaskExecute throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } TaskExecuteParam executeParam = buildExecuteParam(request); - TaskInstanceDTO taskInstanceDTO = taskExecuteService.createTaskInstanceForTask(executeParam); - taskExecuteService.startTask(taskInstanceDTO.getId()); + TaskInstanceDTO taskInstanceDTO = taskExecuteService.executeJobPlan(executeParam); ServiceTaskExecuteResult result = new ServiceTaskExecuteResult(); result.setTaskInstanceId(taskInstanceDTO.getId()); @@ -130,11 +128,7 @@ private ServersDTO convertToServersDTO(ServiceTargetServers servers) { return null; } ServersDTO serversDTO = new ServersDTO(); - if (servers.getIps() != null) { - List staticIpList = new ArrayList<>(); - servers.getIps().forEach(ip -> staticIpList.add(new IpDTO(ip.getCloudAreaId(), ip.getIp()))); - serversDTO.setStaticIpList(staticIpList); - } + serversDTO.setStaticIpList(servers.getIps()); if (servers.getDynamicGroupIds() != null) { List dynamicServerGroups = new ArrayList<>(); servers.getDynamicGroupIds() diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebExecuteTaskResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebExecuteTaskResourceImpl.java index 3bde2f1f04..b6ae6a26d0 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebExecuteTaskResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebExecuteTaskResourceImpl.java @@ -29,7 +29,7 @@ import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.common.model.dto.AppResourceScope; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.util.check.IlegalCharChecker; import com.tencent.bk.job.common.util.check.MaxLengthChecker; import com.tencent.bk.job.common.util.check.NotEmptyChecker; @@ -48,11 +48,13 @@ import com.tencent.bk.job.execute.metrics.ExecuteMetricsConstants; import com.tencent.bk.job.execute.model.DynamicServerGroupDTO; import com.tencent.bk.job.execute.model.DynamicServerTopoNodeDTO; +import com.tencent.bk.job.execute.model.FastTaskDTO; import com.tencent.bk.job.execute.model.FileDetailDTO; import com.tencent.bk.job.execute.model.FileSourceDTO; import com.tencent.bk.job.execute.model.ServersDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.StepOperationDTO; +import com.tencent.bk.job.execute.model.StepRollingConfigDTO; import com.tencent.bk.job.execute.model.TaskExecuteParam; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.web.request.RedoTaskRequest; @@ -118,11 +120,10 @@ public Response executeTask(String username, } List executeVariableValues = buildExecuteVariables(request.getTaskVariables()); - TaskInstanceDTO taskInstanceDTO = taskExecuteService.createTaskInstanceForTask( + TaskInstanceDTO taskInstanceDTO = taskExecuteService.executeJobPlan( TaskExecuteParam.builder().appId(appResourceScope.getAppId()).planId(request.getTaskId()) .operator(username).executeVariableValues(executeVariableValues) .startupMode(TaskStartupModeEnum.NORMAL).build()); - taskExecuteService.startTask(taskInstanceDTO.getId()); TaskExecuteVO result = new TaskExecuteVO(); result.setTaskInstanceId(taskInstanceDTO.getId()); @@ -166,7 +167,6 @@ public Response redoTask(String username, List executeVariableValues = buildExecuteVariables(request.getTaskVariables()); TaskInstanceDTO taskInstanceDTO = taskExecuteService.createTaskInstanceForRedo(appResourceScope.getAppId(), request.getTaskInstanceId(), username, executeVariableValues); - taskExecuteService.startTask(taskInstanceDTO.getId()); TaskExecuteVO result = new TaskExecuteVO(); result.setTaskInstanceId(taskInstanceDTO.getId()); @@ -240,8 +240,12 @@ public Response fastExecuteScript(String username, StepInstanceDTO stepInstance = buildFastScriptStepInstance(username, appResourceScope.getAppId(), request); String decodeScriptContent = new String(Base64.decodeBase64(request.getContent()), StandardCharsets.UTF_8); stepInstance.setScriptContent(decodeScriptContent); + StepRollingConfigDTO rollingConfig = null; + if (request.isRollingEnabled()) { + rollingConfig = StepRollingConfigDTO.fromRollingConfigVO(request.getRollingConfig()); + } - return createAndStartFastTask(request.isRedoTask(), taskInstance, stepInstance); + return createAndStartFastTask(request.isRedoTask(), taskInstance, stepInstance, rollingConfig); } private boolean checkFastExecuteScriptRequest(WebFastExecuteScriptRequest request) { @@ -312,11 +316,11 @@ private TaskInstanceDTO buildFastScriptTaskInstance(String username, Long appId, taskInstance.setTaskTemplateId(-1L); taskInstance.setAppId(appId); taskInstance.setStartupMode(TaskStartupModeEnum.NORMAL.getValue()); - taskInstance.setStatus(RunStatusEnum.BLANK.getValue()); + taskInstance.setStatus(RunStatusEnum.BLANK); taskInstance.setOperator(username); taskInstance.setCreateTime(DateUtils.currentTimeMillis()); taskInstance.setType(TaskTypeEnum.SCRIPT.getValue()); - taskInstance.setCurrentStepId(0L); + taskInstance.setCurrentStepInstanceId(0L); taskInstance.setDebugTask(false); if (request.isRedoTask()) { taskInstance.setId(request.getTaskInstanceId()); @@ -340,7 +344,7 @@ private StepInstanceDTO buildFastScriptStepInstance(String userName, Long appId, stepInstance.setExecuteType(StepExecuteTypeEnum.EXECUTE_SCRIPT.getValue()); } stepInstance.setOperator(userName); - stepInstance.setStatus(RunStatusEnum.BLANK.getValue()); + stepInstance.setStatus(RunStatusEnum.BLANK); stepInstance.setCreateTime(DateUtils.currentTimeMillis()); stepInstance.setScriptSource(request.getScriptSource()); stepInstance.setScriptType(request.getScriptLanguage()); @@ -372,19 +376,26 @@ public Response fastPushFile(String username, TaskInstanceDTO taskInstance = buildFastFileTaskInstance(username, appResourceScope.getAppId(), request); StepInstanceDTO stepInstance = buildFastFileStepInstance(username, appResourceScope.getAppId(), request); + StepRollingConfigDTO rollingConfig = null; + if (request.isRollingEnabled()) { + rollingConfig = StepRollingConfigDTO.fromRollingConfigVO(request.getRollingConfig()); + } - return createAndStartFastTask(false, taskInstance, stepInstance); + return createAndStartFastTask(false, taskInstance, stepInstance, rollingConfig); } - private Response createAndStartFastTask(boolean isRedoTask, TaskInstanceDTO taskInstance, - StepInstanceDTO stepInstance) { + private Response createAndStartFastTask(boolean isRedoTask, + TaskInstanceDTO taskInstance, + StepInstanceDTO stepInstance, + StepRollingConfigDTO rollingConfig) { + FastTaskDTO fastTask = FastTaskDTO.builder().taskInstance(taskInstance).stepInstance(stepInstance) + .rollingConfig(rollingConfig).build(); long taskInstanceId; if (!isRedoTask) { - taskInstanceId = taskExecuteService.createTaskInstanceFast(taskInstance, stepInstance); + taskInstanceId = taskExecuteService.executeFastTask(fastTask); } else { - taskInstanceId = taskExecuteService.createTaskInstanceForFastTaskRedo(taskInstance, stepInstance); + taskInstanceId = taskExecuteService.redoFastTask(fastTask); } - taskExecuteService.startTask(taskInstanceId); StepExecuteVO stepExecuteVO = new StepExecuteVO(); stepExecuteVO.setTaskInstanceId(taskInstanceId); stepExecuteVO.setStepInstanceId(stepInstance.getId()); @@ -461,11 +472,11 @@ private TaskInstanceDTO buildFastFileTaskInstance(String username, Long appId, W taskInstance.setCronTaskId(-1L); taskInstance.setTaskTemplateId(-1L); taskInstance.setAppId(appId); - taskInstance.setStatus(RunStatusEnum.BLANK.getValue()); + taskInstance.setStatus(RunStatusEnum.BLANK); taskInstance.setStartupMode(TaskStartupModeEnum.NORMAL.getValue()); taskInstance.setOperator(username); taskInstance.setCreateTime(DateUtils.currentTimeMillis()); - taskInstance.setCurrentStepId(0L); + taskInstance.setCurrentStepInstanceId(0L); taskInstance.setDebugTask(false); return taskInstance; } @@ -483,7 +494,7 @@ private StepInstanceDTO buildFastFileStepInstance(String userName, Long appId, W stepInstance.setFileSourceList(convertFileSource(request.getFileSourceList())); stepInstance.setAppId(appId); stepInstance.setOperator(userName); - stepInstance.setStatus(RunStatusEnum.BLANK.getValue()); + stepInstance.setStatus(RunStatusEnum.BLANK); stepInstance.setCreateTime(DateUtils.currentTimeMillis()); if (request.getDownloadSpeedLimit() != null && request.getDownloadSpeedLimit() > 0) { // MB->KB @@ -506,8 +517,8 @@ private ServersDTO convertToServersDTO(ExecuteTargetVO target) { ExecuteServersVO hostNode = target.getHostNodeInfo(); ServersDTO serversDTO = new ServersDTO(); if (CollectionUtils.isNotEmpty(hostNode.getIpList())) { - List staticIpList = new ArrayList<>(); - hostNode.getIpList().forEach(host -> staticIpList.add(new IpDTO(host.getCloudAreaInfo().getId(), + List staticIpList = new ArrayList<>(); + hostNode.getIpList().forEach(host -> staticIpList.add(new HostDTO(host.getCloudAreaInfo().getId(), host.getIp()))); serversDTO.setStaticIpList(staticIpList); } @@ -532,15 +543,16 @@ private List convertFileSource(List file } List fileSourceDTOS = new ArrayList<>(); fileSources.forEach(fileSource -> { + TaskFileTypeEnum fileType = TaskFileTypeEnum.valueOf(fileSource.getFileType()); FileSourceDTO fileSourceDTO = new FileSourceDTO(); fileSourceDTO.setAccountId(fileSource.getAccountId()); - fileSourceDTO.setLocalUpload(TaskFileTypeEnum.LOCAL.getType() == fileSource.getFileType()); - fileSourceDTO.setFileType(fileSource.getFileType()); + fileSourceDTO.setLocalUpload(TaskFileTypeEnum.LOCAL == fileType); + fileSourceDTO.setFileType(fileType.getType()); fileSourceDTO.setFileSourceId(fileSource.getFileSourceId()); List files = new ArrayList<>(); if (fileSource.getFileLocation() != null) { for (String file : fileSource.getFileLocation()) { - if (TaskFileTypeEnum.LOCAL.getType() == fileSource.getFileType()) { + if (TaskFileTypeEnum.LOCAL == fileType) { files.add(new FileDetailDTO(true, file, fileSource.getFileHash(), Long.valueOf(fileSource.getFileSize()))); } else { @@ -550,7 +562,10 @@ private List convertFileSource(List file } } fileSourceDTO.setFiles(files); - fileSourceDTO.setServers(convertToServersDTO(fileSource.getHost())); + if (fileType == TaskFileTypeEnum.SERVER) { + // 服务器文件分发才需要解析主机参数 + fileSourceDTO.setServers(convertToServersDTO(fileSource.getHost())); + } fileSourceDTOS.add(fileSourceDTO); }); return fileSourceDTOS; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java index d54a736c8b..db4b98472b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java @@ -43,8 +43,6 @@ import com.tencent.bk.job.common.model.ValidateResult; import com.tencent.bk.job.common.model.dto.AppResourceScope; import com.tencent.bk.job.common.model.dto.HostDTO; -import com.tencent.bk.job.common.model.dto.IpDTO; -import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.CustomCollectionUtils; import com.tencent.bk.job.common.util.JobContextUtil; import com.tencent.bk.job.common.util.date.DateUtils; @@ -59,18 +57,19 @@ import com.tencent.bk.job.execute.common.constants.TaskStartupModeEnum; import com.tencent.bk.job.execute.common.constants.TaskTotalTimeTypeEnum; import com.tencent.bk.job.execute.common.constants.TaskTypeEnum; -import com.tencent.bk.job.execute.engine.consts.IpStatus; +import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; -import com.tencent.bk.job.execute.model.AgentTaskExecutionDTO; -import com.tencent.bk.job.execute.model.ExecutionResultGroupDTO; +import com.tencent.bk.job.execute.model.AgentTaskDetailDTO; +import com.tencent.bk.job.execute.model.AgentTaskResultGroupDTO; import com.tencent.bk.job.execute.model.FileIpLogContent; -import com.tencent.bk.job.execute.model.ScriptIpLogContent; +import com.tencent.bk.job.execute.model.ScriptHostLogContent; import com.tencent.bk.job.execute.model.StepExecutionDTO; import com.tencent.bk.job.execute.model.StepExecutionDetailDTO; import com.tencent.bk.job.execute.model.StepExecutionRecordDTO; import com.tencent.bk.job.execute.model.StepExecutionResultQuery; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; +import com.tencent.bk.job.execute.model.StepInstanceRollingTaskDTO; import com.tencent.bk.job.execute.model.StepInstanceVariableValuesDTO; import com.tencent.bk.job.execute.model.TaskExecuteResultDTO; import com.tencent.bk.job.execute.model.TaskExecutionDTO; @@ -84,6 +83,7 @@ import com.tencent.bk.job.execute.model.web.vo.FileDistributionDetailVO; import com.tencent.bk.job.execute.model.web.vo.IpFileLogContentVO; import com.tencent.bk.job.execute.model.web.vo.IpScriptLogContentVO; +import com.tencent.bk.job.execute.model.web.vo.RollingStepBatchTaskVO; import com.tencent.bk.job.execute.model.web.vo.StepExecutionDetailVO; import com.tencent.bk.job.execute.model.web.vo.StepExecutionRecordVO; import com.tencent.bk.job.execute.model.web.vo.StepExecutionVO; @@ -135,6 +135,7 @@ public class WebTaskExecutionResultResourceImpl implements WebTaskExecutionResul private final ExecuteAuthService executeAuthService; private final WebAuthService webAuthService; + private final LoadingCache> roleCache = CacheBuilder.newBuilder() .maximumSize(10).expireAfterWrite(10, TimeUnit.MINUTES). build(new CacheLoader>() { @@ -185,8 +186,7 @@ public WebTaskExecutionResultResourceImpl(TaskResultService taskResultService, TaskInstanceVariableService taskInstanceVariableService, ServiceNotificationResourceClient notifyResource, ExecuteAuthService executeAuthService, - WebAuthService webAuthService, - AppScopeMappingService appScopeMappingService) { + WebAuthService webAuthService) { this.taskResultService = taskResultService; this.i18nService = i18nService; this.logService = logService; @@ -396,9 +396,7 @@ private TaskExecuteResultVO convertToTaskExecuteResultVO(TaskExecuteResultDTO ta stepExecutionVO.setEndTime(stepExecutionDTO.getEndTime()); stepExecutionVO.setStatus(stepExecutionDTO.getStatus()); RunStatusEnum runStatus = RunStatusEnum.valueOf(stepExecutionDTO.getStatus()); - if (runStatus != null) { - stepExecutionVO.setStatusDesc(i18nService.getI18n(runStatus.getI18nKey())); - } + stepExecutionVO.setStatusDesc(i18nService.getI18n(runStatus.getI18nKey())); stepExecutionVO.setTotalTime(stepExecutionDTO.getTotalTime()); stepExecutionVO.setType(stepExecutionDTO.getType()); stepExecutionVO.setCurrentStepRunning(taskExecuteResultDTO.getTaskInstanceExecutionResult() @@ -470,6 +468,7 @@ public Response getStepExecutionResult(String username, String scopeId, Long stepInstanceId, Integer executeCount, + Integer batch, Integer resultType, String tag, Integer maxIpsPerResultGroup, @@ -477,16 +476,19 @@ public Response getStepExecutionResult(String username, String searchIp, String orderField, Integer order) { - StepExecutionResultQuery query = new StepExecutionResultQuery(); - query.setStepInstanceId(stepInstanceId); - query.setExecuteCount(executeCount); - query.setResultType(resultType); - query.setTag(tag); - query.setLogKeyword(keyword); - query.setSearchIp(searchIp); - query.setMaxAgentTasksForResultGroup(maxIpsPerResultGroup); - query.setOrderField(orderField); - query.setOrder(Order.valueOf(order)); + StepExecutionResultQuery query = StepExecutionResultQuery.builder() + .stepInstanceId(stepInstanceId) + .executeCount(executeCount) + .batch(batch == null ? null : (batch == 0 ? null : batch)) + .filterByLatestBatch(batch == null) + .status(resultType) + .tag(tag) + .logKeyword(keyword) + .searchIp(searchIp) + .maxAgentTasksForResultGroup(maxIpsPerResultGroup) + .orderField(orderField) + .order(Order.valueOf(order)) + .build(); StepExecutionDetailDTO executionResult = taskResultService.getStepExecutionResult(username, appResourceScope.getAppId(), query); @@ -499,18 +501,20 @@ public Response getFastTaskStepExecutionResult(String use String scopeType, String scopeId, Long taskInstanceId, + Integer batch, Integer resultType, String tag, Integer maxIpsPerResultGroup, String orderField, Integer order) { - - StepExecutionResultQuery query = new StepExecutionResultQuery(); - query.setResultType(resultType); - query.setTag(tag); - query.setMaxAgentTasksForResultGroup(maxIpsPerResultGroup); - query.setOrderField(orderField); - query.setOrder(Order.valueOf(order)); + StepExecutionResultQuery query = StepExecutionResultQuery.builder() + .batch(batch) + .status(resultType) + .tag(tag) + .maxAgentTasksForResultGroup(maxIpsPerResultGroup) + .orderField(orderField) + .order(Order.valueOf(order)) + .build(); StepExecutionDetailDTO executionResult = taskResultService.getFastTaskStepExecutionResult(username, appResourceScope.getAppId(), taskInstanceId, query); return Response.buildSuccessResp(convertToStepInstanceExecutionDetailVO(executionResult)); @@ -522,43 +526,43 @@ private StepExecutionDetailVO convertToStepInstanceExecutionDetailVO(StepExecuti stepExecutionDetailVO.setName(executionDetail.getName()); stepExecutionDetailVO.setStepInstanceId(executionDetail.getStepInstanceId()); stepExecutionDetailVO.setRetryCount(executionDetail.getExecuteCount()); - stepExecutionDetailVO.setStatus(executionDetail.getStatus()); + stepExecutionDetailVO.setStatus(executionDetail.getStatus().getValue()); stepExecutionDetailVO.setStatusDesc( - i18nService.getI18n(RunStatusEnum.valueOf(executionDetail.getStatus()).getI18nKey())); + i18nService.getI18n(executionDetail.getStatus().getI18nKey())); stepExecutionDetailVO.setStartTime(executionDetail.getStartTime()); stepExecutionDetailVO.setEndTime(executionDetail.getEndTime()); stepExecutionDetailVO.setTotalTime(executionDetail.getTotalTime()); - stepExecutionDetailVO.setGseTaskId(executionDetail.getGseTaskId()); stepExecutionDetailVO.setIsLastStep(executionDetail.isLastStep()); - stepExecutionDetailVO.setType(executionDetail.getStepType()); + stepExecutionDetailVO.setType(executionDetail.getStepType().getValue()); + stepExecutionDetailVO.setRunMode(executionDetail.getRunMode().getValue()); List resultGroupVOS = new ArrayList<>(); - for (ExecutionResultGroupDTO resultGroup : executionDetail.getResultGroups()) { + for (AgentTaskResultGroupDTO resultGroup : executionDetail.getResultGroups()) { ExecutionResultGroupVO executionResultGroupVO = new ExecutionResultGroupVO(); - executionResultGroupVO.setResultType(resultGroup.getResultType()); + executionResultGroupVO.setResultType(resultGroup.getStatus()); executionResultGroupVO.setResultTypeDesc( - i18nService.getI18n(IpStatus.valueOf(resultGroup.getResultType()).getI18nKey())); + i18nService.getI18n(AgentTaskStatusEnum.valueOf(resultGroup.getStatus()).getI18nKey())); executionResultGroupVO.setTag(resultGroup.getTag()); - executionResultGroupVO.setAgentTaskSize(resultGroup.getAgentTaskSize()); + executionResultGroupVO.setAgentTaskSize(resultGroup.getTotalAgentTasks()); List agentTaskExecutionVOS = new ArrayList<>(); - if (resultGroup.getAgentTaskExecutionDetail() != null) { - for (AgentTaskExecutionDTO agentTaskExecution : resultGroup.getAgentTaskExecutionDetail()) { + if (resultGroup.getAgentTasks() != null) { + for (AgentTaskDetailDTO agentTask : resultGroup.getAgentTasks()) { AgentTaskExecutionVO agentTaskVO = new AgentTaskExecutionVO(); - agentTaskVO.setIp(agentTaskExecution.getCloudIp()); - agentTaskVO.setDisplayIp(agentTaskExecution.getDisplayIp()); - agentTaskVO.setEndTime(agentTaskExecution.getEndTime()); - agentTaskVO.setStartTime(agentTaskExecution.getStartTime()); - agentTaskVO.setStatus(agentTaskExecution.getStatus()); - agentTaskVO.setStatusDesc( - i18nService.getI18n(IpStatus.valueOf(agentTaskExecution.getStatus()).getI18nKey())); - agentTaskVO.setErrorCode(agentTaskExecution.getErrorCode()); - agentTaskVO.setExitCode(agentTaskExecution.getExitCode()); - agentTaskVO.setTag(agentTaskExecution.getTag()); - agentTaskVO.setTotalTime(agentTaskExecution.getTotalTime()); - agentTaskVO.setCloudAreaId(agentTaskExecution.getCloudAreaId()); - agentTaskVO.setCloudAreaName(agentTaskExecution.getCloudAreaName()); - agentTaskVO.setRetryCount(agentTaskExecution.getExecuteCount()); + agentTaskVO.setIp(agentTask.getCloudIp()); + agentTaskVO.setDisplayIp(agentTask.getDisplayIp()); + agentTaskVO.setEndTime(agentTask.getEndTime()); + agentTaskVO.setStartTime(agentTask.getStartTime()); + agentTaskVO.setStatus(agentTask.getStatus().getValue()); + agentTaskVO.setStatusDesc(i18nService.getI18n(agentTask.getStatus().getI18nKey())); + agentTaskVO.setErrorCode(agentTask.getErrorCode()); + agentTaskVO.setExitCode(agentTask.getExitCode()); + agentTaskVO.setTag(agentTask.getTag()); + agentTaskVO.setTotalTime(agentTask.getTotalTime()); + agentTaskVO.setCloudAreaId(agentTask.getBkCloudId()); + agentTaskVO.setCloudAreaName(agentTask.getBkCloudName()); + agentTaskVO.setRetryCount(agentTask.getExecuteCount()); + agentTaskVO.setBatch(agentTask.getBatch()); agentTaskExecutionVOS.add(agentTaskVO); } } @@ -568,9 +572,25 @@ private StepExecutionDetailVO convertToStepInstanceExecutionDetailVO(StepExecuti } stepExecutionDetailVO.setResultGroups(resultGroupVOS); + if (CollectionUtils.isNotEmpty(executionDetail.getRollingTasks())) { + stepExecutionDetailVO.setRollingTasks(toRollingStepBatchTaskVOs(executionDetail.getLatestBatch(), + executionDetail.getRollingTasks())); + } + return stepExecutionDetailVO; } + private List toRollingStepBatchTaskVOs(Integer latestBatch, + List stepInstanceRollingTasks) { + return stepInstanceRollingTasks.stream().map(stepInstanceRollingTask -> { + RollingStepBatchTaskVO vo = new RollingStepBatchTaskVO(); + vo.setBatch(stepInstanceRollingTask.getBatch()); + vo.setStatus(stepInstanceRollingTask.getStatus().getValue()); + vo.setLatestBatch(latestBatch.equals(stepInstanceRollingTask.getBatch())); + return vo; + }).collect(Collectors.toList()); + } + @Override public Response getScriptLogContentByIp(String username, AppResourceScope appResourceScope, @@ -578,26 +598,27 @@ public Response getScriptLogContentByIp(String username, String scopeId, Long stepInstanceId, Integer executeCount, - String ip) { + String ip, + Integer batch) { if (stepInstanceId == null || executeCount == null || ip == null) { log.warn("Get ip log content, param is illegal!"); throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } - if (!IpUtils.checkCloudAreaIdAndIpStr(ip)) { + if (!IpUtils.checkCloudIp(ip)) { log.warn("Get ip log content, param ip is illegal! ip={}", ip); throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } authViewStepInstance(username, appResourceScope, stepInstanceId); - ScriptIpLogContent scriptIpLogContent = logService.getScriptIpLogContent(stepInstanceId, executeCount, - IpDTO.fromCloudAreaIdAndIpStr(ip)); + ScriptHostLogContent scriptHostLogContent = logService.getScriptHostLogContent(stepInstanceId, executeCount, + batch, HostDTO.fromCloudIp(ip)); IpScriptLogContentVO ipScriptLogContentVO = new IpScriptLogContentVO(); ipScriptLogContentVO.setDisplayIp(ip); - if (scriptIpLogContent != null) { - ipScriptLogContentVO.setLogContent(scriptIpLogContent.getContent()); - ipScriptLogContentVO.setFinished(scriptIpLogContent.isFinished()); + if (scriptHostLogContent != null) { + ipScriptLogContentVO.setLogContent(scriptHostLogContent.getContent()); + ipScriptLogContentVO.setFinished(scriptHostLogContent.isFinished()); } return Response.buildSuccessResp(ipScriptLogContentVO); } @@ -735,13 +756,14 @@ public Response getFileLogContentByIp(String username, Long stepInstanceId, Integer executeCount, String ip, - String mode) { + String mode, + Integer batch) { if (stepInstanceId == null || executeCount == null || ip == null) { log.warn("Get ip log content, param is illegal!"); throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } - if (!IpUtils.checkCloudAreaIdAndIpStr(ip)) { + if (!IpUtils.checkCloudIp(ip)) { log.warn("Get ip log content, param ip is illegal! ip={}", ip); throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } @@ -753,8 +775,8 @@ public Response getFileLogContentByIp(String username, result.setFileDistributionDetails(fileDistDetailVOS); if ("download".equals(mode)) { - FileIpLogContent downloadLog = logService.getFileIpLogContent(stepInstanceId, executeCount, - IpDTO.fromCloudAreaIdAndIpStr(ip), FileDistModeEnum.DOWNLOAD.getValue()); + FileIpLogContent downloadLog = logService.getFileIpLogContent(stepInstanceId, executeCount, batch, + HostDTO.fromCloudIp(ip), FileDistModeEnum.DOWNLOAD.getValue()); // downloadLog为null说明步骤还未下发至GSE就被终止 if (downloadLog != null && CollectionUtils.isNotEmpty(downloadLog.getFileTaskLogs())) { downloadLog.getFileTaskLogs().forEach(fileLog -> { @@ -768,7 +790,7 @@ public Response getFileLogContentByIp(String username, Collections.sort(fileDistDetailVOS); } else { List fileTaskLogs = logService.batchGetFileSourceIpLogContent(stepInstanceId, - executeCount); + executeCount, batch); if (CollectionUtils.isNotEmpty(fileTaskLogs)) { fileTaskLogs.forEach(fileTaskLog -> { if (fileTaskLog.getMode().equals(FileDistModeEnum.DOWNLOAD.getValue())) { @@ -838,11 +860,12 @@ public Response> getFileLogContentByFileTaskIds(S String scopeId, Long stepInstanceId, Integer executeCount, + Integer batch, List taskIds) { authViewStepInstance(username, appResourceScope, stepInstanceId); - List fileTaskLogs = logService.getFileLogContentByTaskIds(stepInstanceId, executeCount - , taskIds); + List fileTaskLogs = logService.getFileLogContentByTaskIds(stepInstanceId, executeCount, + batch, taskIds); if (CollectionUtils.isEmpty(fileTaskLogs)) { return Response.buildSuccessResp(null); } @@ -871,13 +894,13 @@ public Response> getHostsByResultType(String username, String scopeId, Long stepInstanceId, Integer executeCount, + Integer batch, Integer resultType, String tag, String keyword) { - List hosts = taskResultService.getHostsByResultType(username, appResourceScope.getAppId(), - stepInstanceId, executeCount, resultType, tag, keyword); - return Response.buildSuccessResp(hosts.stream().map(IpDTO::toHost) - .collect(Collectors.toList())); + List hosts = taskResultService.getHostsByResultType(username, appResourceScope.getAppId(), + stepInstanceId, executeCount, batch, resultType, tag, keyword); + return Response.buildSuccessResp(hosts); } @Override @@ -885,10 +908,11 @@ public Response> listStepExecutionHistory(String use AppResourceScope appResourceScope, String scopeType, String scopeId, - Long stepInstanceId) { + Long stepInstanceId, + Integer batch) { List stepExecutionRecords = taskResultService.listStepExecutionHistory(username, - appResourceScope.getAppId(), stepInstanceId); + appResourceScope.getAppId(), stepInstanceId, batch); return Response.buildSuccessResp(stepExecutionRecords.stream().map(stepExecutionRecord -> { StepExecutionRecordVO vo = new StepExecutionRecordVO(); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskInstanceResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskInstanceResourceImpl.java index 8a1b889f18..ce448fe37f 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskInstanceResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskInstanceResourceImpl.java @@ -36,7 +36,7 @@ import com.tencent.bk.job.common.iam.service.BusinessAuthService; import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.common.model.dto.AppResourceScope; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.util.Base64Util; import com.tencent.bk.job.execute.api.web.WebTaskInstanceResource; import com.tencent.bk.job.execute.auth.ExecuteAuthService; @@ -46,10 +46,13 @@ import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; import com.tencent.bk.job.execute.model.FileSourceDTO; import com.tencent.bk.job.execute.model.OperationLogDTO; +import com.tencent.bk.job.execute.model.RollingConfigDTO; import com.tencent.bk.job.execute.model.ServersDTO; +import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.converter.TaskInstanceConverter; +import com.tencent.bk.job.execute.model.db.RollingConfigDetailDO; import com.tencent.bk.job.execute.model.web.vo.ExecuteApprovalStepVO; import com.tencent.bk.job.execute.model.web.vo.ExecuteCloudAreaInfoVO; import com.tencent.bk.job.execute.model.web.vo.ExecuteFileDestinationInfoVO; @@ -61,10 +64,12 @@ import com.tencent.bk.job.execute.model.web.vo.ExecuteStepVO; import com.tencent.bk.job.execute.model.web.vo.ExecuteTargetVO; import com.tencent.bk.job.execute.model.web.vo.ExecuteVariableVO; +import com.tencent.bk.job.execute.model.web.vo.RollingConfigVO; import com.tencent.bk.job.execute.model.web.vo.TaskInstanceDetailVO; import com.tencent.bk.job.execute.model.web.vo.TaskInstanceVO; import com.tencent.bk.job.execute.model.web.vo.TaskOperationLogVO; import com.tencent.bk.job.execute.service.HostService; +import com.tencent.bk.job.execute.service.RollingConfigService; import com.tencent.bk.job.execute.service.TaskInstanceService; import com.tencent.bk.job.execute.service.TaskInstanceVariableService; import com.tencent.bk.job.execute.service.TaskOperationLogService; @@ -90,6 +95,7 @@ public class WebTaskInstanceResourceImpl implements WebTaskInstanceResource { private final MessageI18nService i18nService; private final ExecuteAuthService executeAuthService; private final BusinessAuthService businessAuthService; + private final RollingConfigService rollingConfigService; @Autowired public WebTaskInstanceResourceImpl(TaskInstanceService taskInstanceService, @@ -98,7 +104,8 @@ public WebTaskInstanceResourceImpl(TaskInstanceService taskInstanceService, TaskOperationLogService taskOperationLogService, MessageI18nService i18nService, ExecuteAuthService executeAuthService, - BusinessAuthService businessAuthService) { + BusinessAuthService businessAuthService, + RollingConfigService rollingConfigService) { this.taskInstanceService = taskInstanceService; this.taskInstanceVariableService = taskInstanceVariableService; this.hostService = hostService; @@ -106,6 +113,7 @@ public WebTaskInstanceResourceImpl(TaskInstanceService taskInstanceService, this.i18nService = i18nService; this.executeAuthService = executeAuthService; this.businessAuthService = businessAuthService; + this.rollingConfigService = rollingConfigService; } @Override @@ -129,9 +137,26 @@ public Response getStepInstanceDetail(String username, } ExecuteStepVO stepVO = convertToStepVO(stepInstance); + fillRollingConfigForRollingStep(stepVO, stepInstance); + return Response.buildSuccessResp(stepVO); } + private void fillRollingConfigForRollingStep(ExecuteStepVO stepVO, StepInstanceBaseDTO stepInstance) { + if (stepInstance.isRollingStep()) { + RollingConfigVO rollingConfigVO = new RollingConfigVO(); + RollingConfigDTO rollingConfigDTO = + rollingConfigService.getRollingConfig(stepInstance.getRollingConfigId()); + RollingConfigDetailDO rollingConfig = rollingConfigDTO.getConfigDetail(); + rollingConfigVO.setMode(rollingConfig.getMode()); + if (rollingConfigDTO.isBatchRollingStep(stepInstance.getId())) { + rollingConfigVO.setExpr(rollingConfig.getExpr()); + } + stepVO.setRollingConfig(rollingConfigVO); + stepVO.setRollingEnabled(true); + } + } + private AuthResult authViewTaskInstance(String username, AppResourceScope appResourceScope, TaskInstanceDTO taskInstance) { return executeAuthService.authViewTaskInstance(username, appResourceScope, taskInstance); @@ -199,12 +224,8 @@ private ExecuteStepVO convertToStepVO(StepInstanceDTO stepInstance) { ExecuteStepVO stepVO = new ExecuteStepVO(); stepVO.setName(stepInstance.getName()); StepExecuteTypeEnum stepType = StepExecuteTypeEnum.valueOf(stepInstance.getExecuteType()); - if (stepType == null) { - log.warn("Invalid step type!"); - return null; - } if (stepType == StepExecuteTypeEnum.EXECUTE_SCRIPT || stepType == StepExecuteTypeEnum.EXECUTE_SQL) { - stepVO.setType(TaskStepTypeEnum.SCRIPT.getType()); + stepVO.setType(TaskStepTypeEnum.SCRIPT.getValue()); ExecuteScriptStepVO scriptStepVO = new ExecuteScriptStepVO(); if (stepType == StepExecuteTypeEnum.EXECUTE_SCRIPT) { scriptStepVO.setAccountId(stepInstance.getAccountId()); @@ -234,7 +255,7 @@ private ExecuteStepVO convertToStepVO(StepInstanceDTO stepInstance) { scriptStepVO.setIgnoreError(stepInstance.isIgnoreError() ? 1 : 0); stepVO.setScriptStepInfo(scriptStepVO); } else if (stepType == StepExecuteTypeEnum.SEND_FILE) { - stepVO.setType(TaskStepTypeEnum.FILE.getType()); + stepVO.setType(TaskStepTypeEnum.FILE.getValue()); ExecuteFileStepVO fileStepVO = new ExecuteFileStepVO(); ExecuteFileDestinationInfoVO fileDestinationInfoVO = new ExecuteFileDestinationInfoVO(); @@ -265,7 +286,7 @@ private ExecuteStepVO convertToStepVO(StepInstanceDTO stepInstance) { } stepVO.setFileStepInfo(fileStepVO); } else if (stepType == StepExecuteTypeEnum.MANUAL_CONFIRM) { - stepVO.setType(TaskStepTypeEnum.APPROVAL.getType()); + stepVO.setType(TaskStepTypeEnum.APPROVAL.getValue()); ExecuteApprovalStepVO approvalStepVO = new ExecuteApprovalStepVO(); approvalStepVO.setApprovalMessage(stepInstance.getConfirmMessage()); stepVO.setApprovalStepInfo(approvalStepVO); @@ -282,9 +303,9 @@ private ExecuteTargetVO convertToServers(ServersDTO serversDTO) { ExecuteServersVO taskHostNodeVO = new ExecuteServersVO(); if (serversDTO.getIpList() != null) { List hosts = new ArrayList<>(); - for (IpDTO ip : serversDTO.getIpList()) { + for (HostDTO ip : serversDTO.getIpList()) { ExecuteHostVO host = new ExecuteHostVO(); - ExecuteCloudAreaInfoVO cloudAreaInfoVO = new ExecuteCloudAreaInfoVO(ip.getCloudAreaId(), ip.getIp()); + ExecuteCloudAreaInfoVO cloudAreaInfoVO = new ExecuteCloudAreaInfoVO(ip.getBkCloudId(), ip.getIp()); host.setIp(ip.getIp()); host.setAlive(ip.getAlive()); host.setCloudAreaInfo(cloudAreaInfoVO); @@ -337,12 +358,12 @@ private ExecuteVariableVO convertToVariableVO(long appId, TaskVariableDTO variab if (servers.getIpList() != null) { ExecuteServersVO taskHostNodeVO = new ExecuteServersVO(); List hosts = new ArrayList<>(servers.getIpList().size()); - for (IpDTO ip : servers.getIpList()) { + for (HostDTO ip : servers.getIpList()) { ExecuteHostVO host = new ExecuteHostVO(); host.setIp(ip.getIp()); host.setAlive(ip.getAlive()); - ExecuteCloudAreaInfoVO cloudAreaInfoVO = new ExecuteCloudAreaInfoVO(ip.getCloudAreaId(), - hostService.getCloudAreaName(ip.getCloudAreaId())); + ExecuteCloudAreaInfoVO cloudAreaInfoVO = new ExecuteCloudAreaInfoVO(ip.getBkCloudId(), + hostService.getCloudAreaName(ip.getBkCloudId())); host.setCloudAreaInfo(cloudAreaInfoVO); hosts.add(host); } @@ -393,6 +414,7 @@ public Response> getTaskInstanceOperationLog(String use vo.setStepInstanceId(detail.getStepInstanceId()); vo.setStepName(detail.getStepName()); vo.setRetry(detail.getExecuteCount()); + vo.setBatch(detail.getBatch()); vo.setDetail(buildDetail(operationLog.getOperationEnum(), operationLog.getDetail())); vos.add(vo); } @@ -462,6 +484,7 @@ private TaskInstanceDetailVO convertToTaskInstanceDetailVO(TaskInstanceDTO taskI List stepVOS = new ArrayList<>(stepInstances.size()); stepInstances.forEach(stepInstance -> { ExecuteStepVO stepVO = convertToStepVO(stepInstance); + fillRollingConfigForRollingStep(stepVO, stepInstance); stepVOS.add(stepVO); }); taskInstanceDetailVO.setSteps(stepVOS); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskLogResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskLogResourceImpl.java index d037eb6f1a..02820cea50 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskLogResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskLogResourceImpl.java @@ -32,7 +32,6 @@ import com.tencent.bk.job.common.exception.NotFoundException; import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.common.model.dto.AppResourceScope; -import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.execute.api.web.WebTaskLogResource; import com.tencent.bk.job.execute.config.ArtifactoryConfig; @@ -79,8 +78,7 @@ public WebTaskLogResourceImpl(TaskInstanceService taskInstanceService, LogExportService logExportService, ArtifactoryClient artifactoryClient, ArtifactoryConfig artifactoryConfig, - LogExportConfig logExportConfig, - AppScopeMappingService appScopeMappingService) { + LogExportConfig logExportConfig) { this.taskInstanceService = taskInstanceService; this.logExportService = logExportService; this.logFileDir = NFSUtils.getFileDir(storageSystemConfig.getJobStorageRootPath(), @@ -119,7 +117,8 @@ public Response requestDownloadLogFile(String username, String scopeType, String scopeId, Long stepInstanceId, - String ip, + Long hostId, + String cloudIp, Boolean repackage) { Long appId = appResourceScope.getAppId(); @@ -134,7 +133,7 @@ public Response requestDownloadLogFile(String username, if (!repackage) { log.debug("Do not need repackage, check exist job"); - LogExportJobInfoDTO exportInfo = logExportService.getExportInfo(appId, stepInstanceId, ip); + LogExportJobInfoDTO exportInfo = logExportService.getExportInfo(appId, stepInstanceId, hostId, cloudIp); if (exportInfo != null) { log.debug("Find exist job info|{}", exportInfo); switch (exportInfo.getStatus()) { @@ -166,18 +165,18 @@ public Response requestDownloadLogFile(String username, int executeCount = stepInstance.getExecuteCount(); - String logFileName = getLogFileName(stepInstanceId, ip, executeCount); + String logFileName = getLogFileName(stepInstanceId, hostId, cloudIp, executeCount); if (StringUtils.isBlank(logFileName)) { throw new InternalException(ErrorCode.EXPORT_STEP_EXECUTION_LOG_FAIL); } - LogExportJobInfoDTO exportInfo = logExportService.packageLogFile(username, appId, stepInstanceId, ip, + LogExportJobInfoDTO exportInfo = logExportService.packageLogFile(username, appId, stepInstanceId, hostId, cloudIp, executeCount, logFileDir, logFileName, repackage); return Response.buildSuccessResp(LogExportJobInfoDTO.toVO(exportInfo)); } - private String getLogFileName(Long stepInstanceId, String ip, int executeCount) { - String fileName = makeExportLogFileName(stepInstanceId, executeCount, ip); + private String getLogFileName(Long stepInstanceId, Long hostId, String cloudIp, int executeCount) { + String fileName = makeExportLogFileName(stepInstanceId, executeCount, hostId, cloudIp); String logFileName = fileName + ".log"; File dir = new File(logFileDir); @@ -244,11 +243,13 @@ public ResponseEntity downloadLogFile(HttpServletResponse String scopeType, String scopeId, Long stepInstanceId, - String ip) { + Long hostId, + String cloudIp) { Long appId = appResourceScope.getAppId(); StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(stepInstanceId); if (!stepInstance.getAppId().equals(appId)) { + log.info("StepInstance: {} is not in app: {}", stepInstance.getId(), appResourceScope.getAppId()); return ResponseEntity.notFound().build(); } @@ -256,16 +257,16 @@ public ResponseEntity downloadLogFile(HttpServletResponse LogExportJobInfoDTO exportInfo; - boolean isGetByIp = StringUtils.isNotBlank(ip); - if (isGetByIp) { - String logFileName = getLogFileName(stepInstanceId, ip, executeCount); + boolean isGetByHost = hostId != null || StringUtils.isNotBlank(cloudIp); + if (isGetByHost) { + String logFileName = getLogFileName(stepInstanceId, hostId, cloudIp, executeCount); if (StringUtils.isBlank(logFileName)) { return ResponseEntity.notFound().build(); } - exportInfo = logExportService.packageLogFile(username, appId, stepInstanceId, ip, executeCount, + exportInfo = logExportService.packageLogFile(username, appId, stepInstanceId, hostId, cloudIp, executeCount, logFileDir, logFileName, false); } else { - exportInfo = logExportService.getExportInfo(appId, stepInstanceId, ip); + exportInfo = logExportService.getExportInfo(appId, stepInstanceId, hostId, cloudIp); } if (exportInfo != null) { @@ -311,12 +312,14 @@ public ResponseEntity downloadLogFile(HttpServletResponse return ResponseEntity.notFound().build(); } - private String makeExportLogFileName(Long stepInstanceId, Integer executeCount, String ip) { + private String makeExportLogFileName(Long stepInstanceId, Integer executeCount, Long hostId, String cloudIp) { StringBuilder fileName = new StringBuilder(); fileName.append("bk_job_export_log_"); fileName.append("step_").append(stepInstanceId).append("_").append(executeCount).append("_"); - if (!StringUtils.isBlank(ip)) { - fileName.append(ip).append("_"); + if (hostId != null) { + fileName.append(hostId).append("_"); + } else if (StringUtils.isNotBlank(cloudIp)) { + fileName.append(cloudIp).append("_"); } fileName.append(DateUtils.formatLocalDateTime(LocalDateTime.now(), "yyyyMMddHHmmssSSS")); return fileName.toString(); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/auth/impl/ExecuteAuthServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/auth/impl/ExecuteAuthServiceImpl.java index b8762adf03..90cc2e3097 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/auth/impl/ExecuteAuthServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/auth/impl/ExecuteAuthServiceImpl.java @@ -24,11 +24,9 @@ package com.tencent.bk.job.execute.auth.impl; -import com.tencent.bk.job.common.cc.model.InstanceTopologyDTO; import com.tencent.bk.job.common.constant.CcNodeTypeEnum; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.FeatureToggleModeEnum; -import com.tencent.bk.job.common.exception.FailedPreconditionException; import com.tencent.bk.job.common.exception.NotFoundException; import com.tencent.bk.job.common.exception.NotImplementedException; import com.tencent.bk.job.common.iam.constant.ActionId; @@ -41,15 +39,11 @@ import com.tencent.bk.job.common.iam.service.ResourceNameQueryService; import com.tencent.bk.job.common.iam.util.IamUtil; import com.tencent.bk.job.common.model.dto.AppResourceScope; -import com.tencent.bk.job.common.model.dto.IpDTO; import com.tencent.bk.job.execute.auth.ExecuteAuthService; import com.tencent.bk.job.execute.config.JobExecuteConfig; -import com.tencent.bk.job.execute.model.DynamicServerTopoNodeDTO; import com.tencent.bk.job.execute.model.ServersDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; -import com.tencent.bk.job.execute.service.HostService; import com.tencent.bk.job.execute.service.TaskInstanceService; -import com.tencent.bk.job.manage.model.inner.ServiceHostDTO; import com.tencent.bk.sdk.iam.constants.SystemId; import com.tencent.bk.sdk.iam.dto.InstanceDTO; import com.tencent.bk.sdk.iam.dto.PathInfoDTO; @@ -64,10 +58,8 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -76,7 +68,6 @@ public class ExecuteAuthServiceImpl implements ExecuteAuthService { private final AuthHelper authHelper; private final ResourceNameQueryService resourceNameQueryService; - private final HostService hostService; private final AuthService authService; private final AppAuthService appAuthService; private final TaskInstanceService taskInstanceService; @@ -85,14 +76,12 @@ public class ExecuteAuthServiceImpl implements ExecuteAuthService { @Autowired public ExecuteAuthServiceImpl(AuthHelper authHelper, ResourceNameQueryService resourceNameQueryService, - HostService hostService, AuthService authService, AppAuthService appAuthService, TaskInstanceService taskInstanceService, JobExecuteConfig jobExecuteConfig) { this.authHelper = authHelper; this.resourceNameQueryService = resourceNameQueryService; - this.hostService = hostService; this.authService = authService; this.appAuthService = appAuthService; this.taskInstanceService = taskInstanceService; @@ -102,10 +91,7 @@ public ExecuteAuthServiceImpl(AuthHelper authHelper, } public AuthResult authFastExecuteScript(String username, AppResourceScope appResourceScope, ServersDTO servers) { - Map ip2HostIdMap = new HashMap<>(); - Map dynamicServerTopoNodeHierarchyMap = new HashMap<>(); - List hostInstanceList = buildHostInstances(appResourceScope, servers, ip2HostIdMap, - dynamicServerTopoNodeHierarchyMap); + List hostInstanceList = buildHostInstances(appResourceScope, servers); log.debug("Auth fast execute script, username:{}, appResourceScope:{}, hostInstances:{}", username, appResourceScope, hostInstanceList); @@ -119,17 +105,14 @@ public AuthResult authFastExecuteScript(String username, AppResourceScope appRes AuthResult authResult = AuthResult.fail(); List hostResources = convertHostsToPermissionResourceList( - appResourceScope, servers, ip2HostIdMap, dynamicServerTopoNodeHierarchyMap); + appResourceScope, servers); authResult.addRequiredPermissions(ActionId.QUICK_EXECUTE_SCRIPT, hostResources); log.debug("Auth execute script, authResult:{}", authResult); return authResult; } public AuthResult authFastPushFile(String username, AppResourceScope appResourceScope, ServersDTO servers) { - Map ip2HostIdMap = new HashMap<>(); - Map dynamicServerTopoNodeHierarchyMap = new HashMap<>(); - List hostInstanceList = buildHostInstances(appResourceScope, servers, ip2HostIdMap, - dynamicServerTopoNodeHierarchyMap); + List hostInstanceList = buildHostInstances(appResourceScope, servers); log.debug("Auth Fast transfer file, username:{}, appResourceScope:{}, hostInstances:{}", username, appResourceScope, hostInstanceList); @@ -142,9 +125,7 @@ public AuthResult authFastPushFile(String username, AppResourceScope appResource AuthResult authResult = AuthResult.fail(); - List hostResources = convertHostsToPermissionResourceList(appResourceScope, servers, - ip2HostIdMap, - dynamicServerTopoNodeHierarchyMap); + List hostResources = convertHostsToPermissionResourceList(appResourceScope, servers); authResult.addRequiredPermissions(ActionId.QUICK_TRANSFER_FILE, hostResources); log.debug("Auth execute script, authResult:{}", authResult); return authResult; @@ -152,10 +133,7 @@ public AuthResult authFastPushFile(String username, AppResourceScope appResource public AuthResult authExecuteAppScript(String username, AppResourceScope appResourceScope, String scriptId, String scriptName, ServersDTO servers) { - Map ip2HostIdMap = new HashMap<>(); - Map dynamicServerTopoNodeHierarchyMap = new HashMap<>(); - List hostInstanceList = buildHostInstances(appResourceScope, servers, ip2HostIdMap, - dynamicServerTopoNodeHierarchyMap); + List hostInstanceList = buildHostInstances(appResourceScope, servers); InstanceDTO scriptInstance = buildExecutableInstance(appResourceScope, ResourceTypeEnum.SCRIPT, scriptId, null); @@ -183,7 +161,7 @@ public AuthResult authExecuteAppScript(String username, AppResourceScope appReso authResult.addRequiredPermission(ActionId.EXECUTE_SCRIPT, scriptResource); List hostResources = convertHostsToPermissionResourceList( - appResourceScope, servers, ip2HostIdMap, dynamicServerTopoNodeHierarchyMap); + appResourceScope, servers); authResult.addRequiredPermissions(ActionId.EXECUTE_SCRIPT, hostResources); log.debug("Auth execute script, authResult:{}", authResult); return authResult; @@ -206,10 +184,7 @@ private InstanceDTO buildExecutableInstance(AppResourceScope appResourceScope, R public AuthResult authExecutePublicScript(String username, AppResourceScope appResourceScope, String scriptId, String scriptName, ServersDTO servers) { - Map ip2HostIdMap = new HashMap<>(); - Map dynamicServerTopoNodeHierarchyMap = new HashMap<>(); - List hostInstanceList = buildHostInstances(appResourceScope, servers, ip2HostIdMap, - dynamicServerTopoNodeHierarchyMap); + List hostInstanceList = buildHostInstances(appResourceScope, servers); InstanceDTO scriptInstance = buildExecutableInstance( appResourceScope, ResourceTypeEnum.PUBLIC_SCRIPT, scriptId, null); @@ -239,7 +214,7 @@ public AuthResult authExecutePublicScript(String username, AppResourceScope appR authResult.addRequiredPermission(ActionId.EXECUTE_PUBLIC_SCRIPT, scriptResource); List hostResources = convertHostsToPermissionResourceList( - appResourceScope, servers, ip2HostIdMap, dynamicServerTopoNodeHierarchyMap); + appResourceScope, servers); authResult.addRequiredPermissions(ActionId.EXECUTE_PUBLIC_SCRIPT, hostResources); log.debug("Auth execute script, authResult:{}", authResult); return authResult; @@ -247,10 +222,7 @@ public AuthResult authExecutePublicScript(String username, AppResourceScope appR public AuthResult authExecutePlan(String username, AppResourceScope appResourceScope, Long templateId, Long planId, String planName, ServersDTO servers) { - Map ip2HostIdMap = new HashMap<>(); - Map dynamicServerTopoNodeHierarchyMap = new HashMap<>(); - List hostInstanceList = buildHostInstances(appResourceScope, servers, ip2HostIdMap, - dynamicServerTopoNodeHierarchyMap); + List hostInstanceList = buildHostInstances(appResourceScope, servers); InstanceDTO planInstance = buildExecutableInstance( appResourceScope, @@ -280,7 +252,7 @@ public AuthResult authExecutePlan(String username, AppResourceScope appResourceS authResult.addRequiredPermission(ActionId.LAUNCH_JOB_PLAN, planResource); List hostResources = convertHostsToPermissionResourceList( - appResourceScope, servers, ip2HostIdMap, dynamicServerTopoNodeHierarchyMap); + appResourceScope, servers); authResult.addRequiredPermissions(ActionId.LAUNCH_JOB_PLAN, hostResources); log.debug("Auth execute plan, authResult:{}", authResult); return authResult; @@ -289,10 +261,7 @@ public AuthResult authExecutePlan(String username, AppResourceScope appResourceS @Override public AuthResult authDebugTemplate(String username, AppResourceScope appResourceScope, Long templateId, ServersDTO servers) { - Map ip2HostIdMap = new HashMap<>(); - Map dynamicServerTopoNodeHierarchyMap = new HashMap<>(); - List hostInstanceList = buildHostInstances(appResourceScope, servers, ip2HostIdMap, - dynamicServerTopoNodeHierarchyMap); + List hostInstanceList = buildHostInstances(appResourceScope, servers); InstanceDTO jobTemplateInstance = buildExecutableInstance(appResourceScope, ResourceTypeEnum.TEMPLATE, templateId.toString(), null); @@ -317,7 +286,7 @@ public AuthResult authDebugTemplate(String username, AppResourceScope appResourc authResult.addRequiredPermission(ActionId.DEBUG_JOB_TEMPLATE, jobTemplateResource); List hostResources = convertHostsToPermissionResourceList( - appResourceScope, servers, ip2HostIdMap, dynamicServerTopoNodeHierarchyMap); + appResourceScope, servers); authResult.addRequiredPermissions(ActionId.DEBUG_JOB_TEMPLATE, hostResources); log.debug("Auth execute job template, authResult:{}", authResult); return authResult; @@ -333,22 +302,11 @@ private List buildAppTopoNodeHostInstances(AppResourceScope appReso return topoNodeInstanceList; } - private List buildBizStaticHostInstances( - AppResourceScope appResourceScope, - ServersDTO servers, - Map ip2HostIdMap - ) { + private List buildBizStaticHostInstances(AppResourceScope appResourceScope, + ServersDTO servers) { List hostInstanceList = new ArrayList<>(); - Map appHosts = - hostService.batchGetHosts(servers.getStaticIpList()); - servers.getStaticIpList().forEach(hostIp -> { + servers.getStaticIpList().forEach(host -> { InstanceDTO hostInstance = new InstanceDTO(); - ServiceHostDTO host = appHosts.get(hostIp); - if (host == null) { - log.warn("Host: {} is not exist!", hostIp); - throw new FailedPreconditionException(ErrorCode.SERVER_UNREGISTERED, - new Object[]{hostIp.getIp()}); - } String hostIdStr = host.getHostId().toString(); hostInstance.setId(hostIdStr); hostInstance.setType(ResourceTypeEnum.HOST.getId()); @@ -357,24 +315,18 @@ private List buildBizStaticHostInstances( hostInstance.setPath( buildAppScopeResourcePath(appResourceScope, ResourceTypeEnum.HOST, hostIdStr)); hostInstanceList.add(hostInstance); - - ip2HostIdMap.put(hostIp.convertToStrIp(), host.getHostId().toString()); }); return hostInstanceList; } - private List buildHostInstances( - AppResourceScope appResourceScope, - ServersDTO servers, - Map ip2HostIdMap, - Map dynamicServerTopoNodeHierarchyMap - ) { + private List buildHostInstances(AppResourceScope appResourceScope, + ServersDTO servers) { List hostInstanceList = new ArrayList<>(); // 静态IP if (!CollectionUtils.isEmpty(servers.getStaticIpList())) { switch (appResourceScope.getType()) { case BIZ: - hostInstanceList.addAll(buildBizStaticHostInstances(appResourceScope, servers, ip2HostIdMap)); + hostInstanceList.addAll(buildBizStaticHostInstances(appResourceScope, servers)); break; case BIZ_SET: InstanceDTO hostInstance = new InstanceDTO(); @@ -411,15 +363,14 @@ private List buildHostInstances( return hostInstanceList; } - private List convertBizStaticIpToPermissionResourceList(ServersDTO servers, - Map ip2HostIdMap) { + private List convertBizStaticIpToPermissionResourceList(ServersDTO servers) { List hostResources = new ArrayList<>(); - servers.getStaticIpList().forEach(ipDTO -> { + servers.getStaticIpList().forEach(host -> { PermissionResource resource = new PermissionResource(); - resource.setResourceId(ip2HostIdMap.get(ipDTO.convertToStrIp())); + resource.setResourceId(String.valueOf(host.getHostId())); resource.setResourceType(ResourceTypeEnum.HOST); resource.setSubResourceType("host"); - resource.setResourceName(ipDTO.getIp()); + resource.setResourceName(host.getIp()); resource.setSystemId(SystemId.CMDB); resource.setType("host"); hostResources.add(resource); @@ -439,9 +390,7 @@ private String getResourceName(AppResourceScope appResourceScope) { return resourceNameQueryService.getResourceName(resourceType, resourceId); } - private List convertBizSetStaticIpToPermissionResourceList(AppResourceScope appResourceScope, - ServersDTO servers, - Map ip2HostIdMap) { + private List convertBizSetStaticIpToPermissionResourceList(AppResourceScope appResourceScope) { List hostResources = new ArrayList<>(); PermissionResource resource = new PermissionResource(); resource.setResourceId(appResourceScope.getId()); @@ -486,26 +435,22 @@ private List convertDynamicGroupsToPermissionResourceList(Ap return hostResources; } - private List convertHostsToPermissionResourceList( - AppResourceScope appResourceScope, - ServersDTO servers, - Map ip2HostIdMap, - Map dynamicServerTopoNodeHierarchyMap - ) { + private List convertHostsToPermissionResourceList(AppResourceScope appResourceScope, + ServersDTO servers) { List hostResources = new ArrayList<>(); if (!CollectionUtils.isEmpty(servers.getStaticIpList())) { switch (appResourceScope.getType()) { case BIZ: - hostResources.addAll(convertBizStaticIpToPermissionResourceList(servers, ip2HostIdMap)); + hostResources.addAll(convertBizStaticIpToPermissionResourceList(servers)); break; case BIZ_SET: hostResources.addAll( - convertBizSetStaticIpToPermissionResourceList(appResourceScope, servers, ip2HostIdMap)); + convertBizSetStaticIpToPermissionResourceList(appResourceScope)); break; default: throw new NotImplementedException( - "Unsupport appScopeType:" + appResourceScope.getType().getValue(), + "Unsupported appScopeType:" + appResourceScope.getType().getValue(), ErrorCode.NOT_SUPPORT_FEATURE); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/GseTaskLogDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/exception/RollingExprParseException.java similarity index 78% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/GseTaskLogDAO.java rename to src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/exception/RollingExprParseException.java index 4d97cad12b..77892a5686 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/GseTaskLogDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/exception/RollingExprParseException.java @@ -22,17 +22,17 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.execute.dao; +package com.tencent.bk.job.execute.common.exception; -import com.tencent.bk.job.execute.model.GseTaskLogDTO; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.FailedPreconditionException; -public interface GseTaskLogDAO { - GseTaskLogDTO getStepLastExecuteLog(long stepInstanceId); - - void saveGseTaskLog(GseTaskLogDTO gseTaskLog); - - GseTaskLogDTO getGseTaskLog(long stepInstanceId, int executeCount); - - void deleteGseTaskLog(long stepInstanceId, int executeCount); +/** + * 滚动执行策略表达式解析异常 + */ +public class RollingExprParseException extends FailedPreconditionException { + public RollingExprParseException() { + super(ErrorCode.INVALID_ROLLING_EXPR); + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/util/JooqDataTypeUtil.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/util/JooqDataTypeUtil.java index d98bcc27af..763a24345e 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/util/JooqDataTypeUtil.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/util/JooqDataTypeUtil.java @@ -25,54 +25,35 @@ package com.tencent.bk.job.execute.common.util; import org.jooq.types.UByte; -import org.jooq.types.UInteger; -import org.jooq.types.ULong; public class JooqDataTypeUtil { - public static ULong getULongFromLong(Long value) { - if (value == null) { - return null; - } - return ULong.valueOf(value); - } - public static UInteger getUIntegerFromInteger(Integer value) { + public static UByte toUByte(Integer value) { if (value == null) { return null; } - return UInteger.valueOf(value); + return UByte.valueOf(value); } - public static UByte getUByteFromInteger(Integer value) { + public static Byte toByte(Integer value) { if (value == null) { return null; } - return UByte.valueOf(value); + return Byte.valueOf(String.valueOf(value)); } - public static Byte getByteFromInteger(Integer value) { + public static Short toShort(Integer value) { if (value == null) { return null; } - return Byte.valueOf(String.valueOf(value)); + return Short.valueOf(String.valueOf(value)); } - public static Integer getIntegerFromByte(Byte value) { + public static Integer toInteger(Byte value) { if (value == null) { return null; } return value.intValue(); } - public static Long getLongFromULong(ULong value) { - return value == null ? null : value.longValue(); - } - - public static Integer getIntegerFromUInteger(UInteger value) { - return value == null ? null : value.intValue(); - } - - public static Double parseDouble(Float value) { - return value == null ? null : value.doubleValue(); - } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/StepOperationEnum.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/StepOperationEnum.java index 85061bf151..aa1c4db510 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/StepOperationEnum.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/StepOperationEnum.java @@ -35,7 +35,8 @@ public enum StepOperationEnum { CONFIRM_TERMINATE(9, "终止确认流程"), CONFIRM_RESTART(10, "重新发起确认"), NEXT_STEP(11, "进入下一步"), - SKIP(12, "强制跳过"); + SKIP(12, "强制跳过"), + ROLLING_CONTINUE(13, "继续滚动"); private final Integer value; private final String name; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileAgentTaskDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileAgentTaskDAO.java new file mode 100644 index 0000000000..c788a5fe46 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileAgentTaskDAO.java @@ -0,0 +1,163 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao; + +import com.tencent.bk.job.common.constant.Order; +import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.model.AgentTaskResultGroupBaseDTO; +import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; + +import java.util.Collection; +import java.util.List; + +/** + * FileAgentTaskDAO + */ +public interface FileAgentTaskDAO { + /** + * 批量新增Agent任务 + * + * @param agentTasks Agent任务列表 + */ + void batchSaveAgentTasks(Collection agentTasks); + + /** + * 批量更新Agent任务 + * + * @param agentTasks Agent任务 + */ + void batchUpdateAgentTasks(Collection agentTasks); + + /** + * 获取步骤成功执行的Agent任务数量 + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @return 步骤成功执行的Agent任务数量 + */ + int getSuccessAgentTaskCount(long stepInstanceId, int executeCount); + + /** + * 查询执行结果分组 + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 + * @return 执行结果分组 + */ + List listResultGroups(long stepInstanceId, int executeCount, Integer batch); + + /** + * 根据执行结果查询Agent任务 + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 + * @param status 任务状态 + * @return Agent任务 + */ + List listAgentTaskByResultGroup(Long stepInstanceId, + Integer executeCount, + Integer batch, + Integer status); + + /** + * 根据执行结果查询Agent任务(排序、限制返回数量) + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 + * @param status 任务状态 + * @param limit 最大返回数量 + * @param orderField 排序字段 + * @param order 排序方式 + * @return Agent任务 + */ + List listAgentTaskByResultGroup(Long stepInstanceId, + Integer executeCount, + Integer batch, + Integer status, + Integer limit, + String orderField, + Order order); + + /** + * 获取agent任务 + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @param batch 滚动执行批次；传入null或者0将忽略该参数 + * @param fileTaskMode 文件分发任务模式;传入null表示忽略该过滤条件 + * @return agent任务 + */ + List listAgentTasks(Long stepInstanceId, + Integer executeCount, + Integer batch, + FileTaskModeEnum fileTaskMode); + + /** + * 根据GSE任务ID获取agent任务 + * + * @param gseTaskId GSE任务ID + * @return agent任务 + */ + List listAgentTasksByGseTaskId(Long gseTaskId); + + /** + * 根据hostId查询Agent任务 + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @param batch 滚动执行批次；传入null或者0将忽略该参数 + * @param mode 文件分发任务模式 + * @param hostId 主机ID + * @return Agent任务 + */ + AgentTaskDTO getAgentTaskByHostId(Long stepInstanceId, Integer executeCount, Integer batch, + FileTaskModeEnum mode, long hostId); + + /** + * 判断步骤实例的Agent Task 记录是否存在 + * + * @param stepInstanceId 步骤实例ID + */ + boolean isStepInstanceRecordExist(long stepInstanceId); + + /** + * 批量更新AgentTask的字段 + * + * @param stepInstanceId 条件 - 步骤实例ID + * @param executeCount 条件 - 重试次数 + * @param batch 条件 - 滚动执行批次；传入null将忽略该条件 + * @param actualExecuteCount 值 - Agent任务实际执行的步骤重试次数；如果传入null，则不更新 + * @param gseTaskId 值 - Agent任务对应的GSE_TASK_ID；如果传入null，则不更新 + */ + void updateAgentTaskFields(long stepInstanceId, + int executeCount, + Integer batch, + Integer actualExecuteCount, + Long gseTaskId); + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/GseTaskDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/GseTaskDAO.java new file mode 100644 index 0000000000..572c693b55 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/GseTaskDAO.java @@ -0,0 +1,68 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao; + +import com.tencent.bk.job.execute.model.GseTaskDTO; + +/** + * GseTaskDAO + */ +public interface GseTaskDAO { + + /** + * 保存 GSE 任务 + * + * @param gseTask GSE 任务 + * @return GSE任务ID + */ + long saveGseTask(GseTaskDTO gseTask); + + /** + * 更新 GSE 任务 + * + * @param gseTask GSE 任务 + * @return 是否更新成功 + */ + boolean updateGseTask(GseTaskDTO gseTask); + + /** + * 获取 GSE 任务 + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 步骤执行次数 + * @param batch 滚动执行批次 + * @return GSE 任务 + */ + GseTaskDTO getGseTask(long stepInstanceId, int executeCount, Integer batch); + + /** + * 获取 GSE 任务 + * + * @param gseTaskId GSE任务ID + * @return GSE 任务 + */ + GseTaskDTO getGseTask(long gseTaskId); + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/GseTaskIpLogDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/GseTaskIpLogDAO.java index c3c5285e86..620e7f15c7 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/GseTaskIpLogDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/GseTaskIpLogDAO.java @@ -24,26 +24,28 @@ package com.tencent.bk.job.execute.dao; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; import com.tencent.bk.job.common.constant.Order; -import com.tencent.bk.job.execute.engine.consts.IpStatus; -import com.tencent.bk.job.execute.model.GseTaskIpLogDTO; -import com.tencent.bk.job.execute.model.ResultGroupBaseDTO; +import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.model.AgentTaskResultGroupBaseDTO; -import java.util.Collection; import java.util.List; /** * GseTaskIpLogDAO */ +@CompatibleImplementation(name = "rolling_execute", explain = "兼容老版本数据，过1-2个大版本之后删除", version = "3.7.x") +@Deprecated public interface GseTaskIpLogDAO { - void batchSaveIpLog(List ipLogList); - void batchUpdateIpLog(long stepInstanceId, int executeCount, Collection cloudAreaAndIps, Long startTime, - Long endTime, IpStatus ipStatus); - - int getSuccessIpCount(long stepInstanceId, int executeCount); - - List getSuccessGseTaskIp(long stepInstanceId, int executeCount); + /** + * 获取步骤成功执行的Agent任务数量 + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @return 步骤成功执行的Agent任务数量 + */ + int getSuccessAgentTaskCount(long stepInstanceId, int executeCount); /** * 查询执行结果分组 @@ -52,50 +54,68 @@ void batchUpdateIpLog(long stepInstanceId, int executeCount, Collection * @param executeCount 执行次数 * @return 执行结果分组 */ - List getResultGroups(long stepInstanceId, int executeCount); - - List getIpLogByResultType(Long stepInstanceId, Integer executeCount, Integer resultType, - String tag); - - List getIpLogByResultType(Long stepInstanceId, Integer executeCount, Integer resultType, - String tag, Integer limit, String orderField, Order order); + List listResultGroups(long stepInstanceId, int executeCount); /** - * 获取agent任务信息 + * 根据执行结果查询Agent任务 * * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 - * @param onlyTargetIp 是否仅返回目标服务器IP - * @return agent任务信息 + * @param status 任务状态 + * @param tag 用户自定义分组标签 + * @return Agent任务 */ - List getIpLog(Long stepInstanceId, Integer executeCount, boolean onlyTargetIp); - - GseTaskIpLogDTO getIpLogByIp(Long stepInstanceId, Integer executeCount, String ip); - - List getIpLogByIps(Long stepInstanceId, Integer executeCount, String[] ipArray); - - void deleteAllIpLog(long stepInstanceId, int executeCount); - - int getSuccessRetryCount(long stepInstanceId, String cloudAreaAndIp); + List listAgentTaskByResultGroup(Long stepInstanceId, + Integer executeCount, + Integer status, + String tag); /** - * 获取文件任务源ip + * 根据执行结果查询Agent任务(排序、限制返回数量) * * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 - * @return 文件任务源ip + * @param status 任务状态 + * @param tag 用户自定义分组标签 + * @param limit 最大返回数量 + * @param orderField 排序字段 + * @param order 排序方式 + * @return Agent任务 */ - List getTaskFileSourceIps(Long stepInstanceId, Integer executeCount); + List listAgentTaskByResultGroup(Long stepInstanceId, + Integer executeCount, + Integer status, + String tag, + Integer limit, + String orderField, + Order order); /** - * 获取任务目标ip - 根据ip模糊匹配 + * 获取Agent任务 * * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 - * @param searchIp 用于检索的IP - * @return 匹配的任务目标IP列表 + * @return Agent任务信息 */ - List fuzzySearchTargetIpsByIp(Long stepInstanceId, Integer executeCount, String searchIp); + List listAgentTasks(Long stepInstanceId, + Integer executeCount); + /** + * 根据hostId查询Agent任务 + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @param ip 主机ip + * @return Agent任务 + */ + AgentTaskDTO getAgentTaskByIp(Long stepInstanceId, Integer executeCount, String ip); + /** + * 获取Agent任务实际执行成功的executeCount值(重试场景) + * + * @param stepInstanceId 步骤实例ID + * @param cloudIp 主机ip + * @return Agent任务实际执行成功的executeCount值 + */ + int getActualSuccessExecuteCount(long stepInstanceId, String cloudIp); } diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BatchUpdateBizSetData.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/RollingConfigDAO.java similarity index 74% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BatchUpdateBizSetData.java rename to src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/RollingConfigDAO.java index 27d0a82b9c..cfdcf2e8cf 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BatchUpdateBizSetData.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/RollingConfigDAO.java @@ -22,31 +22,27 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.common.cc.model.bizset; +package com.tencent.bk.job.execute.dao; -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; +import com.tencent.bk.job.execute.model.RollingConfigDTO; /** - * CMDB接口请求实体类，用于批量更新业务集 + * 作业滚动配置DAO */ -@Setter -@Getter -@ToString -public class BatchUpdateBizSetData { - +public interface RollingConfigDAO { /** - * 业务集字段集合 + * 保存滚动配置 + * + * @param rollingConfig 滚动配置 + * @return 滚动配置ID */ - @JsonProperty("bk_biz_set_attr") - private BizSetAttr attr; + long saveRollingConfig(RollingConfigDTO rollingConfig); /** - * 业务集的业务选择范围 + * 通过滚动配置ID获取滚动配置 + * + * @param rollingConfigId 滚动配置ID + * @return 滚动配置 */ - @JsonProperty("bk_scope") - private BizSetScope scope; - + RollingConfigDTO queryRollingConfigById(Long rollingConfigId); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptAgentTaskDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptAgentTaskDAO.java new file mode 100644 index 0000000000..cd723f3ff4 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptAgentTaskDAO.java @@ -0,0 +1,161 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao; + +import com.tencent.bk.job.common.constant.Order; +import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.model.AgentTaskResultGroupBaseDTO; + +import java.util.Collection; +import java.util.List; + +/** + * ScriptAgentTaskDAO + */ +public interface ScriptAgentTaskDAO { + /** + * 批量保存Agent任务 + * + * @param agentTasks Agent任务 + */ + void batchSaveAgentTasks(Collection agentTasks); + + /** + * 批量更新Agent任务 + * + * @param agentTasks Agent任务 + */ + void batchUpdateAgentTasks(Collection agentTasks); + + /** + * 获取步骤成功执行的Agent任务数量 + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @return 步骤成功执行的Agent任务数量 + */ + int getSuccessAgentTaskCount(long stepInstanceId, int executeCount); + + /** + * 查询执行结果分组 + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 + * @return 执行结果分组 + */ + List listResultGroups(long stepInstanceId, int executeCount, Integer batch); + + /** + * 根据执行结果查询Agent任务 + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 + * @param status 任务状态 + * @param tag 用户自定义分组标签 + * @return Agent任务 + */ + List listAgentTaskByResultGroup(Long stepInstanceId, + Integer executeCount, + Integer batch, + Integer status, + String tag); + + /** + * 根据执行结果查询Agent任务(排序、限制返回数量) + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 + * @param status 任务状态 + * @param tag 用户自定义分组标签 + * @param limit 最大返回数量 + * @param orderField 排序字段 + * @param order 排序方式 + * @return Agent任务 + */ + List listAgentTaskByResultGroup(Long stepInstanceId, + Integer executeCount, + Integer batch, + Integer status, + String tag, + Integer limit, + String orderField, + Order order); + + /** + * 获取Agent任务 + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @param batch 滚动执行批次；传入null或者0将忽略该参数 + * @return Agent任务信息 + */ + List listAgentTasks(Long stepInstanceId, + Integer executeCount, + Integer batch); + + /** + * 根据GSE任务ID获取Agent任务 + * + * @param gseTaskId GSE任务ID + * @return Agent任务 + */ + List listAgentTasksByGseTaskId(Long gseTaskId); + + /** + * 根据hostId查询Agent任务 + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @param batch 滚动执行批次；传入null或者0将忽略该参数 + * @param hostId 主机ID + * @return Agent任务 + */ + AgentTaskDTO getAgentTaskByHostId(Long stepInstanceId, Integer executeCount, Integer batch, long hostId); + + /** + * 判断步骤实例的Agent Task 记录是否存在 + * + * @param stepInstanceId 步骤实例ID + */ + boolean isStepInstanceRecordExist(long stepInstanceId); + + /** + * 批量更新AgentTask的字段 + * + * @param stepInstanceId 条件 - 步骤实例ID + * @param executeCount 条件 - 重试次数 + * @param batch 条件 - 滚动执行批次；传入null将忽略该条件 + * @param actualExecuteCount 值 - Agent任务实际执行的步骤重试次数；如果传入null，则不更新 + * @param gseTaskId 值 - Agent任务对应的GSE_TASK_ID；如果传入null，则不更新 + */ + void updateAgentTaskFields(long stepInstanceId, + int executeCount, + Integer batch, + Integer actualExecuteCount, + Long gseTaskId); +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceDAO.java index 71e4b24cdb..88a7a2970b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceDAO.java @@ -28,7 +28,12 @@ import com.tencent.bk.job.common.constant.NotExistPathHandlerEnum; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; -import com.tencent.bk.job.execute.model.*; +import com.tencent.bk.job.execute.model.ConfirmStepInstanceDTO; +import com.tencent.bk.job.execute.model.FileSourceDTO; +import com.tencent.bk.job.execute.model.FileStepInstanceDTO; +import com.tencent.bk.job.execute.model.ScriptStepInstanceDTO; +import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; +import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; import java.util.List; @@ -44,6 +49,23 @@ public interface StepInstanceDAO { StepInstanceBaseDTO getStepInstanceBase(long stepInstanceId); + /** + * 获取作业的第一个步骤实例 + * + * @param taskInstanceId 作业实例ID + * @return 作业第一个步骤实例 + */ + StepInstanceBaseDTO getFirstStepInstanceBase(long taskInstanceId); + + /** + * 获取下一个步骤实例 + * + * @param taskInstanceId 作业实例ID + * @param currentStepOrder 当前步骤的顺序 + * @return 步骤实例；如果当前为最后一个步骤实例，那么返回null + */ + StepInstanceBaseDTO getNextStepInstance(long taskInstanceId, int currentStepOrder); + ScriptStepInstanceDTO getScriptStepInstance(long stepInstanceId); FileStepInstanceDTO getFileStepInstance(long stepInstanceId); @@ -72,18 +94,10 @@ public interface StepInstanceDAO { void updateStepEndTime(long stepInstanceId, Long endTime); - void addTaskExecuteCount(long taskInstanceId); + void addStepInstanceExecuteCount(long stepInstanceId); void updateStepTotalTime(long stepInstanceId, long totalTime); - void updateStepStatInfo(long stepInstanceId, int runIPNum, int successIPNum, int failIPNum); - - Long getFirstStepStartTime(long taskInstanceId); - - Long getLastStepEndTime(long taskInstanceId); - - long getAllStepTotalTime(long taskInstanceId); - /** * 更新步骤的执行信息 * @@ -96,21 +110,6 @@ public interface StepInstanceDAO { void updateStepExecutionInfo(long stepInstanceId, RunStatusEnum status, Long startTime, Long endTime, Long totalTime); - /** - * 更新步骤的执行信息 - * - * @param stepInstanceId 步骤实例ID - * @param status 步骤执行状态 - * @param startTime 开始时间 - * @param endTime 结束时间 - * @param totalTime 总耗时 - * @param runIPNum 运行中的ip - * @param successIPNum 执行成功的ip - * @param failIPNum 失败的ip - */ - void updateStepExecutionInfo(long stepInstanceId, RunStatusEnum status, Long startTime, Long endTime, - Long totalTime, Integer runIPNum, Integer successIPNum, Integer failIPNum); - /** * 更新解析之后的脚本参数 * @@ -188,4 +187,28 @@ List> listFastPushFileSource(Long appId, DuplicateHandlerEnu * @return */ Byte getScriptTypeByStepInstanceId(long stepInstanceId); + + /** + * 更新步骤实例的当前滚动执行批次 + * + * @param stepInstanceId 步骤实例ID + * @param batch 滚动执行批次 + */ + void updateStepCurrentBatch(long stepInstanceId, int batch); + + /** + * 更新步骤实例的当前滚动执行批次 + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + */ + void updateStepCurrentExecuteCount(long stepInstanceId, int executeCount); + + /** + * 更新步骤实例的滚动配置ID + * + * @param stepInstanceId 步骤实例ID + * @param rollingConfigId 滚动配置ID + */ + void updateStepRollingConfigId(long stepInstanceId, long rollingConfigId); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceRollingTaskDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceRollingTaskDAO.java new file mode 100644 index 0000000000..d7a0f9f3e6 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceRollingTaskDAO.java @@ -0,0 +1,88 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao; + +import com.tencent.bk.job.execute.common.constants.RunStatusEnum; +import com.tencent.bk.job.execute.model.StepInstanceRollingTaskDTO; + +import java.util.List; + +/** + * 步骤滚动任务DAO + */ +public interface StepInstanceRollingTaskDAO { + + /** + * 查询步骤滚动任务 + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 步骤执行次数 + * @param batch 滚动批次 + * @return 步骤滚动任务 + */ + StepInstanceRollingTaskDTO queryRollingTask(long stepInstanceId, + int executeCount, + int batch); + + /** + * 查询步骤滚动任务 + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数;如果不为null，会根据executeCount过滤 + * @param batch 滚动执行批次;如果不为null，会根据batch过滤 + * @return 步骤滚动任务 + */ + List listRollingTasks(long stepInstanceId, + Integer executeCount, + Integer batch); + + /** + * 保存步骤滚动任务 + * + * @param rollingTask 滚动任务 + * @return 步骤滚动任务ID + */ + long saveRollingTask(StepInstanceRollingTaskDTO rollingTask); + + /** + * 更新滚动任务 + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 步骤执行次数 + * @param batch 滚动执行批次 + * @param status 任务状态；如果不更新传入null + * @param startTime 任务开始时间；如果不更新传入null + * @param endTime 任务结束时间；如果不更新传入null + * @param totalTime 任务执行总时间；如果不更新传入null + */ + void updateRollingTask(long stepInstanceId, + int executeCount, + int batch, + RunStatusEnum status, + Long startTime, + Long endTime, Long totalTime); + + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/TaskInstanceDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/TaskInstanceDAO.java index feee900d36..210360024f 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/TaskInstanceDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/TaskInstanceDAO.java @@ -99,11 +99,11 @@ void updateTaskExecutionInfo(long taskInstanceId, RunStatusEnum status, Long cur Long startTime, Long endTime, Long totalTime); /** - * 作业恢复执行-重置作业执行状态 + * 重置作业执行状态 * * @param taskInstanceId 作业实例ID */ - void resetTaskExecuteInfoForResume(long taskInstanceId); + void resetTaskExecuteInfoForRetry(long taskInstanceId); /** * 根据条件统计执行过的任务 diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/DangerousRecordDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/DangerousRecordDAOImpl.java index 810392c4de..197276e50e 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/DangerousRecordDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/DangerousRecordDAOImpl.java @@ -99,10 +99,10 @@ private List buildSearchCondition(DangerousRecordDTO query, BaseSearc conditions.add(T.OPERATOR.eq(query.getOperator())); } if (query.getStartupMode() != null) { - conditions.add(T.STARTUP_MODE.eq(JooqDataTypeUtil.getByteFromInteger(query.getStartupMode()))); + conditions.add(T.STARTUP_MODE.eq(JooqDataTypeUtil.toByte(query.getStartupMode()))); } if (query.getAction() != null) { - conditions.add(T.ACTION.eq(JooqDataTypeUtil.getByteFromInteger(query.getAction()))); + conditions.add(T.ACTION.eq(JooqDataTypeUtil.toByte(query.getAction()))); } if (StringUtils.isNotEmpty(query.getClient())) { conditions.add(T.CLIENT.eq(query.getClient())); @@ -138,10 +138,10 @@ private DangerousRecordDTO extractInfo(Record record) { dangerousRecord.setAppName(record.get(T.APP_NAME)); dangerousRecord.setOperator(record.get(T.OPERATOR)); dangerousRecord.setCreateTime(record.get(T.CREATE_TIME)); - dangerousRecord.setStartupMode(JooqDataTypeUtil.getIntegerFromByte(record.get(T.STARTUP_MODE))); + dangerousRecord.setStartupMode(JooqDataTypeUtil.toInteger(record.get(T.STARTUP_MODE))); dangerousRecord.setClient(record.get(T.CLIENT)); - dangerousRecord.setAction(JooqDataTypeUtil.getIntegerFromByte(record.get(T.ACTION))); - dangerousRecord.setScriptLanguage(JooqDataTypeUtil.getIntegerFromByte(record.get(T.SCRIPT_LANGUAGE))); + dangerousRecord.setAction(JooqDataTypeUtil.toInteger(record.get(T.ACTION))); + dangerousRecord.setScriptLanguage(JooqDataTypeUtil.toInteger(record.get(T.SCRIPT_LANGUAGE))); dangerousRecord.setScriptContent(record.get(T.SCRIPT_CONTENT)); dangerousRecord.setCheckResult(JsonUtils.fromJson(record.get(T.CHECK_RESULT), new TypeReference() { @@ -163,12 +163,12 @@ public boolean saveDangerousRecord(DangerousRecordDTO record) { record.getAppId(), record.getAppName(), record.getOperator(), - JooqDataTypeUtil.getByteFromInteger(record.getScriptLanguage()), + JooqDataTypeUtil.toByte(record.getScriptLanguage()), record.getScriptContent(), record.getCreateTime(), - JooqDataTypeUtil.getByteFromInteger(record.getStartupMode()), + JooqDataTypeUtil.toByte(record.getStartupMode()), record.getClient(), - JooqDataTypeUtil.getByteFromInteger(record.getAction()), JsonUtils.toJson(record.getCheckResult()), + JooqDataTypeUtil.toByte(record.getAction()), JsonUtils.toJson(record.getCheckResult()), record.getExtData() == null ? "" : JsonUtils.toJson(record.getExtData())) .execute(); return count > 0; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java new file mode 100644 index 0000000000..80c3038a14 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java @@ -0,0 +1,393 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao.impl; + +import com.tencent.bk.job.common.constant.Order; +import com.tencent.bk.job.execute.dao.FileAgentTaskDAO; +import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; +import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.model.AgentTaskResultGroupBaseDTO; +import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.jooq.Condition; +import org.jooq.DSLContext; +import org.jooq.OrderField; +import org.jooq.Record; +import org.jooq.Result; +import org.jooq.SelectConditionStep; +import org.jooq.SelectLimitPercentStep; +import org.jooq.SelectSeekStep1; +import org.jooq.TableField; +import org.jooq.UpdateConditionStep; +import org.jooq.UpdateSetMoreStep; +import org.jooq.UpdateSetStep; +import org.jooq.generated.tables.GseFileAgentTask; +import org.jooq.generated.tables.records.GseFileAgentTaskRecord; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Repository; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import static com.tencent.bk.job.common.constant.Order.DESCENDING; +import static org.jooq.impl.DSL.count; + +@Repository +public class FileAgentTaskDAOImpl implements FileAgentTaskDAO { + + private static final GseFileAgentTask T_GSE_FILE_AGENT_TASK = GseFileAgentTask.GSE_FILE_AGENT_TASK; + private static final TableField[] ALL_FIELDS = { + T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID, + T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT, + T_GSE_FILE_AGENT_TASK.ACTUAL_EXECUTE_COUNT, + T_GSE_FILE_AGENT_TASK.BATCH, + T_GSE_FILE_AGENT_TASK.MODE, + T_GSE_FILE_AGENT_TASK.HOST_ID, + T_GSE_FILE_AGENT_TASK.AGENT_ID, + T_GSE_FILE_AGENT_TASK.GSE_TASK_ID, + T_GSE_FILE_AGENT_TASK.STATUS, + T_GSE_FILE_AGENT_TASK.START_TIME, + T_GSE_FILE_AGENT_TASK.END_TIME, + T_GSE_FILE_AGENT_TASK.TOTAL_TIME, + T_GSE_FILE_AGENT_TASK.ERROR_CODE + }; + + private final DSLContext CTX; + + @Autowired + public FileAgentTaskDAOImpl(@Qualifier("job-execute-dsl-context") DSLContext CTX) { + this.CTX = CTX; + } + + @Override + public void batchSaveAgentTasks(Collection agentTasks) { + String sql = "insert into gse_file_agent_task (step_instance_id, execute_count, actual_execute_count, batch," + + "mode, host_id, agent_id ,gse_task_id,status, start_time, end_time, total_time, error_code)" + + " values (?,?,?,?,?,?,?,?,?,?,?,?,?)"; + Object[][] params = new Object[agentTasks.size()][13]; + int batchCount = 0; + for (AgentTaskDTO agentTask : agentTasks) { + Object[] param = new Object[13]; + param[0] = agentTask.getStepInstanceId(); + param[1] = agentTask.getExecuteCount(); + param[2] = agentTask.getActualExecuteCount(); + param[3] = agentTask.getBatch(); + param[4] = agentTask.getFileTaskMode().getValue(); + param[5] = agentTask.getHostId(); + param[6] = agentTask.getAgentId(); + param[7] = agentTask.getGseTaskId(); + param[8] = agentTask.getStatus().getValue(); + param[9] = agentTask.getStartTime(); + param[10] = agentTask.getEndTime(); + param[11] = agentTask.getTotalTime(); + param[12] = agentTask.getErrorCode(); + params[batchCount++] = param; + } + CTX.batch(sql, params).execute(); + } + + @Override + public void batchUpdateAgentTasks(Collection agentTasks) { + if (CollectionUtils.isEmpty(agentTasks)) { + return; + } + String sql = "update gse_file_agent_task set gse_task_id = ?, status = ?, start_time = ?, end_time = ?" + + ", total_time = ?, error_code = ?" + + " where step_instance_id = ? and execute_count = ? and batch = ? and mode = ? and host_id = ?"; + Object[][] params = new Object[agentTasks.size()][11]; + int batchCount = 0; + for (AgentTaskDTO agentTask : agentTasks) { + Object[] param = new Object[11]; + param[0] = agentTask.getGseTaskId(); + param[1] = agentTask.getStatus().getValue(); + param[2] = agentTask.getStartTime(); + param[3] = agentTask.getEndTime(); + param[4] = agentTask.getTotalTime(); + param[5] = agentTask.getErrorCode(); + param[6] = agentTask.getStepInstanceId(); + param[7] = agentTask.getExecuteCount(); + param[8] = agentTask.getBatch(); + param[9] = agentTask.getFileTaskMode().getValue(); + param[10] = agentTask.getHostId(); + params[batchCount++] = param; + } + CTX.batch(sql, params).execute(); + } + + @Override + public int getSuccessAgentTaskCount(long stepInstanceId, int executeCount) { + Integer count = CTX.selectCount() + .from(T_GSE_FILE_AGENT_TASK) + .where(T_GSE_FILE_AGENT_TASK.STATUS.in(AgentTaskStatusEnum.LAST_SUCCESS.getValue(), + AgentTaskStatusEnum.SUCCESS.getValue())) + .and(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq((short) executeCount)) + .and(T_GSE_FILE_AGENT_TASK.MODE.eq(FileTaskModeEnum.DOWNLOAD.getValue().byteValue())) + .fetchOne(0, Integer.class); + return count == null ? 0 : count; + } + + @Override + public List listResultGroups(long stepInstanceId, int executeCount, Integer batch) { + SelectConditionStep selectConditionStep = + CTX.select(T_GSE_FILE_AGENT_TASK.STATUS, count().as("ip_count")) + .from(T_GSE_FILE_AGENT_TASK) + .where(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq((short) executeCount)) + .and(T_GSE_FILE_AGENT_TASK.MODE.eq(FileTaskModeEnum.DOWNLOAD.getValue().byteValue())); + if (batch != null && batch > 0) { + selectConditionStep.and(T_GSE_FILE_AGENT_TASK.BATCH.eq(batch.shortValue())); + } + + Result result = selectConditionStep.groupBy(T_GSE_FILE_AGENT_TASK.STATUS) + .orderBy(T_GSE_FILE_AGENT_TASK.STATUS.asc()) + .fetch(); + + List resultGroups = new ArrayList<>(); + result.forEach(record -> { + AgentTaskResultGroupBaseDTO resultGroup = new AgentTaskResultGroupBaseDTO(); + resultGroup.setStatus(record.get(T_GSE_FILE_AGENT_TASK.STATUS)); + resultGroup.setTag(""); + Object ipCount = record.get("ip_count"); + resultGroup.setTotalAgentTasks(ipCount == null ? 0 : (int) ipCount); + resultGroups.add(resultGroup); + }); + return resultGroups; + } + + @Override + public List listAgentTaskByResultGroup(Long stepInstanceId, + Integer executeCount, + Integer batch, + Integer status) { + SelectConditionStep selectConditionStep = CTX.select(ALL_FIELDS) + .from(T_GSE_FILE_AGENT_TASK) + .where(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())) + .and(T_GSE_FILE_AGENT_TASK.STATUS.eq(status)) + .and(T_GSE_FILE_AGENT_TASK.MODE.eq(FileTaskModeEnum.DOWNLOAD.getValue().byteValue())); + if (batch != null && batch > 0) { + selectConditionStep.and(T_GSE_FILE_AGENT_TASK.BATCH.eq(batch.shortValue())); + } + Result result = selectConditionStep.fetch(); + + List agentTasks = new ArrayList<>(); + if (result.size() > 0) { + result.forEach(record -> agentTasks.add(extract(record))); + } + return agentTasks; + } + + @Override + public List listAgentTaskByResultGroup(Long stepInstanceId, + Integer executeCount, + Integer batch, + Integer status, + Integer limit, + String orderField, + Order order) { + List conditions = new ArrayList<>(); + conditions.add(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)); + conditions.add(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())); + conditions.add(T_GSE_FILE_AGENT_TASK.STATUS.eq(status)); + conditions.add(T_GSE_FILE_AGENT_TASK.MODE.eq(FileTaskModeEnum.DOWNLOAD.getValue().byteValue())); + + SelectConditionStep select = CTX.select(ALL_FIELDS) + .from(T_GSE_FILE_AGENT_TASK) + .where(conditions); + + if (batch != null && batch > 0) { + select.and(T_GSE_FILE_AGENT_TASK.BATCH.eq(batch.shortValue())); + } + + SelectSeekStep1 selectSeekStep = null; + OrderField orderFieldEntity = buildOrderField(orderField, order); + if (orderFieldEntity != null) { + selectSeekStep = select.orderBy(orderFieldEntity); + } + + SelectLimitPercentStep selectLimitPercentStep = null; + if (limit != null && limit > 0) { + if (selectSeekStep != null) { + selectLimitPercentStep = selectSeekStep.limit(limit); + } else { + selectLimitPercentStep = select.limit(limit); + } + } + + List agentTasks = new ArrayList<>(); + Result result; + if (selectLimitPercentStep != null) { + result = selectLimitPercentStep.fetch(); + } else if (selectSeekStep != null) { + result = selectSeekStep.fetch(); + } else { + result = select.fetch(); + } + + if (result.size() > 0) { + result.into(record -> agentTasks.add(extract(record))); + } + return agentTasks; + } + + private OrderField buildOrderField(String field, Order order) { + OrderField orderField = null; + if (StringUtils.isNotBlank(field)) { + if (field.equals(T_GSE_FILE_AGENT_TASK.TOTAL_TIME.getName())) { + if (order == DESCENDING) { + orderField = T_GSE_FILE_AGENT_TASK.TOTAL_TIME.desc(); + } else { + orderField = T_GSE_FILE_AGENT_TASK.TOTAL_TIME.asc(); + } + } + } + return orderField; + } + + @Override + public List listAgentTasks(Long stepInstanceId, + Integer executeCount, + Integer batch, + FileTaskModeEnum fileTaskMode) { + SelectConditionStep selectConditionStep = CTX.select(ALL_FIELDS) + .from(T_GSE_FILE_AGENT_TASK) + .where(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())); + if (batch != null && batch > 0) { + selectConditionStep.and(T_GSE_FILE_AGENT_TASK.BATCH.eq(batch.shortValue())); + } + if (fileTaskMode != null) { + selectConditionStep.and(T_GSE_FILE_AGENT_TASK.MODE.eq(fileTaskMode.getValue().byteValue())); + } + Result result = selectConditionStep.fetch(); + List agentTaskList = new ArrayList<>(); + if (result.size() != 0) { + result.map(record -> { + agentTaskList.add(extract(record)); + return null; + }); + } + return agentTaskList; + } + + private AgentTaskDTO extract(Record record) { + if (record == null) { + return null; + } + AgentTaskDTO agentTask = new AgentTaskDTO(); + agentTask.setStepInstanceId(record.get(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID)); + agentTask.setExecuteCount(record.get(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT)); + Short actualExecuteCount = record.get(T_GSE_FILE_AGENT_TASK.ACTUAL_EXECUTE_COUNT); + agentTask.setActualExecuteCount(actualExecuteCount != null ? actualExecuteCount.intValue() : null); + agentTask.setBatch(record.get(T_GSE_FILE_AGENT_TASK.BATCH)); + agentTask.setFileTaskMode(FileTaskModeEnum.getFileTaskMode(record.get(T_GSE_FILE_AGENT_TASK.MODE).intValue())); + agentTask.setHostId(record.get(T_GSE_FILE_AGENT_TASK.HOST_ID)); + agentTask.setAgentId(record.get(T_GSE_FILE_AGENT_TASK.AGENT_ID)); + agentTask.setGseTaskId(record.get(T_GSE_FILE_AGENT_TASK.GSE_TASK_ID)); + agentTask.setStatus(AgentTaskStatusEnum.valueOf(record.get(T_GSE_FILE_AGENT_TASK.STATUS))); + agentTask.setStartTime(record.get(T_GSE_FILE_AGENT_TASK.START_TIME)); + agentTask.setEndTime(record.get(T_GSE_FILE_AGENT_TASK.END_TIME)); + agentTask.setTotalTime(record.get(T_GSE_FILE_AGENT_TASK.TOTAL_TIME)); + agentTask.setErrorCode(record.get(T_GSE_FILE_AGENT_TASK.ERROR_CODE)); + return agentTask; + } + + @Override + public List listAgentTasksByGseTaskId(Long gseTaskId) { + if (gseTaskId == null || gseTaskId <= 0) { + return Collections.emptyList(); + } + + List agentTaskList = new ArrayList<>(); + + Result result = CTX.select(ALL_FIELDS) + .from(T_GSE_FILE_AGENT_TASK) + .where(T_GSE_FILE_AGENT_TASK.GSE_TASK_ID.eq(gseTaskId)) + .fetch(); + if (result.size() > 0) { + result.forEach(record -> agentTaskList.add(extract(record))); + } + return agentTaskList; + } + + @Override + public AgentTaskDTO getAgentTaskByHostId(Long stepInstanceId, Integer executeCount, Integer batch, + FileTaskModeEnum mode, long hostId) { + Record record = CTX.select(ALL_FIELDS) + .from(T_GSE_FILE_AGENT_TASK) + .where(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())) + .and(T_GSE_FILE_AGENT_TASK.BATCH.eq(batch == null ? 0 : batch.shortValue())) + .and(T_GSE_FILE_AGENT_TASK.MODE.eq(mode.getValue().byteValue())) + .and(T_GSE_FILE_AGENT_TASK.HOST_ID.eq(hostId)) + .fetchOne(); + return extract(record); + } + + @Override + public boolean isStepInstanceRecordExist(long stepInstanceId) { + return CTX.fetchExists(T_GSE_FILE_AGENT_TASK, T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)); + } + + @Override + public void updateAgentTaskFields(long stepInstanceId, + int executeCount, + Integer batch, + Integer actualExecuteCount, + Long gseTaskId) { + UpdateSetStep updateSetStep = CTX.update(T_GSE_FILE_AGENT_TASK); + boolean needUpdate = false; + if (actualExecuteCount != null) { + updateSetStep = updateSetStep.set(T_GSE_FILE_AGENT_TASK.ACTUAL_EXECUTE_COUNT, + actualExecuteCount.shortValue()); + needUpdate = true; + } + if (gseTaskId != null) { + updateSetStep = updateSetStep.set(T_GSE_FILE_AGENT_TASK.GSE_TASK_ID, gseTaskId); + needUpdate = true; + } + + if (!needUpdate) { + return; + } + + UpdateSetMoreStep updateSetMoreStep = + (UpdateSetMoreStep) updateSetStep; + + UpdateConditionStep updateConditionStep = + updateSetMoreStep + .where(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq((short) executeCount)); + if (batch != null) { + updateConditionStep.and(T_GSE_FILE_AGENT_TASK.BATCH.eq(batch.shortValue())); + } + updateConditionStep.execute(); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileSourceTaskLogDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileSourceTaskLogDAOImpl.java index fa2485e490..42daf5870d 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileSourceTaskLogDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileSourceTaskLogDAOImpl.java @@ -87,13 +87,13 @@ public void saveFileSourceTaskLog(FileSourceTaskLogDTO fileSourceTaskLog) { fileSourceTaskLog.getStartTime(), fileSourceTaskLog.getEndTime(), fileSourceTaskLog.getTotalTime(), - JooqDataTypeUtil.getByteFromInteger(fileSourceTaskLog.getStatus()), + JooqDataTypeUtil.toByte(fileSourceTaskLog.getStatus()), fileSourceTaskLog.getFileSourceBatchTaskId()) .onDuplicateKeyUpdate() .set(t.START_TIME, fileSourceTaskLog.getStartTime()) .set(t.END_TIME, fileSourceTaskLog.getEndTime()) .set(t.TOTAL_TIME, fileSourceTaskLog.getTotalTime()) - .set(t.STATUS, JooqDataTypeUtil.getByteFromInteger(fileSourceTaskLog.getStatus())).set(t.FILE_SOURCE_BATCH_TASK_ID, fileSourceTaskLog.getFileSourceBatchTaskId()) + .set(t.STATUS, JooqDataTypeUtil.toByte(fileSourceTaskLog.getStatus())).set(t.FILE_SOURCE_BATCH_TASK_ID, fileSourceTaskLog.getFileSourceBatchTaskId()) .execute(); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/GseTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/GseTaskDAOImpl.java new file mode 100644 index 0000000000..ac88f92822 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/GseTaskDAOImpl.java @@ -0,0 +1,126 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao.impl; + +import com.tencent.bk.job.execute.common.util.JooqDataTypeUtil; +import com.tencent.bk.job.execute.dao.GseTaskDAO; +import com.tencent.bk.job.execute.model.GseTaskDTO; +import org.jooq.DSLContext; +import org.jooq.Record; +import org.jooq.TableField; +import org.jooq.generated.tables.GseTask; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Repository; + +@Repository +public class GseTaskDAOImpl implements GseTaskDAO { + private final DSLContext dslContext; + + private static final GseTask TABLE = GseTask.GSE_TASK; + private static final TableField[] ALL_FIELDS = {TABLE.ID, TABLE.STEP_INSTANCE_ID, TABLE.EXECUTE_COUNT, + TABLE.BATCH, TABLE.START_TIME, TABLE.END_TIME, TABLE.TOTAL_TIME, TABLE.STATUS, TABLE.GSE_TASK_ID}; + + @Autowired + public GseTaskDAOImpl(@Qualifier("job-execute-dsl-context") DSLContext dslContext) { + this.dslContext = dslContext; + } + + private GseTaskDTO extractInfo(Record record) { + if (record == null) { + return null; + } + GseTaskDTO gseTaskDTO = new GseTaskDTO(); + + gseTaskDTO.setId(record.get(TABLE.ID)); + gseTaskDTO.setStepInstanceId(record.get(TABLE.STEP_INSTANCE_ID)); + gseTaskDTO.setExecuteCount(record.get(TABLE.EXECUTE_COUNT).intValue()); + gseTaskDTO.setBatch(record.get(TABLE.BATCH)); + gseTaskDTO.setStartTime(record.get(TABLE.START_TIME)); + gseTaskDTO.setEndTime(record.get(TABLE.END_TIME)); + gseTaskDTO.setTotalTime(record.get(TABLE.TOTAL_TIME)); + gseTaskDTO.setStatus(record.get(TABLE.STATUS).intValue()); + gseTaskDTO.setGseTaskId(record.get(TABLE.GSE_TASK_ID)); + return gseTaskDTO; + } + + @Override + public long saveGseTask(GseTaskDTO gseTask) { + Record record = dslContext.insertInto( + TABLE, + TABLE.STEP_INSTANCE_ID, + TABLE.EXECUTE_COUNT, + TABLE.BATCH, + TABLE.START_TIME, + TABLE.END_TIME, + TABLE.TOTAL_TIME, + TABLE.STATUS, + TABLE.GSE_TASK_ID) + .values( + gseTask.getStepInstanceId(), + gseTask.getExecuteCount().shortValue(), + (short) gseTask.getBatch(), + gseTask.getStartTime(), + gseTask.getEndTime(), + gseTask.getTotalTime(), + JooqDataTypeUtil.toByte(gseTask.getStatus()), + gseTask.getGseTaskId()) + .returning(TABLE.ID) + .fetchOne(); + + return record == null ? 0 : record.get(TABLE.ID); + } + + @Override + public boolean updateGseTask(GseTaskDTO gseTask) { + int affectRows = dslContext.update(TABLE) + .set(TABLE.START_TIME, gseTask.getStartTime()) + .set(TABLE.END_TIME, gseTask.getEndTime()) + .set(TABLE.TOTAL_TIME, gseTask.getTotalTime()) + .set(TABLE.STATUS, gseTask.getStatus().byteValue()) + .set(TABLE.GSE_TASK_ID, gseTask.getGseTaskId()) + .where(TABLE.ID.eq(gseTask.getId())) + .execute(); + return affectRows > 0; + } + + @Override + public GseTaskDTO getGseTask(long stepInstanceId, int executeCount, Integer batch) { + Record record = dslContext.select(ALL_FIELDS).from(TABLE) + .where(TABLE.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(TABLE.EXECUTE_COUNT.eq((short) executeCount)) + .and(TABLE.BATCH.eq(batch == null ? 0 : batch.shortValue())) + .fetchOne(); + return extractInfo(record); + } + + @Override + public GseTaskDTO getGseTask(long gseTaskId) { + Record record = dslContext.select(ALL_FIELDS).from(TABLE) + .where(TABLE.ID.eq(gseTaskId)) + .fetchOne(); + return extractInfo(record); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/GseTaskIpLogDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/GseTaskIpLogDAOImpl.java index 3fd93dd547..347350ad62 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/GseTaskIpLogDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/GseTaskIpLogDAOImpl.java @@ -24,12 +24,14 @@ package com.tencent.bk.job.execute.dao.impl; -import com.tencent.bk.job.common.constant.JobConstants; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.Bool; import com.tencent.bk.job.common.constant.Order; import com.tencent.bk.job.execute.dao.GseTaskIpLogDAO; -import com.tencent.bk.job.execute.engine.consts.IpStatus; -import com.tencent.bk.job.execute.model.GseTaskIpLogDTO; -import com.tencent.bk.job.execute.model.ResultGroupBaseDTO; +import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; +import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.model.AgentTaskResultGroupBaseDTO; +import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; import org.apache.commons.lang3.StringUtils; import org.jooq.Condition; import org.jooq.DSLContext; @@ -45,159 +47,98 @@ import org.springframework.stereotype.Repository; import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; import java.util.List; import static com.tencent.bk.job.common.constant.Order.DESCENDING; import static org.jooq.impl.DSL.count; +@CompatibleImplementation(name = "rolling_execute", explain = "兼容老版本数据，过1-2个大版本之后删除", version = "3.7.x") @Repository public class GseTaskIpLogDAOImpl implements GseTaskIpLogDAO { - private DSLContext create; + private final DSLContext CTX; @Autowired - public GseTaskIpLogDAOImpl(@Qualifier("job-execute-dsl-context") DSLContext create) { - this.create = create; + public GseTaskIpLogDAOImpl(@Qualifier("job-execute-dsl-context") DSLContext CTX) { + this.CTX = CTX; } @Override - public void batchSaveIpLog(List ipLogList) { - String sql = "replace into gse_task_ip_log (step_instance_id, execute_count, ip, status, start_time, " + - "end_time, total_time, error_code, exit_code, tag, log_offset, display_ip, is_target,is_source) values " + - "(?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; - Object[][] params = new Object[ipLogList.size()][14]; - int batchCount = 0; - for (GseTaskIpLogDTO ipLog : ipLogList) { - Object[] param = new Object[14]; - param[0] = ipLog.getStepInstanceId(); - param[1] = ipLog.getExecuteCount(); - param[2] = ipLog.getCloudAreaAndIp(); - param[3] = ipLog.getStatus(); - param[4] = ipLog.getStartTime(); - param[5] = ipLog.getEndTime(); - param[6] = ipLog.getTotalTime(); - param[7] = ipLog.getErrCode(); - param[8] = ipLog.getExitCode(); - param[9] = StringUtils.truncate(ipLog.getTag(), JobConstants.RESULT_GROUP_TAG_MAX_LENGTH); - param[10] = ipLog.getOffset(); - param[11] = ipLog.getDisplayIp(); - param[12] = ipLog.isTargetServer() ? 1 : 0; - param[13] = ipLog.isSourceServer() ? 1 : 0; - params[batchCount++] = param; - } - create.batch(sql, params).execute(); - } - - @Override - public void batchUpdateIpLog(long stepInstanceId, int executeCount, Collection cloudAreaAndIps, - Long startTime, Long endTime, IpStatus ipStatus) { - String sql = "update gse_task_ip_log set start_time = ?,end_time = ?,status = ? where step_instance_id = ? " + - "and execute_count = ? and ip = ?"; - Object[][] params = new Object[cloudAreaAndIps.size()][6]; - int batchCount = 0; - for (String ip : cloudAreaAndIps) { - Object[] param = new Object[6]; - param[0] = startTime; - param[1] = endTime; - param[2] = ipStatus.getValue(); - param[3] = stepInstanceId; - param[4] = executeCount; - param[5] = ip; - params[batchCount++] = param; - } - create.batch(sql, params).execute(); - } - - @Override - public int getSuccessIpCount(long stepInstanceId, int executeCount) { + public int getSuccessAgentTaskCount(long stepInstanceId, int executeCount) { GseTaskIpLog t = GseTaskIpLog.GSE_TASK_IP_LOG; - return create.selectCount().from(t) - .where(t.STATUS.in(3, 9)) + return CTX.selectCount().from(t) + .where(t.STATUS.in(AgentTaskStatusEnum.LAST_SUCCESS.getValue(), + AgentTaskStatusEnum.SUCCESS.getValue())) .and(t.STEP_INSTANCE_ID.eq(stepInstanceId)) .and(t.EXECUTE_COUNT.eq(executeCount)) - .and(t.IS_TARGET.eq(Byte.valueOf("1"))) + .and(t.IS_TARGET.eq(Bool.TRUE.getValue())) .fetchOne(0, Integer.class); } @Override - public List getSuccessGseTaskIp(long stepInstanceId, int executeCount) { + public List listResultGroups(long stepInstanceId, int executeCount) { GseTaskIpLog t = GseTaskIpLog.GSE_TASK_IP_LOG; - Result result = create.select(t.STEP_INSTANCE_ID, t.EXECUTE_COUNT, t.IP, t.STATUS, t.START_TIME, t.END_TIME, - t.TOTAL_TIME, t.ERROR_CODE, t.EXIT_CODE, t.TAG, t.LOG_OFFSET, t.DISPLAY_IP, t.IS_TARGET, t.IS_SOURCE) - .from(t) - .where(t.STATUS.in(3, 9)) - .and(t.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(t.EXECUTE_COUNT.eq(executeCount)) - .and(t.IS_TARGET.eq(Byte.valueOf("1"))) - .fetch(); - List successGseIpList = new ArrayList<>(); - result.into(record -> { - successGseIpList.add(extract(record)); - }); - return successGseIpList; - } - - @Override - public List getResultGroups(long stepInstanceId, int executeCount) { - GseTaskIpLog t = GseTaskIpLog.GSE_TASK_IP_LOG; - Result result = create.select(t.STATUS, t.TAG, count().as("ip_count")).from(t) + Result result = CTX.select(t.STATUS, t.TAG, count().as("ip_count")).from(t) .where(t.STEP_INSTANCE_ID.eq(stepInstanceId)) .and(t.EXECUTE_COUNT.eq(executeCount)) - .and(t.IS_TARGET.eq(Byte.valueOf("1"))) + .and(t.IS_TARGET.eq(Bool.TRUE.getValue())) .groupBy(t.STATUS, t.TAG) .orderBy(t.STATUS.asc()) .fetch(); - List resultGroups = new ArrayList<>(); + List resultGroups = new ArrayList<>(); result.into(record -> { - ResultGroupBaseDTO resultGroup = new ResultGroupBaseDTO(); - resultGroup.setResultType(record.get(t.STATUS)); + AgentTaskResultGroupBaseDTO resultGroup = new AgentTaskResultGroupBaseDTO(); + resultGroup.setStatus(record.get(t.STATUS)); resultGroup.setTag(record.get(t.TAG)); - resultGroup.setAgentTaskCount((int) record.get("ip_count")); + resultGroup.setTotalAgentTasks((int) record.get("ip_count")); resultGroups.add(resultGroup); }); return resultGroups; } @Override - public List getIpLogByResultType(Long stepInstanceId, Integer executeCount, Integer resultType, - String tag) { + public List listAgentTaskByResultGroup(Long stepInstanceId, + Integer executeCount, + Integer status, + String tag) { GseTaskIpLog t = GseTaskIpLog.GSE_TASK_IP_LOG; - Result result = create.select(t.STEP_INSTANCE_ID, t.EXECUTE_COUNT, t.IP, t.STATUS, t.START_TIME, t.END_TIME, + Result result = CTX.select(t.STEP_INSTANCE_ID, t.EXECUTE_COUNT, t.IP, t.STATUS, t.START_TIME, t.END_TIME, t.TOTAL_TIME, t.ERROR_CODE, t.EXIT_CODE, t.TAG, t.LOG_OFFSET, t.DISPLAY_IP, t.IS_TARGET, t.IS_SOURCE) .from(t) .where(t.STEP_INSTANCE_ID.eq(stepInstanceId)) .and(t.EXECUTE_COUNT.eq(executeCount)) - .and(t.STATUS.eq(resultType)) + .and(t.STATUS.eq(status)) .and(t.TAG.eq(tag == null ? "" : tag)) - .and(t.IS_TARGET.eq(Byte.valueOf("1"))) + .and(t.IS_TARGET.eq(Bool.TRUE.getValue())) .fetch(); - List ipLogs = new ArrayList<>(); + List agentTasks = new ArrayList<>(); if (result.size() > 0) { - result.into(record -> { - ipLogs.add(extract(record)); - }); + result.into(record -> agentTasks.add(extract(record))); } - return ipLogs; + return agentTasks; } @Override - public List getIpLogByResultType(Long stepInstanceId, Integer executeCount, Integer resultType, - String tag, Integer limit, String orderField, Order order) { + public List listAgentTaskByResultGroup(Long stepInstanceId, + Integer executeCount, + Integer status, + String tag, + Integer limit, + String orderField, + Order order) { GseTaskIpLog t = GseTaskIpLog.GSE_TASK_IP_LOG; List conditions = new ArrayList<>(); conditions.add(t.STEP_INSTANCE_ID.eq(stepInstanceId)); conditions.add(t.EXECUTE_COUNT.eq(executeCount)); - conditions.add(t.STATUS.eq(resultType)); + conditions.add(t.STATUS.eq(status)); conditions.add(t.TAG.eq(tag == null ? "" : tag)); - conditions.add(t.IS_TARGET.eq(Byte.valueOf("1"))); + conditions.add(t.IS_TARGET.eq(Bool.TRUE.getValue())); - SelectConditionStep select = create.select(t.STEP_INSTANCE_ID, t.EXECUTE_COUNT, t.IP, t.STATUS, t.START_TIME, + SelectConditionStep select = CTX.select(t.STEP_INSTANCE_ID, t.EXECUTE_COUNT, t.IP, t.STATUS, t.START_TIME, t.END_TIME, t.TOTAL_TIME, t.ERROR_CODE, t.EXIT_CODE, t.TAG, t.LOG_OFFSET, t.DISPLAY_IP, t.IS_TARGET, t.IS_SOURCE) .from(t) @@ -218,7 +159,7 @@ public List getIpLogByResultType(Long stepInstanceId, Integer e } } - List ipLogs = new ArrayList<>(); + List agentTasks = new ArrayList<>(); Result result; if (selectLimitPercentStep != null) { result = selectLimitPercentStep.fetch(); @@ -230,10 +171,10 @@ public List getIpLogByResultType(Long stepInstanceId, Integer e if (result.size() > 0) { result.into(record -> { - ipLogs.add(extract(record)); + agentTasks.add(extract(record)); }); } - return ipLogs; + return agentTasks; } private OrderField buildOrderField(String field, Order order) { @@ -262,59 +203,55 @@ private OrderField buildOrderField(String field, Order order) { return orderField; } + @Override - public List getIpLog(Long stepInstanceId, Integer executeCount, boolean onlyTargetIp) { + public List listAgentTasks(Long stepInstanceId, Integer executeCount) { GseTaskIpLog t = GseTaskIpLog.GSE_TASK_IP_LOG; - SelectConditionStep selectConditionStep = create.select(t.STEP_INSTANCE_ID, t.EXECUTE_COUNT, t.IP, t.STATUS, + Result result = CTX.select(t.STEP_INSTANCE_ID, t.EXECUTE_COUNT, t.IP, t.STATUS, t.START_TIME, t.END_TIME, t.TOTAL_TIME, t.ERROR_CODE, t.EXIT_CODE, t.TAG, t.LOG_OFFSET, t.DISPLAY_IP, t.IS_TARGET, t.IS_SOURCE) .from(t) .where(t.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(t.EXECUTE_COUNT.eq(executeCount)); - if (onlyTargetIp) { - selectConditionStep.and(t.IS_TARGET.eq(Byte.valueOf("1"))); - } - Result result = selectConditionStep.fetch(); - List ipLogList = new ArrayList<>(); + .and(t.EXECUTE_COUNT.eq(executeCount)) + .fetch(); + List agentTaskList = new ArrayList<>(); if (result.size() != 0) { result.map(record -> { - ipLogList.add(extract(record)); + agentTaskList.add(extract(record)); return null; }); } - return ipLogList; + return agentTaskList; } - private GseTaskIpLogDTO extract(Record record) { + private AgentTaskDTO extract(Record record) { if (record == null) { return null; } GseTaskIpLog t = GseTaskIpLog.GSE_TASK_IP_LOG; - GseTaskIpLogDTO ipLog = new GseTaskIpLogDTO(); - ipLog.setStepInstanceId(record.get(t.STEP_INSTANCE_ID)); - ipLog.setExecuteCount(record.get(t.EXECUTE_COUNT)); - ipLog.setCloudAreaAndIp(record.get(t.IP)); - ipLog.setStatus(record.get(t.STATUS)); - ipLog.setStartTime(record.get(t.START_TIME)); - ipLog.setEndTime(record.get(t.END_TIME)); - ipLog.setTotalTime(record.get(t.TOTAL_TIME)); - ipLog.setErrCode(record.get(t.ERROR_CODE)); - ipLog.setExitCode(record.get(t.EXIT_CODE, Integer.class)); - ipLog.setTag(record.get(t.TAG)); - ipLog.setOffset(record.get(t.LOG_OFFSET)); - ipLog.setDisplayIp(record.get(t.DISPLAY_IP)); - String[] cloudAreaIdAndIpArray = record.get(t.IP).split(":"); - ipLog.setCloudAreaId(Long.valueOf(cloudAreaIdAndIpArray[0])); - ipLog.setIp(cloudAreaIdAndIpArray[1]); - ipLog.setTargetServer(record.get(t.IS_TARGET) == 1); - ipLog.setSourceServer(record.get(t.IS_SOURCE) == 1); - return ipLog; + AgentTaskDTO agentTask = new AgentTaskDTO(); + agentTask.setStepInstanceId(record.get(t.STEP_INSTANCE_ID)); + agentTask.setExecuteCount(record.get(t.EXECUTE_COUNT)); + agentTask.setCloudIp(record.get(t.IP)); + agentTask.setDisplayIp(record.get(t.DISPLAY_IP)); + agentTask.setAgentId(record.get(t.IP)); + agentTask.setStatus(AgentTaskStatusEnum.valueOf(record.get(t.STATUS))); + agentTask.setStartTime(record.get(t.START_TIME)); + agentTask.setEndTime(record.get(t.END_TIME)); + agentTask.setTotalTime(record.get(t.TOTAL_TIME)); + agentTask.setErrorCode(record.get(t.ERROR_CODE)); + agentTask.setExitCode(record.get(t.EXIT_CODE, Integer.class)); + agentTask.setTag(record.get(t.TAG)); + agentTask.setScriptLogOffset(record.get(t.LOG_OFFSET)); + boolean isUploadMode = record.get(t.IS_SOURCE) == Bool.TRUE.getValue(); + agentTask.setFileTaskMode(isUploadMode ? FileTaskModeEnum.UPLOAD : FileTaskModeEnum.DOWNLOAD); + return agentTask; } @Override - public GseTaskIpLogDTO getIpLogByIp(Long stepInstanceId, Integer executeCount, String ip) { + public AgentTaskDTO getAgentTaskByIp(Long stepInstanceId, Integer executeCount, String ip) { GseTaskIpLog t = GseTaskIpLog.GSE_TASK_IP_LOG; - Record record = create.select(t.STEP_INSTANCE_ID, t.EXECUTE_COUNT, t.IP, t.STATUS, t.START_TIME, t.END_TIME, + Record record = CTX.select(t.STEP_INSTANCE_ID, t.EXECUTE_COUNT, t.IP, t.STATUS, t.START_TIME, t.END_TIME, t.TOTAL_TIME, t.ERROR_CODE, t.EXIT_CODE, t.TAG, t.LOG_OFFSET, t.DISPLAY_IP, t.IS_TARGET, t.IS_SOURCE) .from(t) .where(t.STEP_INSTANCE_ID.eq(stepInstanceId)) @@ -325,39 +262,11 @@ public GseTaskIpLogDTO getIpLogByIp(Long stepInstanceId, Integer executeCount, S } @Override - public List getIpLogByIps(Long stepInstanceId, Integer executeCount, String[] ipArray) { - GseTaskIpLog t = GseTaskIpLog.GSE_TASK_IP_LOG; - Result result = create.select(t.STEP_INSTANCE_ID, t.EXECUTE_COUNT, t.IP, t.STATUS, t.START_TIME, t.END_TIME, - t.TOTAL_TIME, t.ERROR_CODE, t.EXIT_CODE, t.TAG, t.LOG_OFFSET, t.DISPLAY_IP, t.IS_TARGET, t.IS_SOURCE) - .from(t) - .where(t.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(t.EXECUTE_COUNT.eq(executeCount)) - .and(t.IP.in(ipArray)) - .and(t.IS_TARGET.eq(Byte.valueOf("1"))) - .fetch(); - List ipLogList = new ArrayList<>(); - if (result.size() != 0) { - result.into(record -> { - ipLogList.add(extract(record)); - }); - } - return ipLogList; - } - - @Override - public void deleteAllIpLog(long stepInstanceId, int executeCount) { + public int getActualSuccessExecuteCount(long stepInstanceId, String cloudIp) { GseTaskIpLog t = GseTaskIpLog.GSE_TASK_IP_LOG; - create.deleteFrom(t).where(t.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(t.EXECUTE_COUNT.eq((executeCount))) - .execute(); - } - - @Override - public int getSuccessRetryCount(long stepInstanceId, String cloudAreaAndIp) { - GseTaskIpLog t = GseTaskIpLog.GSE_TASK_IP_LOG; - Record record = create.select(t.EXECUTE_COUNT).from(t).where(t.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(t.IP.eq(cloudAreaAndIp)) - .and(t.STATUS.eq(IpStatus.SUCCESS.getValue())) + Record record = CTX.select(t.EXECUTE_COUNT).from(t).where(t.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(t.IP.eq(cloudIp)) + .and(t.STATUS.eq(AgentTaskStatusEnum.SUCCESS.getValue())) .orderBy(t.EXECUTE_COUNT.desc()) .limit(1) .fetchOne(); @@ -367,36 +276,4 @@ public int getSuccessRetryCount(long stepInstanceId, String cloudAreaAndIp) { return 0; } } - - @Override - public List getTaskFileSourceIps(Long stepInstanceId, Integer executeCount) { - GseTaskIpLog t = GseTaskIpLog.GSE_TASK_IP_LOG; - Result result = create.select(t.IP).from(t).where(t.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(t.EXECUTE_COUNT.eq(executeCount)) - .and(t.IS_SOURCE.eq(Byte.valueOf("1"))) - .fetch(); - List ips = new ArrayList<>(); - if (result != null && result.size() > 0) { - result.into(record -> ips.add(record.getValue(t.IP))); - } - return ips; - } - - @Override - public List fuzzySearchTargetIpsByIp(Long stepInstanceId, Integer executeCount, String searchIp) { - GseTaskIpLog t = GseTaskIpLog.GSE_TASK_IP_LOG; - Result result = create.select(t.IP) - .from(t) - .where(t.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(t.EXECUTE_COUNT.eq(executeCount)) - .and(t.DISPLAY_IP.like("%" + searchIp + "%")) - .and(t.IS_TARGET.eq(Byte.valueOf("1"))) - .fetch(); - if (result == null || result.size() == 0) { - return Collections.emptyList(); - } - List cloudIps = new ArrayList<>(); - result.into(record -> cloudIps.add(record.getValue(t.IP))); - return cloudIps; - } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/GseTaskLogDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/GseTaskLogDAOImpl.java deleted file mode 100644 index 49652ee60c..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/GseTaskLogDAOImpl.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.dao.impl; - -import com.tencent.bk.job.execute.common.util.JooqDataTypeUtil; -import com.tencent.bk.job.execute.dao.GseTaskLogDAO; -import com.tencent.bk.job.execute.model.GseTaskLogDTO; -import org.jooq.DSLContext; -import org.jooq.Record; -import org.jooq.generated.tables.GseTaskLog; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.stereotype.Repository; - -@Repository -public class GseTaskLogDAOImpl implements GseTaskLogDAO { - private DSLContext create; - - @Autowired - public GseTaskLogDAOImpl(@Qualifier("job-execute-dsl-context") DSLContext create) { - this.create = create; - } - - @Override - public GseTaskLogDTO getStepLastExecuteLog(long stepInstanceId) { - GseTaskLog t = GseTaskLog.GSE_TASK_LOG; - Record record = create.select(t.STEP_INSTANCE_ID, t.EXECUTE_COUNT, t.START_TIME, t.END_TIME, t.TOTAL_TIME, - t.STATUS, t.GSE_TASK_ID) - .from(t) - .where(t.STEP_INSTANCE_ID.eq(stepInstanceId)) - .orderBy(t.EXECUTE_COUNT.desc()) - .limit(1) - .fetchOne(); - return extractInfo(record); - } - - private GseTaskLogDTO extractInfo(Record record) { - if (record == null) { - return null; - } - GseTaskLogDTO gseTaskLogDTO = new GseTaskLogDTO(); - GseTaskLog t = GseTaskLog.GSE_TASK_LOG; - - gseTaskLogDTO.setStepInstanceId(record.get(t.STEP_INSTANCE_ID)); - gseTaskLogDTO.setExecuteCount(record.get(t.EXECUTE_COUNT)); - gseTaskLogDTO.setStartTime(record.get(t.START_TIME)); - gseTaskLogDTO.setEndTime(record.get(t.END_TIME)); - gseTaskLogDTO.setTotalTime(record.get(t.TOTAL_TIME)); - gseTaskLogDTO.setStatus(record.get(t.STATUS).intValue()); - gseTaskLogDTO.setGseTaskId(record.get(t.GSE_TASK_ID)); - return gseTaskLogDTO; - } - - @Override - public void saveGseTaskLog(GseTaskLogDTO gseTaskLog) { - GseTaskLog t = GseTaskLog.GSE_TASK_LOG; - create.insertInto(t, t.STEP_INSTANCE_ID, t.EXECUTE_COUNT, t.START_TIME, t.END_TIME, t.TOTAL_TIME, t.STATUS, - t.GSE_TASK_ID) - .values(gseTaskLog.getStepInstanceId(), - gseTaskLog.getExecuteCount(), - gseTaskLog.getStartTime(), - gseTaskLog.getEndTime(), - gseTaskLog.getTotalTime(), - JooqDataTypeUtil.getByteFromInteger(gseTaskLog.getStatus()), - gseTaskLog.getGseTaskId()) - .onDuplicateKeyUpdate() - .set(t.START_TIME, gseTaskLog.getStartTime()) - .set(t.END_TIME, gseTaskLog.getEndTime()) - .set(t.TOTAL_TIME, gseTaskLog.getTotalTime()) - .set(t.STATUS, JooqDataTypeUtil.getByteFromInteger(gseTaskLog.getStatus())).set(t.GSE_TASK_ID, - gseTaskLog.getGseTaskId()) - .execute(); - } - - @Override - public GseTaskLogDTO getGseTaskLog(long stepInstanceId, int executeCount) { - GseTaskLog t = GseTaskLog.GSE_TASK_LOG; - Record record = create.select(t.STEP_INSTANCE_ID, t.EXECUTE_COUNT, t.START_TIME, t.END_TIME, t.TOTAL_TIME, - t.STATUS, t.GSE_TASK_ID).from(t) - .where(t.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(t.EXECUTE_COUNT.eq(executeCount)) - .fetchOne(); - return extractInfo(record); - } - - @Override - public void deleteGseTaskLog(long stepInstanceId, int executeCount) { - GseTaskLog t = GseTaskLog.GSE_TASK_LOG; - create.deleteFrom(t).where(t.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(t.EXECUTE_COUNT.eq(executeCount)) - .execute(); - } - -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/OperationLogDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/OperationLogDAOImpl.java index 36caf3f934..d4237635bd 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/OperationLogDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/OperationLogDAOImpl.java @@ -56,7 +56,7 @@ public long saveOperationLog(OperationLogDTO operationLog) { Record record = ctx.insertInto(TABLE, TABLE.TASK_INSTANCE_ID, TABLE.OP_CODE, TABLE.OPERATOR, TABLE.CREATE_TIME, TABLE.DETAIL) .values(operationLog.getTaskInstanceId(), - JooqDataTypeUtil.getByteFromInteger(operationLog.getOperationEnum().getValue()), + JooqDataTypeUtil.toByte(operationLog.getOperationEnum().getValue()), operationLog.getOperator(), operationLog.getCreateTime(), JsonUtils.toJson(operationLog.getDetail())) @@ -85,7 +85,7 @@ private OperationLogDTO extractInfo(Record record) { OperationLogDTO opLog = new OperationLogDTO(); opLog.setId(record.get(TABLE.ID)); opLog.setTaskInstanceId(record.get(TABLE.TASK_INSTANCE_ID)); - opLog.setOperationEnum(UserOperationEnum.valueOf(JooqDataTypeUtil.getIntegerFromByte(record.get(TABLE.OP_CODE)))); + opLog.setOperationEnum(UserOperationEnum.valueOf(JooqDataTypeUtil.toInteger(record.get(TABLE.OP_CODE)))); opLog.setCreateTime(record.get(TABLE.CREATE_TIME)); opLog.setOperator(record.get(TABLE.OPERATOR)); String detail = record.get(TABLE.DETAIL); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/RollingConfigDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/RollingConfigDAOImpl.java new file mode 100644 index 0000000000..e3db47f84a --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/RollingConfigDAOImpl.java @@ -0,0 +1,90 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao.impl; + +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.execute.dao.RollingConfigDAO; +import com.tencent.bk.job.execute.model.RollingConfigDTO; +import com.tencent.bk.job.execute.model.db.RollingConfigDetailDO; +import org.jooq.DSLContext; +import org.jooq.Record; +import org.jooq.generated.tables.RollingConfig; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Repository; + +@Repository +public class RollingConfigDAOImpl implements RollingConfigDAO { + + private static final RollingConfig TABLE = RollingConfig.ROLLING_CONFIG; + private final DSLContext CTX; + + @Autowired + public RollingConfigDAOImpl(@Qualifier("job-execute-dsl-context") DSLContext ctx) { + this.CTX = ctx; + } + + @Override + public long saveRollingConfig(RollingConfigDTO rollingConfig) { + Record record = CTX.insertInto( + TABLE, + TABLE.TASK_INSTANCE_ID, + TABLE.CONFIG_NAME, + TABLE.CONFIG) + .values( + rollingConfig.getTaskInstanceId(), + rollingConfig.getConfigName(), + JsonUtils.toJson(rollingConfig.getConfigDetail())) + .returning(TABLE.ID) + .fetchOne(); + assert record != null; + return record.get(TABLE.ID); + } + + @Override + public RollingConfigDTO queryRollingConfigById(Long rollingConfigId) { + Record record = CTX.select( + TABLE.ID, + TABLE.TASK_INSTANCE_ID, + TABLE.CONFIG_NAME, + TABLE.CONFIG) + .from(TABLE) + .where(TABLE.ID.eq(rollingConfigId)) + .fetchOne(); + return extract(record); + } + + private RollingConfigDTO extract(Record record) { + if (record == null) { + return null; + } + RollingConfigDTO rollingConfig = new RollingConfigDTO(); + rollingConfig.setId(record.get(TABLE.ID)); + rollingConfig.setTaskInstanceId(record.get(TABLE.TASK_INSTANCE_ID)); + rollingConfig.setConfigName(record.get(TABLE.CONFIG_NAME)); + rollingConfig.setConfigDetail(JsonUtils.fromJson(record.get(TABLE.CONFIG), RollingConfigDetailDO.class)); + return rollingConfig; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java new file mode 100644 index 0000000000..43674c6265 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java @@ -0,0 +1,403 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao.impl; + +import com.tencent.bk.job.common.constant.JobConstants; +import com.tencent.bk.job.common.constant.Order; +import com.tencent.bk.job.execute.dao.ScriptAgentTaskDAO; +import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; +import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.model.AgentTaskResultGroupBaseDTO; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.jooq.Condition; +import org.jooq.DSLContext; +import org.jooq.OrderField; +import org.jooq.Record; +import org.jooq.Result; +import org.jooq.SelectConditionStep; +import org.jooq.SelectLimitPercentStep; +import org.jooq.SelectSeekStep1; +import org.jooq.TableField; +import org.jooq.UpdateConditionStep; +import org.jooq.UpdateSetMoreStep; +import org.jooq.UpdateSetStep; +import org.jooq.generated.tables.GseScriptAgentTask; +import org.jooq.generated.tables.records.GseScriptAgentTaskRecord; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Repository; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import static com.tencent.bk.job.common.constant.Order.DESCENDING; +import static org.jooq.impl.DSL.count; + +@Repository +public class ScriptAgentTaskDAOImpl implements ScriptAgentTaskDAO { + + private static final GseScriptAgentTask T_GSE_SCRIPT_AGENT_TASK = GseScriptAgentTask.GSE_SCRIPT_AGENT_TASK; + private static final TableField[] ALL_FIELDS = { + T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID, + T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT, + T_GSE_SCRIPT_AGENT_TASK.ACTUAL_EXECUTE_COUNT, + T_GSE_SCRIPT_AGENT_TASK.BATCH, + T_GSE_SCRIPT_AGENT_TASK.HOST_ID, + T_GSE_SCRIPT_AGENT_TASK.AGENT_ID, + T_GSE_SCRIPT_AGENT_TASK.GSE_TASK_ID, + T_GSE_SCRIPT_AGENT_TASK.STATUS, + T_GSE_SCRIPT_AGENT_TASK.START_TIME, + T_GSE_SCRIPT_AGENT_TASK.END_TIME, + T_GSE_SCRIPT_AGENT_TASK.TOTAL_TIME, + T_GSE_SCRIPT_AGENT_TASK.ERROR_CODE, + T_GSE_SCRIPT_AGENT_TASK.EXIT_CODE, + T_GSE_SCRIPT_AGENT_TASK.TAG, + T_GSE_SCRIPT_AGENT_TASK.LOG_OFFSET + }; + + private final DSLContext CTX; + + @Autowired + public ScriptAgentTaskDAOImpl(@Qualifier("job-execute-dsl-context") DSLContext CTX) { + this.CTX = CTX; + } + + @Override + public void batchSaveAgentTasks(Collection agentTasks) { + String sql = "insert into gse_script_agent_task (step_instance_id, execute_count, actual_execute_count, batch," + + " host_id, agent_id, gse_task_id, status, start_time, end_time, total_time, error_code, exit_code, tag," + + " log_offset)" + + " values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; + Object[][] params = new Object[agentTasks.size()][15]; + int batchCount = 0; + for (AgentTaskDTO agentTask : agentTasks) { + Object[] param = new Object[15]; + param[0] = agentTask.getStepInstanceId(); + param[1] = agentTask.getExecuteCount(); + param[2] = agentTask.getActualExecuteCount(); + param[3] = agentTask.getBatch(); + param[4] = agentTask.getHostId(); + param[5] = agentTask.getAgentId(); + param[6] = agentTask.getGseTaskId(); + param[7] = agentTask.getStatus().getValue(); + param[8] = agentTask.getStartTime(); + param[9] = agentTask.getEndTime(); + param[10] = agentTask.getTotalTime(); + param[11] = agentTask.getErrorCode(); + param[12] = agentTask.getExitCode(); + param[13] = StringUtils.truncate(agentTask.getTag(), JobConstants.RESULT_GROUP_TAG_MAX_LENGTH); + param[14] = agentTask.getScriptLogOffset(); + params[batchCount++] = param; + } + CTX.batch(sql, params).execute(); + } + + @Override + public void batchUpdateAgentTasks(Collection agentTasks) { + if (CollectionUtils.isEmpty(agentTasks)) { + return; + } + String sql = "update gse_script_agent_task set gse_task_id = ?, status = ?, start_time = ?, end_time = ?" + + ", total_time = ?, error_code = ?, exit_code = ?, tag = ?, log_offset = ?" + + " where step_instance_id = ? and execute_count = ? and batch = ? and host_id = ?"; + Object[][] params = new Object[agentTasks.size()][13]; + int batchCount = 0; + for (AgentTaskDTO agentTask : agentTasks) { + Object[] param = new Object[13]; + param[0] = agentTask.getGseTaskId(); + param[1] = agentTask.getStatus().getValue(); + param[2] = agentTask.getStartTime(); + param[3] = agentTask.getEndTime(); + param[4] = agentTask.getTotalTime(); + param[5] = agentTask.getErrorCode(); + param[6] = agentTask.getExitCode(); + param[7] = StringUtils.truncate(agentTask.getTag(), JobConstants.RESULT_GROUP_TAG_MAX_LENGTH); + param[8] = agentTask.getScriptLogOffset(); + param[9] = agentTask.getStepInstanceId(); + param[10] = agentTask.getExecuteCount(); + param[11] = agentTask.getBatch(); + param[12] = agentTask.getHostId(); + params[batchCount++] = param; + } + CTX.batch(sql, params).execute(); + } + + @Override + public int getSuccessAgentTaskCount(long stepInstanceId, int executeCount) { + Integer count = CTX.selectCount() + .from(T_GSE_SCRIPT_AGENT_TASK) + .where(T_GSE_SCRIPT_AGENT_TASK.STATUS.in(AgentTaskStatusEnum.LAST_SUCCESS.getValue(), + AgentTaskStatusEnum.SUCCESS.getValue())) + .and(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq((short) executeCount)) + .fetchOne(0, Integer.class); + return count == null ? 0 : count; + } + + @Override + public List listResultGroups(long stepInstanceId, int executeCount, Integer batch) { + SelectConditionStep selectConditionStep = + CTX.select(T_GSE_SCRIPT_AGENT_TASK.STATUS, T_GSE_SCRIPT_AGENT_TASK.TAG, count().as("ip_count")) + .from(T_GSE_SCRIPT_AGENT_TASK) + .where(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq((short) executeCount)); + if (batch != null && batch > 0) { + selectConditionStep.and(T_GSE_SCRIPT_AGENT_TASK.BATCH.eq(batch.shortValue())); + } + + Result result = selectConditionStep.groupBy(T_GSE_SCRIPT_AGENT_TASK.STATUS, T_GSE_SCRIPT_AGENT_TASK.TAG) + .orderBy(T_GSE_SCRIPT_AGENT_TASK.STATUS.asc()) + .fetch(); + + List resultGroups = new ArrayList<>(); + result.forEach(record -> { + AgentTaskResultGroupBaseDTO resultGroup = new AgentTaskResultGroupBaseDTO(); + resultGroup.setStatus(record.get(T_GSE_SCRIPT_AGENT_TASK.STATUS)); + resultGroup.setTag(record.get(T_GSE_SCRIPT_AGENT_TASK.TAG)); + Object ipCount = record.get("ip_count"); + resultGroup.setTotalAgentTasks(ipCount == null ? 0 : (int) ipCount); + resultGroups.add(resultGroup); + }); + return resultGroups; + } + + @Override + public List listAgentTaskByResultGroup(Long stepInstanceId, + Integer executeCount, + Integer batch, + Integer status, + String tag) { + SelectConditionStep selectConditionStep = CTX.select(ALL_FIELDS) + .from(T_GSE_SCRIPT_AGENT_TASK) + .where(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())) + .and(T_GSE_SCRIPT_AGENT_TASK.STATUS.eq(status)) + .and(T_GSE_SCRIPT_AGENT_TASK.TAG.eq(tag == null ? "" : tag)); + if (batch != null && batch > 0) { + selectConditionStep.and(T_GSE_SCRIPT_AGENT_TASK.BATCH.eq(batch.shortValue())); + } + Result result = selectConditionStep.fetch(); + + List agentTasks = new ArrayList<>(); + if (result.size() > 0) { + result.forEach(record -> agentTasks.add(extract(record))); + } + return agentTasks; + } + + @Override + public List listAgentTaskByResultGroup(Long stepInstanceId, + Integer executeCount, + Integer batch, + Integer status, + String tag, + Integer limit, + String orderField, + Order order) { + List conditions = new ArrayList<>(); + conditions.add(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)); + conditions.add(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())); + conditions.add(T_GSE_SCRIPT_AGENT_TASK.STATUS.eq(status)); + conditions.add(T_GSE_SCRIPT_AGENT_TASK.TAG.eq(tag == null ? "" : tag)); + + SelectConditionStep select = CTX.select(ALL_FIELDS) + .from(T_GSE_SCRIPT_AGENT_TASK) + .where(conditions); + + if (batch != null && batch > 0) { + select.and(T_GSE_SCRIPT_AGENT_TASK.BATCH.eq(batch.shortValue())); + } + + SelectSeekStep1 selectSeekStep = null; + OrderField orderFieldEntity = buildOrderField(orderField, order); + if (orderFieldEntity != null) { + selectSeekStep = select.orderBy(orderFieldEntity); + } + + SelectLimitPercentStep selectLimitPercentStep = null; + if (limit != null && limit > 0) { + if (selectSeekStep != null) { + selectLimitPercentStep = selectSeekStep.limit(limit); + } else { + selectLimitPercentStep = select.limit(limit); + } + } + + List agentTasks = new ArrayList<>(); + Result result; + if (selectLimitPercentStep != null) { + result = selectLimitPercentStep.fetch(); + } else if (selectSeekStep != null) { + result = selectSeekStep.fetch(); + } else { + result = select.fetch(); + } + + if (result.size() > 0) { + result.into(record -> agentTasks.add(extract(record))); + } + return agentTasks; + } + + private OrderField buildOrderField(String field, Order order) { + OrderField orderField = null; + if (StringUtils.isNotBlank(field)) { + if (field.equals(T_GSE_SCRIPT_AGENT_TASK.TOTAL_TIME.getName())) { + if (order == DESCENDING) { + orderField = T_GSE_SCRIPT_AGENT_TASK.TOTAL_TIME.desc(); + } else { + orderField = T_GSE_SCRIPT_AGENT_TASK.TOTAL_TIME.asc(); + } + } else if (field.equals(T_GSE_SCRIPT_AGENT_TASK.EXIT_CODE.getName())) { + if (order == DESCENDING) { + orderField = T_GSE_SCRIPT_AGENT_TASK.EXIT_CODE.desc(); + } else { + orderField = T_GSE_SCRIPT_AGENT_TASK.EXIT_CODE.asc(); + } + } + } + return orderField; + } + + @Override + public List listAgentTasks(Long stepInstanceId, + Integer executeCount, + Integer batch) { + SelectConditionStep selectConditionStep = CTX.select(ALL_FIELDS) + .from(T_GSE_SCRIPT_AGENT_TASK) + .where(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())); + if (batch != null && batch > 0) { + selectConditionStep.and(T_GSE_SCRIPT_AGENT_TASK.BATCH.eq(batch.shortValue())); + } + Result result = selectConditionStep.fetch(); + List agentTaskList = new ArrayList<>(); + if (result.size() != 0) { + result.map(record -> { + agentTaskList.add(extract(record)); + return null; + }); + } + return agentTaskList; + } + + private AgentTaskDTO extract(Record record) { + if (record == null) { + return null; + } + AgentTaskDTO agentTask = new AgentTaskDTO(); + agentTask.setStepInstanceId(record.get(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID)); + agentTask.setExecuteCount(record.get(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT)); + Short actualExecuteCount = record.get(T_GSE_SCRIPT_AGENT_TASK.ACTUAL_EXECUTE_COUNT); + agentTask.setActualExecuteCount(actualExecuteCount != null ? actualExecuteCount.intValue() : null); + agentTask.setBatch(record.get(T_GSE_SCRIPT_AGENT_TASK.BATCH)); + agentTask.setHostId(record.get(T_GSE_SCRIPT_AGENT_TASK.HOST_ID)); + agentTask.setAgentId(record.get(T_GSE_SCRIPT_AGENT_TASK.AGENT_ID)); + agentTask.setGseTaskId(record.get(T_GSE_SCRIPT_AGENT_TASK.GSE_TASK_ID)); + agentTask.setStatus(AgentTaskStatusEnum.valueOf(record.get(T_GSE_SCRIPT_AGENT_TASK.STATUS))); + agentTask.setStartTime(record.get(T_GSE_SCRIPT_AGENT_TASK.START_TIME)); + agentTask.setEndTime(record.get(T_GSE_SCRIPT_AGENT_TASK.END_TIME)); + agentTask.setTotalTime(record.get(T_GSE_SCRIPT_AGENT_TASK.TOTAL_TIME)); + agentTask.setErrorCode(record.get(T_GSE_SCRIPT_AGENT_TASK.ERROR_CODE)); + agentTask.setExitCode(record.get(T_GSE_SCRIPT_AGENT_TASK.EXIT_CODE, Integer.class)); + agentTask.setTag(record.get(T_GSE_SCRIPT_AGENT_TASK.TAG)); + agentTask.setScriptLogOffset(record.get(T_GSE_SCRIPT_AGENT_TASK.LOG_OFFSET)); + return agentTask; + } + + @Override + public List listAgentTasksByGseTaskId(Long gseTaskId) { + if (gseTaskId == null || gseTaskId <= 0) { + return Collections.emptyList(); + } + + List agentTaskList = new ArrayList<>(); + + Result result = CTX.select(ALL_FIELDS) + .from(T_GSE_SCRIPT_AGENT_TASK) + .where(T_GSE_SCRIPT_AGENT_TASK.GSE_TASK_ID.eq(gseTaskId)) + .fetch(); + if (result.size() > 0) { + result.forEach(record -> agentTaskList.add(extract(record))); + } + return agentTaskList; + } + + @Override + public AgentTaskDTO getAgentTaskByHostId(Long stepInstanceId, Integer executeCount, Integer batch, long hostId) { + Record record = CTX.select(ALL_FIELDS) + .from(T_GSE_SCRIPT_AGENT_TASK) + .where(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())) + .and(T_GSE_SCRIPT_AGENT_TASK.BATCH.eq(batch == null ? 0 : batch.shortValue())) + .and(T_GSE_SCRIPT_AGENT_TASK.HOST_ID.eq(hostId)) + .fetchOne(); + return extract(record); + } + + @Override + public boolean isStepInstanceRecordExist(long stepInstanceId) { + return CTX.fetchExists(T_GSE_SCRIPT_AGENT_TASK, T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)); + } + + + @Override + public void updateAgentTaskFields(long stepInstanceId, + int executeCount, + Integer batch, + Integer actualExecuteCount, + Long gseTaskId) { + UpdateSetStep updateSetStep = CTX.update(T_GSE_SCRIPT_AGENT_TASK); + boolean needUpdate = false; + if (actualExecuteCount != null) { + updateSetStep = updateSetStep.set(T_GSE_SCRIPT_AGENT_TASK.ACTUAL_EXECUTE_COUNT, + actualExecuteCount.shortValue()); + needUpdate = true; + } + if (gseTaskId != null) { + updateSetStep = updateSetStep.set(T_GSE_SCRIPT_AGENT_TASK.GSE_TASK_ID, gseTaskId); + needUpdate = true; + } + + if (!needUpdate) { + return; + } + + UpdateSetMoreStep updateSetMoreStep = + (UpdateSetMoreStep) updateSetStep; + + UpdateConditionStep updateConditionStep = + updateSetMoreStep + .where(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq((short) executeCount)); + if (batch != null) { + updateConditionStep.and(T_GSE_SCRIPT_AGENT_TASK.BATCH.eq(batch.shortValue())); + } + updateConditionStep.execute(); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImpl.java index a1460df530..25128a2302 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImpl.java @@ -49,13 +49,13 @@ import org.jooq.Record; import org.jooq.Record1; import org.jooq.Result; +import org.jooq.TableField; import org.jooq.UpdateSetMoreStep; import org.jooq.conf.ParamType; import org.jooq.generated.tables.StepInstance; import org.jooq.generated.tables.StepInstanceConfirm; import org.jooq.generated.tables.StepInstanceFile; import org.jooq.generated.tables.StepInstanceScript; -import org.jooq.generated.tables.TaskInstance; import org.jooq.generated.tables.records.StepInstanceRecord; import org.jooq.types.UByte; import org.springframework.beans.factory.annotation.Autowired; @@ -67,55 +67,66 @@ import java.util.Collections; import java.util.List; -import static org.jooq.impl.DSL.max; -import static org.jooq.impl.DSL.min; -import static org.jooq.impl.DSL.sum; - @Slf4j @Repository public class StepInstanceDAOImpl implements StepInstanceDAO { - private static final TaskInstance TABLE_TASK_INSTANCE = TaskInstance.TASK_INSTANCE; - private static final StepInstance TABLE_STEP_INSTANCE = StepInstance.STEP_INSTANCE; + private static final StepInstance T_STEP_INSTANCE = StepInstance.STEP_INSTANCE; private static final StepInstanceScript TABLE_STEP_INSTANCE_SCRIPT = StepInstanceScript.STEP_INSTANCE_SCRIPT; private static final StepInstanceFile TABLE_STEP_INSTANCE_FILE = StepInstanceFile.STEP_INSTANCE_FILE; - private DSLContext ctx; + private static final TableField[] T_STEP_INSTANCE_ALL_FIELDS = { + T_STEP_INSTANCE.ID, + T_STEP_INSTANCE.STEP_ID, + T_STEP_INSTANCE.TASK_INSTANCE_ID, + T_STEP_INSTANCE.APP_ID, + T_STEP_INSTANCE.NAME, + T_STEP_INSTANCE.TYPE, + T_STEP_INSTANCE.OPERATOR, + T_STEP_INSTANCE.STATUS, + T_STEP_INSTANCE.EXECUTE_COUNT, + T_STEP_INSTANCE.TARGET_SERVERS, + T_STEP_INSTANCE.START_TIME, + T_STEP_INSTANCE.END_TIME, + T_STEP_INSTANCE.TOTAL_TIME, + T_STEP_INSTANCE.CREATE_TIME, + T_STEP_INSTANCE.IGNORE_ERROR, + T_STEP_INSTANCE.STEP_NUM, + T_STEP_INSTANCE.STEP_ORDER, + T_STEP_INSTANCE.BATCH, + T_STEP_INSTANCE.ROLLING_CONFIG_ID + }; + + private final DSLContext CTX; @Autowired - public StepInstanceDAOImpl(@Qualifier("job-execute-dsl-context") DSLContext ctx) { - this.ctx = ctx; + public StepInstanceDAOImpl(@Qualifier("job-execute-dsl-context") DSLContext CTX) { + this.CTX = CTX; } @Override public Long addStepInstanceBase(StepInstanceBaseDTO stepInstance) { StepInstance t = StepInstance.STEP_INSTANCE; - Record record = ctx.insertInto(t, t.STEP_ID, t.TASK_INSTANCE_ID, t.APP_ID, t.NAME, t.TYPE, + Record record = CTX.insertInto(t, t.STEP_ID, t.TASK_INSTANCE_ID, t.APP_ID, t.NAME, t.TYPE, t.OPERATOR, t.STATUS, t.EXECUTE_COUNT, t.START_TIME, t.END_TIME, t.TOTAL_TIME, - t.TARGET_SERVERS, t.ABNORMAL_AGENT_IP_LIST, t.TOTAL_IP_NUM, t.ABNORMAL_AGENT_NUM, - t.RUN_IP_NUM, t.FAIL_IP_NUM, t.SUCCESS_IP_NUM, t.CREATE_TIME, t.IGNORE_ERROR, t.STEP_NUM, - t.STEP_ORDER) + t.TARGET_SERVERS, t.CREATE_TIME, t.IGNORE_ERROR, t.STEP_NUM, + t.STEP_ORDER, t.BATCH) .values(stepInstance.getStepId(), stepInstance.getTaskInstanceId(), stepInstance.getAppId(), stepInstance.getName(), - JooqDataTypeUtil.getByteFromInteger(stepInstance.getExecuteType()), + JooqDataTypeUtil.toByte(stepInstance.getExecuteType()), stepInstance.getOperator(), - JooqDataTypeUtil.getByteFromInteger(stepInstance.getStatus()), + stepInstance.getStatus().getValue().byteValue(), stepInstance.getExecuteCount(), stepInstance.getStartTime(), stepInstance.getEndTime(), stepInstance.getTotalTime(), stepInstance.getTargetServers() == null ? null : JsonUtils.toJson(stepInstance.getTargetServers()), - stepInstance.getBadIpList(), - stepInstance.getTotalIPNum(), - stepInstance.getBadIPNum(), - stepInstance.getRunIPNum(), - stepInstance.getFailIPNum(), - stepInstance.getSuccessIPNum(), stepInstance.getCreateTime(), stepInstance.isIgnoreError() ? Byte.valueOf("1") : Byte.valueOf("0"), stepInstance.getStepNum(), - stepInstance.getStepOrder()) + stepInstance.getStepOrder(), + (short) stepInstance.getBatch()) .returning(t.ID).fetchOne(); return record.getValue(t.ID); } @@ -128,35 +139,35 @@ public void addScriptStepInstance(StepInstanceDTO stepInstance) { scriptSourceByteValue = scriptSource.byteValue(); } StepInstanceScript t = StepInstanceScript.STEP_INSTANCE_SCRIPT; - ctx.insertInto(t, t.STEP_INSTANCE_ID, t.SCRIPT_CONTENT, t.SCRIPT_TYPE, t.SCRIPT_PARAM, t.RESOLVED_SCRIPT_PARAM, + CTX.insertInto(t, t.STEP_INSTANCE_ID, t.SCRIPT_CONTENT, t.SCRIPT_TYPE, t.SCRIPT_PARAM, t.RESOLVED_SCRIPT_PARAM, t.EXECUTION_TIMEOUT, t.SYSTEM_ACCOUNT_ID, t.SYSTEM_ACCOUNT, t.DB_ACCOUNT_ID, t.DB_TYPE, t.DB_ACCOUNT, t.DB_PASSWORD, t.DB_PORT, t.SCRIPT_SOURCE, t.SCRIPT_ID, t.SCRIPT_VERSION_ID, t.IS_SECURE_PARAM) .values(stepInstance.getId(), stepInstance.getScriptContent(), - JooqDataTypeUtil.getByteFromInteger(stepInstance.getScriptType()), + JooqDataTypeUtil.toByte(stepInstance.getScriptType()), stepInstance.getScriptParam(), stepInstance.getResolvedScriptParam(), stepInstance.getTimeout(), stepInstance.getAccountId(), stepInstance.getAccount(), stepInstance.getDbAccountId(), - JooqDataTypeUtil.getByteFromInteger(stepInstance.getDbType()), + JooqDataTypeUtil.toByte(stepInstance.getDbType()), stepInstance.getDbAccount(), stepInstance.getDbPass(), stepInstance.getDbPort(), scriptSourceByteValue, stepInstance.getScriptId(), stepInstance.getScriptVersionId(), - stepInstance.isSecureParam() ? JooqDataTypeUtil.getByteFromInteger(1) : - JooqDataTypeUtil.getByteFromInteger(0) + stepInstance.isSecureParam() ? JooqDataTypeUtil.toByte(1) : + JooqDataTypeUtil.toByte(0) ).execute(); } @Override public void addFileStepInstance(StepInstanceDTO stepInstance) { StepInstanceFile t = StepInstanceFile.STEP_INSTANCE_FILE; - ctx.insertInto(t, t.STEP_INSTANCE_ID, t.FILE_SOURCE, t.FILE_TARGET_PATH, t.FILE_TARGET_NAME, + CTX.insertInto(t, t.STEP_INSTANCE_ID, t.FILE_SOURCE, t.FILE_TARGET_PATH, t.FILE_TARGET_NAME, t.FILE_UPLOAD_SPEED_LIMIT, t.FILE_DOWNLOAD_SPEED_LIMIT, t.FILE_DUPLICATE_HANDLE, t.NOT_EXIST_PATH_HANDLER, t.EXECUTION_TIMEOUT, t.SYSTEM_ACCOUNT_ID, t.SYSTEM_ACCOUNT) .values(stepInstance.getId(), @@ -165,8 +176,8 @@ public void addFileStepInstance(StepInstanceDTO stepInstance) { stepInstance.getFileTargetName(), stepInstance.getFileUploadSpeedLimit(), stepInstance.getFileDownloadSpeedLimit(), - JooqDataTypeUtil.getByteFromInteger(stepInstance.getFileDuplicateHandle()), - JooqDataTypeUtil.getUByteFromInteger(stepInstance.getNotExistPathHandler()), + JooqDataTypeUtil.toByte(stepInstance.getFileDuplicateHandle()), + JooqDataTypeUtil.toUByte(stepInstance.getNotExistPathHandler()), stepInstance.getTimeout(), stepInstance.getAccountId(), stepInstance.getAccount() @@ -176,7 +187,7 @@ public void addFileStepInstance(StepInstanceDTO stepInstance) { @Override public void addConfirmStepInstance(StepInstanceDTO stepInstance) { StepInstanceConfirm t = StepInstanceConfirm.STEP_INSTANCE_CONFIRM; - ctx.insertInto(t, t.STEP_INSTANCE_ID, t.CONFIRM_MESSAGE, t.CONFIRM_USERS, t.CONFIRM_ROLES, t.NOTIFY_CHANNELS) + CTX.insertInto(t, t.STEP_INSTANCE_ID, t.CONFIRM_MESSAGE, t.CONFIRM_USERS, t.CONFIRM_ROLES, t.NOTIFY_CHANNELS) .values(stepInstance.getId(), stepInstance.getConfirmMessage(), stepInstance.getConfirmUsers() == null ? null : @@ -191,7 +202,7 @@ public void addConfirmStepInstance(StepInstanceDTO stepInstance) { @Override public ScriptStepInstanceDTO getScriptStepInstance(long stepInstanceId) { StepInstanceScript t = StepInstanceScript.STEP_INSTANCE_SCRIPT; - Record record = ctx.select(t.STEP_INSTANCE_ID, t.SCRIPT_CONTENT, t.SCRIPT_TYPE, t.SCRIPT_PARAM, + Record record = CTX.select(t.STEP_INSTANCE_ID, t.SCRIPT_CONTENT, t.SCRIPT_TYPE, t.SCRIPT_PARAM, t.RESOLVED_SCRIPT_PARAM, t.EXECUTION_TIMEOUT, t.SYSTEM_ACCOUNT_ID, t.SYSTEM_ACCOUNT, t.DB_ACCOUNT_ID, t.DB_ACCOUNT, t.DB_TYPE, t.DB_PASSWORD, t.DB_PORT, t.SCRIPT_SOURCE, t.SCRIPT_ID, t.SCRIPT_VERSION_ID, t.IS_SECURE_PARAM @@ -208,14 +219,14 @@ private ScriptStepInstanceDTO extractScriptInfo(Record record) { ScriptStepInstanceDTO stepInstance = new ScriptStepInstanceDTO(); stepInstance.setStepInstanceId(record.get(t.STEP_INSTANCE_ID)); stepInstance.setScriptContent(record.get(t.SCRIPT_CONTENT)); - stepInstance.setScriptType(JooqDataTypeUtil.getIntegerFromByte(record.get(t.SCRIPT_TYPE))); + stepInstance.setScriptType(JooqDataTypeUtil.toInteger(record.get(t.SCRIPT_TYPE))); stepInstance.setScriptParam(record.get(t.SCRIPT_PARAM)); stepInstance.setResolvedScriptParam(record.get(t.RESOLVED_SCRIPT_PARAM)); stepInstance.setTimeout(record.get(t.EXECUTION_TIMEOUT)); stepInstance.setAccountId(record.get(t.SYSTEM_ACCOUNT_ID)); stepInstance.setAccount(record.get(t.SYSTEM_ACCOUNT)); stepInstance.setDbAccountId(record.get(t.DB_ACCOUNT_ID)); - stepInstance.setDbType(JooqDataTypeUtil.getIntegerFromByte(record.get(t.DB_TYPE))); + stepInstance.setDbType(JooqDataTypeUtil.toInteger(record.get(t.DB_TYPE))); stepInstance.setDbAccount(record.get(t.DB_ACCOUNT)); stepInstance.setDbPass(record.get(t.DB_PASSWORD)); stepInstance.setDbPort(record.get(t.DB_PORT)); @@ -234,7 +245,7 @@ private ScriptStepInstanceDTO extractScriptInfo(Record record) { @Override public FileStepInstanceDTO getFileStepInstance(long stepInstanceId) { StepInstanceFile t = StepInstanceFile.STEP_INSTANCE_FILE; - Record record = ctx.select(t.STEP_INSTANCE_ID, t.FILE_SOURCE, t.RESOLVED_FILE_SOURCE, t.FILE_TARGET_PATH, + Record record = CTX.select(t.STEP_INSTANCE_ID, t.FILE_SOURCE, t.RESOLVED_FILE_SOURCE, t.FILE_TARGET_PATH, t.FILE_TARGET_NAME, t.RESOLVED_FILE_TARGET_PATH, t.FILE_UPLOAD_SPEED_LIMIT, t.FILE_DOWNLOAD_SPEED_LIMIT, t.FILE_DUPLICATE_HANDLE, t.NOT_EXIST_PATH_HANDLER, t.EXECUTION_TIMEOUT, t.SYSTEM_ACCOUNT_ID, t.SYSTEM_ACCOUNT) @@ -253,12 +264,12 @@ private FileStepInstanceDTO extractFileInfo(Record record) { stepInstance.setTimeout(record.get(t.EXECUTION_TIMEOUT)); List fileSourceList = JsonUtils.fromJson(record.get(t.FILE_SOURCE), new TypeReference>() { - }); + }); stepInstance.setFileSourceList(fileSourceList); if (StringUtils.isNotEmpty(record.get(t.RESOLVED_FILE_SOURCE))) { List resolvedFileSourceList = JsonUtils.fromJson(record.get(t.RESOLVED_FILE_SOURCE), new TypeReference>() { - }); + }); stepInstance.setResolvedFileSourceList(resolvedFileSourceList); } stepInstance.setFileTargetPath(record.get(t.FILE_TARGET_PATH)); @@ -268,7 +279,7 @@ private FileStepInstanceDTO extractFileInfo(Record record) { stepInstance.setAccount(record.get(t.SYSTEM_ACCOUNT)); stepInstance.setFileUploadSpeedLimit(record.get(t.FILE_UPLOAD_SPEED_LIMIT)); stepInstance.setFileDownloadSpeedLimit(record.get(t.FILE_DOWNLOAD_SPEED_LIMIT)); - stepInstance.setFileDuplicateHandle(JooqDataTypeUtil.getIntegerFromByte(record.get(t.FILE_DUPLICATE_HANDLE))); + stepInstance.setFileDuplicateHandle(JooqDataTypeUtil.toInteger(record.get(t.FILE_DUPLICATE_HANDLE))); UByte notExistPathHandlerRecordValue = record.get(t.NOT_EXIST_PATH_HANDLER); if (notExistPathHandlerRecordValue == null) { // 默认为创建目录 @@ -282,7 +293,7 @@ private FileStepInstanceDTO extractFileInfo(Record record) { @Override public ConfirmStepInstanceDTO getConfirmStepInstance(long stepInstanceId) { StepInstanceConfirm t = StepInstanceConfirm.STEP_INSTANCE_CONFIRM; - Record record = ctx.select(t.STEP_INSTANCE_ID, t.CONFIRM_MESSAGE, t.CONFIRM_REASON, t.CONFIRM_USERS, + Record record = CTX.select(t.STEP_INSTANCE_ID, t.CONFIRM_MESSAGE, t.CONFIRM_REASON, t.CONFIRM_USERS, t.CONFIRM_ROLES, t.NOTIFY_CHANNELS).from(t) .where(t.STEP_INSTANCE_ID.eq(stepInstanceId)).fetchOne(); return extractConfirmInfo(record); @@ -308,13 +319,11 @@ private ConfirmStepInstanceDTO extractConfirmInfo(Record record) { @Override public StepInstanceBaseDTO getStepInstanceBase(long stepInstanceId) { - StepInstance t = StepInstance.STEP_INSTANCE; - Record record = ctx.select(t.ID, t.STEP_ID, t.TASK_INSTANCE_ID, t.APP_ID, t.NAME, t.TYPE, t.OPERATOR, - t.STATUS, t.EXECUTE_COUNT, t.TARGET_SERVERS, t.ABNORMAL_AGENT_IP_LIST, t.START_TIME, t.END_TIME, - t.TOTAL_TIME, t.TOTAL_IP_NUM, t.ABNORMAL_AGENT_NUM, t.RUN_IP_NUM, t.FAIL_IP_NUM, t.SUCCESS_IP_NUM, - t.CREATE_TIME, t.IGNORE_ERROR, t.STEP_NUM, t.STEP_ORDER) - .from(t) - .where(t.ID.eq(stepInstanceId)).fetchOne(); + Record record = CTX + .select(T_STEP_INSTANCE_ALL_FIELDS) + .from(T_STEP_INSTANCE) + .where(T_STEP_INSTANCE.ID.eq(stepInstanceId)) + .fetchOne(); return extractBaseInfo(record); } @@ -329,97 +338,108 @@ private StepInstanceBaseDTO extractBaseInfo(Record record) { stepInstance.setStepId(record.get(t.STEP_ID)); stepInstance.setTaskInstanceId(record.get(t.TASK_INSTANCE_ID)); stepInstance.setName(record.get(t.NAME)); - stepInstance.setExecuteType(JooqDataTypeUtil.getIntegerFromByte(record.get(t.TYPE))); + stepInstance.setExecuteType(JooqDataTypeUtil.toInteger(record.get(t.TYPE))); stepInstance.setOperator(record.get(t.OPERATOR)); - stepInstance.setStatus(JooqDataTypeUtil.getIntegerFromByte(record.get(t.STATUS))); + stepInstance.setStatus(RunStatusEnum.valueOf(record.get(t.STATUS))); stepInstance.setExecuteCount(record.get(t.EXECUTE_COUNT)); stepInstance.setStartTime(record.get(t.START_TIME)); stepInstance.setEndTime(record.get(t.END_TIME)); stepInstance.setTotalTime(record.get(t.TOTAL_TIME)); - stepInstance.setTotalIPNum(record.get(t.TOTAL_IP_NUM)); if (StringUtils.isNotBlank(record.get(t.TARGET_SERVERS))) { ServersDTO targetServers = JsonUtils.fromJson(record.get(t.TARGET_SERVERS), ServersDTO.class); stepInstance.setTargetServers(targetServers); stepInstance.setIpList(targetServers.buildIpListStr()); } - stepInstance.setBadIpList(record.get(t.ABNORMAL_AGENT_IP_LIST)); - stepInstance.setBadIPNum(record.get(t.ABNORMAL_AGENT_NUM)); - stepInstance.setRunIPNum(record.get(t.RUN_IP_NUM)); - stepInstance.setFailIPNum(record.get(t.FAIL_IP_NUM)); - stepInstance.setSuccessIPNum(record.get(t.SUCCESS_IP_NUM)); stepInstance.setCreateTime(record.get(t.CREATE_TIME)); - stepInstance.setIgnoreError(JooqDataTypeUtil.getIntegerFromByte(record.get(t.IGNORE_ERROR)) != null - && JooqDataTypeUtil.getIntegerFromByte(record.get(t.IGNORE_ERROR)).equals(1)); + stepInstance.setIgnoreError(JooqDataTypeUtil.toInteger(record.get(t.IGNORE_ERROR)) != null + && JooqDataTypeUtil.toInteger(record.get(t.IGNORE_ERROR)).equals(1)); stepInstance.setStepNum(record.get(t.STEP_NUM)); stepInstance.setStepOrder(record.get(t.STEP_ORDER)); + stepInstance.setBatch(record.get(t.BATCH)); + stepInstance.setRollingConfigId(record.get(t.ROLLING_CONFIG_ID)); return stepInstance; } + @Override + public StepInstanceBaseDTO getFirstStepInstanceBase(long taskInstanceId) { + Record record = CTX + .select(T_STEP_INSTANCE_ALL_FIELDS) + .from(T_STEP_INSTANCE) + .where(T_STEP_INSTANCE.TASK_INSTANCE_ID.eq(taskInstanceId)) + .orderBy(T_STEP_INSTANCE.ID.asc()) + .limit(1) + .fetchOne(); + return extractBaseInfo(record); + } + + @Override + public StepInstanceBaseDTO getNextStepInstance(long taskInstanceId, int currentStepOrder) { + Record record = CTX + .select(T_STEP_INSTANCE_ALL_FIELDS) + .from(T_STEP_INSTANCE) + .where(T_STEP_INSTANCE.TASK_INSTANCE_ID.eq(taskInstanceId)) + .and(T_STEP_INSTANCE.STEP_ORDER.gt(currentStepOrder)) + .orderBy(T_STEP_INSTANCE.STEP_ORDER.asc()) + .limit(1) + .fetchOne(); + return extractBaseInfo(record); + } + @Override public List listStepInstanceBaseByTaskInstanceId(long taskInstanceId) { - StepInstance t = StepInstance.STEP_INSTANCE; - Result result = ctx.select(t.ID, t.STEP_ID, t.TASK_INSTANCE_ID, t.APP_ID, t.NAME, t.TYPE, t.OPERATOR, - t.STATUS, t.EXECUTE_COUNT, t.TARGET_SERVERS, t.ABNORMAL_AGENT_IP_LIST, t.START_TIME, t.END_TIME, - t.TOTAL_TIME, t.TOTAL_IP_NUM, t.ABNORMAL_AGENT_NUM, t.RUN_IP_NUM, t.FAIL_IP_NUM, t.SUCCESS_IP_NUM, - t.CREATE_TIME, t.IGNORE_ERROR, t.STEP_NUM, t.STEP_ORDER) - .from(t) - .where(StepInstance.STEP_INSTANCE.TASK_INSTANCE_ID.eq(taskInstanceId)) - .orderBy(StepInstance.STEP_INSTANCE.ID.asc()) + Result result = CTX + .select(T_STEP_INSTANCE_ALL_FIELDS) + .from(T_STEP_INSTANCE) + .where(T_STEP_INSTANCE.TASK_INSTANCE_ID.eq(taskInstanceId)) + .orderBy(T_STEP_INSTANCE.ID.asc()) .fetch(); List stepInstanceList = new ArrayList<>(); - result.into(record -> { - stepInstanceList.add(extractBaseInfo(record)); - }); + result.into(record -> stepInstanceList.add(extractBaseInfo(record))); return stepInstanceList; } @Override public void resetStepStatus(long stepInstanceId) { StepInstance t = StepInstance.STEP_INSTANCE; - ctx.update(t).setNull(t.START_TIME).setNull(t.END_TIME).setNull(t.TOTAL_TIME).set(t.SUCCESS_IP_NUM, 0) - .set(t.FAIL_IP_NUM, 0).set(t.RUN_IP_NUM, 0) + CTX.update(t).setNull(t.START_TIME).setNull(t.END_TIME).setNull(t.TOTAL_TIME) .where(t.ID.eq(stepInstanceId)).execute(); } @Override public void resetStepExecuteInfoForRetry(long stepInstanceId) { StepInstance t = StepInstance.STEP_INSTANCE; - ctx.update(t).set(t.START_TIME, System.currentTimeMillis()) - .set(t.EXECUTE_COUNT, t.EXECUTE_COUNT.plus(1)) + CTX.update(t) .set(t.STATUS, RunStatusEnum.RUNNING.getValue().byteValue()) .setNull(t.END_TIME) .setNull(t.TOTAL_TIME) - .set(t.SUCCESS_IP_NUM, 0) - .set(t.FAIL_IP_NUM, 0) - .set(t.RUN_IP_NUM, 0) .where(t.ID.eq(stepInstanceId)).execute(); } @Override public void addStepExecuteCount(long stepInstanceId) { StepInstance t = StepInstance.STEP_INSTANCE; - ctx.update(t).set(t.EXECUTE_COUNT, t.EXECUTE_COUNT.plus(1)) + CTX.update(t).set(t.EXECUTE_COUNT, t.EXECUTE_COUNT.plus(1)) .where(t.ID.eq(stepInstanceId)).execute(); } @Override public void updateStepStatus(long stepInstanceId, int status) { StepInstance t = StepInstance.STEP_INSTANCE; - ctx.update(t).set(t.STATUS, JooqDataTypeUtil.getByteFromInteger(status)) + CTX.update(t).set(t.STATUS, JooqDataTypeUtil.toByte(status)) .where(t.ID.eq(stepInstanceId)).execute(); } @Override public void updateStepStartTime(long stepInstanceId, Long startTime) { StepInstance t = StepInstance.STEP_INSTANCE; - ctx.update(t).set(t.START_TIME, startTime) + CTX.update(t).set(t.START_TIME, startTime) .where(t.ID.eq(stepInstanceId)).execute(); } @Override public void updateStepStartTimeIfNull(long stepInstanceId, Long startTime) { StepInstance t = StepInstance.STEP_INSTANCE; - ctx.update(t).set(t.START_TIME, startTime) + CTX.update(t).set(t.START_TIME, startTime) .where(t.ID.eq(stepInstanceId)) .and(t.START_TIME.isNull()) .execute(); @@ -428,66 +448,25 @@ public void updateStepStartTimeIfNull(long stepInstanceId, Long startTime) { @Override public void updateStepEndTime(long stepInstanceId, Long endTime) { StepInstance t = StepInstance.STEP_INSTANCE; - ctx.update(t).set(t.END_TIME, endTime) + CTX.update(t).set(t.END_TIME, endTime) .where(t.ID.eq(stepInstanceId)) .execute(); } @Override - public void addTaskExecuteCount(long taskInstanceId) { + public void addStepInstanceExecuteCount(long stepInstanceId) { StepInstance t = StepInstance.STEP_INSTANCE; - ctx.update(t).set(t.EXECUTE_COUNT, t.EXECUTE_COUNT.plus(1)) - .where(t.TASK_INSTANCE_ID.eq(taskInstanceId) - .and(t.STATUS.eq(JooqDataTypeUtil.getByteFromInteger(RunStatusEnum.BLANK.getValue())))).execute(); + CTX.update(t).set(t.EXECUTE_COUNT, t.EXECUTE_COUNT.plus(1)) + .where(t.ID.eq(stepInstanceId)).execute(); } @Override public void updateStepTotalTime(long stepInstanceId, long totalTime) { StepInstance t = StepInstance.STEP_INSTANCE; - ctx.update(t).set(t.TOTAL_TIME, totalTime) + CTX.update(t).set(t.TOTAL_TIME, totalTime) .where(t.ID.eq(stepInstanceId)).execute(); } - @Override - public void updateStepStatInfo(long stepInstanceId, int runIPNum, int successIPNum, int failIPNum) { - StepInstance t = StepInstance.STEP_INSTANCE; - ctx.update(t).set(t.RUN_IP_NUM, runIPNum) - .set(t.SUCCESS_IP_NUM, successIPNum) - .set(t.FAIL_IP_NUM, failIPNum) - .where(t.ID.eq(stepInstanceId)) - .execute(); - } - - @Override - public Long getFirstStepStartTime(long taskInstanceId) { - StepInstance t = StepInstance.STEP_INSTANCE; - Record record = ctx.select(min(t.START_TIME).as("time")).from(t) - .where(t.TASK_INSTANCE_ID.eq(taskInstanceId)) - .and(t.START_TIME.isNotNull()) - .fetchOne(); - return record.get("time", Long.class); - } - - @Override - public Long getLastStepEndTime(long taskInstanceId) { - StepInstance t = StepInstance.STEP_INSTANCE; - Record record = ctx.select(max(t.END_TIME).as("time")).from(t) - .where(t.TASK_INSTANCE_ID.eq(taskInstanceId)) - .and(t.END_TIME.isNotNull()) - .fetchOne(); - return record.get("time", Long.class); - } - - @Override - public long getAllStepTotalTime(long taskInstanceId) { - StepInstance t = StepInstance.STEP_INSTANCE; - Record record = ctx.select(sum(t.TOTAL_TIME).as("total_time")) - .from(t) - .where(t.TASK_INSTANCE_ID.eq(taskInstanceId)) - .fetchOne(); - return record.getValue("total_time", Long.class); - } - @Override public void updateStepExecutionInfo(long stepInstanceId, RunStatusEnum status, Long startTime, Long endTime, Long totalTime) { @@ -505,25 +484,25 @@ private UpdateSetMoreStep buildBasicUpdateSetMoreStep(RunSta StepInstance t = StepInstance.STEP_INSTANCE; UpdateSetMoreStep updateSetMoreStep = null; if (status != null) { - updateSetMoreStep = ctx.update(t).set(t.STATUS, JooqDataTypeUtil.getByteFromInteger(status.getValue())); + updateSetMoreStep = CTX.update(t).set(t.STATUS, JooqDataTypeUtil.toByte(status.getValue())); } if (startTime != null) { if (updateSetMoreStep == null) { - updateSetMoreStep = ctx.update(t).set(t.START_TIME, startTime); + updateSetMoreStep = CTX.update(t).set(t.START_TIME, startTime); } else { updateSetMoreStep.set(t.START_TIME, startTime); } } if (endTime != null) { if (updateSetMoreStep == null) { - updateSetMoreStep = ctx.update(t).set(t.END_TIME, endTime); + updateSetMoreStep = CTX.update(t).set(t.END_TIME, endTime); } else { updateSetMoreStep.set(t.END_TIME, endTime); } } if (totalTime != null) { if (updateSetMoreStep == null) { - updateSetMoreStep = ctx.update(t).set(t.TOTAL_TIME, totalTime); + updateSetMoreStep = CTX.update(t).set(t.TOTAL_TIME, totalTime); } else { updateSetMoreStep.set(t.TOTAL_TIME, totalTime); } @@ -531,43 +510,10 @@ private UpdateSetMoreStep buildBasicUpdateSetMoreStep(RunSta return updateSetMoreStep; } - @Override - public void updateStepExecutionInfo(long stepInstanceId, RunStatusEnum status, Long startTime, Long endTime, - Long totalTime, Integer runIPNum, Integer successIPNum, Integer failIPNum) { - StepInstance t = StepInstance.STEP_INSTANCE; - UpdateSetMoreStep updateSetMoreStep = buildBasicUpdateSetMoreStep(status, - startTime, endTime, totalTime); - if (runIPNum != null) { - if (updateSetMoreStep == null) { - updateSetMoreStep = ctx.update(t).set(t.RUN_IP_NUM, runIPNum); - } else { - updateSetMoreStep.set(t.RUN_IP_NUM, runIPNum); - } - } - if (successIPNum != null) { - if (updateSetMoreStep == null) { - updateSetMoreStep = ctx.update(t).set(t.SUCCESS_IP_NUM, successIPNum); - } else { - updateSetMoreStep.set(t.SUCCESS_IP_NUM, successIPNum); - } - } - if (failIPNum != null) { - if (updateSetMoreStep == null) { - updateSetMoreStep = ctx.update(t).set(t.FAIL_IP_NUM, failIPNum); - } else { - updateSetMoreStep.set(t.FAIL_IP_NUM, failIPNum); - } - } - if (updateSetMoreStep == null) { - return; - } - updateSetMoreStep.where(t.ID.eq(stepInstanceId)).execute(); - } - @Override public void updateResolvedScriptParam(long stepInstanceId, String resolvedScriptParam) { StepInstanceScript t = StepInstanceScript.STEP_INSTANCE_SCRIPT; - ctx.update(t).set(t.RESOLVED_SCRIPT_PARAM, resolvedScriptParam) + CTX.update(t).set(t.RESOLVED_SCRIPT_PARAM, resolvedScriptParam) .where(t.STEP_INSTANCE_ID.eq(stepInstanceId)) .execute(); } @@ -575,7 +521,7 @@ public void updateResolvedScriptParam(long stepInstanceId, String resolvedScript @Override public void updateResolvedSourceFile(long stepInstanceId, List resolvedFileSources) { StepInstanceFile t = StepInstanceFile.STEP_INSTANCE_FILE; - ctx.update(t).set(t.FILE_SOURCE, JsonUtils.toJson(resolvedFileSources)) + CTX.update(t).set(t.FILE_SOURCE, JsonUtils.toJson(resolvedFileSources)) .where(t.STEP_INSTANCE_ID.eq(stepInstanceId)) .execute(); } @@ -583,7 +529,7 @@ public void updateResolvedSourceFile(long stepInstanceId, List re @Override public void updateResolvedTargetPath(long stepInstanceId, String resolvedTargetPath) { StepInstanceFile t = StepInstanceFile.STEP_INSTANCE_FILE; - ctx.update(t).set(t.RESOLVED_FILE_TARGET_PATH, resolvedTargetPath) + CTX.update(t).set(t.RESOLVED_FILE_TARGET_PATH, resolvedTargetPath) .where(t.STEP_INSTANCE_ID.eq(stepInstanceId)) .execute(); } @@ -591,7 +537,7 @@ public void updateResolvedTargetPath(long stepInstanceId, String resolvedTargetP @Override public void updateConfirmReason(long stepInstanceId, String confirmReason) { StepInstanceConfirm t = StepInstanceConfirm.STEP_INSTANCE_CONFIRM; - ctx.update(t).set(t.CONFIRM_REASON, confirmReason) + CTX.update(t).set(t.CONFIRM_REASON, confirmReason) .where(t.STEP_INSTANCE_ID.eq(stepInstanceId)) .execute(); } @@ -599,23 +545,20 @@ public void updateConfirmReason(long stepInstanceId, String confirmReason) { @Override public void updateStepOperator(long stepInstanceId, String operator) { StepInstance t = StepInstance.STEP_INSTANCE; - ctx.update(t).set(t.OPERATOR, operator) + CTX.update(t).set(t.OPERATOR, operator) .where(t.ID.eq(stepInstanceId)) .execute(); } @Override public StepInstanceBaseDTO getPreExecutableStepInstance(long taskInstanceId, long stepInstanceId) { - StepInstance t = StepInstance.STEP_INSTANCE; - Record record = ctx.select(t.ID, t.STEP_ID, t.TASK_INSTANCE_ID, t.APP_ID, t.NAME, t.TYPE, t.OPERATOR, - t.STATUS, t.EXECUTE_COUNT, t.TARGET_SERVERS, t.ABNORMAL_AGENT_IP_LIST, t.START_TIME, t.END_TIME, - t.TOTAL_TIME, t.TOTAL_IP_NUM, t.ABNORMAL_AGENT_NUM, t.RUN_IP_NUM, t.FAIL_IP_NUM, t.SUCCESS_IP_NUM, - t.CREATE_TIME, t.IGNORE_ERROR, t.STEP_NUM, t.STEP_ORDER) - .from(t) - .where(t.TASK_INSTANCE_ID.eq(taskInstanceId)) - .and(t.ID.lt(stepInstanceId)) - .and(t.TYPE.notIn(StepExecuteTypeEnum.MANUAL_CONFIRM.getValue().byteValue())) - .orderBy(t.ID.desc()) + Record record = CTX + .select(T_STEP_INSTANCE_ALL_FIELDS) + .from(T_STEP_INSTANCE) + .where(T_STEP_INSTANCE.TASK_INSTANCE_ID.eq(taskInstanceId)) + .and(T_STEP_INSTANCE.ID.lt(stepInstanceId)) + .and(T_STEP_INSTANCE.TYPE.notIn(StepExecuteTypeEnum.MANUAL_CONFIRM.getValue().byteValue())) + .orderBy(T_STEP_INSTANCE.ID.desc()) .limit(1) .fetchOne(); return extractBaseInfo(record); @@ -623,21 +566,21 @@ public StepInstanceBaseDTO getPreExecutableStepInstance(long taskInstanceId, lon @Override public Long getStepInstanceId(long taskInstanceId) { - Result> records = ctx.select(TABLE_STEP_INSTANCE.ID) - .from(TABLE_STEP_INSTANCE) - .where(TABLE_STEP_INSTANCE.TASK_INSTANCE_ID.eq(taskInstanceId)) + Result> records = CTX.select(T_STEP_INSTANCE.ID) + .from(T_STEP_INSTANCE) + .where(T_STEP_INSTANCE.TASK_INSTANCE_ID.eq(taskInstanceId)) .limit(1) .fetch(); if (records.isEmpty()) { return null; } else { - return records.get(0).get(TABLE_STEP_INSTANCE.ID); + return records.get(0).get(T_STEP_INSTANCE.ID); } } @Override public Byte getScriptTypeByStepInstanceId(long stepInstanceId) { - Result> records = ctx.select(TABLE_STEP_INSTANCE_SCRIPT.SCRIPT_TYPE) + Result> records = CTX.select(TABLE_STEP_INSTANCE_SCRIPT.SCRIPT_TYPE) .from(TABLE_STEP_INSTANCE_SCRIPT) .where(TABLE_STEP_INSTANCE_SCRIPT.STEP_INSTANCE_ID.eq(stepInstanceId)) .limit(1) @@ -650,17 +593,17 @@ public Byte getScriptTypeByStepInstanceId(long stepInstanceId) { } public Integer countStepInstanceByConditions(Collection conditions) { - return ctx.selectCount().from(TABLE_STEP_INSTANCE) + return CTX.selectCount().from(T_STEP_INSTANCE) .where(conditions).fetchOne().value1(); } public Integer countStepInstanceScriptByConditions(Collection conditions) { - return ctx.selectCount().from(TABLE_STEP_INSTANCE_SCRIPT) + return CTX.selectCount().from(TABLE_STEP_INSTANCE_SCRIPT) .where(conditions).fetchOne().value1(); } public Integer countStepInstanceFileByConditions(Collection conditions) { - return ctx.selectCount().from(TABLE_STEP_INSTANCE_FILE) + return CTX.selectCount().from(TABLE_STEP_INSTANCE_FILE) .where(conditions).fetchOne().value1(); } @@ -694,15 +637,15 @@ public Integer count(Long appId, List stepIdList, StepExecuteTypeEnum step } private List listStepInstanceIds(Collection conditions, int offset, int limit) { - Result> records = ctx.select(TABLE_STEP_INSTANCE.ID) - .from(TABLE_STEP_INSTANCE) + Result> records = CTX.select(T_STEP_INSTANCE.ID) + .from(T_STEP_INSTANCE) .where(conditions) .limit(offset, limit) .fetch(); - if (records == null || records.isEmpty()) { + if (records.isEmpty()) { return Collections.emptyList(); } else { - return records.map(it -> it.get(TABLE_STEP_INSTANCE.ID)); + return records.map(it -> it.get(T_STEP_INSTANCE.ID)); } } @@ -711,22 +654,22 @@ private List genStepInstanceConditions(Long appId, List stepIdL Long fromTime, Long toTime) { List conditions = new ArrayList<>(); if (appId != null) { - conditions.add(TABLE_STEP_INSTANCE.APP_ID.eq(appId)); + conditions.add(T_STEP_INSTANCE.APP_ID.eq(appId)); } if (stepIdList != null && !stepIdList.isEmpty()) { - conditions.add(TABLE_STEP_INSTANCE.STEP_ID.in(stepIdList)); + conditions.add(T_STEP_INSTANCE.STEP_ID.in(stepIdList)); } if (stepExecuteType != null) { - conditions.add(TABLE_STEP_INSTANCE.TYPE.eq(stepExecuteType.getValue().byteValue())); + conditions.add(T_STEP_INSTANCE.TYPE.eq(stepExecuteType.getValue().byteValue())); } if (runStatus != null) { - conditions.add(TABLE_STEP_INSTANCE.STATUS.eq(runStatus.getValue().byteValue())); + conditions.add(T_STEP_INSTANCE.STATUS.eq(runStatus.getValue().byteValue())); } if (fromTime != null) { - conditions.add(TABLE_STEP_INSTANCE.CREATE_TIME.greaterOrEqual(fromTime)); + conditions.add(T_STEP_INSTANCE.CREATE_TIME.greaterOrEqual(fromTime)); } if (toTime != null) { - conditions.add(TABLE_STEP_INSTANCE.CREATE_TIME.lessThan(toTime)); + conditions.add(T_STEP_INSTANCE.CREATE_TIME.lessThan(toTime)); } return conditions; } @@ -774,7 +717,7 @@ private List genConditions(Long appId, DuplicateHandlerEnum fileDupli , Boolean notExistPathHandlerNull, RunStatusEnum runStatus, Long fromTime, Long toTime) { List conditions = new ArrayList<>(); if (appId != null) { - conditions.add(TABLE_STEP_INSTANCE.APP_ID.eq(appId)); + conditions.add(T_STEP_INSTANCE.APP_ID.eq(appId)); } if (fileDupliateHandle != null) { conditions.add(TABLE_STEP_INSTANCE_FILE.FILE_DUPLICATE_HANDLE.eq((byte) fileDupliateHandle.getId())); @@ -803,13 +746,13 @@ private List genConditions(Long appId, DuplicateHandlerEnum fileDupli } } if (runStatus != null) { - conditions.add(TABLE_STEP_INSTANCE.STATUS.eq(runStatus.getValue().byteValue())); + conditions.add(T_STEP_INSTANCE.STATUS.eq(runStatus.getValue().byteValue())); } if (fromTime != null) { - conditions.add(TABLE_STEP_INSTANCE.CREATE_TIME.greaterOrEqual(fromTime)); + conditions.add(T_STEP_INSTANCE.CREATE_TIME.greaterOrEqual(fromTime)); } if (toTime != null) { - conditions.add(TABLE_STEP_INSTANCE.CREATE_TIME.lessThan(toTime)); + conditions.add(T_STEP_INSTANCE.CREATE_TIME.lessThan(toTime)); } return conditions; } @@ -823,9 +766,9 @@ public List> listFastPushFileSource(Long appId, DuplicateHan List conditions = genConditions(appId, fileDupliateHandle, fileDupliateHandleNull, notExistPathHandler, notExistPathHandlerNull, runStatus, fromTime, toTime); val query = - ctx.select(TABLE_STEP_INSTANCE_FILE.FILE_SOURCE).from(TABLE_STEP_INSTANCE).join(TABLE_STEP_INSTANCE_FILE) - .on(TABLE_STEP_INSTANCE.ID.eq(TABLE_STEP_INSTANCE_FILE.STEP_INSTANCE_ID)) - .where(conditions); + CTX.select(TABLE_STEP_INSTANCE_FILE.FILE_SOURCE).from(T_STEP_INSTANCE).join(TABLE_STEP_INSTANCE_FILE) + .on(T_STEP_INSTANCE.ID.eq(TABLE_STEP_INSTANCE_FILE.STEP_INSTANCE_ID)) + .where(conditions); String sql = query.getSQL(ParamType.INLINED); try { Result> records = query.fetch(); @@ -869,4 +812,25 @@ private List convertStringToFileSourceDTO(String str) { return JsonUtils.fromJson(str, new TypeReference>() { }); } + + @Override + public void updateStepCurrentBatch(long stepInstanceId, int batch) { + CTX.update(T_STEP_INSTANCE).set(T_STEP_INSTANCE.BATCH, JooqDataTypeUtil.toShort(batch)) + .where(T_STEP_INSTANCE.ID.eq(stepInstanceId)) + .execute(); + } + + @Override + public void updateStepCurrentExecuteCount(long stepInstanceId, int executeCount) { + CTX.update(T_STEP_INSTANCE).set(T_STEP_INSTANCE.EXECUTE_COUNT, executeCount) + .where(T_STEP_INSTANCE.ID.eq(stepInstanceId)) + .execute(); + } + + @Override + public void updateStepRollingConfigId(long stepInstanceId, long rollingConfigId) { + CTX.update(T_STEP_INSTANCE).set(T_STEP_INSTANCE.ROLLING_CONFIG_ID, rollingConfigId) + .where(T_STEP_INSTANCE.ID.eq(stepInstanceId)) + .execute(); + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImpl.java new file mode 100644 index 0000000000..2d7c1d7339 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImpl.java @@ -0,0 +1,187 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao.impl; + +import com.tencent.bk.job.execute.common.constants.RunStatusEnum; +import com.tencent.bk.job.execute.common.util.JooqDataTypeUtil; +import com.tencent.bk.job.execute.dao.StepInstanceRollingTaskDAO; +import com.tencent.bk.job.execute.model.StepInstanceRollingTaskDTO; +import lombok.extern.slf4j.Slf4j; +import org.jooq.DSLContext; +import org.jooq.Record; +import org.jooq.Result; +import org.jooq.SelectConditionStep; +import org.jooq.TableField; +import org.jooq.UpdateSetMoreStep; +import org.jooq.generated.tables.StepInstanceRollingTask; +import org.jooq.generated.tables.records.StepInstanceRollingTaskRecord; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Repository; + +import java.util.ArrayList; +import java.util.List; + +@Repository +@Slf4j +public class StepInstanceRollingTaskDAOImpl implements StepInstanceRollingTaskDAO { + + private static final StepInstanceRollingTask TABLE = StepInstanceRollingTask.STEP_INSTANCE_ROLLING_TASK; + private static final TableField[] ALL_FIELDS = { + TABLE.ID, + TABLE.STEP_INSTANCE_ID, + TABLE.EXECUTE_COUNT, + TABLE.BATCH, + TABLE.STATUS, + TABLE.START_TIME, + TABLE.END_TIME, + TABLE.TOTAL_TIME + }; + private final DSLContext CTX; + + @Autowired + public StepInstanceRollingTaskDAOImpl(@Qualifier("job-execute-dsl-context") DSLContext CTX) { + this.CTX = CTX; + } + + @Override + public StepInstanceRollingTaskDTO queryRollingTask(long stepInstanceId, int executeCount, int batch) { + Record record = CTX.select(ALL_FIELDS) + .from(TABLE) + .where(TABLE.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(TABLE.EXECUTE_COUNT.eq(JooqDataTypeUtil.toShort(executeCount))) + .and(TABLE.BATCH.eq(JooqDataTypeUtil.toShort(batch))) + .fetchOne(); + return extract(record); + } + + private StepInstanceRollingTaskDTO extract(Record record) { + if (record == null) { + return null; + } + StepInstanceRollingTaskDTO stepInstanceRollingTask = new StepInstanceRollingTaskDTO(); + stepInstanceRollingTask.setId(record.get(TABLE.ID)); + stepInstanceRollingTask.setStepInstanceId(record.get(TABLE.STEP_INSTANCE_ID)); + stepInstanceRollingTask.setExecuteCount(record.get(TABLE.EXECUTE_COUNT).intValue()); + stepInstanceRollingTask.setBatch(record.get(TABLE.BATCH).intValue()); + stepInstanceRollingTask.setStatus(RunStatusEnum.valueOf(record.get(TABLE.STATUS))); + stepInstanceRollingTask.setStartTime(record.get(TABLE.START_TIME)); + stepInstanceRollingTask.setEndTime(record.get(TABLE.END_TIME)); + stepInstanceRollingTask.setTotalTime(record.get(TABLE.TOTAL_TIME)); + return stepInstanceRollingTask; + } + + @Override + public List listRollingTasks(long stepInstanceId, + Integer executeCount, + Integer batch) { + SelectConditionStep selectConditionStep = CTX.select(ALL_FIELDS) + .from(TABLE) + .where(TABLE.STEP_INSTANCE_ID.eq(stepInstanceId)); + if (executeCount != null) { + selectConditionStep.and(TABLE.EXECUTE_COUNT.eq(executeCount.shortValue())); + } + if (batch != null && batch > 0) { + selectConditionStep.and(TABLE.BATCH.eq(batch.shortValue())); + } + + Result result = selectConditionStep.orderBy(TABLE.BATCH.asc()).fetch(); + + List stepInstanceRollingTasks = new ArrayList<>(); + if (result.size() > 0) { + stepInstanceRollingTasks = result.map(this::extract); + } + return stepInstanceRollingTasks; + } + + @Override + public long saveRollingTask(StepInstanceRollingTaskDTO rollingTask) { + Record record = CTX.insertInto( + TABLE, + TABLE.STEP_INSTANCE_ID, + TABLE.EXECUTE_COUNT, + TABLE.BATCH, + TABLE.STATUS, + TABLE.START_TIME, + TABLE.END_TIME, + TABLE.TOTAL_TIME) + .values( + rollingTask.getStepInstanceId(), + JooqDataTypeUtil.toShort(rollingTask.getExecuteCount()), + JooqDataTypeUtil.toShort(rollingTask.getBatch()), + JooqDataTypeUtil.toByte(rollingTask.getStatus().getValue()), + rollingTask.getStartTime(), + rollingTask.getEndTime(), + rollingTask.getTotalTime()) + .returning(TABLE.ID) + .fetchOne(); + assert record != null; + return record.get(TABLE.ID); + } + + @Override + public void updateRollingTask(long stepInstanceId, + int executeCount, + int batch, + RunStatusEnum status, + Long startTime, + Long endTime, + Long totalTime) { + UpdateSetMoreStep updateSetMoreStep = null; + if (status != null) { + updateSetMoreStep = CTX.update(TABLE).set(TABLE.STATUS, JooqDataTypeUtil.toByte(status.getValue())); + } + if (startTime != null) { + if (updateSetMoreStep == null) { + updateSetMoreStep = CTX.update(TABLE).set(TABLE.START_TIME, startTime); + } else { + updateSetMoreStep.set(TABLE.START_TIME, startTime); + } + } + if (endTime != null) { + if (updateSetMoreStep == null) { + updateSetMoreStep = CTX.update(TABLE).set(TABLE.END_TIME, endTime); + } else { + updateSetMoreStep.set(TABLE.END_TIME, endTime); + } + } + if (totalTime != null) { + if (updateSetMoreStep == null) { + updateSetMoreStep = CTX.update(TABLE).set(TABLE.TOTAL_TIME, totalTime); + } else { + updateSetMoreStep.set(TABLE.TOTAL_TIME, totalTime); + } + } + if (updateSetMoreStep == null) { + log.error("Invalid update rolling task param, do nothing!"); + return; + } + updateSetMoreStep.where(TABLE.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(TABLE.EXECUTE_COUNT.eq(JooqDataTypeUtil.toByte(executeCount).shortValue())) + .and(TABLE.BATCH.eq(JooqDataTypeUtil.toShort(batch))) + .execute(); + + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceVariableDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceVariableDAOImpl.java index ee77127737..8bb05d97d8 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceVariableDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceVariableDAOImpl.java @@ -59,7 +59,7 @@ public StepInstanceVariableDAOImpl(@Qualifier("job-execute-dsl-context") DSLCont public void saveVariableValues(StepInstanceVariableValuesDTO variableValues) { ctx.insertInto(TABLE, FIELDS) .values(variableValues.getTaskInstanceId(), variableValues.getStepInstanceId(), - variableValues.getExecuteCount(), JooqDataTypeUtil.getByteFromInteger(variableValues.getType()), + variableValues.getExecuteCount(), JooqDataTypeUtil.toByte(variableValues.getType()), JsonUtils.toJson(variableValues)) .execute(); } @@ -71,7 +71,7 @@ public StepInstanceVariableValuesDTO getStepVariableValues(long stepInstanceId, .from(TABLE) .where(TABLE.STEP_INSTANCE_ID.eq(stepInstanceId)) .and(TABLE.EXECUTE_COUNT.eq(executeCount)) - .and(TABLE.TYPE.eq(JooqDataTypeUtil.getByteFromInteger(variableValueType.getValue()))) + .and(TABLE.TYPE.eq(JooqDataTypeUtil.toByte(variableValueType.getValue()))) .limit(1) .fetchOne(); return extract(record); @@ -102,7 +102,7 @@ public List listStepOutputVariableValuesByTaskIns Result result = ctx.select(FIELDS) .from(TABLE) .where(TABLE.TASK_INSTANCE_ID.eq(taskInstanceId)) - .and(TABLE.TYPE.eq(JooqDataTypeUtil.getByteFromInteger(VariableValueTypeEnum.OUTPUT.getValue()))) + .and(TABLE.TYPE.eq(JooqDataTypeUtil.toByte(VariableValueTypeEnum.OUTPUT.getValue()))) .orderBy(TABLE.STEP_INSTANCE_ID.asc(), TABLE.EXECUTE_COUNT.asc()) .fetch(); @@ -120,7 +120,7 @@ public List listSortedPreStepOutputVariableValues .from(TABLE) .where(TABLE.TASK_INSTANCE_ID.eq(taskInstanceId)) .and(TABLE.STEP_INSTANCE_ID.lt(stepInstanceId)) - .and(TABLE.TYPE.eq(JooqDataTypeUtil.getByteFromInteger(VariableValueTypeEnum.OUTPUT.getValue()))) + .and(TABLE.TYPE.eq(JooqDataTypeUtil.toByte(VariableValueTypeEnum.OUTPUT.getValue()))) .orderBy(TABLE.STEP_INSTANCE_ID.asc(), TABLE.EXECUTE_COUNT.asc()) .fetch(); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceDAOImpl.java index a63418462d..298f2cb800 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceDAOImpl.java @@ -65,7 +65,7 @@ @Repository public class TaskInstanceDAOImpl implements TaskInstanceDAO { private static final TaskInstance TABLE = TaskInstance.TASK_INSTANCE; - private DSLContext ctx; + private final DSLContext ctx; @Autowired public TaskInstanceDAOImpl(@Qualifier("job-execute-dsl-context") DSLContext ctx) { @@ -85,15 +85,15 @@ public Long addTaskInstance(TaskInstanceDTO taskInstance) { taskInstance.getAppId(), taskInstance.getName(), taskInstance.getOperator(), - JooqDataTypeUtil.getByteFromInteger(taskInstance.getStartupMode()), - taskInstance.getCurrentStepId(), - JooqDataTypeUtil.getByteFromInteger(taskInstance.getStatus()), + JooqDataTypeUtil.toByte(taskInstance.getStartupMode()), + taskInstance.getCurrentStepInstanceId(), + taskInstance.getStatus().getValue().byteValue(), taskInstance.getStartTime(), taskInstance.getEndTime(), taskInstance.getTotalTime(), taskInstance.getCreateTime(), taskInstance.getCallbackUrl(), - JooqDataTypeUtil.getByteFromInteger(taskInstance.getType()), + JooqDataTypeUtil.toByte(taskInstance.getType()), taskInstance.getAppCode()) .returning(TABLE.ID).fetchOne(); return record.getValue(TABLE.ID); @@ -122,11 +122,11 @@ private TaskInstanceDTO extractInfo(Record record) { taskInstance.setDebugTask(record.get(TaskInstance.TASK_INSTANCE.IS_DEBUG_TASK) == 1); taskInstance.setAppId(record.get(TaskInstance.TASK_INSTANCE.APP_ID)); taskInstance.setName(record.get(TaskInstance.TASK_INSTANCE.NAME)); - taskInstance.setType(JooqDataTypeUtil.getIntegerFromByte(record.get(TaskInstance.TASK_INSTANCE.TYPE))); + taskInstance.setType(JooqDataTypeUtil.toInteger(record.get(TaskInstance.TASK_INSTANCE.TYPE))); taskInstance.setOperator(record.get(TaskInstance.TASK_INSTANCE.OPERATOR)); - taskInstance.setStartupMode(JooqDataTypeUtil.getIntegerFromByte(record.get(TaskInstance.TASK_INSTANCE.STARTUP_MODE))); - taskInstance.setCurrentStepId(record.get(TaskInstance.TASK_INSTANCE.CURRENT_STEP_ID)); - taskInstance.setStatus(JooqDataTypeUtil.getIntegerFromByte(record.get(TaskInstance.TASK_INSTANCE.STATUS))); + taskInstance.setStartupMode(JooqDataTypeUtil.toInteger(record.get(TaskInstance.TASK_INSTANCE.STARTUP_MODE))); + taskInstance.setCurrentStepInstanceId(record.get(TaskInstance.TASK_INSTANCE.CURRENT_STEP_ID)); + taskInstance.setStatus(RunStatusEnum.valueOf(record.get(TaskInstance.TASK_INSTANCE.STATUS))); taskInstance.setStartTime(record.get(TaskInstance.TASK_INSTANCE.START_TIME)); taskInstance.setEndTime(record.get(TaskInstance.TASK_INSTANCE.END_TIME)); taskInstance.setTotalTime(record.get(TaskInstance.TASK_INSTANCE.TOTAL_TIME)); @@ -202,7 +202,7 @@ public void updateTaskCurrentStepId(Long taskInstanceId, Long stepInstanceId) { public void resetTaskStatus(Long taskInstanceId) { ctx.update(TABLE).setNull(TABLE.START_TIME).setNull(TABLE.END_TIME).setNull(TABLE.TOTAL_TIME) .setNull(TABLE.CURRENT_STEP_ID) - .set(TABLE.STATUS, JooqDataTypeUtil.getByteFromInteger(RunStatusEnum.BLANK.getValue())) + .set(TABLE.STATUS, JooqDataTypeUtil.toByte(RunStatusEnum.BLANK.getValue())) .where(TABLE.ID.eq(taskInstanceId)) .execute(); } @@ -314,17 +314,17 @@ private List buildSearchCondition(TaskInstanceQuery taskQuery) { conditions.add(TABLE.NAME.like("%" + taskQuery.getTaskName() + "%")); } if (taskQuery.getStatus() != null) { - conditions.add(TABLE.STATUS.eq(JooqDataTypeUtil.getByteFromInteger(taskQuery.getStatus().getValue()))); + conditions.add(TABLE.STATUS.eq(JooqDataTypeUtil.toByte(taskQuery.getStatus().getValue()))); } if (CollectionUtils.isNotEmpty(taskQuery.getStartupModes())) { if (taskQuery.getStartupModes().size() == 0) { - conditions.add(TABLE.STARTUP_MODE.eq(JooqDataTypeUtil.getByteFromInteger(taskQuery.getStartupModes().get(0).getValue()))); + conditions.add(TABLE.STARTUP_MODE.eq(JooqDataTypeUtil.toByte(taskQuery.getStartupModes().get(0).getValue()))); } else { conditions.add(TABLE.STARTUP_MODE.in(taskQuery.getStartupModeValues())); } } if (taskQuery.getTaskType() != null) { - conditions.add(TABLE.TYPE.eq(JooqDataTypeUtil.getByteFromInteger(taskQuery.getTaskType().getValue()))); + conditions.add(TABLE.TYPE.eq(JooqDataTypeUtil.toByte(taskQuery.getTaskType().getValue()))); } if (taskQuery.getStartTime() != null) { conditions.add(TABLE.CREATE_TIME.ge(taskQuery.getStartTime())); @@ -398,7 +398,7 @@ public void updateTaskExecutionInfo(long taskInstanceId, RunStatusEnum status, L UpdateSetMoreStep updateSetMoreStep = null; if (status != null) { updateSetMoreStep = ctx.update(TABLE).set(TABLE.STATUS, - JooqDataTypeUtil.getByteFromInteger(status.getValue())); + JooqDataTypeUtil.toByte(status.getValue())); } if (currentStepId != null) { if (updateSetMoreStep == null) { @@ -435,7 +435,7 @@ public void updateTaskExecutionInfo(long taskInstanceId, RunStatusEnum status, L } @Override - public void resetTaskExecuteInfoForResume(long taskInstanceId) { + public void resetTaskExecuteInfoForRetry(long taskInstanceId) { ctx.update(TABLE) .setNull(TABLE.END_TIME) .setNull(TABLE.TOTAL_TIME) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceVariableDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceVariableDAOImpl.java index a4ec81ad03..a9651f0ef7 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceVariableDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceVariableDAOImpl.java @@ -72,9 +72,9 @@ private TaskVariableDTO extract(Record record) { taskVariable.setId(record.get(TABLE.ID)); taskVariable.setTaskInstanceId(record.get(TABLE.TASK_INSTANCE_ID)); taskVariable.setName(record.get(TABLE.NAME)); - taskVariable.setType(JooqDataTypeUtil.getIntegerFromByte(record.get(TABLE.TYPE))); + taskVariable.setType(JooqDataTypeUtil.toInteger(record.get(TABLE.TYPE))); taskVariable.setValue(record.get(TABLE.VALUE)); - taskVariable.setChangeable(JooqDataTypeUtil.getIntegerFromByte(record.get(TABLE.IS_CHANGEABLE)).equals(1)); + taskVariable.setChangeable(JooqDataTypeUtil.toInteger(record.get(TABLE.IS_CHANGEABLE)).equals(1)); return taskVariable; } @@ -95,8 +95,8 @@ public void saveTaskInstanceVariables(List taskVarList) { TABLE.IS_CHANGEABLE); taskVarList.forEach(taskVar -> insertStep.values(taskVar.getTaskInstanceId(), - taskVar.getName(), JooqDataTypeUtil.getByteFromInteger(taskVar.getType()), - taskVar.getValue(), JooqDataTypeUtil.getByteFromInteger(taskVar.isChangeable() ? 1 : 0))); + taskVar.getName(), JooqDataTypeUtil.toByte(taskVar.getType()), + taskVar.getValue(), JooqDataTypeUtil.toByte(taskVar.isChangeable() ? 1 : 0))); insertStep.execute(); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/GseTaskManager.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/GseTaskManager.java deleted file mode 100644 index a074f380f1..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/GseTaskManager.java +++ /dev/null @@ -1,671 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.engine; - -import brave.Tracing; -import com.tencent.bk.job.common.redis.util.LockUtils; -import com.tencent.bk.job.common.util.date.DateUtils; -import com.tencent.bk.job.execute.common.constants.RunStatusEnum; -import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; -import com.tencent.bk.job.execute.common.exception.MessageHandlerUnavailableException; -import com.tencent.bk.job.execute.common.ha.DestroyOrder; -import com.tencent.bk.job.execute.config.JobExecuteConfig; -import com.tencent.bk.job.execute.config.StorageSystemConfig; -import com.tencent.bk.job.execute.engine.consts.IpStatus; -import com.tencent.bk.job.execute.engine.evict.TaskEvictPolicyExecutor; -import com.tencent.bk.job.execute.engine.exception.ExceptionStatusManager; -import com.tencent.bk.job.execute.engine.executor.AbstractGseTaskExecutor; -import com.tencent.bk.job.execute.engine.executor.FileTaskExecutor; -import com.tencent.bk.job.execute.engine.executor.SQLScriptTaskExecutor; -import com.tencent.bk.job.execute.engine.executor.ScriptTaskExecutor; -import com.tencent.bk.job.execute.engine.model.GseTaskExecuteResult; -import com.tencent.bk.job.execute.engine.result.ResultHandleManager; -import com.tencent.bk.job.execute.engine.result.ha.ResultHandleTaskKeepaliveManager; -import com.tencent.bk.job.execute.engine.util.RunningTaskCounter; -import com.tencent.bk.job.execute.engine.variable.JobBuildInVariableResolver; -import com.tencent.bk.job.execute.model.GseTaskIpLogDTO; -import com.tencent.bk.job.execute.model.GseTaskLogDTO; -import com.tencent.bk.job.execute.model.StepInstanceDTO; -import com.tencent.bk.job.execute.model.TaskInstanceDTO; -import com.tencent.bk.job.execute.monitor.ExecuteMetricNames; -import com.tencent.bk.job.execute.monitor.metrics.ExecuteMonitor; -import com.tencent.bk.job.execute.monitor.metrics.GseTasksExceptionCounter; -import com.tencent.bk.job.execute.service.AccountService; -import com.tencent.bk.job.execute.service.AgentService; -import com.tencent.bk.job.execute.service.GseTaskLogService; -import com.tencent.bk.job.execute.service.LogService; -import com.tencent.bk.job.execute.service.StepInstanceVariableValueService; -import com.tencent.bk.job.execute.service.TaskInstanceService; -import com.tencent.bk.job.execute.service.TaskInstanceVariableService; -import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Triple; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.SmartLifecycle; -import org.springframework.stereotype.Component; -import org.springframework.util.StopWatch; - -import javax.annotation.PostConstruct; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; - -/** - * GSE任务执行管理 - */ -@Component -@Slf4j -public class GseTaskManager implements SmartLifecycle { - private final ResultHandleManager resultHandleManager; - private final TaskInstanceService taskInstanceService; - private final GseTaskLogService gseTaskLogService; - private final TaskExecuteControlMsgSender taskManager; - private final AccountService accountService; - private final LogService logService; - private final TaskInstanceVariableService taskInstanceVariableService; - private final StepInstanceVariableValueService stepInstanceVariableValueService; - private final AgentService agentService; - private final ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager; - private final JobBuildInVariableResolver jobBuildInVariableResolver; - private final ExceptionStatusManager exceptionStatusManager; - private final Tracing tracing; - private final ExecuteMonitor executeMonitor; - private final StorageSystemConfig storageSystemConfig; - private final JobExecuteConfig jobExecuteConfig; - private final Object lifecycleMonitor = new Object(); - private final RunningTaskCounter counter = new RunningTaskCounter<>("GseTask-Counter"); - private final TaskEvictPolicyExecutor taskEvictPolicyExecutor; - /** - * 正在执行中的任务 - */ - private Map executorMap = new ConcurrentHashMap<>(); - /** - * 本地服务器IP - */ - private String serverIp = null; - private volatile boolean running = false; - private volatile boolean active = false; - - private GseTasksExceptionCounter gseTasksExceptionCounter; - /** - * 正在处理的gse任务数 - */ - private AtomicInteger runningTasks = new AtomicInteger(0); - /** - * 正在处理的gse文件任务数 - */ - private AtomicInteger runningFileTasks = new AtomicInteger(0); - /** - * 正在处理的gse脚本任务数 - */ - private AtomicInteger runningScriptTasks = new AtomicInteger(0); - /** - * 正在处理的gse文件任务数 - */ - private AtomicInteger fileTaskCounter = new AtomicInteger(0); - /** - * 正在处理的gse脚本任务数 - */ - private AtomicInteger scriptTaskCounter = new AtomicInteger(0); - - /** - * GseTaskManager Constructor - */ - @Autowired - public GseTaskManager(ResultHandleManager resultHandleManager, - TaskInstanceService taskInstanceService, - GseTaskLogService gseTaskLogService, - TaskExecuteControlMsgSender taskManager, - AccountService accountService, - LogService logService, - TaskInstanceVariableService taskInstanceVariableService, - StepInstanceVariableValueService stepInstanceVariableValueService, - JobBuildInVariableResolver jobBuildInVariableResolver, - StorageSystemConfig storageSystemConfig, - AgentService agentService, - ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager, - ExceptionStatusManager exceptionStatusManager, - GseTasksExceptionCounter gseTasksExceptionCounter, - Tracing tracing, - ExecuteMonitor executeMonitor, - JobExecuteConfig jobExecuteConfig, - TaskEvictPolicyExecutor taskEvictPolicyExecutor) { - this.resultHandleManager = resultHandleManager; - this.taskInstanceService = taskInstanceService; - this.gseTaskLogService = gseTaskLogService; - this.taskManager = taskManager; - this.accountService = accountService; - this.logService = logService; - this.taskInstanceVariableService = taskInstanceVariableService; - this.stepInstanceVariableValueService = stepInstanceVariableValueService; - this.jobBuildInVariableResolver = jobBuildInVariableResolver; - this.storageSystemConfig = storageSystemConfig; - this.agentService = agentService; - this.resultHandleTaskKeepaliveManager = resultHandleTaskKeepaliveManager; - this.exceptionStatusManager = exceptionStatusManager; - this.gseTasksExceptionCounter = gseTasksExceptionCounter; - this.tracing = tracing; - this.executeMonitor = executeMonitor; - this.jobExecuteConfig = jobExecuteConfig; - this.taskEvictPolicyExecutor = taskEvictPolicyExecutor; - } - - /** - * 判定GSE任务管理器是否活跃 - * - * @param stepInstanceId 当前请求调度的步骤Id - */ - private void checkActiveStatus(long stepInstanceId) { - if (!isActive()) { - log.warn("GseTaskManager is not active, reject! stepInstanceId: {}", stepInstanceId); - throw new MessageHandlerUnavailableException(); - } - } - - /** - * 检查并获取任务与步骤实例，并判断后续是否应当下发GSE任务 - * - * @param stepInstanceId 步骤Id - * @return <是否应下发GSE任务，任务实例，步骤实例> - */ - private Triple checkTaskAndStep(long stepInstanceId) { - StepInstanceDTO stepInstance = taskInstanceService.getStepInstanceDetail(stepInstanceId); - TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(stepInstance.getTaskInstanceId()); - - // 如果任务应当被驱逐，直接置为被丢弃状态 - if (taskEvictPolicyExecutor.shouldEvictTask(taskInstance)) { - taskEvictPolicyExecutor.updateEvictedTaskStatus(taskInstance, stepInstance); - return Triple.of(false, taskInstance, stepInstance); - } - - // 如果任务处于“终止中”状态，直接终止 - if (taskInstance.getStatus().equals(RunStatusEnum.STOPPING.getValue())) { - log.info("Task instance status is stopping, stop executing the step! taskInstanceId:{}, " - + "stepInstanceId:{}", - taskInstance.getId(), stepInstance.getId()); - taskManager.refreshTask(stepInstance.getTaskInstanceId()); - return Triple.of(false, taskInstance, stepInstance); - } - return Triple.of(true, taskInstance, stepInstance); - } - - private Set getTargetIpListWithCloudId(StepInstanceDTO stepInstance, StopWatch watch) { - watch.start("init-task-executor"); - Set executeIps = new HashSet<>(); - stepInstance.getTargetServers().getIpList().forEach(ipDTO -> { - executeIps.add(ipDTO.getCloudAreaId() + ":" + ipDTO.getIp()); - }); - watch.stop(); - return executeIps; - } - - /** - * 启动任务(首次执行/继续执行异常中断的任务) - * - * @param stepInstanceId 步骤实例ID - */ - public void startStep(long stepInstanceId, String requestId) { - checkActiveStatus(stepInstanceId); - - boolean success = false; - String taskName = "start-gse-task:" + stepInstanceId; - StopWatch watch = new StopWatch("GseTaskManager-start-step-" + stepInstanceId); - String startTaskRequestId = requestId; - if (StringUtils.isEmpty(startTaskRequestId)) { - startTaskRequestId = UUID.randomUUID().toString(); - } - - AbstractGseTaskExecutor gseTaskExecutor = null; - try { - watch.start("get-running-lock"); - // 可重入锁，如果任务正在执行，则放弃 - if (!LockUtils.tryGetReentrantLock( - "job:running:gse:task:" + stepInstanceId, startTaskRequestId, 30000L)) { - log.info("Fail to get running lock, stepInstanceId: {}", stepInstanceId); - return; - } - watch.stop(); - - watch.start("get-task-and-step-from-db-and-check-task"); - Triple triple = checkTaskAndStep(stepInstanceId); - Boolean shouldStartTask = triple.getLeft(); - if (!shouldStartTask) return; - TaskInstanceDTO taskInstance = triple.getMiddle(); - StepInstanceDTO stepInstance = triple.getRight(); - watch.stop(); - - Set executeIps = getTargetIpListWithCloudId(stepInstance, watch); - - watch.start("init-gse-task-executor"); - gseTaskExecutor = initGseTaskExecutor(startTaskRequestId, stepInstance, taskInstance, executeIps); - if (gseTaskExecutor == null) return; - watch.stop(); - - counter.add(taskName); - watch.start("execute-task"); - executeTask(gseTaskExecutor, stepInstance); - watch.stop(); - success = true; - } finally { - if (!watch.isRunning()) { - watch.start("release-running-lock"); - } - LockUtils.releaseDistributedLock("job:running:gse:task:", String.valueOf(stepInstanceId), - startTaskRequestId); - counter.release(taskName); - watch.stop(); - if (watch.getTotalTimeMillis() > 2000L) { - log.warn("GseTaskManager-> start gse step is slow, run statistics:{}", watch.prettyPrint()); - } - executeMonitor.getMeterRegistry().timer(ExecuteMetricNames.EXECUTE_TASK_PREFIX, - "task_type", getTaskTypeDesc(gseTaskExecutor), "status", success ? "ok" : "error") - .record(watch.getTotalTimeNanos(), TimeUnit.NANOSECONDS); - } - } - - private String getTaskTypeDesc(AbstractGseTaskExecutor executor) { - if (executor == null) { - return "none"; - } - if (executor instanceof ScriptTaskExecutor) { - return "script"; - } else if (executor instanceof FileTaskExecutor) { - return "file"; - } else { - return "none"; - } - } - - private void incrementRunningTasksCount(AbstractGseTaskExecutor executor) { - this.runningTasks.incrementAndGet(); - if (executor instanceof ScriptTaskExecutor) { - this.runningScriptTasks.incrementAndGet(); - } else { - this.runningFileTasks.incrementAndGet(); - } - } - - private void decrementRunningTasksCount(AbstractGseTaskExecutor executor) { - this.runningTasks.decrementAndGet(); - if (executor instanceof ScriptTaskExecutor) { - this.runningScriptTasks.decrementAndGet(); - } else { - this.runningFileTasks.decrementAndGet(); - } - } - - /** - * 初始化步骤对应的executor - * - * @param stepInstance 步骤 - * @param taskInstance 作业 - * @param executeIps 目标ip - * @return executor - */ - private AbstractGseTaskExecutor initGseTaskExecutor(String requestId, StepInstanceDTO stepInstance, - TaskInstanceDTO taskInstance, - Set executeIps) { - AbstractGseTaskExecutor gseTaskExecutor = null; - int executeType = stepInstance.getExecuteType(); - if (executeType == StepExecuteTypeEnum.EXECUTE_SCRIPT.getValue()) { - scriptTaskCounter.incrementAndGet(); - gseTaskExecutor = new ScriptTaskExecutor(requestId, gseTasksExceptionCounter, taskInstance, stepInstance, - executeIps, jobBuildInVariableResolver); - } else if (executeType == StepExecuteTypeEnum.EXECUTE_SQL.getValue()) { - gseTaskExecutor = new SQLScriptTaskExecutor(requestId, gseTasksExceptionCounter, taskInstance, - stepInstance, executeIps); - scriptTaskCounter.incrementAndGet(); - } else if (executeType == TaskStepTypeEnum.FILE.getValue()) { - gseTaskExecutor = new FileTaskExecutor(requestId, gseTasksExceptionCounter, - storageSystemConfig.getJobStorageRootPath(), - agentService.getLocalAgentBindIp(), taskInstance, stepInstance, executeIps); - fileTaskCounter.incrementAndGet(); - } - - if (gseTaskExecutor == null) { - log.warn("No match GseTaskExecutor, stepInstanceId:{}", stepInstance.getId()); - return null; - } - - gseTaskExecutor.initDependentService(resultHandleManager, taskInstanceService, gseTaskLogService, - accountService, taskInstanceVariableService, stepInstanceVariableValueService, agentService, logService, - taskManager, resultHandleTaskKeepaliveManager, executeMonitor, jobExecuteConfig); - gseTaskExecutor.setExceptionStatusManager(exceptionStatusManager); - gseTaskExecutor.setTaskEvictPolicyExecutor(taskEvictPolicyExecutor); - gseTaskExecutor.setTracing(tracing); - return gseTaskExecutor; - } - - /** - * 终止步骤 - * - * @param stepInstanceId 步骤实例ID - */ - public void stopStep(long stepInstanceId, String requestId) { - log.info("Stop gse task, stepInstanceId:" + stepInstanceId); - - StepInstanceDTO stepInstance = taskInstanceService.getStepInstanceDetail(stepInstanceId); - if (stepInstance == null || !stepInstance.getStatus().equals(RunStatusEnum.RUNNING.getValue())) { - log.info("StepInstance: {} is null or is not running, should not stop!", stepInstanceId); - return; - } - - TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(stepInstance.getTaskInstanceId()); - int executeCount = stepInstance.getExecuteCount(); - GseTaskLogDTO gseTaskLog = gseTaskLogService.getGseTaskLog(stepInstanceId, executeCount); - if (null == gseTaskLog) { - log.info("Get gseTaskLog return null, stepInstanceId: {}, executeCount:{}", stepInstanceId, executeCount); - return; - } - Set stopIps = new HashSet<>(); - stepInstance.getTargetServers().getIpList().forEach(ipDTO -> { - stopIps.add(ipDTO.getCloudAreaId() + ":" + ipDTO.getIp()); - }); - - AbstractGseTaskExecutor gseTaskExecutor = initGseTaskExecutor(createRequestIdIfEmpty(requestId), - stepInstance, taskInstance, stopIps); - if (gseTaskExecutor == null) { - log.warn("TaskExecutor is not found!"); - return; - } - - GseTaskExecuteResult stopResult = gseTaskExecutor.stopGseTask(); - // 处理GSE任务执行结果 - if (stopResult.getResultCode().equals(GseTaskExecuteResult.RESULT_CODE_STOP_SUCCESS)) { - taskInstanceService.updateStepStatus(stepInstanceId, RunStatusEnum.STOPPING.getValue()); - } - } - - private String createRequestIdIfEmpty(String requestId) { - String reqId = requestId; - if (StringUtils.isEmpty(requestId)) { - reqId = UUID.randomUUID().toString(); - } - return reqId; - } - - @PostConstruct - public void init() { - this.serverIp = agentService.getLocalAgentBindIp(); - log.info("Server ip: {}", serverIp); - } - - /** - * 重试执行失败的IP - * - * @param stepInstanceId 步骤实例ID - */ - public void retryFail(long stepInstanceId, String requestId) { - if (!isActive()) { - log.warn("GseTaskManager is not active, reject!"); - throw new MessageHandlerUnavailableException(); - } - - String taskName = "retry-fail:" + stepInstanceId; - try { - StepInstanceDTO stepInstance = taskInstanceService.getStepInstanceDetail(stepInstanceId); - if (stepInstance == null) { - log.warn("StepInstance is not exist, stop retry! stepInstanceId:{}", stepInstanceId); - return; - } - TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(stepInstance.getTaskInstanceId()); - if (taskInstance == null) { - log.warn("TaskInstance is not exist, stop retry! taskInstanceId:{}", stepInstance.getTaskInstanceId()); - return; - } - - int executeCount = stepInstance.getExecuteCount(); - List successGseTaskIpLogs = gseTaskLogService.getSuccessGseTaskIp(stepInstanceId, - executeCount - 1); - Set lastSuccessIpSet = new HashSet<>(); - if (successGseTaskIpLogs != null) { - lastSuccessIpSet.addAll(successGseTaskIpLogs.parallelStream().map(GseTaskIpLogDTO::getCloudAreaAndIp) - .collect(Collectors.toSet())); - } - Set executeIps = new HashSet<>(); - stepInstance.getTargetServers().getIpList().forEach(ipDTO -> { - String fullIp = ipDTO.getCloudAreaId() + ":" + ipDTO.getIp(); - if (!lastSuccessIpSet.contains(fullIp)) { - executeIps.add(fullIp); - } - }); - log.info("Get execute ips, stepInstanceId:{}, executeIps:{}", stepInstanceId, executeIps); - AbstractGseTaskExecutor gseTaskExecutor = initGseTaskExecutor(createRequestIdIfEmpty(requestId), - stepInstance, taskInstance, executeIps); - if (gseTaskExecutor == null) { - log.warn("No match GseTaskExecutor for task, stepInstanceId:{}", stepInstanceId); - return; - } - - // GseTaskLog初始状态 - dealGseTaskLog(stepInstance); - // 已成功执行过的IP无需执行，仅保存记录 - if (successGseTaskIpLogs != null && !successGseTaskIpLogs.isEmpty()) { - dealLastSuccessIp(executeCount, successGseTaskIpLogs); - } - - counter.add(taskName); - executeTask(gseTaskExecutor, stepInstance); - } finally { - counter.release(taskName); - } - - } - - private void dealGseTaskLog(StepInstanceDTO stepInstance) { - // 初始化GseTaskLog - GseTaskLogDTO gseTaskLog = new GseTaskLogDTO(); - gseTaskLog.setStepInstanceId(stepInstance.getId()); - gseTaskLog.setExecuteCount(stepInstance.getExecuteCount()); - gseTaskLog.setStatus(RunStatusEnum.RUNNING.getValue()); - gseTaskLog.setStartTime(DateUtils.currentTimeMillis()); - gseTaskLogService.saveGseTaskLog(gseTaskLog); - } - - /** - * 处理已执行成功的IP - * - * @param executeCount 执行次数 - * @param lastSuccessGseTaskIpLogs 已执行成功IP - */ - private void dealLastSuccessIp(int executeCount, List lastSuccessGseTaskIpLogs) { - List ipLogList = new ArrayList<>(); - for (GseTaskIpLogDTO gseTaskIpLog : lastSuccessGseTaskIpLogs) { - GseTaskIpLogDTO ipLog = setGseTaskIpLogForRetry(gseTaskIpLog, executeCount, IpStatus.LAST_SUCCESS); - ipLogList.add(ipLog); - } - if (ipLogList.size() > 0) { - gseTaskLogService.batchSaveIpLog(ipLogList); - } - } - - private GseTaskIpLogDTO setGseTaskIpLogForRetry(GseTaskIpLogDTO gseTaskIpLog, int executeCount, IpStatus ipStatus) { - gseTaskIpLog.setExecuteCount(executeCount); - gseTaskIpLog.setStatus(ipStatus.getValue()); - return gseTaskIpLog; - } - - /** - * 重试执行所有的IP,包括已执行成功的 - * - * @param stepInstanceId 步骤实例ID - */ - public void retryAll(long stepInstanceId, String requestId) { - if (!isActive()) { - log.warn("GseTaskManager is not active, reject!"); - throw new MessageHandlerUnavailableException(); - } - String taskName = "retry-all:" + stepInstanceId; - try { - StepInstanceDTO stepInstance = taskInstanceService.getStepInstanceDetail(stepInstanceId); - if (stepInstance == null) { - log.warn("StepInstance is not exist, stop retry! stepInstanceId:{}", stepInstanceId); - return; - } - TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(stepInstance.getTaskInstanceId()); - if (taskInstance == null) { - log.warn("TaskInstance is not exist, stop retry! taskInstanceId:{}", stepInstance.getTaskInstanceId()); - return; - } - - Set executeIps = new HashSet<>(); - stepInstance.getTargetServers().getIpList().forEach(ipDTO -> { - String fullIp = ipDTO.getCloudAreaId() + ":" + ipDTO.getIp(); - executeIps.add(fullIp); - }); - log.info("Get execute ips, stepInstanceId:{}, executeIps:{}", stepInstanceId, executeIps); - AbstractGseTaskExecutor gseTaskExecutor = initGseTaskExecutor(createRequestIdIfEmpty(requestId), - stepInstance, taskInstance, executeIps); - if (gseTaskExecutor == null) { - log.warn("No match GseTaskExecutor for task, stepInstanceId:{}", stepInstanceId); - return; - } - - counter.add(taskName); - executeTask(gseTaskExecutor, stepInstance); - } finally { - counter.release(taskName); - } - - } - - private void executeTask(AbstractGseTaskExecutor gseTaskExecutor, StepInstanceDTO stepInstance) { - long stepInstanceId = stepInstance.getId(); - int executeCount = stepInstance.getExecuteCount(); - String taskKey = stepInstanceId + "_" + executeCount; - try { - executorMap.put(taskKey, gseTaskExecutor); - incrementRunningTasksCount(gseTaskExecutor); - gseTaskExecutor.execute(); - } finally { - executorMap.remove(taskKey); - decrementRunningTasksCount(gseTaskExecutor); - } - } - - - private boolean isActive() { - synchronized (this.lifecycleMonitor) { - return this.active; - } - } - - @Override - public void start() { - if (isRunning()) { - return; - } - log.info("GseTaskManager starting."); - counter.start(); - synchronized (lifecycleMonitor) { - this.running = true; - this.active = true; - } - } - - @Override - public void stop() { - log.info("GseTaskManager stopping."); - synchronized (this.lifecycleMonitor) { - this.active = false; - } - try { - counter.stop(); - counter.waitUntilTaskDone(5, TimeUnit.SECONDS); - } finally { - synchronized (this.lifecycleMonitor) { - this.running = false; - } - } - log.info("Save unfinished task snapshot, tasks: {}", executorMap.keySet()); - executorMap.values().parallelStream().forEach(AbstractGseTaskExecutor::interrupt); - log.info("Save unfinished task snapshot successfully."); - } - - @Override - public boolean isRunning() { - synchronized (this.lifecycleMonitor) { - return (this.running); - } - } - - @Override - public int getPhase() { - return DestroyOrder.GSE_TASK_HANDLER; - } - - /** - * 返回正在执行的任务数量 - * - * @return 任务数量 - */ - public int getRunningTaskCount() { - return this.runningTasks.get(); - } - - /** - * 返回正在执行的文件任务数量 - * - * @return 任务数量 - */ - public int getRunningFileTaskCount() { - return this.runningFileTasks.get(); - } - - /** - * 返回正在执行的脚本任务数量 - * - * @return 任务数量 - */ - public int getRunningScriptTaskCount() { - return this.runningScriptTasks.get(); - } - - /** - * 返回累计处理的文件任务数量 - * - * @return 任务数量 - */ - public int getFileTaskCount() { - return this.fileTaskCounter.get(); - } - - /** - * 返回累计处理的脚本任务数量 - * - * @return 任务数量 - */ - public int getScriptTaskCount() { - return this.scriptTaskCounter.get(); - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/TaskExecuteControlMsgSender.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/TaskExecuteControlMsgSender.java deleted file mode 100644 index e52c6675d3..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/TaskExecuteControlMsgSender.java +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.engine; - -import com.tencent.bk.job.execute.engine.model.JobCallbackDTO; -import com.tencent.bk.job.execute.model.TaskNotifyDTO; - -/** - * 作业执行控制消息发送 - */ -public interface TaskExecuteControlMsgSender { - - /** - * 发送启动作业的作业控制消息 - * - * @param taskInstanceId 作业实例ID - */ - void startTask(long taskInstanceId); - - /** - * 发送停止作业的作业控制消息 - * - * @param taskInstanceId 作业实例ID - */ - void stopTask(long taskInstanceId); - - /** - * 发送重头执行作业的作业控制消息 - * - * @param taskInstanceId 作业实例ID - */ - void restartTask(long taskInstanceId); - - /** - * 触发作业继续后续步骤的作业控制消息 - * - * @param taskInstanceId 作业实例ID - */ - void refreshTask(long taskInstanceId); - - /** - * 触发忽略错误的作业控制消息 - * - * @param stepInstanceId 步骤实例ID - */ - void ignoreStepError(long stepInstanceId); - - /** - * 发送进入下一步骤的作业控制消息 - * - * @param stepInstanceId 步骤实例ID - */ - void nextStep(long stepInstanceId); - - /** - * 发送进入下一步骤的作业控制消息 - * - * @param stepInstanceId 步骤实例ID - */ - void confirmStepContinue(long stepInstanceId); - - /** - * 人工确认-终止流程 - * - * @param stepInstanceId 步骤实例ID - */ - void confirmStepTerminate(long stepInstanceId); - - /** - * 人工确认-重新发起确认 - * - * @param stepInstanceId 步骤实例ID - */ - void confirmStepRestart(long stepInstanceId); - - /** - * 发送启动步骤的步骤控制消息 - * - * @param stepInstanceId 步骤实例ID - */ - void startStep(long stepInstanceId); - - /** - * 发送跳过步骤的步骤控制消息 - * - * @param stepInstanceId 步骤实例ID - */ - void skipStep(long stepInstanceId); - - /** - * 发送强制终止步骤的步骤控制消息 - * - * @param stepInstanceId 步骤实例ID - */ - void stopStep(long stepInstanceId); - - /** - * 重新执行步骤中失败的ip - * - * @param stepInstanceId 步骤实例ID - */ - void retryStepFail(long stepInstanceId); - - /** - * 重新执行步骤 - * - * @param stepInstanceId 步骤实例ID - */ - void retryStepAll(long stepInstanceId); - - /** - * 发送继续GSE文件分发步骤的步骤控制消息 - * - * @param stepInstanceId 步骤实例ID - */ - void continueGseFileStep(long stepInstanceId); - - /** - * 发送清理步骤的步骤控制消息：清理步骤中产生的临时文件等 - * - * @param stepInstanceId 步骤实例ID - */ - void clearStep(long stepInstanceId); - - /** - * 发送执行gse步骤的消息 - * - * @param stepInstanceId 步骤实例ID - */ - void startGseStep(long stepInstanceId); - - /** - * 恢复GSE任务执行 - * - * @param stepInstanceId 步骤实例ID - * @param executeCount 执行次数 - * @param requestId 请求ID - */ - void resumeGseStep(long stepInstanceId, int executeCount, String requestId); - - /** - * 重新执行步骤中失败的ip - * - * @param stepInstanceId 步骤实例ID - */ - void retryGseStepFail(long stepInstanceId); - - /** - * 重新执行步骤 - * - * @param stepInstanceId 步骤实例ID - */ - void retryGseStepAll(long stepInstanceId); - - /** - * 发送强制终止TSC步骤的消息 - * - * @param stepInstanceId 步骤实例ID - */ - void stopGseStep(long stepInstanceId); - - /** - * 异步发送消息通知 - * - * @param notification 消息内容 - */ - void asyncSendNotifyMsg(TaskNotifyDTO notification); - - /** - * 发送回调信息 - * - * @param jobCallbackDto - */ - void sendCallback(JobCallbackDTO jobCallbackDto); -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/TaskExecuteControlMsgSenderImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/TaskExecuteControlMsgSenderImpl.java deleted file mode 100644 index 01b28cbb75..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/TaskExecuteControlMsgSenderImpl.java +++ /dev/null @@ -1,342 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.engine; - -import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.execute.engine.consts.GseStepActionEnum; -import com.tencent.bk.job.execute.engine.consts.JobActionEnum; -import com.tencent.bk.job.execute.engine.consts.StepActionEnum; -import com.tencent.bk.job.execute.engine.message.*; -import com.tencent.bk.job.execute.engine.model.JobCallbackDTO; -import com.tencent.bk.job.execute.engine.model.StepControlMessage; -import com.tencent.bk.job.execute.engine.model.TaskControlMessage; -import com.tencent.bk.job.execute.model.TaskNotifyDTO; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.messaging.MessageChannel; -import org.springframework.messaging.support.MessageBuilder; -import org.springframework.stereotype.Service; - -import java.time.LocalDateTime; -import java.util.UUID; - -@Service -@Slf4j -public class TaskExecuteControlMsgSenderImpl implements TaskExecuteControlMsgSender { - /** - * 消息通道-作业 - */ - private final MessageChannel taskOutput; - /** - * 消息通道-步骤 - */ - private final MessageChannel stepOutput; - /** - * 消息通道-GSE任务 - */ - private final MessageChannel gseTaskOutput; - - /** - * 消息通道-消息通知 - */ - private final MessageChannel notifyMsgOutput; - /** - * 消息通道-作业执行完成回调 - */ - private final MessageChannel callbackOutput; - - /** - * 消息通道-作业执行完成回调 - */ - private final MessageChannel resultHandleTaskResumeOutput; - - @Autowired - public TaskExecuteControlMsgSenderImpl(@Qualifier(TaskProcessor.OUTPUT) MessageChannel taskOutput, - @Qualifier(StepProcessor.OUTPUT) MessageChannel stepOutput, - @Qualifier(GseTaskProcessor.OUTPUT) MessageChannel gseTaskOutput, - @Qualifier(NotifyMsgProcessor.OUTPUT) MessageChannel notifyMsgOutput, - @Qualifier(CallbackProcessor.OUTPUT) MessageChannel callbackOutput, - @Qualifier(TaskResultHandleResumeProcessor.OUTPUT) MessageChannel resultHandleTaskResumeOutput) { - this.taskOutput = taskOutput; - this.stepOutput = stepOutput; - this.gseTaskOutput = gseTaskOutput; - this.notifyMsgOutput = notifyMsgOutput; - this.callbackOutput = callbackOutput; - this.resultHandleTaskResumeOutput = resultHandleTaskResumeOutput; - } - - @Override - public void startTask(long taskInstanceId) { - log.info("Begin to send start task control message, taskInstanceId={}", taskInstanceId); - TaskControlMessage msg = new TaskControlMessage(); - msg.setTaskInstanceId(taskInstanceId); - msg.setAction(JobActionEnum.START.getValue()); - msg.setTime(LocalDateTime.now()); - taskOutput.send(MessageBuilder.withPayload(msg).build()); - log.info("Send start task control message successfully, taskInstanceId={}", taskInstanceId); - } - - @Override - public void stopTask(long taskInstanceId) { - log.info("Begin to send stop task control message, taskInstanceId={}", taskInstanceId); - TaskControlMessage msg = new TaskControlMessage(); - msg.setTaskInstanceId(taskInstanceId); - msg.setAction(JobActionEnum.STOP.getValue()); - msg.setTime(LocalDateTime.now()); - taskOutput.send(MessageBuilder.withPayload(msg).build()); - log.info("Send stop task control message successfully, taskInstanceId={}", taskInstanceId); - } - - @Override - public void restartTask(long taskInstanceId) { - log.info("Begin to send restart task control message, taskInstanceId={}", taskInstanceId); - TaskControlMessage msg = new TaskControlMessage(); - msg.setTaskInstanceId(taskInstanceId); - msg.setAction(JobActionEnum.RESTART.getValue()); - msg.setTime(LocalDateTime.now()); - taskOutput.send(MessageBuilder.withPayload(msg).build()); - log.info("Send restart task control message successfully, taskInstanceId={}", taskInstanceId); - } - - @Override - public void refreshTask(long taskInstanceId) { - log.info("Begin to send refresh task control message, taskInstanceId={}", taskInstanceId); - TaskControlMessage msg = new TaskControlMessage(); - msg.setTaskInstanceId(taskInstanceId); - msg.setAction(JobActionEnum.REFRESH.getValue()); - msg.setTime(LocalDateTime.now()); - taskOutput.send(MessageBuilder.withPayload(msg).build()); - log.info("Send refresh task control message successfully, taskInstanceId={}", taskInstanceId); - } - - @Override - public void ignoreStepError(long stepInstanceId) { - log.info("Begin to send ignore-error step control message, stepInstanceId={}", stepInstanceId); - StepControlMessage msg = new StepControlMessage(); - msg.setStepInstanceId(stepInstanceId); - msg.setAction(StepActionEnum.IGNORE_ERROR.getValue()); - msg.setTime(LocalDateTime.now()); - stepOutput.send(MessageBuilder.withPayload(msg).build()); - log.info("Send ignore-error step control message successfully, stepInstanceId={}", stepInstanceId); - } - - @Override - public void nextStep(long stepInstanceId) { - log.info("Begin to send next-step step control message, stepInstanceId={}", stepInstanceId); - StepControlMessage msg = new StepControlMessage(); - msg.setStepInstanceId(stepInstanceId); - msg.setAction(StepActionEnum.NEXT_STEP.getValue()); - msg.setTime(LocalDateTime.now()); - stepOutput.send(MessageBuilder.withPayload(msg).build()); - log.info("Send next-step step control message successfully, stepInstanceId={}", stepInstanceId); - } - - @Override - public void confirmStepContinue(long stepInstanceId) { - log.info("Begin to send confirm-continue step control message, stepInstanceId={}", stepInstanceId); - StepControlMessage msg = new StepControlMessage(); - msg.setStepInstanceId(stepInstanceId); - msg.setAction(StepActionEnum.CONFIRM_CONTINUE.getValue()); - msg.setTime(LocalDateTime.now()); - stepOutput.send(MessageBuilder.withPayload(msg).build()); - log.info("Send confirm-continue step control message successfully, stepInstanceId={}", stepInstanceId); - } - - @Override - public void confirmStepTerminate(long stepInstanceId) { - log.info("Begin to send confirm-terminate step control message, stepInstanceId={}", stepInstanceId); - StepControlMessage msg = new StepControlMessage(); - msg.setStepInstanceId(stepInstanceId); - msg.setAction(StepActionEnum.CONFIRM_TERMINATE.getValue()); - msg.setTime(LocalDateTime.now()); - stepOutput.send(MessageBuilder.withPayload(msg).build()); - log.info("Send confirm-terminate step control message successfully, stepInstanceId={}", stepInstanceId); - } - - @Override - public void confirmStepRestart(long stepInstanceId) { - log.info("Begin to send confirm-restart step control message, stepInstanceId={}", stepInstanceId); - StepControlMessage msg = new StepControlMessage(); - msg.setStepInstanceId(stepInstanceId); - msg.setAction(StepActionEnum.CONFIRM_RESTART.getValue()); - msg.setTime(LocalDateTime.now()); - stepOutput.send(MessageBuilder.withPayload(msg).build()); - log.info("Send confirm-restart step control message successfully, stepInstanceId={}", stepInstanceId); - } - - @Override - public void startStep(long stepInstanceId) { - StepControlMessage msg = new StepControlMessage(); - msg.setStepInstanceId(stepInstanceId); - msg.setAction(StepActionEnum.START.getValue()); - msg.setTime(LocalDateTime.now()); - stepOutput.send(MessageBuilder.withPayload(msg).build()); - log.info("Send start step control message successfully, stepInstanceId={}", stepInstanceId); - } - - @Override - public void skipStep(long stepInstanceId) { - log.info("Begin to send skip step control message successfully, stepInstanceId={}", stepInstanceId); - StepControlMessage msg = new StepControlMessage(); - msg.setStepInstanceId(stepInstanceId); - msg.setAction(StepActionEnum.SKIP.getValue()); - msg.setTime(LocalDateTime.now()); - stepOutput.send(MessageBuilder.withPayload(msg).build()); - log.info("Send skip step control message successfully, stepInstanceId={}", stepInstanceId); - } - - @Override - public void stopStep(long stepInstanceId) { - log.info("Begin to send stop step control message successfully, stepInstanceId={}", stepInstanceId); - StepControlMessage msg = new StepControlMessage(); - msg.setStepInstanceId(stepInstanceId); - msg.setAction(StepActionEnum.STOP.getValue()); - msg.setTime(LocalDateTime.now()); - stepOutput.send(MessageBuilder.withPayload(msg).build()); - log.info("Send stop step control message successfully, stepInstanceId={}", stepInstanceId); - } - - @Override - public void retryStepFail(long stepInstanceId) { - log.info("Begin to send retry-step-fail step control message successfully, stepInstanceId={}", stepInstanceId); - StepControlMessage msg = new StepControlMessage(); - msg.setStepInstanceId(stepInstanceId); - msg.setAction(StepActionEnum.RETRY_FAIL.getValue()); - msg.setTime(LocalDateTime.now()); - stepOutput.send(MessageBuilder.withPayload(msg).build()); - log.info("Send retry-step-fail step control message successfully, stepInstanceId={}", stepInstanceId); - } - - @Override - public void retryStepAll(long stepInstanceId) { - log.info("Begin to send retry-step-all step control message successfully, stepInstanceId={}", stepInstanceId); - StepControlMessage msg = new StepControlMessage(); - msg.setStepInstanceId(stepInstanceId); - msg.setAction(StepActionEnum.RETRY_ALL.getValue()); - msg.setTime(LocalDateTime.now()); - stepOutput.send(MessageBuilder.withPayload(msg).build()); - log.info("Send retry-step-all step control message successfully, stepInstanceId={}", stepInstanceId); - } - - @Override - public void continueGseFileStep(long stepInstanceId) { - log.info("Begin to send continue-gse-file-step step control message successfully, stepInstanceId={}", - stepInstanceId); - StepControlMessage msg = new StepControlMessage(); - msg.setStepInstanceId(stepInstanceId); - msg.setAction(StepActionEnum.CONTINUE_FILE_PUSH.getValue()); - msg.setTime(LocalDateTime.now()); - stepOutput.send(MessageBuilder.withPayload(msg).build()); - log.info("Send continue-gse-file-step step control message successfully, stepInstanceId={}", stepInstanceId); - } - - @Override - public void clearStep(long stepInstanceId) { - log.info("Begin to send clear-step step control message successfully, stepInstanceId={}", stepInstanceId); - StepControlMessage msg = new StepControlMessage(); - msg.setStepInstanceId(stepInstanceId); - msg.setAction(StepActionEnum.CLEAR.getValue()); - msg.setTime(LocalDateTime.now()); - stepOutput.send(MessageBuilder.withPayload(msg).build()); - log.info("Send clear-step step control message successfully, stepInstanceId={}", stepInstanceId); - } - - @Override - public void startGseStep(long stepInstanceId) { - log.info("Begin to send start gse step control message successfully, stepInstanceId={}", stepInstanceId); - StepControlMessage msg = new StepControlMessage(); - msg.setStepInstanceId(stepInstanceId); - msg.setAction(GseStepActionEnum.START.getValue()); - msg.setTime(LocalDateTime.now()); - msg.setRequestId(UUID.randomUUID().toString()); - gseTaskOutput.send(MessageBuilder.withPayload(msg).build()); - log.info("Send start gse step control message successfully, stepInstanceId={}", stepInstanceId); - } - - @Override - public void resumeGseStep(long stepInstanceId, int executeCount, String requestId) { - log.info("Begin to send resume gse step control message successfully, stepInstanceId={}, executeCount={}, " + - "requestId={}", - stepInstanceId, executeCount, requestId); - StepControlMessage msg = new StepControlMessage(); - msg.setStepInstanceId(stepInstanceId); - msg.setExecuteCount(executeCount); - msg.setAction(StepActionEnum.RESUME.getValue()); - msg.setTime(LocalDateTime.now()); - msg.setRequestId(requestId); - resultHandleTaskResumeOutput.send(MessageBuilder.withPayload(msg).build()); - log.info("Send resume gse step control message successfully, stepInstanceId={}, executeCount={}, requestId={}", - stepInstanceId, executeCount, requestId); - } - - @Override - public void retryGseStepFail(long stepInstanceId) { - log.info("Begin to send retry gse step fail control message successfully, stepInstanceId={}", stepInstanceId); - StepControlMessage msg = new StepControlMessage(); - msg.setStepInstanceId(stepInstanceId); - msg.setAction(GseStepActionEnum.RETRY_FAIL.getValue()); - msg.setTime(LocalDateTime.now()); - msg.setRequestId(UUID.randomUUID().toString()); - gseTaskOutput.send(MessageBuilder.withPayload(msg).build()); - log.info("Send start gse step fail control message successfully, stepInstanceId={}", stepInstanceId); - } - - @Override - public void retryGseStepAll(long stepInstanceId) { - log.info("Begin to send retry gse step all control message successfully, stepInstanceId={}", stepInstanceId); - StepControlMessage msg = new StepControlMessage(); - msg.setStepInstanceId(stepInstanceId); - msg.setAction(GseStepActionEnum.RETRY_ALL.getValue()); - msg.setTime(LocalDateTime.now()); - msg.setRequestId(UUID.randomUUID().toString()); - gseTaskOutput.send(MessageBuilder.withPayload(msg).build()); - log.info("Send start gse step all control message successfully, stepInstanceId={}", stepInstanceId); - } - - @Override - public void stopGseStep(long stepInstanceId) { - log.info("Begin to send stop gse step control message successfully, stepInstanceId={}", stepInstanceId); - StepControlMessage msg = new StepControlMessage(); - msg.setStepInstanceId(stepInstanceId); - msg.setAction(GseStepActionEnum.STOP.getValue()); - msg.setTime(LocalDateTime.now()); - msg.setRequestId(UUID.randomUUID().toString()); - gseTaskOutput.send(MessageBuilder.withPayload(msg).build()); - log.info("Send stop gse step control message successfully, stepInstanceId={}", stepInstanceId); - } - - @Override - public void asyncSendNotifyMsg(TaskNotifyDTO notification) { - log.info("Async send notification msg:{}", JsonUtils.toJson(notification)); - notifyMsgOutput.send(MessageBuilder.withPayload(JsonUtils.toJson(notification)).build()); - } - - @Override - public void sendCallback(JobCallbackDTO jobCallback) { - log.info("Async invoke callback url, callback:{}", JsonUtils.toJson(jobCallback)); - callbackOutput.send(MessageBuilder.withPayload(JsonUtils.toJson(jobCallback)).build()); - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/IpStatus.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/AgentTaskStatusEnum.java similarity index 81% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/IpStatus.java rename to src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/AgentTaskStatusEnum.java index 3989a29cb8..d0e112cb21 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/IpStatus.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/AgentTaskStatusEnum.java @@ -24,7 +24,10 @@ package com.tencent.bk.job.execute.engine.consts; -public enum IpStatus { +/** + * Agent 任务状态 + */ +public enum AgentTaskStatusEnum { /** * 未知错误 */ @@ -153,17 +156,17 @@ public enum IpStatus { private final int status; - IpStatus(int status) { + AgentTaskStatusEnum(int status) { this.status = status; } - public static IpStatus valueOf(Integer status) { + public static AgentTaskStatusEnum valueOf(Integer status) { if (status == null) { return null; } - for (IpStatus ipStatus : values()) { - if (ipStatus.status == status) { - return ipStatus; + for (AgentTaskStatusEnum agentTaskStatus : values()) { + if (agentTaskStatus.status == status) { + return agentTaskStatus; } } return UNKNOWN_ERROR; @@ -176,4 +179,26 @@ public final int getValue() { public String getI18nKey() { return "agent.task.status." + this.name().toLowerCase(); } + + public static boolean isSuccess(AgentTaskStatusEnum status) { + if (status == null) { + return false; + } + return status == SUCCESS || status == LAST_SUCCESS; + } + + public static boolean isSuccess(Integer status) { + if (status == null) { + return false; + } + return status.equals(SUCCESS.getValue()) || status.equals(LAST_SUCCESS.getValue()); + } + + /** + * 是否完成状态 + */ + public boolean isFinished() { + return this != AgentTaskStatusEnum.RUNNING && this != AgentTaskStatusEnum.WAITING; + } + } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/Consts.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/Consts.java index 73037ecc2b..f00227b021 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/Consts.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/Consts.java @@ -28,60 +28,73 @@ import java.util.HashMap; import java.util.Map; +import static com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum.GSE_AGENT_ERROR; +import static com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum.GSE_FILE_ERROR; +import static com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum.GSE_FILE_SIZE_EXCEED; +import static com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum.GSE_FILE_TASK_ERROR; +import static com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum.GSE_TASK_ERROR; +import static com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum.GSE_TASK_TERMINATE_SUCCESS; +import static com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum.GSE_TIMEOUT; +import static com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum.GSE_USER_ERROR; +import static com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum.GSE_USER_PWD_ERROR; + public class Consts { - public static final Map GSE_ERROR_CODE_2_STATUS_MAP; + /** + * GSE 错误码与 AgentTask 执行结果的映射关系 + */ + public static final Map GSE_ERROR_CODE_2_STATUS_MAP; /** * 直连云区域ID */ public static final int DEFAULT_CLOUD_ID = 0; static { - Map gseErrorCode2StatusMap = new HashMap<>(); + Map gseErrorCode2StatusMap = new HashMap<>(); // 文件任务超时 - gseErrorCode2StatusMap.put(120, 303); + gseErrorCode2StatusMap.put(120, GSE_TIMEOUT); // agent异常 - gseErrorCode2StatusMap.put(117, 310); + gseErrorCode2StatusMap.put(117, GSE_AGENT_ERROR); // 用户名不存在 - gseErrorCode2StatusMap.put(122, 311); + gseErrorCode2StatusMap.put(122, GSE_USER_ERROR); // 用户密码错误 - gseErrorCode2StatusMap.put(1326, 312); + gseErrorCode2StatusMap.put(1326, GSE_USER_PWD_ERROR); // 文件获取失败 - gseErrorCode2StatusMap.put(104, 320); - gseErrorCode2StatusMap.put(123, 320); - gseErrorCode2StatusMap.put(705, 320); - gseErrorCode2StatusMap.put(706, 320); - gseErrorCode2StatusMap.put(708, 320); - gseErrorCode2StatusMap.put(709, 320); - gseErrorCode2StatusMap.put(127, 320); + gseErrorCode2StatusMap.put(104, GSE_FILE_ERROR); + gseErrorCode2StatusMap.put(123, GSE_FILE_ERROR); + gseErrorCode2StatusMap.put(705, GSE_FILE_ERROR); + gseErrorCode2StatusMap.put(706, GSE_FILE_ERROR); + gseErrorCode2StatusMap.put(708, GSE_FILE_ERROR); + gseErrorCode2StatusMap.put(709, GSE_FILE_ERROR); + gseErrorCode2StatusMap.put(127, GSE_FILE_ERROR); // 文件超出限制 - gseErrorCode2StatusMap.put(119, 321); + gseErrorCode2StatusMap.put(119, GSE_FILE_SIZE_EXCEED); // 文件传输错误 - gseErrorCode2StatusMap.put(701, 329); - gseErrorCode2StatusMap.put(702, 329); - gseErrorCode2StatusMap.put(703, 329); - gseErrorCode2StatusMap.put(704, 329); - gseErrorCode2StatusMap.put(707, 329); - gseErrorCode2StatusMap.put(710, 329); + gseErrorCode2StatusMap.put(701, GSE_FILE_TASK_ERROR); + gseErrorCode2StatusMap.put(702, GSE_FILE_TASK_ERROR); + gseErrorCode2StatusMap.put(703, GSE_FILE_TASK_ERROR); + gseErrorCode2StatusMap.put(704, GSE_FILE_TASK_ERROR); + gseErrorCode2StatusMap.put(707, GSE_FILE_TASK_ERROR); + gseErrorCode2StatusMap.put(710, GSE_FILE_TASK_ERROR); // 任务执行出错 - gseErrorCode2StatusMap.put(101, 399); - gseErrorCode2StatusMap.put(102, 399); - gseErrorCode2StatusMap.put(103, 399); - gseErrorCode2StatusMap.put(105, 399); - gseErrorCode2StatusMap.put(106, 399); - gseErrorCode2StatusMap.put(107, 399); - gseErrorCode2StatusMap.put(108, 399); - gseErrorCode2StatusMap.put(109, 399); - gseErrorCode2StatusMap.put(111, 399); - gseErrorCode2StatusMap.put(112, 399); - gseErrorCode2StatusMap.put(114, 399); - gseErrorCode2StatusMap.put(118, 399); + gseErrorCode2StatusMap.put(101, GSE_TASK_ERROR); + gseErrorCode2StatusMap.put(102, GSE_TASK_ERROR); + gseErrorCode2StatusMap.put(103, GSE_TASK_ERROR); + gseErrorCode2StatusMap.put(105, GSE_TASK_ERROR); + gseErrorCode2StatusMap.put(106, GSE_TASK_ERROR); + gseErrorCode2StatusMap.put(107, GSE_TASK_ERROR); + gseErrorCode2StatusMap.put(108, GSE_TASK_ERROR); + gseErrorCode2StatusMap.put(109, GSE_TASK_ERROR); + gseErrorCode2StatusMap.put(111, GSE_TASK_ERROR); + gseErrorCode2StatusMap.put(112, GSE_TASK_ERROR); + gseErrorCode2StatusMap.put(114, GSE_TASK_ERROR); + gseErrorCode2StatusMap.put(118, GSE_TASK_ERROR); // 任务强制终止 - gseErrorCode2StatusMap.put(126, 403); + gseErrorCode2StatusMap.put(126, GSE_TASK_TERMINATE_SUCCESS); GSE_ERROR_CODE_2_STATUS_MAP = Collections.unmodifiableMap(gseErrorCode2StatusMap); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/GseStepActionEnum.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/EventSourceTypeEnum.java similarity index 84% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/GseStepActionEnum.java rename to src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/EventSourceTypeEnum.java index 9e00c16fa7..ec344d17e7 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/GseStepActionEnum.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/EventSourceTypeEnum.java @@ -24,31 +24,30 @@ package com.tencent.bk.job.execute.engine.consts; +import com.fasterxml.jackson.annotation.JsonCreator; + /** - * GSE步骤执行Action + * Job 执行引擎任务调度事件源类型 */ -public enum GseStepActionEnum { - /** - * 启动任务 - */ - START(1), +public enum EventSourceTypeEnum { /** - * 终止任务 + * 作业 */ - STOP(2), + JOB(1), /** - * 重试失败IP + * 步骤 */ - RETRY_FAIL(3), + STEP(2), /** - * 重试所有IP + * GSE 任务 */ - RETRY_ALL(4); + GSE_TASK(3); private final int value; - GseStepActionEnum(int val) { - this.value = val; + @JsonCreator + EventSourceTypeEnum(int value) { + this.value = value; } public int getValue() { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/GSECode.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/GSECode.java index da573719f1..b7b10ccb01 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/GSECode.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/GSECode.java @@ -32,7 +32,7 @@ public interface GSECode { /** * Agent异常 */ - public static final Integer AGENT_DOWN = 117; + Integer AGENT_DOWN = 117; /** * 运行脚本执行状态 diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BatchUpdateBizSetReq.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/GseTaskActionEnum.java similarity index 69% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BatchUpdateBizSetReq.java rename to src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/GseTaskActionEnum.java index 98ffb19002..46373f7a2a 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BatchUpdateBizSetReq.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/GseTaskActionEnum.java @@ -22,34 +22,37 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.common.cc.model.bizset; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.tencent.bk.job.common.esb.model.EsbReq; -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; - -import java.util.List; +package com.tencent.bk.job.execute.engine.consts; /** - * CMDB接口请求实体类，用于批量更新业务集 + * GSE任务Action */ -@Setter -@Getter -@ToString -public class BatchUpdateBizSetReq extends EsbReq { - +public enum GseTaskActionEnum { /** - * 业务集ID列表 + * 启动 GSE 任务 */ - @JsonProperty("bk_biz_set_ids") - private List bizSetIds; - + START(1), /** - * 要更新的业务集数据 + * 停止GSE任务 */ - @JsonProperty("data") - private BatchUpdateBizSetData data; + STOP(2); + + private final int value; + + GseTaskActionEnum(int val) { + this.value = val; + } + + public int getValue() { + return value; + } + public static GseTaskActionEnum valueOf(int value) { + for (GseTaskActionEnum gseTaskAction : values()) { + if (gseTaskAction.getValue() == value) { + return gseTaskAction; + } + } + throw new IllegalArgumentException("No GseTaskActionEnum constant: " + value); + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/JobActionEnum.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/JobActionEnum.java index 3af54ff4d2..73c1f2537f 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/JobActionEnum.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/JobActionEnum.java @@ -59,6 +59,15 @@ public enum JobActionEnum { this.value = val; } + public static JobActionEnum valueOf(int value) { + for (JobActionEnum jobAction : values()) { + if (jobAction.getValue() == value) { + return jobAction; + } + } + throw new IllegalArgumentException("No JobActionEnum constant: " + value); + } + public int getValue() { return value; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/StepActionEnum.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/StepActionEnum.java index 0100098d97..c58fdc9f59 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/StepActionEnum.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/StepActionEnum.java @@ -79,7 +79,11 @@ public enum StepActionEnum { /** * 恢复步骤执行 */ - RESUME(13); + RESUME(13), + /** + * 刷新步骤状态 + */ + REFRESH(14); private final int value; @@ -90,4 +94,13 @@ public enum StepActionEnum { public int getValue() { return value; } + + public static StepActionEnum valueOf(int value) { + for (StepActionEnum stepAction : values()) { + if (stepAction.getValue() == value) { + return stepAction; + } + } + throw new IllegalArgumentException("No StepActionEnum constant: " + value); + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/TaskEvictPolicyExecutor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/TaskEvictPolicyExecutor.java index 5483e43495..b424e9924b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/TaskEvictPolicyExecutor.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/TaskEvictPolicyExecutor.java @@ -73,7 +73,7 @@ public boolean shouldEvictTask(TaskInstanceDTO taskInstance) { public boolean updateEvictedTaskStatus(TaskInstanceDTO taskInstance, StepInstanceBaseDTO stepInstance) { long endTime = System.currentTimeMillis(); Long taskInstanceId = stepInstance.getTaskInstanceId(); - if (RunStatusEnum.isFinishedStatus(RunStatusEnum.valueOf(stepInstance.getStatus()))) { + if (RunStatusEnum.isFinishedStatus(stepInstance.getStatus())) { long totalTime = TaskCostCalculator.calculate(stepInstance.getStartTime(), endTime, stepInstance.getTotalTime()); taskInstanceService.updateStepExecutionInfo( @@ -90,7 +90,7 @@ public boolean updateEvictedTaskStatus(TaskInstanceDTO taskInstance, StepInstanc stepInstance.getStatus() ); } - if (RunStatusEnum.isFinishedStatus(RunStatusEnum.valueOf(taskInstance.getStatus()))) { + if (RunStatusEnum.isFinishedStatus(taskInstance.getStatus())) { long totalTime = TaskCostCalculator.calculate(taskInstance.getStartTime(), endTime, taskInstance.getTotalTime()); taskInstanceService.updateTaskExecutionInfo( diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/exception/ExceptionStatusManager.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/exception/ExceptionStatusManager.java deleted file mode 100644 index 0d15b1b634..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/exception/ExceptionStatusManager.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.engine.exception; - -import com.tencent.bk.job.execute.common.constants.RunStatusEnum; -import com.tencent.bk.job.execute.common.util.TaskCostCalculator; -import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; -import com.tencent.bk.job.execute.model.TaskInstanceDTO; -import com.tencent.bk.job.execute.service.TaskInstanceService; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -@Slf4j -@Service -public class ExceptionStatusManager { - - private final TaskInstanceService taskInstanceService; - - @Autowired - public ExceptionStatusManager(TaskInstanceService taskInstanceService) { - this.taskInstanceService = taskInstanceService; - } - - public void setAbnormalStatusForStep(long stepInstanceId) { - StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(stepInstanceId); - long endTime = System.currentTimeMillis(); - Long taskInstanceId = stepInstance.getTaskInstanceId(); - if (RunStatusEnum.isFinishedStatus(RunStatusEnum.valueOf(stepInstance.getStatus()))) { - long totalTime = TaskCostCalculator.calculate(stepInstance.getStartTime(), endTime, - stepInstance.getTotalTime()); - taskInstanceService.updateStepExecutionInfo( - stepInstanceId, - RunStatusEnum.ABNORMAL_STATE, - null, - endTime, - totalTime - ); - } else { - log.info( - "stepInstance {} already enter a final state:{}", - stepInstance.getId(), - stepInstance.getStatus() - ); - } - TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(taskInstanceId); - if (RunStatusEnum.isFinishedStatus(RunStatusEnum.valueOf(taskInstance.getStatus()))) { - long totalTime = TaskCostCalculator.calculate(taskInstance.getStartTime(), endTime, - taskInstance.getTotalTime()); - taskInstanceService.updateTaskExecutionInfo( - taskInstanceId, - RunStatusEnum.ABNORMAL_STATE, - null, - null, - endTime, - totalTime - ); - } else { - log.info( - "taskInstance {} already enter a final state:{}", - taskInstanceId, - taskInstance.getStatus() - ); - } - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/AbstractGseTaskCommand.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/AbstractGseTaskCommand.java new file mode 100644 index 0000000000..0a1a832e41 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/AbstractGseTaskCommand.java @@ -0,0 +1,166 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.executor; + +import com.tencent.bk.job.execute.model.AccountDTO; +import com.tencent.bk.job.execute.model.GseTaskDTO; +import com.tencent.bk.job.execute.model.StepInstanceDTO; +import com.tencent.bk.job.execute.model.TaskInstanceDTO; +import com.tencent.bk.job.execute.service.AccountService; +import com.tencent.bk.job.execute.service.AgentService; +import com.tencent.bk.job.execute.service.AgentTaskService; +import com.tencent.bk.job.execute.service.GseTaskService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.cloud.sleuth.Tracer; + +import java.util.HashMap; +import java.util.Map; + +/** + * GSE 任务命令基础实现 + */ +@Slf4j +public abstract class AbstractGseTaskCommand implements GseTaskCommand { + + protected final AgentService agentService; + protected final AccountService accountService; + protected final GseTaskService gseTaskService; + protected final AgentTaskService agentTaskService; + protected final Tracer tracer; + + /** + * 步骤实例 + */ + protected StepInstanceDTO stepInstance; + /** + * 作业实例 + */ + protected TaskInstanceDTO taskInstance; + /** + * GSE任务 + */ + protected GseTaskDTO gseTask; + /** + * 作业实例ID + */ + protected long taskInstanceId; + /** + * 步骤实例ID + */ + protected long stepInstanceId; + /** + * 执行次数 + */ + protected int executeCount; + /** + * 滚动执行批次 + */ + protected int batch; + /** + * GSE 任务唯一名称 + */ + protected String gseTaskUniqueName; + + + public AbstractGseTaskCommand(AgentService agentService, + AccountService accountService, + GseTaskService gseTaskService, + AgentTaskService agentTaskService, + Tracer tracer, + TaskInstanceDTO taskInstance, + StepInstanceDTO stepInstance, + GseTaskDTO gseTask) { + this.agentService = agentService; + this.accountService = accountService; + this.gseTaskService = gseTaskService; + this.agentTaskService = agentTaskService; + this.tracer = tracer; + this.taskInstance = taskInstance; + this.stepInstance = stepInstance; + this.gseTask = gseTask; + this.taskInstanceId = taskInstance.getId(); + this.stepInstanceId = gseTask.getStepInstanceId(); + this.executeCount = gseTask.getExecuteCount(); + this.batch = gseTask.getBatch(); + this.gseTaskUniqueName = gseTask.getTaskUniqueName(); + } + + /** + * 生成GSE trace 信息 + */ + protected Map buildGSETraceInfo() { + // 捕获所有异常，避免影响任务下发主流程 + Map traceInfoMap = new HashMap<>(); + try { + traceInfoMap.put("CALLER_NAME", "JOB"); + traceInfoMap.put("JOB_ID", stepInstance.getTaskInstanceId().toString()); + traceInfoMap.put("STEP_ID", stepInstance.getId().toString()); + traceInfoMap.put("EXECUTE_COUNT", String.valueOf(stepInstance.getExecuteCount())); + traceInfoMap.put("JOB_BIZ_ID", taskInstance.getAppId().toString()); + if (tracer != null) { + traceInfoMap.put("REQUEST_ID", tracer.currentSpan().context().traceId()); + } + if (StringUtils.isNotEmpty(taskInstance.getAppCode())) { + traceInfoMap.put("APP_CODE", taskInstance.getAppCode()); + } + traceInfoMap.put("CALLER_IP", agentService.getLocalAgentHost().getIp()); + traceInfoMap.put("TASK_ACCOUNT", stepInstance.getOperator()); + } catch (Throwable e) { + log.error("Build trace info map for gse failed"); + } + return traceInfoMap; + } + + /** + * 获取账号信息 + * + * @param accountId 账号ID + * @param accountAlias 账号别名 + * @param appId 业务ID + * @return 账号 + */ + protected AccountDTO getAccountBean(Long accountId, String accountAlias, Long appId) { + AccountDTO accountInfo = null; + if (accountId != null && accountId > 0) { + accountInfo = accountService.getAccountById(accountId); + } else if (StringUtils.isNotBlank(accountAlias)) { //原account传的是account,改为支持alias，减少用户API调用增加参数的成本 + accountInfo = accountService.getSystemAccountByAlias(accountAlias, appId); + } + // 可能帐号已经被删除了的情况：如从执行历史中点重做/克隆的方式。 + if (accountInfo == null && StringUtils.isNotBlank(accountAlias)) {//兼容老的传参，直接传递没有密码的只有帐号名称的认证 + accountInfo = new AccountDTO(); + accountInfo.setAccount(accountAlias); + accountInfo.setAlias(accountAlias); + } + return accountInfo; + } + + + /** + * 执行命令 + */ + public abstract void execute(); +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/AbstractGseTaskExecutor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/AbstractGseTaskExecutor.java deleted file mode 100644 index 4c9d51456a..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/AbstractGseTaskExecutor.java +++ /dev/null @@ -1,525 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.engine.executor; - -import brave.Tracing; -import com.tencent.bk.job.common.constant.JobConstants; -import com.tencent.bk.job.common.constant.TaskVariableTypeEnum; -import com.tencent.bk.job.common.util.date.DateUtils; -import com.tencent.bk.job.execute.common.constants.RunStatusEnum; -import com.tencent.bk.job.execute.config.JobExecuteConfig; -import com.tencent.bk.job.execute.engine.TaskExecuteControlMsgSender; -import com.tencent.bk.job.execute.engine.consts.IpStatus; -import com.tencent.bk.job.execute.engine.evict.TaskEvictPolicyExecutor; -import com.tencent.bk.job.execute.engine.exception.ExceptionStatusManager; -import com.tencent.bk.job.execute.engine.model.GseTaskExecuteResult; -import com.tencent.bk.job.execute.engine.model.GseTaskResponse; -import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; -import com.tencent.bk.job.execute.engine.model.TaskVariablesAnalyzeResult; -import com.tencent.bk.job.execute.engine.result.ResultHandleManager; -import com.tencent.bk.job.execute.engine.result.ha.ResultHandleTaskKeepaliveManager; -import com.tencent.bk.job.execute.model.AccountDTO; -import com.tencent.bk.job.execute.model.GseTaskIpLogDTO; -import com.tencent.bk.job.execute.model.GseTaskLogDTO; -import com.tencent.bk.job.execute.model.StepInstanceDTO; -import com.tencent.bk.job.execute.model.StepInstanceVariableValuesDTO; -import com.tencent.bk.job.execute.model.TaskInstanceDTO; -import com.tencent.bk.job.execute.monitor.metrics.ExecuteMonitor; -import com.tencent.bk.job.execute.monitor.metrics.GseTasksExceptionCounter; -import com.tencent.bk.job.execute.service.AccountService; -import com.tencent.bk.job.execute.service.AgentService; -import com.tencent.bk.job.execute.service.GseTaskLogService; -import com.tencent.bk.job.execute.service.LogService; -import com.tencent.bk.job.execute.service.StepInstanceVariableValueService; -import com.tencent.bk.job.execute.service.TaskInstanceService; -import com.tencent.bk.job.execute.service.TaskInstanceVariableService; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.springframework.util.StopWatch; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * GSE任务执行基础类 - */ -@Slf4j -public abstract class AbstractGseTaskExecutor implements ResumableTask { - /** - * GSE任务异常Counter - */ - private final GseTasksExceptionCounter gseTasksExceptionCounter; - protected ResultHandleManager resultHandleManager; - protected TaskInstanceService taskInstanceService; - protected GseTaskLogService gseTaskLogService; - protected AccountService accountService; - protected TaskInstanceVariableService taskInstanceVariableService; - protected StepInstanceVariableValueService stepInstanceVariableValueService; - protected AgentService agentService; - protected LogService logService; - protected TaskExecuteControlMsgSender taskManager; - protected ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager; - protected ExecuteMonitor executeMonitor; - protected ExceptionStatusManager exceptionStatusManager; - protected TaskEvictPolicyExecutor taskEvictPolicyExecutor; - protected JobExecuteConfig jobExecuteConfig; - protected String requestId; - protected Tracing tracing; - /** - * 步骤实例 - */ - protected StepInstanceDTO stepInstance; - /** - * 作业实例 - */ - protected TaskInstanceDTO taskInstance; - /** - * GSE任务信息 - */ - protected GseTaskLogDTO gseTaskLog; - /** - * 步骤实例ID - */ - protected long stepInstanceId; - /** - * 执行次数 - */ - protected int executeCount; - /** - * 目标主机 - */ - protected Set jobIpSet = new HashSet<>(); - /** - * 不合法的主机 - */ - protected Set invalidIpSet; - /** - * 文件源主机 - */ - protected Set fileSourceIPSet = new HashSet<>(); - /** - * gse 原子任务信息 - */ - protected Map ipLogMap = new HashMap<>(); - /** - * 全局参数分析结果 - */ - protected TaskVariablesAnalyzeResult taskVariablesAnalyzeResult; - /** - * 步骤输入参数 - */ - protected StepInstanceVariableValuesDTO stepInputVariables; - /** - * 全局变量参数定义与初始值 - */ - protected Map globalVariables = new HashMap<>(); - /** - * 执行任务的所有主机 - */ - private Set allJobIpSet = new HashSet<>(); - /** - * 未开始执行任务的主机 - */ - private Set notStartedJobIPSet = new HashSet<>(); - - /** - * GSETaskExecutor Constructor - * - * @param requestId 请求ID - * @param gseTasksExceptionCounter GSE任务异常Counter - * @param taskInstance 作业实例 - * @param stepInstance 步骤实例 - * @param executeIps 目标IP - */ - AbstractGseTaskExecutor(String requestId, GseTasksExceptionCounter gseTasksExceptionCounter, - TaskInstanceDTO taskInstance, StepInstanceDTO stepInstance, Set executeIps) { - this.requestId = requestId; - this.gseTasksExceptionCounter = gseTasksExceptionCounter; - this.taskInstance = taskInstance; - this.stepInstance = stepInstance; - this.stepInstanceId = stepInstance.getId(); - this.executeCount = stepInstance.getExecuteCount(); - this.invalidIpSet = stepInstance.getInvalidIps(); - if (!this.invalidIpSet.isEmpty()) { - stepInstance.setInvalidIps(this.invalidIpSet); - log.warn("Init gse task info, task contains invalid hosts: {}", this.invalidIpSet); - } - - this.allJobIpSet.addAll(executeIps); - executeIps.removeAll(this.invalidIpSet); - this.jobIpSet.addAll(executeIps); - this.notStartedJobIPSet.addAll(executeIps); - } - - /** - * 初始化依赖的服务 - */ - public void initDependentService(ResultHandleManager resultHandleManager, - TaskInstanceService taskInstanceService, - GseTaskLogService gseTaskLogService, - AccountService accountService, - TaskInstanceVariableService taskInstanceVariableService, - StepInstanceVariableValueService stepInstanceVariableValueService, - AgentService agentService, - LogService logService, - TaskExecuteControlMsgSender taskManager, - ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager, - ExecuteMonitor executeMonitor, - JobExecuteConfig jobExecuteConfig) { - this.resultHandleManager = resultHandleManager; - this.taskInstanceService = taskInstanceService; - this.gseTaskLogService = gseTaskLogService; - this.accountService = accountService; - this.taskInstanceVariableService = taskInstanceVariableService; - this.stepInstanceVariableValueService = stepInstanceVariableValueService; - this.agentService = agentService; - this.logService = logService; - this.taskManager = taskManager; - this.resultHandleTaskKeepaliveManager = resultHandleTaskKeepaliveManager; - this.executeMonitor = executeMonitor; - this.jobExecuteConfig = jobExecuteConfig; - } - - public void setExceptionStatusManager(ExceptionStatusManager exceptionStatusManager) { - this.exceptionStatusManager = exceptionStatusManager; - } - - public void setTaskEvictPolicyExecutor(TaskEvictPolicyExecutor taskEvictPolicyExecutor) { - this.taskEvictPolicyExecutor = taskEvictPolicyExecutor; - } - - private void analyseAndSetTaskVariables() { - List taskVariables = - taskInstanceVariableService.getByTaskInstanceId(stepInstance.getTaskInstanceId()); - if (taskVariables != null && !taskVariables.isEmpty()) { - taskVariables.forEach(var -> globalVariables.put(var.getName(), var)); - } - taskVariablesAnalyzeResult = new TaskVariablesAnalyzeResult(taskVariables); - if (!taskVariablesAnalyzeResult.isExistAnyVar()) { - return; - } - stepInputVariables = stepInstanceVariableValueService.computeInputStepInstanceVariableValues( - taskInstance.getId(), stepInstance.getId(), taskVariables); - log.info("Compute step input variable, stepInputVariables:{}", stepInputVariables); - } - - /** - * 保存将要执行的gse原子任务初始状态 - */ - protected void initAndSaveGseIpLogsToBeStarted() { - for (String cloudAreaIdAndIp : notStartedJobIPSet) { - GseTaskIpLogDTO ipLog = buildGseTaskIpLog(cloudAreaIdAndIp, IpStatus.WAITING, true, false); - ipLogMap.put(cloudAreaIdAndIp, ipLog); - } - gseTaskLogService.batchSaveIpLog(new ArrayList<>(ipLogMap.values())); - } - - /** - * 保存不合法主机的错误信息 - */ - private void saveInvalidGseIpLogs() { - if (!invalidIpSet.isEmpty()) { - List ipLogList = new ArrayList<>(); - invalidIpSet.forEach(cloudAreaIdAndIp -> { - boolean isTargetServer = this.allJobIpSet.contains(cloudAreaIdAndIp); - GseTaskIpLogDTO ipLog = buildGseTaskIpLog(cloudAreaIdAndIp, IpStatus.HOST_NOT_EXIST, isTargetServer, - !isTargetServer); - ipLogList.add(ipLog); - }); - gseTaskLogService.batchSaveIpLog(ipLogList); - logService.batchWriteJobSystemScriptLog(taskInstance.getCreateTime(), stepInstanceId, - stepInstance.getExecuteCount(), - buildIpAndLogOffsetMap(invalidIpSet), "The host(s) is not belongs to the Business, or doesn't exists" + - ".", System.currentTimeMillis()); - } - } - - private Map buildIpAndLogOffsetMap(Collection ips) { - Map ipAndLogOffsetMap = new HashMap<>(); - ips.forEach(ip -> { - GseTaskIpLogDTO ipLog = ipLogMap.get(ip); - if (ipLog != null) { - ipAndLogOffsetMap.put(ip, ipLog.getOffset()); - } else { - ipAndLogOffsetMap.put(ip, 0); - } - }); - return ipAndLogOffsetMap; - } - - protected GseTaskIpLogDTO buildGseTaskIpLog(String cloudAreaIdAndIp, IpStatus status, boolean isTargetServer, - boolean isSourceServer) { - String[] cloudAreaIdAndIpArray = cloudAreaIdAndIp.split(":"); - GseTaskIpLogDTO ipLog = new GseTaskIpLogDTO(); - ipLog.setStepInstanceId(stepInstanceId); - ipLog.setExecuteCount(executeCount); - ipLog.setStatus(status.getValue()); - ipLog.setTargetServer(isTargetServer); - ipLog.setIp(cloudAreaIdAndIpArray[1]); - ipLog.setCloudAreaAndIp(cloudAreaIdAndIp); - ipLog.setCloudAreaId(Long.valueOf(cloudAreaIdAndIpArray[0])); - ipLog.setDisplayIp(cloudAreaIdAndIpArray[1]); - ipLog.setSourceServer(isSourceServer); - return ipLog; - } - - protected Map buildReferenceGlobalVarValueMap(StepInstanceVariableValuesDTO stepInputVariables) { - Map globalVarValueMap = new HashMap<>(); - if (stepInputVariables == null || CollectionUtils.isEmpty(stepInputVariables.getGlobalParams())) { - return globalVarValueMap; - } - stepInputVariables.getGlobalParams().forEach(globalParam -> { - if (TaskVariableTypeEnum.valOf(globalParam.getType()) == TaskVariableTypeEnum.STRING) { - globalVarValueMap.put(globalParam.getName(), globalParam.getValue()); - } - }); - return globalVarValueMap; - } - - - /** - * 执行GSE任务 - */ - public void execute() { - StopWatch watch = new StopWatch("GseTaskExecutor-execute-" + stepInstanceId); - - // 检查步骤超时设置是否合法；理论上View/Service层已经完全处理了不合法的参数，此处日志输入用于发现问题。后续版本观察正常之后需要删除 - if (this.stepInstance.getTimeout() == null - || this.stepInstance.getTimeout() < JobConstants.MIN_JOB_TIMEOUT_SECONDS - || this.stepInstance.getTimeout() > JobConstants.MAX_JOB_TIMEOUT_SECONDS) { - log.warn("Invalid step timeout, timeout: {}", stepInstance.getTimeout()); - } - - watch.start("init-execution-context"); - initExecutionContext(); - watch.stop(); - - watch.start("check-executable"); - if (!checkHostExecutable()) { - log.warn("Task is not executable, stepInstanceId:{}", stepInstanceId); - handleStartGseTaskError(System.currentTimeMillis(), IpStatus.HOST_NOT_EXIST, "The host(s) is not belongs " + - "to the Business, or doesn't exists."); - return; - } - watch.stop(); - - watch.start("get-gse-task-log-from-db"); - gseTaskLog = gseTaskLogService.getGseTaskLog(stepInstanceId, executeCount); - watch.stop(); - - boolean shouldSendTaskToGseServer = (gseTaskLog == null || StringUtils.isEmpty(gseTaskLog.getGseTaskId())); - - String gseTaskId; - if (shouldSendTaskToGseServer) { - watch.start("send-task-to-gse-server"); - log.info("Sending task to gse server, stepInstanceId:{}", stepInstanceId); - long startTime = DateUtils.currentTimeMillis(); - GseTaskResponse gseTaskResponse = startGseTask(stepInstance); - watch.stop(); - watch.start("analyse-gse-response"); - if (GseTaskResponse.ERROR_CODE_SUCCESS != gseTaskResponse.getErrorCode()) { - handleStartGseTaskError(startTime, IpStatus.SUBMIT_FAILED, - "GSE Job failed:" + gseTaskResponse.getErrorMessage()); - gseTasksExceptionCounter.increment(); - return; - } else { - // 如果gseTaskLog不存在，需要填充一些基本信息；如果gseTaskLog存在，则只需要更新本次执行的一些状态信息 - if (gseTaskLog == null) { - gseTaskLog = new GseTaskLogDTO(); - gseTaskLog.setStepInstanceId(stepInstanceId); - gseTaskLog.setExecuteCount(executeCount); - } - gseTaskLog.setStartTime(startTime); - gseTaskLog.setGseTaskId(gseTaskResponse.getGseTaskId()); - gseTaskLog.setStatus(RunStatusEnum.RUNNING.getValue()); - gseTaskLogService.saveGseTaskLog(gseTaskLog); - } - watch.stop(); - } else { - gseTaskId = gseTaskLog.getGseTaskId(); - log.debug("Init FileTaskExecutor,get from db,gseTaskId={},executeCount={}", gseTaskId, executeCount); - } - - // 添加执行结果处理后台任务 - watch.start("add-result-handle-task"); - if (stepInstance.getStatus().equals(RunStatusEnum.RUNNING.getValue())) { - addExecutionResultHandleTask(); - } - watch.stop(); - if (watch.getTotalTimeMillis() > 1000L) { - log.warn("GseTaskExecutor-> execute task is slow, run statistics:{}", watch.prettyPrint()); - } - } - - /** - * 添加执行结果处理任务 - */ - abstract void addExecutionResultHandleTask(); - - /** - * 处理gse任务下发失败 - * - * @param startTime 启动时间 - * @param status 失败状态 - * @param msg 错误信息 - */ - private void handleStartGseTaskError(long startTime, IpStatus status, String msg) { - gseTaskLog = new GseTaskLogDTO(); - gseTaskLog.setStepInstanceId(stepInstanceId); - gseTaskLog.setExecuteCount(executeCount); - long endTime = DateUtils.currentTimeMillis(); - - gseTaskLog.setStatus(RunStatusEnum.FAIL.getValue()); - gseTaskLog.setEndTime(endTime); - gseTaskLog.setTotalTime(endTime - startTime); - gseTaskLogService.saveGseTaskLog(gseTaskLog); - - // 处理未完成的任务 - handleNotStartedIPResult(startTime, endTime, status, msg); - - taskInstanceService.updateStepEndTime(stepInstanceId, endTime); - int invalidIpNum = this.invalidIpSet == null ? 0 : this.invalidIpSet.size(); - taskInstanceService.updateStepStatInfo(stepInstanceId, invalidIpNum + jobIpSet.size(), 0, - invalidIpNum + jobIpSet.size()); - taskInstanceService.updateStepTotalTime(stepInstanceId, endTime - stepInstance.getStartTime()); - exceptionStatusManager.setAbnormalStatusForStep(stepInstanceId); - } - - private void handleNotStartedIPResult(Long startTime, Long endTime, IpStatus status, String errorMsg) { - log.info("[{}]: handleNotStartedIPResult| noStartJobIPSet={}", stepInstanceId, this.notStartedJobIPSet); - - Set unfinishedIPSet = new HashSet<>(); - unfinishedIPSet.addAll(notStartedJobIPSet); - unfinishedIPSet.addAll(this.fileSourceIPSet); - if (unfinishedIPSet.isEmpty()) { - log.debug("unfinishedIPSet is empty"); - return; - } - if (StringUtils.isNotEmpty(errorMsg)) { - logService.batchWriteJobSystemScriptLog(taskInstance.getCreateTime(), stepInstanceId, - stepInstance.getExecuteCount(), - buildIpAndLogOffsetMap(unfinishedIPSet), errorMsg, endTime); - } - - gseTaskLogService.batchUpdateIpLog(stepInstanceId, executeCount, unfinishedIPSet, startTime, endTime, status); - } - - /** - * 获取账号信息 - * - * @param accountId 账号ID - * @param accountAlias 账号别名 - * @param appId 业务ID - * @return 账号 - */ - protected AccountDTO getAccountBean(Long accountId, String accountAlias, Long appId) { - AccountDTO accountInfo = null; - if (accountId != null && accountId > 0) { - accountInfo = accountService.getAccountById(accountId); - } else if (StringUtils.isNotBlank(accountAlias)) { //原account传的是account,改为支持alias，减少用户API调用增加参数的成本 - accountInfo = accountService.getSystemAccountByAlias(accountAlias, appId); - } - // 可能帐号已经被删除了的情况：如从执行历史中点重做/克隆的方式。 - if (accountInfo == null && StringUtils.isNotBlank(accountAlias)) {//兼容老的传参，直接传递没有密码的只有帐号名称的认证 - accountInfo = new AccountDTO(); - accountInfo.setAccount(accountAlias); - accountInfo.setAlias(accountAlias); - } - return accountInfo; - } - - /** - * 初始化执行上下文，在GSE任务下发前调用 - */ - protected void initExecutionContext() { - this.initAndSaveGseIpLogsToBeStarted(); - this.saveInvalidGseIpLogs(); - this.analyseAndSetTaskVariables(); - } - - /** - * 生成GSE trace 信息 - */ - protected Map buildTraceInfoMap() { - // 捕获所有异常，避免影响任务下发主流程 - Map traceInfoMap = new HashMap<>(); - try { - traceInfoMap.put("CALLER_NAME", "JOB"); - traceInfoMap.put("JOB_ID", stepInstance.getTaskInstanceId().toString()); - traceInfoMap.put("STEP_ID", stepInstance.getId().toString()); - traceInfoMap.put("EXECUTE_COUNT", String.valueOf(stepInstance.getExecuteCount())); - traceInfoMap.put("JOB_BIZ_ID", taskInstance.getAppId().toString()); - if (tracing != null) { - traceInfoMap.put("REQUEST_ID", tracing.currentTraceContext().get().traceIdString()); - } - if (org.apache.commons.lang3.StringUtils.isNotEmpty(taskInstance.getAppCode())) { - traceInfoMap.put("APP_CODE", taskInstance.getAppCode()); - } - traceInfoMap.put("CALLER_IP", agentService.getLocalAgentBindIp()); - traceInfoMap.put("TASK_ACCOUNT", stepInstance.getOperator()); - } catch (Throwable e) { - log.warn("Build trace info map for gse failed"); - } - return traceInfoMap; - } - - /** - * 检查任务是否可被执行 - * - * @return 是否可执行 - */ - protected abstract boolean checkHostExecutable(); - - /** - * 下发GSE任务 - * - * @param stepInstance 步骤实例 - * @return GSE任务下发请求结果 - */ - protected abstract GseTaskResponse startGseTask(StepInstanceDTO stepInstance); - - /** - * 终止GSE任务 - * - * @return GSE任务终止请求结果 - */ - public abstract GseTaskExecuteResult stopGseTask(); - - @Override - public void interrupt() { - // 利用mq的ack机制即可，此处仅打印日志 - log.info("Interrupt running, stepInstanceId: {}", stepInstanceId); - } - - public void setTracing(Tracing tracing) { - this.tracing = tracing; - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/AbstractGseTaskStartCommand.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/AbstractGseTaskStartCommand.java new file mode 100644 index 0000000000..cdc2ecc25d --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/AbstractGseTaskStartCommand.java @@ -0,0 +1,336 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.executor; + +import com.tencent.bk.job.common.constant.TaskVariableTypeEnum; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.common.constants.RunStatusEnum; +import com.tencent.bk.job.execute.config.JobExecuteConfig; +import com.tencent.bk.job.execute.engine.evict.TaskEvictPolicyExecutor; +import com.tencent.bk.job.execute.engine.listener.event.EventSource; +import com.tencent.bk.job.execute.engine.listener.event.StepEvent; +import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; +import com.tencent.bk.job.execute.engine.model.GseTaskResponse; +import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; +import com.tencent.bk.job.execute.engine.model.TaskVariablesAnalyzeResult; +import com.tencent.bk.job.execute.engine.result.ResultHandleManager; +import com.tencent.bk.job.execute.engine.result.ha.ResultHandleTaskKeepaliveManager; +import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.model.GseTaskDTO; +import com.tencent.bk.job.execute.model.StepInstanceDTO; +import com.tencent.bk.job.execute.model.StepInstanceVariableValuesDTO; +import com.tencent.bk.job.execute.model.TaskInstanceDTO; +import com.tencent.bk.job.execute.monitor.metrics.ExecuteMonitor; +import com.tencent.bk.job.execute.monitor.metrics.GseTasksExceptionCounter; +import com.tencent.bk.job.execute.service.AccountService; +import com.tencent.bk.job.execute.service.AgentService; +import com.tencent.bk.job.execute.service.AgentTaskService; +import com.tencent.bk.job.execute.service.GseTaskService; +import com.tencent.bk.job.execute.service.LogService; +import com.tencent.bk.job.execute.service.StepInstanceService; +import com.tencent.bk.job.execute.service.StepInstanceVariableValueService; +import com.tencent.bk.job.execute.service.TaskInstanceService; +import com.tencent.bk.job.execute.service.TaskInstanceVariableService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.cloud.sleuth.Tracer; +import org.springframework.util.StopWatch; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Slf4j +public abstract class AbstractGseTaskStartCommand extends AbstractGseTaskCommand { + + protected final GseTasksExceptionCounter gseTasksExceptionCounter; + protected final ResultHandleManager resultHandleManager; + protected final TaskInstanceService taskInstanceService; + protected final TaskInstanceVariableService taskInstanceVariableService; + protected final StepInstanceVariableValueService stepInstanceVariableValueService; + protected final LogService logService; + protected final TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher; + protected final ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager; + protected final ExecuteMonitor executeMonitor; + protected final TaskEvictPolicyExecutor taskEvictPolicyExecutor; + protected final JobExecuteConfig jobExecuteConfig; + protected final StepInstanceService stepInstanceService; + + /** + * 任务下发请求ID,防止重复下发任务 + */ + protected String requestId; + + + /** + * GSE 目标 Agent 任务, Map + */ + protected Map targetAgentTaskMap = new HashMap<>(); + + /** + * 全局参数分析结果 + */ + protected TaskVariablesAnalyzeResult taskVariablesAnalyzeResult; + /** + * 步骤输入参数 + */ + protected StepInstanceVariableValuesDTO stepInputVariables; + /** + * 全局变量参数定义与初始值 + */ + protected Map globalVariables = new HashMap<>(); + + /** + * Agent ID 与 host 映射关系 + */ + protected Map agentIdHostMap; + + + AbstractGseTaskStartCommand(ResultHandleManager resultHandleManager, + TaskInstanceService taskInstanceService, + GseTaskService gseTaskService, + AgentTaskService agentTaskService, + AccountService accountService, + TaskInstanceVariableService taskInstanceVariableService, + StepInstanceVariableValueService stepInstanceVariableValueService, + AgentService agentService, + LogService logService, + TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, + ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager, + ExecuteMonitor executeMonitor, + JobExecuteConfig jobExecuteConfig, + TaskEvictPolicyExecutor taskEvictPolicyExecutor, + GseTasksExceptionCounter gseTasksExceptionCounter, + Tracer tracer, + String requestId, + TaskInstanceDTO taskInstance, + StepInstanceDTO stepInstance, + GseTaskDTO gseTask, + StepInstanceService stepInstanceService) { + super(agentService, + accountService, + gseTaskService, + agentTaskService, + tracer, + taskInstance, + stepInstance, + gseTask); + this.resultHandleManager = resultHandleManager; + this.taskInstanceService = taskInstanceService; + this.taskInstanceVariableService = taskInstanceVariableService; + this.stepInstanceVariableValueService = stepInstanceVariableValueService; + this.logService = logService; + this.taskExecuteMQEventDispatcher = taskExecuteMQEventDispatcher; + this.resultHandleTaskKeepaliveManager = resultHandleTaskKeepaliveManager; + this.executeMonitor = executeMonitor; + this.jobExecuteConfig = jobExecuteConfig; + this.taskEvictPolicyExecutor = taskEvictPolicyExecutor; + this.gseTasksExceptionCounter = gseTasksExceptionCounter; + this.requestId = requestId; + this.stepInstanceService = stepInstanceService; + + this.agentIdHostMap = stepInstanceService.computeStepHosts(stepInstance, + host -> host.getAgentId() != null ? host.getAgentId() : host.toCloudIp()); + } + + + /** + * 执行GSE任务 + */ + @Override + public void execute() { + StopWatch watch = new StopWatch("startGseTask-" + this.gseTaskUniqueName); + + watch.start("initExecutionContext"); + initExecutionContext(); + watch.stop(); + + startGseTaskIfNotAvailable(watch); + + // 添加执行结果处理后台任务 + watch.start("addResultHandleTask"); + if (stepInstance.getStatus() == RunStatusEnum.RUNNING) { + addResultHandleTask(); + } + watch.stop(); + + if (watch.getTotalTimeMillis() > 1000L) { + log.warn("Start gse task slow , statistics:{}", watch.prettyPrint()); + } + } + + private void startGseTaskIfNotAvailable(StopWatch watch) { + boolean isGseTaskStarted = StringUtils.isNotEmpty(gseTask.getGseTaskId()); + if (!isGseTaskStarted) { + watch.start("sendGseTask"); + gseTask.setStartTime(System.currentTimeMillis()); + log.info("[{}] Sending task to gse server", this.gseTaskUniqueName); + GseTaskResponse gseTaskResponse = startGseTask(); + watch.stop(); + + watch.start("handleGseResponse"); + if (GseTaskResponse.ERROR_CODE_SUCCESS != gseTaskResponse.getErrorCode()) { + handleStartGseTaskError(gseTaskResponse); + gseTasksExceptionCounter.increment(); + taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.refreshStep(stepInstanceId, + EventSource.buildGseTaskEventSource(stepInstanceId, executeCount, batch, gseTask.getId()))); + watch.stop(); + return; + } else { + updateGseTaskExecutionInfo(gseTaskResponse.getGseTaskId(), RunStatusEnum.RUNNING, + System.currentTimeMillis(), null, null); + } + watch.stop(); + } else { + // GSE 任务已经下发过，不做处理 + log.info("[{}] Gse Task had already started!", this.gseTaskUniqueName); + } + } + + /** + * 初始化执行上下文，在GSE任务下发前调用 + */ + private void initExecutionContext() { + initTargetAgentTasks(); + initVariables(); + preExecute(); + } + + private void initTargetAgentTasks() { + List agentTasks = agentTaskService.listAgentTasksByGseTaskId(gseTask.getId()); + fillIpForAgentTasks(agentTasks); + agentTasks.stream() + .filter(AgentTaskDTO::isTarget) + .forEach(agentTask -> this.targetAgentTaskMap.put(agentTask.getAgentId(), agentTask)); + } + + protected void fillIpForAgentTasks(List agentTasks) { + Map hosts = stepInstanceService.computeStepHosts(stepInstance, HostDTO::getHostId); + agentTasks.forEach(agentTask -> { + if (agentTask.getHostId() != null) { + HostDTO host = hosts.get(agentTask.getHostId()); + if (host != null) { + agentTask.setCloudIp(host.toCloudIp()); + } + } + }); + } + + + private void initVariables() { + if (taskInstance.isPlanInstance()) { + List taskVariables = + taskInstanceVariableService.getByTaskInstanceId(stepInstance.getTaskInstanceId()); + if (taskVariables != null && !taskVariables.isEmpty()) { + taskVariables.forEach(var -> globalVariables.put(var.getName(), var)); + } + taskVariablesAnalyzeResult = new TaskVariablesAnalyzeResult(taskVariables); + if (!taskVariablesAnalyzeResult.isExistAnyVar()) { + return; + } + stepInputVariables = stepInstanceVariableValueService.computeInputStepInstanceVariableValues( + taskInstance.getId(), stepInstance.getId(), taskVariables); + log.info("Compute step input variable, stepInputVariables:{}", stepInputVariables); + } else { + taskVariablesAnalyzeResult = new TaskVariablesAnalyzeResult(null); + } + } + + + /** + * 更新GSE任务执行情况 + * + * @param gseTaskId GSE返回的任务ID;如果不需要更新，传入null + * @param status 任务状态;如果不需要更新，传入null + * @param startTime 任务开始时间;如果不需要更新，传入null + * @param endTime 任务结束时间;如果不需要更新，传入null + * @param totalTime 任务耗时;如果不需要更新，传入null + */ + protected void updateGseTaskExecutionInfo(String gseTaskId, + RunStatusEnum status, + Long startTime, + Long endTime, + Long totalTime) { + if (StringUtils.isNotEmpty(gseTaskId)) { + gseTask.setGseTaskId(gseTaskId); + } + if (status != null) { + gseTask.setStatus(status.getValue()); + } + if (startTime != null) { + gseTask.setStartTime(System.currentTimeMillis()); + } + if (endTime != null) { + gseTask.setEndTime(endTime); + } + if (totalTime != null) { + gseTask.setTotalTime(totalTime); + } + gseTaskService.updateGseTask(gseTask); + } + + /** + * 获取字符类型全局变量的变量名和值 + * + * @param stepInputVariables 步骤入参 + * @return 字符类型全局变量的变量名和值 + */ + protected Map buildStringGlobalVarKV(StepInstanceVariableValuesDTO stepInputVariables) { + Map globalVarValueMap = new HashMap<>(); + if (stepInputVariables == null || CollectionUtils.isEmpty(stepInputVariables.getGlobalParams())) { + return globalVarValueMap; + } + stepInputVariables.getGlobalParams().forEach(globalParam -> { + if (TaskVariableTypeEnum.valOf(globalParam.getType()) == TaskVariableTypeEnum.STRING) { + globalVarValueMap.put(globalParam.getName(), globalParam.getValue()); + } + }); + return globalVarValueMap; + } + + /** + * GSE任务下发前调用 + */ + protected abstract void preExecute(); + + /** + * 下发GSE任务 + * + * @return GSE任务下发请求结果 + */ + protected abstract GseTaskResponse startGseTask(); + + /** + * 处理GSE任务下发失败 + */ + protected abstract void handleStartGseTaskError(GseTaskResponse gseTaskResponse); + + /** + * 添加执行结果处理任务 + */ + protected abstract void addResultHandleTask(); + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/FileTaskExecutor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/FileGseTaskStartCommand.java similarity index 52% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/FileTaskExecutor.java rename to src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/FileGseTaskStartCommand.java index 861ab3b30f..2d65d9f701 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/FileTaskExecutor.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/FileGseTaskStartCommand.java @@ -26,39 +26,52 @@ import com.tencent.bk.gse.taskapi.api_agent; import com.tencent.bk.gse.taskapi.api_copy_fileinfoV2; -import com.tencent.bk.gse.taskapi.api_stop_task_request; import com.tencent.bk.job.common.constant.NotExistPathHandlerEnum; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.common.constants.FileDistModeEnum; import com.tencent.bk.job.execute.common.constants.FileDistStatusEnum; -import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; import com.tencent.bk.job.execute.common.util.VariableValueResolver; +import com.tencent.bk.job.execute.config.JobExecuteConfig; +import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; import com.tencent.bk.job.execute.engine.consts.FileDirTypeConf; -import com.tencent.bk.job.execute.engine.consts.IpStatus; +import com.tencent.bk.job.execute.engine.evict.TaskEvictPolicyExecutor; import com.tencent.bk.job.execute.engine.gse.GseRequestUtils; +import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; import com.tencent.bk.job.execute.engine.model.FileDest; -import com.tencent.bk.job.execute.engine.model.GseTaskExecuteResult; import com.tencent.bk.job.execute.engine.model.GseTaskResponse; import com.tencent.bk.job.execute.engine.model.JobFile; import com.tencent.bk.job.execute.engine.result.FileResultHandleTask; +import com.tencent.bk.job.execute.engine.result.ResultHandleManager; +import com.tencent.bk.job.execute.engine.result.ha.ResultHandleTaskKeepaliveManager; import com.tencent.bk.job.execute.engine.util.FilePathUtils; -import com.tencent.bk.job.execute.engine.util.IpHelper; import com.tencent.bk.job.execute.engine.util.JobSrcFileUtils; import com.tencent.bk.job.execute.engine.util.MacroUtil; import com.tencent.bk.job.execute.engine.util.NFSUtils; import com.tencent.bk.job.execute.model.AccountDTO; +import com.tencent.bk.job.execute.model.AgentTaskDTO; import com.tencent.bk.job.execute.model.FileDetailDTO; import com.tencent.bk.job.execute.model.FileSourceDTO; -import com.tencent.bk.job.execute.model.GseTaskIpLogDTO; -import com.tencent.bk.job.execute.model.GseTaskLogDTO; +import com.tencent.bk.job.execute.model.GseTaskDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; +import com.tencent.bk.job.execute.monitor.metrics.ExecuteMonitor; import com.tencent.bk.job.execute.monitor.metrics.GseTasksExceptionCounter; +import com.tencent.bk.job.execute.service.AccountService; +import com.tencent.bk.job.execute.service.AgentService; +import com.tencent.bk.job.execute.service.FileAgentTaskService; +import com.tencent.bk.job.execute.service.GseTaskService; +import com.tencent.bk.job.execute.service.LogService; +import com.tencent.bk.job.execute.service.StepInstanceService; +import com.tencent.bk.job.execute.service.StepInstanceVariableValueService; +import com.tencent.bk.job.execute.service.TaskInstanceService; +import com.tencent.bk.job.execute.service.TaskInstanceVariableService; +import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; import com.tencent.bk.job.logsvr.model.service.ServiceFileTaskLogDTO; -import com.tencent.bk.job.logsvr.model.service.ServiceIpLogDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceHostLogDTO; import com.tencent.bk.job.manage.common.consts.account.AccountCategoryEnum; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; +import org.springframework.cloud.sleuth.Tracer; import java.util.ArrayList; import java.util.HashMap; @@ -67,20 +80,18 @@ import java.util.Map; import java.util.Set; -/** - * GSE-文件分发执行 - */ @Slf4j -public class FileTaskExecutor extends AbstractGseTaskExecutor { - /* - * 本地Agent ip +public class FileGseTaskStartCommand extends AbstractGseTaskStartCommand { + + /** + * 本地文件服务器 Agent */ - private final String localAgentIp; - /* + private HostDTO localAgentHost; + /** * 待分发文件，文件传输的源文件 */ private Set sendFiles; - /* + /** * 本地文件的存储根目录 */ private final String fileStorageRootPath; @@ -89,92 +100,110 @@ public class FileTaskExecutor extends AbstractGseTaskExecutor { * 格式： Map<源IP:源文件路径，目标路径> */ private final Map sourceDestPathMap = new HashMap<>(); - + /** + * 源文件原始文件路径与展示文件路径的映射 + */ private Map sourceFileDisplayMap = new HashMap<>(); - + /** + * 本地文件存储目录 + */ private final String localUploadDir; - /** - * FileTaskExecutor Constructor - * - * @param requestId 请求ID - * @param gseTasksExceptionCounter GSE任务异常计数 - * @param fileStorageRootPath 本地文件存储根路径 - * @param localAgentIp 本地agent bind ip - * @param taskInstance 作业实例 - * @param stepInstance 步骤实例 - * @param executeIps 目标IP + * GSE 源 Agent 任务, Map */ - public FileTaskExecutor(String requestId, GseTasksExceptionCounter gseTasksExceptionCounter, - String fileStorageRootPath, String localAgentIp, - TaskInstanceDTO taskInstance, StepInstanceDTO stepInstance, Set executeIps) { - super(requestId, gseTasksExceptionCounter, taskInstance, stepInstance, executeIps); - this.localAgentIp = localAgentIp; + protected Map sourceAgentTaskMap = new HashMap<>(); + + private final FileAgentTaskService fileAgentTaskService; + + + public FileGseTaskStartCommand(ResultHandleManager resultHandleManager, + TaskInstanceService taskInstanceService, + GseTaskService gseTaskService, + FileAgentTaskService fileAgentTaskService, + AccountService accountService, + TaskInstanceVariableService taskInstanceVariableService, + StepInstanceVariableValueService stepInstanceVariableValueService, + AgentService agentService, + LogService logService, + TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, + ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager, + ExecuteMonitor executeMonitor, + JobExecuteConfig jobExecuteConfig, + TaskEvictPolicyExecutor taskEvictPolicyExecutor, + GseTasksExceptionCounter gseTasksExceptionCounter, + StepInstanceService stepInstanceService, + Tracer tracer, + String requestId, + TaskInstanceDTO taskInstance, + StepInstanceDTO stepInstance, + GseTaskDTO gseTask, + String fileStorageRootPath) { + super(resultHandleManager, + taskInstanceService, + gseTaskService, + fileAgentTaskService, + accountService, + taskInstanceVariableService, + stepInstanceVariableValueService, + agentService, + logService, + taskExecuteMQEventDispatcher, + resultHandleTaskKeepaliveManager, + executeMonitor, + jobExecuteConfig, + taskEvictPolicyExecutor, + gseTasksExceptionCounter, + tracer, + requestId, + taskInstance, + stepInstance, + gseTask, + stepInstanceService); + + this.fileAgentTaskService = fileAgentTaskService; this.fileStorageRootPath = fileStorageRootPath; this.localUploadDir = NFSUtils.getFileDir(fileStorageRootPath, FileDirTypeConf.UPLOAD_FILE_DIR); } @Override - protected void initExecutionContext() { - super.initExecutionContext(); - // 解析文件传输的源文件, 得到List + protected void preExecute() { + // 设置本地文件服务器的Agent Ip + this.localAgentHost = agentService.getLocalAgentHost(); + // 解析文件传输的源文件, 得到List parseSendFileList(); resolvedTargetPathWithVariable(); - initSourceServerIp(); - initAndSaveGseIpLogsToBeStarted(); - } - - @Override - protected void initAndSaveGseIpLogsToBeStarted() { - super.initAndSaveGseIpLogsToBeStarted(); - initFileSourceIpLog(); - } - - /* - * 初始化源文件服务器任务状态 - */ - private void initFileSourceIpLog() { - List fileSourceIpLogs = new ArrayList<>(); - for (String cloudAreaIdAndIp : fileSourceIPSet) { - if (jobIpSet.contains(cloudAreaIdAndIp) && ipLogMap.get(cloudAreaIdAndIp) != null) { - GseTaskIpLogDTO ipLog = ipLogMap.get(cloudAreaIdAndIp); - ipLog.setSourceServer(true); - fileSourceIpLogs.add(ipLog); - } else { - GseTaskIpLogDTO ipLog = buildGseTaskIpLog(cloudAreaIdAndIp, IpStatus.WAITING, false, true); - ipLogMap.put(cloudAreaIdAndIp, ipLog); - fileSourceIpLogs.add(ipLog); - } - } - if (!fileSourceIpLogs.isEmpty()) { - gseTaskLogService.batchSaveIpLog(fileSourceIpLogs); - } + initFileSourceGseAgentTasks(); } - - /* - * 初始化文件源服务器IP - */ - private void initSourceServerIp() { - if (sendFiles != null) { - for (JobFile sendFile : sendFiles) { - fileSourceIPSet.add(sendFile.getCloudAreaIdAndIp()); - } - } - } - - /* + /** * 解析源文件 */ private void parseSendFileList() { resolveVariableForSourceFilePath(stepInstance.getFileSourceList(), - buildReferenceGlobalVarValueMap(stepInputVariables)); - sendFiles = JobSrcFileUtils.parseSendFileList(stepInstance, localAgentIp, fileStorageRootPath); + buildStringGlobalVarKV(stepInputVariables)); + sendFiles = JobSrcFileUtils.parseSendFileList(stepInstance, localAgentHost, fileStorageRootPath); setAccountInfoForSourceFiles(sendFiles); // 初始化显示名称映射Map sourceFileDisplayMap = JobSrcFileUtils.buildSourceFileDisplayMapping(sendFiles, localUploadDir); } + private void setAccountInfoForSourceFiles(Set sendFiles) { + Map accounts = new HashMap<>(); + sendFiles.forEach(sendFile -> { + String accountKey = sendFile.getAccountId() == null ? ("id_" + sendFile.getAccountId()) + : ("alias_" + sendFile.getAccountAlias()); + AccountDTO account = accounts.computeIfAbsent(accountKey, + k -> accountService.getAccount(sendFile.getAccountId(), AccountCategoryEnum.SYSTEM, + sendFile.getAccountAlias(), stepInstance.getAppId())); + if (account != null) { + sendFile.setAccountId(account.getId()); + sendFile.setAccount(account.getAccount()); + sendFile.setAccountAlias(account.getAlias()); + sendFile.setPassword(account.getPassword()); + } + }); + } + private void resolveVariableForSourceFilePath(List fileSources, Map stepInputGlobalVariableValueMap) { if (stepInputGlobalVariableValueMap == null || stepInputGlobalVariableValueMap.isEmpty()) { @@ -198,83 +227,12 @@ private void resolveVariableForSourceFilePath(List fileSources, } } - private void setAccountInfoForSourceFiles(Set sendFiles) { - Map accounts = new HashMap<>(); - sendFiles.forEach(sendFile -> { - String accountKey = sendFile.getAccountId() == null ? ("id_" + sendFile.getAccountId()) - : ("alias_" + sendFile.getAccountAlias()); - AccountDTO account = accounts.computeIfAbsent(accountKey, - k -> accountService.getAccount(sendFile.getAccountId(), AccountCategoryEnum.SYSTEM, - sendFile.getAccountAlias(), stepInstance.getAppId())); - if (account != null) { - sendFile.setAccountId(account.getId()); - sendFile.setAccount(account.getAccount()); - sendFile.setAccountAlias(account.getAlias()); - sendFile.setPassword(account.getPassword()); - } - }); - } - /** - * 保存文件执行日志初始状态 - */ - private void saveInitialFileTaskLogs(Map sourceDestPathMap) { - try { - log.debug("[{}] SourceDestPathMap: {}", stepInstanceId, sourceDestPathMap); - Map logs = new HashMap<>(); - // 每个要分发的源文件一条上传日志 - for (JobFile file : sendFiles) { - String fileSourceIp = file.isLocalUploadFile() ? IpHelper.fix1To0(localAgentIp) : - file.getCloudAreaIdAndIp(); - ServiceIpLogDTO ipTaskLog = initServiceLogDTOIfAbsent(logs, stepInstanceId, executeCount, fileSourceIp); - ipTaskLog.addFileTaskLog(new ServiceFileTaskLogDTO(FileDistModeEnum.UPLOAD.getValue(), null, null, - fileSourceIp, fileSourceIp, file.getStandardFilePath(), file.getDisplayFilePath(), "--", - FileDistStatusEnum.WAITING.getValue(), FileDistStatusEnum.WAITING.getName(), "--", "--", null)); - } - // 每个目标IP从每个要分发的源文件下载的一条下载日志 - for (String fileTargetIp : jobIpSet) { - ServiceIpLogDTO ipTaskLog = initServiceLogDTOIfAbsent(logs, stepInstanceId, executeCount, fileTargetIp); - for (JobFile file : sendFiles) { - String fileSourceIp = file.isLocalUploadFile() ? IpHelper.fix1To0(localAgentIp) : - file.getCloudAreaIdAndIp(); - ipTaskLog.addFileTaskLog(new ServiceFileTaskLogDTO(FileDistModeEnum.DOWNLOAD.getValue(), - fileTargetIp, - getDestPath(file), fileSourceIp, fileSourceIp, file.getStandardFilePath(), - file.getDisplayFilePath(), "--", - FileDistStatusEnum.WAITING.getValue(), FileDistStatusEnum.WAITING.getName(), "--", "--", null)); - } - } - // 调用logService写入MongoDB - writeLogs(logs); - } catch (Throwable e) { - log.warn("Save Initial File Task logs fail", e); - } - } - - private ServiceIpLogDTO initServiceLogDTOIfAbsent(Map logs, long stepInstanceId, - int executeCount, String ip) { - ServiceIpLogDTO ipTaskLog = logs.get(ip); - if (ipTaskLog == null) { - ipTaskLog = new ServiceIpLogDTO(); - ipTaskLog.setStepInstanceId(stepInstanceId); - ipTaskLog.setIp(ip); - ipTaskLog.setExecuteCount(executeCount); - logs.put(ip, ipTaskLog); - } - return ipTaskLog; - } - - private void writeLogs(Map executionLogs) { - log.debug("Write file task initial logs, executionLogs: {}", executionLogs); - logService.writeFileLogs(taskInstance.getCreateTime(), new ArrayList<>(executionLogs.values())); - } - - /* * 解析文件分发目标路径，替换变量 */ private void resolvedTargetPathWithVariable() { String resolvedTargetPath = VariableValueResolver.resolve(stepInstance.getFileTargetPath(), - buildReferenceGlobalVarValueMap(stepInputVariables)); + buildStringGlobalVarKV(stepInputVariables)); resolvedTargetPath = MacroUtil.resolveDateWithStrfTime(resolvedTargetPath); stepInstance.setResolvedFileTargetPath(resolvedTargetPath); if (!resolvedTargetPath.equals(stepInstance.getFileTargetPath())) { @@ -282,8 +240,36 @@ private void resolvedTargetPathWithVariable() { } } + /** + * 初始化源文件服务器上传任务状态 + */ + private void initFileSourceGseAgentTasks() { + Set sourceHosts = new HashSet<>(); + if (sendFiles != null) { + for (JobFile sendFile : sendFiles) { + if (sendFile.getHost() != null) { + sourceHosts.add(sendFile.getHost()); + } + } + } + List fileSourceGseAgentTasks = new ArrayList<>(); + for (HostDTO sourceHost : sourceHosts) { + AgentTaskDTO agentTask = new AgentTaskDTO(stepInstanceId, executeCount, batch, sourceHost.getHostId(), + sourceHost.getAgentId()); + agentTask.setActualExecuteCount(executeCount); + agentTask.setCloudIp(sourceHost.toCloudIp()); + agentTask.setDisplayIp(sourceHost.getDisplayIp()); + agentTask.setFileTaskMode(FileTaskModeEnum.UPLOAD); + agentTask.setStatus(AgentTaskStatusEnum.WAITING); + agentTask.setGseTaskId(gseTask.getId()); + sourceAgentTaskMap.put(sourceHost.getAgentId(), agentTask); + fileSourceGseAgentTasks.add(agentTask); + } + fileAgentTaskService.batchSaveAgentTasks(fileSourceGseAgentTasks); + } + @Override - protected GseTaskResponse startGseTask(StepInstanceDTO stepInstance) { + protected GseTaskResponse startGseTask() { // 账号信息查询与填充 AccountDTO accountInfo = accountService.getAccount(stepInstance.getAccountId(), AccountCategoryEnum.SYSTEM, stepInstance.getAccountAlias(), stepInstance.getAppId()); @@ -291,7 +277,7 @@ protected GseTaskResponse startGseTask(StepInstanceDTO stepInstance) { log.error("Start gse task fail, account is null!"); return GseTaskResponse.fail(GseTaskResponse.ERROR_CODE_FAIL, "account is empty"); } - List dst = GseRequestUtils.buildAgentList(jobIpSet, accountInfo.getAccount(), + List dst = GseRequestUtils.buildAgentList(targetAgentTaskMap.keySet(), accountInfo.getAccount(), accountInfo.getPassword()); List copyFileInfoList = new ArrayList<>(); @@ -305,8 +291,8 @@ protected GseTaskResponse startGseTask(StepInstanceDTO stepInstance) { // 构造GSE文件分发请求 for (JobFile file : sendFiles) { - FileDest fileDest = srcAndDestMap.get(file.getFileUniqueKey()); - api_agent src = GseRequestUtils.buildAgent(file.getCloudAreaIdAndIp(), file.getAccount(), + FileDest fileDest = srcAndDestMap.get(file.getUniqueKey()); + api_agent src = GseRequestUtils.buildAgent(file.getHost().toCloudIp(), file.getAccount(), file.getPassword()); api_copy_fileinfoV2 copyFileInfo = GseRequestUtils.buildCopyFileInfo(src, file.getDir(), file.getFileName(), dst, fileDest.getDestDirPath(), fileDest.getDestName(), @@ -327,94 +313,133 @@ protected GseTaskResponse startGseTask(StepInstanceDTO stepInstance) { copyFileInfoList.add(copyFileInfo); } if (CollectionUtils.isNotEmpty(copyFileInfoList)) { - copyFileInfoList.get(0).setM_caller(buildTraceInfoMap()); + copyFileInfoList.get(0).setM_caller(buildGSETraceInfo()); } saveInitialFileTaskLogs(sourceDestPathMap); return GseRequestUtils.sendCopyFileTaskRequest(stepInstanceId, copyFileInfoList); } - private void initSourceDestPathMap(Map srcAndDestMap) { - srcAndDestMap.forEach((fileKey, dest) -> this.sourceDestPathMap.put(fileKey, dest.getDestPath())); - } - - private String getDestPath(JobFile sourceFile) { - return sourceDestPathMap.get(sourceFile.getFileUniqueKey()); - } - - @Override - public GseTaskExecuteResult stopGseTask() { - AccountDTO accountInfo = accountService.getAccount(stepInstance.getAccountId(), AccountCategoryEnum.SYSTEM, - stepInstance.getAccount(), stepInstance.getAppId()); - if (accountInfo == null) { - log.error("Start gse task fail, account is null!"); - return new GseTaskExecuteResult(GseTaskExecuteResult.RESULT_CODE_STOP_FAILED, "Account is empty"); - } - //目标机器的agent - List agentList = GseRequestUtils.buildAgentList(jobIpSet, accountInfo.getAccount(), - accountInfo.getPassword()); - - //源机器agent - Set srcIpSet = new HashSet<>(); - this.parseSendFileList(); - for (JobFile file : sendFiles) { - if (srcIpSet.contains(file.getCloudAreaIdAndIp())) { - continue; + /** + * 保存文件执行日志初始状态 + */ + private void saveInitialFileTaskLogs(Map sourceDestPathMap) { + try { + log.debug("[{}] SourceDestPathMap: {}", stepInstanceId, sourceDestPathMap); + Map logs = new HashMap<>(); + // 每个要分发的源文件一条上传日志 + for (JobFile file : sendFiles) { + Long sourceHostId = file.getHost().getHostId(); + ServiceHostLogDTO hostTaskLog = initServiceLogDTOIfAbsent(logs, stepInstanceId, executeCount, + sourceHostId, file.getHost().toCloudIp()); + hostTaskLog.addFileTaskLog( + new ServiceFileTaskLogDTO( + FileDistModeEnum.UPLOAD.getValue(), + null, + null, + null, + sourceHostId, + file.getHost().toCloudIp(), + file.getHost().toCloudIp(), + file.getStandardFilePath(), + file.getDisplayFilePath(), + "--", + FileDistStatusEnum.WAITING.getValue(), + FileDistStatusEnum.WAITING.getName(), + "--", + "--", + null) + ); } - srcIpSet.add(file.getCloudAreaIdAndIp()); - api_agent src = GseRequestUtils.buildAgent(file.getCloudAreaIdAndIp(), file.getAccount(), - file.getPassword()); - agentList.add(src); + // 每个目标IP从每个要分发的源文件下载的一条下载日志 + for (AgentTaskDTO targetAgentTask : targetAgentTaskMap.values()) { + HostDTO targetHost = agentIdHostMap.get(targetAgentTask.getAgentId()); + ServiceHostLogDTO ipTaskLog = initServiceLogDTOIfAbsent(logs, stepInstanceId, executeCount, + targetHost.getHostId(), targetHost.toCloudIp()); + for (JobFile file : sendFiles) { + Long sourceHostId = file.getHost().getHostId(); + ipTaskLog.addFileTaskLog( + new ServiceFileTaskLogDTO( + FileDistModeEnum.DOWNLOAD.getValue(), + targetAgentTask.getHostId(), + targetHost.toCloudIp(), + getDestPath(file), + sourceHostId, + file.getHost().toCloudIp(), + file.getHost().toCloudIp(), + file.getStandardFilePath(), + file.getDisplayFilePath(), + "--", + FileDistStatusEnum.WAITING.getValue(), + FileDistStatusEnum.WAITING.getName(), + "--", + "--", + null) + ); + } + } + // 调用logService写入MongoDB + writeLogs(logs); + } catch (Throwable e) { + log.warn("Save Initial File Task logs fail", e); } + } - GseTaskLogDTO gseTaskLog = gseTaskLogService.getGseTaskLog(stepInstanceId, executeCount); - if (gseTaskLog == null || StringUtils.isEmpty(gseTaskLog.getGseTaskId())) { - log.warn("Gse Task not send to gse server, not support stop"); - return new GseTaskExecuteResult(GseTaskExecuteResult.RESULT_CODE_STOP_FAILED, "Termination failed"); + private ServiceHostLogDTO initServiceLogDTOIfAbsent(Map logs, long stepInstanceId, + int executeCount, Long hostId, String cloudIp) { + ServiceHostLogDTO hostTaskLog = logs.get(hostId); + if (hostTaskLog == null) { + hostTaskLog = new ServiceHostLogDTO(); + hostTaskLog.setStepInstanceId(stepInstanceId); + hostTaskLog.setExecuteCount(executeCount); + hostTaskLog.setHostId(hostId); + hostTaskLog.setIp(cloudIp); + logs.put(hostId, hostTaskLog); } + return hostTaskLog; + } - api_stop_task_request stopTaskRequest = new api_stop_task_request(); - stopTaskRequest.setStop_task_id(gseTaskLog.getGseTaskId()); - stopTaskRequest.setAgents(agentList); - stopTaskRequest.setType(StepExecuteTypeEnum.SEND_FILE.getValue()); - stopTaskRequest.setM_caller(buildTraceInfoMap()); - - GseTaskResponse gseTaskResponse = GseRequestUtils.sendForceStopTaskRequest(stepInstance.getId(), - stopTaskRequest); - if (GseTaskResponse.ERROR_CODE_SUCCESS != gseTaskResponse.getErrorCode()) { - log.info("[{}]: sendForceStopTaskRequest response failed!", stepInstanceId); - return new GseTaskExecuteResult(GseTaskExecuteResult.RESULT_CODE_STOP_FAILED, - "Termination failed, msg:" + gseTaskResponse.getErrorMessage()); - } else { - log.info("[{}]: sendForceStopTaskRequest response success!", stepInstanceId); - return new GseTaskExecuteResult(GseTaskExecuteResult.RESULT_CODE_STOP_SUCCESS, "Termination successfully"); - } + private void writeLogs(Map executionLogs) { + log.debug("Write file task initial logs, executionLogs: {}", executionLogs); + logService.writeFileLogs(taskInstance.getCreateTime(), new ArrayList<>(executionLogs.values())); } - @Override - void addExecutionResultHandleTask() { - FileResultHandleTask fileResultHandleTask = - new FileResultHandleTask(taskInstance, stepInstance, taskVariablesAnalyzeResult, ipLogMap, - gseTaskLog, jobIpSet, sendFiles, fileStorageRootPath, sourceDestPathMap, sourceFileDisplayMap, - requestId); - fileResultHandleTask.initDependentService(taskInstanceService, gseTaskLogService, logService, - taskInstanceVariableService, stepInstanceVariableValueService, - taskManager, resultHandleTaskKeepaliveManager, exceptionStatusManager, taskEvictPolicyExecutor); - resultHandleManager.handleDeliveredTask(fileResultHandleTask); + private void initSourceDestPathMap(Map srcAndDestMap) { + srcAndDestMap.forEach((fileKey, dest) -> this.sourceDestPathMap.put(fileKey, dest.getDestPath())); } - @Override - public void resume() { - log.info("Resume file task from snapshot, stepInstanceId: {}", stepInstanceId); + private String getDestPath(JobFile sourceFile) { + return sourceDestPathMap.get(sourceFile.getUniqueKey()); } @Override - public void saveSnapshot() { - log.info("Save file task snapshot, stepInstanceId: {}", stepInstanceId); + protected void handleStartGseTaskError(GseTaskResponse gseTaskResponse) { + } @Override - protected boolean checkHostExecutable() { - // 同时包含合法的源和目标，才可以被执行 - return (!jobIpSet.isEmpty() && !fileSourceIPSet.isEmpty()); + protected void addResultHandleTask() { + FileResultHandleTask fileResultHandleTask = + new FileResultHandleTask(taskInstanceService, + gseTaskService, + logService, + taskInstanceVariableService, + stepInstanceVariableValueService, + taskExecuteMQEventDispatcher, + resultHandleTaskKeepaliveManager, + taskEvictPolicyExecutor, + fileAgentTaskService, + stepInstanceService, + taskInstance, + stepInstance, + taskVariablesAnalyzeResult, + targetAgentTaskMap, + sourceAgentTaskMap, + gseTask, + sendFiles, + fileStorageRootPath, + sourceDestPathMap, + sourceFileDisplayMap, + requestId); + resultHandleManager.handleDeliveredTask(fileResultHandleTask); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/FileGseTaskStopCommand.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/FileGseTaskStopCommand.java new file mode 100644 index 0000000000..572e625b82 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/FileGseTaskStopCommand.java @@ -0,0 +1,99 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.executor; + +import com.tencent.bk.gse.taskapi.api_agent; +import com.tencent.bk.gse.taskapi.api_stop_task_request; +import com.tencent.bk.job.execute.common.constants.RunStatusEnum; +import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; +import com.tencent.bk.job.execute.engine.gse.GseRequestUtils; +import com.tencent.bk.job.execute.engine.model.GseTaskResponse; +import com.tencent.bk.job.execute.model.AccountDTO; +import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.model.GseTaskDTO; +import com.tencent.bk.job.execute.model.StepInstanceDTO; +import com.tencent.bk.job.execute.model.TaskInstanceDTO; +import com.tencent.bk.job.execute.service.AccountService; +import com.tencent.bk.job.execute.service.AgentService; +import com.tencent.bk.job.execute.service.AgentTaskService; +import com.tencent.bk.job.execute.service.GseTaskService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.cloud.sleuth.Tracer; + +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +@Slf4j +public class FileGseTaskStopCommand extends AbstractGseTaskCommand { + + public FileGseTaskStopCommand(AgentService agentService, + AccountService accountService, + GseTaskService gseTaskService, + AgentTaskService agentTaskService, + Tracer tracer, + TaskInstanceDTO taskInstance, + StepInstanceDTO stepInstance, + GseTaskDTO gseTask) { + super(agentService, + accountService, + gseTaskService, + agentTaskService, + tracer, + taskInstance, + stepInstance, + gseTask); + } + + @Override + public void execute() { + log.info("Stop gse task, gseTask:" + gseTaskUniqueName); + List agentTasks = agentTaskService.listAgentTasksByGseTaskId(gseTask.getId()); + AccountDTO targetAccount = getAccountBean(stepInstance.getAccountId(), stepInstance.getAccount(), + stepInstance.getAppId()); + Set agentIds = agentTasks.stream() + .map(AgentTaskDTO::getAgentId) + .collect(Collectors.toSet()); + //目标机器的agent + List agentList = GseRequestUtils.buildAgentList(agentIds, targetAccount.getAccount(), + targetAccount.getPassword()); + + api_stop_task_request stopTaskRequest = new api_stop_task_request(); + stopTaskRequest.setStop_task_id(gseTask.getGseTaskId()); + stopTaskRequest.setAgents(agentList); + stopTaskRequest.setType(StepExecuteTypeEnum.SEND_FILE.getValue()); + stopTaskRequest.setM_caller(buildGSETraceInfo()); + + GseTaskResponse gseTaskResponse = GseRequestUtils.sendForceStopTaskRequest(stepInstance.getId(), + stopTaskRequest); + if (GseTaskResponse.ERROR_CODE_SUCCESS != gseTaskResponse.getErrorCode()) { + log.error("Terminate gse task failed! gseTask: {}", gseTaskUniqueName); + } else { + log.info("Terminate gse task response success! gseTask: {}", gseTaskUniqueName); + gseTask.setStatus(RunStatusEnum.STOPPING.getValue()); + gseTaskService.updateGseTask(gseTask); + } + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/ResumableTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/GseTaskCommand.java similarity index 88% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/ResumableTask.java rename to src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/GseTaskCommand.java index 6cbb19cd0b..d3bc6444a9 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/ResumableTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/GseTaskCommand.java @@ -24,19 +24,12 @@ package com.tencent.bk.job.execute.engine.executor; -public interface ResumableTask { - /** - * 中断任务 - */ - void interrupt(); - - /** - * 保存任务快照 - */ - void saveSnapshot(); - +/** + * GSE 任务命令 + */ +public interface GseTaskCommand { /** - * 恢复任务执行 + * 执行命令 */ - void resume(); + void execute(); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/GseTaskManager.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/GseTaskManager.java new file mode 100644 index 0000000000..9014922a60 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/GseTaskManager.java @@ -0,0 +1,581 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.executor; + +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.common.redis.util.LockUtils; +import com.tencent.bk.job.execute.common.constants.RunStatusEnum; +import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; +import com.tencent.bk.job.execute.common.exception.MessageHandlerUnavailableException; +import com.tencent.bk.job.execute.common.ha.DestroyOrder; +import com.tencent.bk.job.execute.config.JobExecuteConfig; +import com.tencent.bk.job.execute.config.StorageSystemConfig; +import com.tencent.bk.job.execute.engine.evict.TaskEvictPolicyExecutor; +import com.tencent.bk.job.execute.engine.listener.event.EventSource; +import com.tencent.bk.job.execute.engine.listener.event.StepEvent; +import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; +import com.tencent.bk.job.execute.engine.result.ResultHandleManager; +import com.tencent.bk.job.execute.engine.result.ha.ResultHandleTaskKeepaliveManager; +import com.tencent.bk.job.execute.engine.util.RunningTaskCounter; +import com.tencent.bk.job.execute.engine.variable.JobBuildInVariableResolver; +import com.tencent.bk.job.execute.model.GseTaskDTO; +import com.tencent.bk.job.execute.model.StepInstanceDTO; +import com.tencent.bk.job.execute.model.TaskInstanceDTO; +import com.tencent.bk.job.execute.monitor.ExecuteMetricNames; +import com.tencent.bk.job.execute.monitor.metrics.ExecuteMonitor; +import com.tencent.bk.job.execute.monitor.metrics.GseTasksExceptionCounter; +import com.tencent.bk.job.execute.service.AccountService; +import com.tencent.bk.job.execute.service.AgentService; +import com.tencent.bk.job.execute.service.FileAgentTaskService; +import com.tencent.bk.job.execute.service.GseTaskService; +import com.tencent.bk.job.execute.service.LogService; +import com.tencent.bk.job.execute.service.ScriptAgentTaskService; +import com.tencent.bk.job.execute.service.StepInstanceService; +import com.tencent.bk.job.execute.service.StepInstanceVariableValueService; +import com.tencent.bk.job.execute.service.TaskInstanceService; +import com.tencent.bk.job.execute.service.TaskInstanceVariableService; +import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cloud.sleuth.Tracer; +import org.springframework.context.SmartLifecycle; +import org.springframework.stereotype.Component; +import org.springframework.util.StopWatch; + +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * GSE任务执行管理 + */ +@Component +@Slf4j +public class GseTaskManager implements SmartLifecycle { + private final ResultHandleManager resultHandleManager; + private final TaskInstanceService taskInstanceService; + private final GseTaskService gseTaskService; + private final ScriptAgentTaskService scriptAgentTaskService; + private final FileAgentTaskService fileAgentTaskService; + private final TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher; + private final AccountService accountService; + private final LogService logService; + private final TaskInstanceVariableService taskInstanceVariableService; + private final StepInstanceVariableValueService stepInstanceVariableValueService; + private final AgentService agentService; + private final ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager; + private final JobBuildInVariableResolver jobBuildInVariableResolver; + private final Tracer tracer; + private final ExecuteMonitor executeMonitor; + private final StorageSystemConfig storageSystemConfig; + private final JobExecuteConfig jobExecuteConfig; + private final TaskEvictPolicyExecutor taskEvictPolicyExecutor; + private final GseTasksExceptionCounter gseTasksExceptionCounter; + private final StepInstanceService stepInstanceService; + + private final Object lifecycleMonitor = new Object(); + private final RunningTaskCounter counter = new RunningTaskCounter<>("GseTask-Counter"); + /** + * 正在执行中的任务 + */ + private final Map startingGseTasks = new ConcurrentHashMap<>(); + private volatile boolean running = false; + private volatile boolean active = false; + + /** + * 正在处理的gse任务数 + */ + private final AtomicInteger runningTasks = new AtomicInteger(0); + /** + * 正在处理的gse文件任务数 + */ + private final AtomicInteger runningFileTasks = new AtomicInteger(0); + /** + * 正在处理的gse脚本任务数 + */ + private final AtomicInteger runningScriptTasks = new AtomicInteger(0); + /** + * 正在处理的gse文件任务数 + */ + private final AtomicInteger fileTaskCounter = new AtomicInteger(0); + /** + * 正在处理的gse脚本任务数 + */ + private final AtomicInteger scriptTaskCounter = new AtomicInteger(0); + + /** + * GseTaskManager Constructor + */ + @Autowired + public GseTaskManager(ResultHandleManager resultHandleManager, + TaskInstanceService taskInstanceService, + GseTaskService gseTaskService, + TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, + AccountService accountService, + LogService logService, + TaskInstanceVariableService taskInstanceVariableService, + StepInstanceVariableValueService stepInstanceVariableValueService, + JobBuildInVariableResolver jobBuildInVariableResolver, + StorageSystemConfig storageSystemConfig, + AgentService agentService, + ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager, + GseTasksExceptionCounter gseTasksExceptionCounter, + Tracer tracer, + ExecuteMonitor executeMonitor, + JobExecuteConfig jobExecuteConfig, + TaskEvictPolicyExecutor taskEvictPolicyExecutor, + ScriptAgentTaskService scriptAgentTaskService, + FileAgentTaskService fileAgentTaskService, + StepInstanceService stepInstanceService) { + this.resultHandleManager = resultHandleManager; + this.taskInstanceService = taskInstanceService; + this.gseTaskService = gseTaskService; + this.scriptAgentTaskService = scriptAgentTaskService; + this.fileAgentTaskService = fileAgentTaskService; + this.taskExecuteMQEventDispatcher = taskExecuteMQEventDispatcher; + this.accountService = accountService; + this.logService = logService; + this.taskInstanceVariableService = taskInstanceVariableService; + this.stepInstanceVariableValueService = stepInstanceVariableValueService; + this.jobBuildInVariableResolver = jobBuildInVariableResolver; + this.storageSystemConfig = storageSystemConfig; + this.agentService = agentService; + this.resultHandleTaskKeepaliveManager = resultHandleTaskKeepaliveManager; + this.gseTasksExceptionCounter = gseTasksExceptionCounter; + this.tracer = tracer; + this.executeMonitor = executeMonitor; + this.jobExecuteConfig = jobExecuteConfig; + this.taskEvictPolicyExecutor = taskEvictPolicyExecutor; + this.stepInstanceService = stepInstanceService; + } + + /** + * 启动任务(首次执行/继续执行异常中断的任务) + * + * @param gseTask GSE任务 + * @param requestId 请求ID + */ + public void startTask(GseTaskDTO gseTask, String requestId) { + long stepInstanceId = gseTask.getStepInstanceId(); + checkActiveStatus(stepInstanceId); + + boolean success = false; + String taskName = gseTask.getTaskUniqueName(); + + StopWatch watch = new StopWatch("startGseTask"); + String startTaskRequestId = requestId; + if (StringUtils.isEmpty(startTaskRequestId)) { + startTaskRequestId = UUID.randomUUID().toString(); + } + + AbstractGseTaskStartCommand startCommand = null; + String lockKey = buildGseTaskLockKey(gseTask); + try { + watch.start("getRunningLock"); + // 可重入锁，如果任务正在执行，则放弃 + if (!LockUtils.tryGetReentrantLock(lockKey, startTaskRequestId, 30000L)) { + log.error("Fail to get running lock, lockKey: {}", lockKey); + return; + } + watch.stop(); + + watch.start("loadTaskAndCheck"); + StepInstanceDTO stepInstance = taskInstanceService.getStepInstanceDetail(stepInstanceId); + TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(stepInstance.getTaskInstanceId()); + + // 如果任务应当被驱逐，直接置为被丢弃状态 + if (taskEvictPolicyExecutor.shouldEvictTask(taskInstance)) { + log.warn("Evict job, taskInstanceId: {}, gseTask: {}", taskInstance.getId(), taskName); + taskEvictPolicyExecutor.updateEvictedTaskStatus(taskInstance, stepInstance); + watch.stop(); + return; + } + + // 如果任务处于“终止中”状态，直接终止,不需要下发任务给GSE + if (taskInstance.getStatus() == RunStatusEnum.STOPPING) { + log.info("Task instance status is stopping, stop executing the step! taskInstanceId:{}, " + + "stepInstanceId:{}", taskInstance.getId(), stepInstance.getId()); + gseTask.setStatus(RunStatusEnum.STOP_SUCCESS.getValue()); + gseTaskService.updateGseTask(gseTask); + taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.refreshStep(stepInstanceId, + EventSource.buildGseTaskEventSource(stepInstanceId, stepInstance.getExecuteCount(), + stepInstance.getBatch(), gseTask.getId()))); + watch.stop(); + return; + } + watch.stop(); + + watch.start("initStarCommand"); + startCommand = initGseTaskStartCommand(startTaskRequestId, stepInstance, taskInstance, gseTask); + watch.stop(); + + watch.start("executeTask"); + counter.add(taskName); + executeTask(startCommand, gseTask); + watch.stop(); + success = true; + } finally { + if (!watch.isRunning()) { + watch.start("release-running-lock"); + } + LockUtils.releaseDistributedLock(lockKey, startTaskRequestId); + counter.release(taskName); + watch.stop(); + if (watch.getTotalTimeMillis() > 2000L) { + log.warn("GseTaskManager-> start gse task is slow, statistics:{}", watch.prettyPrint()); + } + executeMonitor.getMeterRegistry().timer(ExecuteMetricNames.EXECUTE_TASK_PREFIX, + "task_type", getTaskTypeDesc(startCommand), "status", success ? "ok" : "error") + .record(watch.getTotalTimeNanos(), TimeUnit.NANOSECONDS); + } + } + + private String buildGseTaskLockKey(GseTaskDTO gseTask) { + return "job:running:gse:task:" + gseTask.getId(); + } + + /** + * 判定GSE任务管理器是否活跃 + * + * @param stepInstanceId 当前请求调度的步骤Id + */ + private void checkActiveStatus(long stepInstanceId) { + if (!isActive()) { + log.warn("GseTaskManager is not active, reject! stepInstanceId: {}", stepInstanceId); + throw new MessageHandlerUnavailableException(); + } + } + + + private String getTaskTypeDesc(AbstractGseTaskStartCommand command) { + if (command == null) { + return "none"; + } + if (command instanceof ScriptGseTaskStartCommand) { + return "script"; + } else if (command instanceof FileGseTaskStartCommand) { + return "file"; + } else { + return "none"; + } + } + + private void incrementRunningTasksCount(AbstractGseTaskStartCommand gseTaskStartCommand) { + this.runningTasks.incrementAndGet(); + if (gseTaskStartCommand instanceof ScriptGseTaskStartCommand) { + this.runningScriptTasks.incrementAndGet(); + } else if (gseTaskStartCommand instanceof FileGseTaskStartCommand) { + this.runningFileTasks.incrementAndGet(); + } + } + + private void decrementRunningTasksCount(AbstractGseTaskStartCommand gseTaskStartCommand) { + this.runningTasks.decrementAndGet(); + if (gseTaskStartCommand instanceof ScriptGseTaskStartCommand) { + this.runningScriptTasks.decrementAndGet(); + } else if (gseTaskStartCommand instanceof FileGseTaskStartCommand) { + this.runningFileTasks.decrementAndGet(); + } + } + + private AbstractGseTaskStartCommand initGseTaskStartCommand(String requestId, + StepInstanceDTO stepInstance, + TaskInstanceDTO taskInstance, + GseTaskDTO gseTask) { + AbstractGseTaskStartCommand gseTaskStartCommand = null; + int executeType = stepInstance.getExecuteType(); + if (executeType == StepExecuteTypeEnum.EXECUTE_SCRIPT.getValue()) { + gseTaskStartCommand = new ScriptGseTaskStartCommand( + resultHandleManager, + taskInstanceService, + stepInstanceService, + gseTaskService, + scriptAgentTaskService, + accountService, + taskInstanceVariableService, + stepInstanceVariableValueService, + agentService, + logService, + taskExecuteMQEventDispatcher, + resultHandleTaskKeepaliveManager, + executeMonitor, + jobExecuteConfig, + taskEvictPolicyExecutor, + gseTasksExceptionCounter, + jobBuildInVariableResolver, + tracer, + requestId, + taskInstance, + stepInstance, + gseTask + ); + scriptTaskCounter.incrementAndGet(); + } else if (executeType == StepExecuteTypeEnum.EXECUTE_SQL.getValue()) { + gseTaskStartCommand = new SQLScriptGseTaskStartCommand( + resultHandleManager, + taskInstanceService, + stepInstanceService, + gseTaskService, + scriptAgentTaskService, + accountService, + taskInstanceVariableService, + stepInstanceVariableValueService, + agentService, + logService, + taskExecuteMQEventDispatcher, + resultHandleTaskKeepaliveManager, + executeMonitor, + jobExecuteConfig, + taskEvictPolicyExecutor, + gseTasksExceptionCounter, + jobBuildInVariableResolver, + tracer, + requestId, + taskInstance, + stepInstance, + gseTask + ); + scriptTaskCounter.incrementAndGet(); + } else if (executeType == TaskStepTypeEnum.FILE.getValue()) { + gseTaskStartCommand = new FileGseTaskStartCommand( + resultHandleManager, + taskInstanceService, + gseTaskService, + fileAgentTaskService, + accountService, + taskInstanceVariableService, + stepInstanceVariableValueService, + agentService, + logService, + taskExecuteMQEventDispatcher, + resultHandleTaskKeepaliveManager, + executeMonitor, + jobExecuteConfig, + taskEvictPolicyExecutor, + gseTasksExceptionCounter, + stepInstanceService, + tracer, + requestId, + taskInstance, + stepInstance, + gseTask, + storageSystemConfig.getJobStorageRootPath() + ); + fileTaskCounter.incrementAndGet(); + } + + if (gseTaskStartCommand == null) { + log.error("No match GseTaskStartCommand, gseTask: {}", gseTask.getTaskUniqueName()); + throw new InternalException("No match GseTaskStartCommand", ErrorCode.INTERNAL_ERROR); + } + + return gseTaskStartCommand; + } + + private void executeTask(AbstractGseTaskStartCommand startCommand, + GseTaskDTO gseTask) { + try { + startingGseTasks.put(gseTask.getTaskUniqueName(), startCommand); + incrementRunningTasksCount(startCommand); + startCommand.execute(); + } finally { + startingGseTasks.remove(gseTask.getTaskUniqueName()); + decrementRunningTasksCount(startCommand); + } + } + + /** + * 停止任务 + * + * @param gseTask GSE任务 + */ + public void stopTask(GseTaskDTO gseTask) { + long stepInstanceId = gseTask.getStepInstanceId(); + checkActiveStatus(stepInstanceId); + + String taskName = gseTask.getTaskUniqueName(); + + StopWatch watch = new StopWatch("stopGseTask"); + GseTaskCommand stopCommand; + try { + watch.start("loadTaskAndCheck"); + StepInstanceDTO stepInstance = taskInstanceService.getStepInstanceDetail(stepInstanceId); + TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(stepInstance.getTaskInstanceId()); + watch.stop(); + + watch.start("initStopCommand"); + stopCommand = initGseTaskStopCommand(stepInstance, taskInstance, gseTask); + watch.stop(); + + watch.start("stopTask"); + counter.add(taskName); + stopCommand.execute(); + watch.stop(); + } finally { + if (watch.isRunning()) { + watch.stop(); + } + counter.release(taskName); + if (watch.getTotalTimeMillis() > 2000L) { + log.warn("GseTaskManager-> stop gse task is slow, statistics: {}", watch.prettyPrint()); + } + } + } + + private GseTaskCommand initGseTaskStopCommand(StepInstanceDTO stepInstance, + TaskInstanceDTO taskInstance, + GseTaskDTO gseTask) { + GseTaskCommand gseTaskStopCommand = null; + if (stepInstance.isScriptStep()) { + gseTaskStopCommand = new ScriptGseTaskStopCommand( + agentService, + accountService, + gseTaskService, + scriptAgentTaskService, + tracer, + taskInstance, + stepInstance, + gseTask + ); + scriptTaskCounter.incrementAndGet(); + } else if (stepInstance.isFileStep()) { + gseTaskStopCommand = new FileGseTaskStopCommand( + agentService, + accountService, + gseTaskService, + fileAgentTaskService, + tracer, + taskInstance, + stepInstance, + gseTask + ); + scriptTaskCounter.incrementAndGet(); + } + + if (gseTaskStopCommand == null) { + log.error("No match GseTaskStopCommand, gseTask: {}", gseTask.getTaskUniqueName()); + throw new InternalException("No match GseTaskStopCommand", ErrorCode.INTERNAL_ERROR); + } + + return gseTaskStopCommand; + } + + private boolean isActive() { + synchronized (this.lifecycleMonitor) { + return this.active; + } + } + + @Override + public void start() { + if (isRunning()) { + return; + } + log.info("GseTaskManager starting."); + counter.start(); + synchronized (lifecycleMonitor) { + this.running = true; + this.active = true; + } + } + + @Override + public void stop() { + log.info("GseTaskManager stopping"); + synchronized (this.lifecycleMonitor) { + this.active = false; + } + try { + counter.stop(); + counter.waitUntilTaskDone(5, TimeUnit.SECONDS); + } finally { + synchronized (this.lifecycleMonitor) { + this.running = false; + } + } + log.info("GseTaskManager is stopped"); + } + + @Override + public boolean isRunning() { + synchronized (this.lifecycleMonitor) { + return (this.running); + } + } + + @Override + public int getPhase() { + return DestroyOrder.GSE_TASK_HANDLER; + } + + /** + * 返回正在执行的任务数量 + * + * @return 任务数量 + */ + public int getRunningTaskCount() { + return this.runningTasks.get(); + } + + /** + * 返回正在执行的文件任务数量 + * + * @return 任务数量 + */ + public int getRunningFileTaskCount() { + return this.runningFileTasks.get(); + } + + /** + * 返回正在执行的脚本任务数量 + * + * @return 任务数量 + */ + public int getRunningScriptTaskCount() { + return this.runningScriptTasks.get(); + } + + /** + * 返回累计处理的文件任务数量 + * + * @return 任务数量 + */ + public int getFileTaskCount() { + return this.fileTaskCounter.get(); + } + + /** + * 返回累计处理的脚本任务数量 + * + * @return 任务数量 + */ + public int getScriptTaskCount() { + return this.scriptTaskCounter.get(); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/SQLScriptTaskExecutor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/SQLScriptGseTaskStartCommand.java similarity index 65% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/SQLScriptTaskExecutor.java rename to src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/SQLScriptGseTaskStartCommand.java index c204f95123..53a107d1e2 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/SQLScriptTaskExecutor.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/SQLScriptGseTaskStartCommand.java @@ -31,31 +31,45 @@ import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.util.Base64Util; import com.tencent.bk.job.common.util.crypto.AESUtils; +import com.tencent.bk.job.execute.config.JobExecuteConfig; +import com.tencent.bk.job.execute.engine.evict.TaskEvictPolicyExecutor; import com.tencent.bk.job.execute.engine.gse.GseRequestUtils; +import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; import com.tencent.bk.job.execute.engine.model.RunSQLScriptFile; +import com.tencent.bk.job.execute.engine.result.ResultHandleManager; +import com.tencent.bk.job.execute.engine.result.ha.ResultHandleTaskKeepaliveManager; import com.tencent.bk.job.execute.engine.util.TimeoutUtils; +import com.tencent.bk.job.execute.engine.variable.JobBuildInVariableResolver; import com.tencent.bk.job.execute.model.AccountDTO; +import com.tencent.bk.job.execute.model.GseTaskDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; +import com.tencent.bk.job.execute.monitor.metrics.ExecuteMonitor; import com.tencent.bk.job.execute.monitor.metrics.GseTasksExceptionCounter; +import com.tencent.bk.job.execute.service.AccountService; +import com.tencent.bk.job.execute.service.AgentService; +import com.tencent.bk.job.execute.service.GseTaskService; +import com.tencent.bk.job.execute.service.LogService; +import com.tencent.bk.job.execute.service.ScriptAgentTaskService; +import com.tencent.bk.job.execute.service.StepInstanceService; +import com.tencent.bk.job.execute.service.StepInstanceVariableValueService; +import com.tencent.bk.job.execute.service.TaskInstanceService; +import com.tencent.bk.job.execute.service.TaskInstanceVariableService; import com.tencent.bk.job.manage.common.consts.account.AccountTypeEnum; import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; +import org.springframework.cloud.sleuth.Tracer; import java.io.InputStream; import java.io.StringWriter; import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; -import java.util.Set; -/** - * SQL执行 - */ @Slf4j -public class SQLScriptTaskExecutor extends ScriptTaskExecutor { +public class SQLScriptGseTaskStartCommand extends ScriptGseTaskStartCommand { private static final Map sqlMap = Maps.newHashMap(); static { @@ -103,21 +117,54 @@ public class SQLScriptTaskExecutor extends ScriptTaskExecutor { } } - /** - * SQLScriptTaskExecutor Constructor - * - * @param requestId 请求ID - * @param gseTasksExceptionCounter GSE任务异常计数 - * @param taskInstance 作业实例 - * @param stepInstance 步骤实例 - * @param executeIps 目标IP - */ - public SQLScriptTaskExecutor(String requestId, GseTasksExceptionCounter gseTasksExceptionCounter, - TaskInstanceDTO taskInstance, StepInstanceDTO stepInstance, Set executeIps) { - super(requestId, gseTasksExceptionCounter, taskInstance, stepInstance, executeIps, null); + public SQLScriptGseTaskStartCommand(ResultHandleManager resultHandleManager, + TaskInstanceService taskInstanceService, + StepInstanceService stepInstanceService, + GseTaskService gseTaskService, + ScriptAgentTaskService scriptAgentTaskService, + AccountService accountService, + TaskInstanceVariableService taskInstanceVariableService, + StepInstanceVariableValueService stepInstanceVariableValueService, + AgentService agentService, + LogService logService, + TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, + ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager, + ExecuteMonitor executeMonitor, + JobExecuteConfig jobExecuteConfig, + TaskEvictPolicyExecutor taskEvictPolicyExecutor, + GseTasksExceptionCounter gseTasksExceptionCounter, + JobBuildInVariableResolver jobBuildInVariableResolver, + Tracer tracer, + String requestId, + TaskInstanceDTO taskInstance, + StepInstanceDTO stepInstance, + GseTaskDTO gseTask) { + super(resultHandleManager, + taskInstanceService, + stepInstanceService, + gseTaskService, + scriptAgentTaskService, + accountService, + taskInstanceVariableService, + stepInstanceVariableValueService, + agentService, + logService, + taskExecuteMQEventDispatcher, + resultHandleTaskKeepaliveManager, + executeMonitor, + jobExecuteConfig, + taskEvictPolicyExecutor, + gseTasksExceptionCounter, + jobBuildInVariableResolver, + tracer, + requestId, + taskInstance, + stepInstance, + gseTask); } - protected api_script_request getScriptRequest(StepInstanceDTO stepInstance) { + @Override + protected api_script_request buildScriptRequest() { String sqlScriptContent = stepInstance.getScriptContent(); String sqlScriptFileName = buildScriptFileName(stepInstance); @@ -159,8 +206,8 @@ protected api_script_request getScriptRequest(StepInstanceDTO stepInstance) { AccountDTO accountInfo = getAccountBean(stepInstance.getAccountId(), stepInstance.getAccount(), stepInstance.getAppId()); - List agentList = GseRequestUtils.buildAgentList(jobIpSet, accountInfo.getAccount(), - accountInfo.getPassword()); + List agentList = GseRequestUtils.buildAgentList(targetAgentTaskMap.keySet(), + accountInfo.getAccount(), accountInfo.getPassword()); return GseRequestUtils.buildScriptRequestWithSQL(agentList, param); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/ScriptTaskExecutor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/ScriptGseTaskStartCommand.java similarity index 78% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/ScriptTaskExecutor.java rename to src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/ScriptGseTaskStartCommand.java index 2071dbffa0..d1a50499ed 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/ScriptTaskExecutor.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/ScriptGseTaskStartCommand.java @@ -26,18 +26,24 @@ import com.tencent.bk.gse.taskapi.api_agent; import com.tencent.bk.gse.taskapi.api_script_request; -import com.tencent.bk.gse.taskapi.api_stop_task_request; import com.tencent.bk.job.common.constant.TaskVariableTypeEnum; import com.tencent.bk.job.common.service.VariableResolver; -import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; +import com.tencent.bk.job.common.util.date.DateUtils; +import com.tencent.bk.job.execute.common.constants.RunStatusEnum; +import com.tencent.bk.job.execute.common.util.TaskCostCalculator; import com.tencent.bk.job.execute.common.util.VariableValueResolver; +import com.tencent.bk.job.execute.config.JobExecuteConfig; +import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; +import com.tencent.bk.job.execute.engine.evict.TaskEvictPolicyExecutor; import com.tencent.bk.job.execute.engine.gse.GseRequestUtils; import com.tencent.bk.job.execute.engine.gse.ScriptRequestBuilder; -import com.tencent.bk.job.execute.engine.model.GseTaskExecuteResult; +import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; import com.tencent.bk.job.execute.engine.model.GseTaskResponse; import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; import com.tencent.bk.job.execute.engine.model.TaskVariablesAnalyzeResult; +import com.tencent.bk.job.execute.engine.result.ResultHandleManager; import com.tencent.bk.job.execute.engine.result.ScriptResultHandleTask; +import com.tencent.bk.job.execute.engine.result.ha.ResultHandleTaskKeepaliveManager; import com.tencent.bk.job.execute.engine.util.MacroUtil; import com.tencent.bk.job.execute.engine.util.TimeoutUtils; import com.tencent.bk.job.execute.engine.variable.JobBuildInVariableResolver; @@ -45,28 +51,43 @@ import com.tencent.bk.job.execute.engine.variable.VariableResolveResult; import com.tencent.bk.job.execute.engine.variable.VariableResolveUtils; import com.tencent.bk.job.execute.model.AccountDTO; -import com.tencent.bk.job.execute.model.GseTaskLogDTO; +import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.model.GseTaskDTO; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.VariableValueDTO; +import com.tencent.bk.job.execute.monitor.metrics.ExecuteMonitor; import com.tencent.bk.job.execute.monitor.metrics.GseTasksExceptionCounter; +import com.tencent.bk.job.execute.service.AccountService; +import com.tencent.bk.job.execute.service.AgentService; +import com.tencent.bk.job.execute.service.GseTaskService; +import com.tencent.bk.job.execute.service.LogService; +import com.tencent.bk.job.execute.service.ScriptAgentTaskService; +import com.tencent.bk.job.execute.service.StepInstanceService; +import com.tencent.bk.job.execute.service.StepInstanceVariableValueService; +import com.tencent.bk.job.execute.service.TaskInstanceService; +import com.tencent.bk.job.execute.service.TaskInstanceVariableService; +import com.tencent.bk.job.logsvr.model.service.ServiceScriptLogDTO; import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.springframework.cloud.sleuth.Tracer; import java.io.File; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; -/** - * GSE-脚本任务执行 - */ @Slf4j -public class ScriptTaskExecutor extends AbstractGseTaskExecutor { +public class ScriptGseTaskStartCommand extends AbstractGseTaskStartCommand { + + private final ScriptAgentTaskService scriptAgentTaskService; + + private final JobBuildInVariableResolver jobBuildInVariableResolver; + /** * 下发到gse的脚本文件根目录 */ @@ -78,27 +99,51 @@ public class ScriptTaskExecutor extends AbstractGseTaskExecutor { private final String GSE_SCRIPT_FILE_NAME_PREFIX = "bk_gse_script_"; - private final JobBuildInVariableResolver jobBuildInVariableResolver; - - /** - * ScriptTaskExecutor Constructor - * - * @param requestId 请求ID - * @param gseTasksExceptionCounter GSE任务异常计数 - * @param taskInstance 作业实例 - * @param stepInstance 步骤实例 - * @param executeIps 目标IP - * @param jobBuildInVariableResolver 内置变量解析 - */ - public ScriptTaskExecutor(String requestId, GseTasksExceptionCounter gseTasksExceptionCounter, - TaskInstanceDTO taskInstance, - StepInstanceDTO stepInstance, - Set executeIps, - JobBuildInVariableResolver jobBuildInVariableResolver) { - super(requestId, gseTasksExceptionCounter, taskInstance, stepInstance, executeIps); - this.scriptFileNamePrefix = buildScriptFileNamePrefix(stepInstance); + public ScriptGseTaskStartCommand(ResultHandleManager resultHandleManager, + TaskInstanceService taskInstanceService, + StepInstanceService stepInstanceService, + GseTaskService gseTaskService, + ScriptAgentTaskService scriptAgentTaskService, + AccountService accountService, + TaskInstanceVariableService taskInstanceVariableService, + StepInstanceVariableValueService stepInstanceVariableValueService, + AgentService agentService, + LogService logService, + TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, + ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager, + ExecuteMonitor executeMonitor, + JobExecuteConfig jobExecuteConfig, + TaskEvictPolicyExecutor taskEvictPolicyExecutor, + GseTasksExceptionCounter gseTasksExceptionCounter, + JobBuildInVariableResolver jobBuildInVariableResolver, + Tracer tracer, + String requestId, + TaskInstanceDTO taskInstance, + StepInstanceDTO stepInstance, + GseTaskDTO gseTask) { + super(resultHandleManager, + taskInstanceService, + gseTaskService, + scriptAgentTaskService, + accountService, + taskInstanceVariableService, + stepInstanceVariableValueService, + agentService, + logService, + taskExecuteMQEventDispatcher, + resultHandleTaskKeepaliveManager, + executeMonitor, + jobExecuteConfig, + taskEvictPolicyExecutor, + gseTasksExceptionCounter, + tracer, + requestId, + taskInstance, + stepInstance, + gseTask, stepInstanceService); + this.scriptAgentTaskService = scriptAgentTaskService; this.jobBuildInVariableResolver = jobBuildInVariableResolver; - + this.scriptFileNamePrefix = buildScriptFileNamePrefix(stepInstance); } private String buildScriptFileNamePrefix(StepInstanceDTO stepInstance) { @@ -107,6 +152,12 @@ private String buildScriptFileNamePrefix(StepInstanceDTO stepInstance) { "_" + stepInstance.getId(); } + @Override + protected void preExecute() { + scriptFilePath = buildScriptFilePath(jobExecuteConfig.getGseScriptFileRootPath(), + stepInstance.getAccount()); + } + private String buildScriptFilePath(String gseScriptFileRootPath, String account) { if (gseScriptFileRootPath.endsWith("/")) { return gseScriptFileRootPath + account; @@ -115,26 +166,26 @@ private String buildScriptFilePath(String gseScriptFileRootPath, String account) } } - @Override - protected void initExecutionContext() { - scriptFilePath = buildScriptFilePath(jobExecuteConfig.getGseScriptFileRootPath(), - stepInstance.getAccount()); - super.initExecutionContext(); - } - @Override - protected GseTaskResponse startGseTask(StepInstanceDTO stepInstance) { - return GseRequestUtils.sendScriptTaskRequest(stepInstanceId, getScriptRequest(stepInstance)); + protected GseTaskResponse startGseTask() { + return GseRequestUtils.sendScriptTaskRequest(stepInstanceId, buildScriptRequest()); } - protected api_script_request getScriptRequest(StepInstanceDTO stepInstance) { + protected api_script_request buildScriptRequest() { + api_script_request request; // shell 脚本需要支持全局变量传参，需要特殊的处理逻辑 if (stepInstance.getScriptType().equals(ScriptTypeEnum.SHELL.getValue())) { - api_script_request request = buildShellScriptRequest(stepInstance); - request.setM_caller(buildTraceInfoMap()); - return request; + request = buildShellScriptRequest(); + } else { + request = buildNonShellScriptRequest(); } + request.setM_caller(buildGSETraceInfo()); + + return request; + } + + private api_script_request buildNonShellScriptRequest() { String scriptContent = stepInstance.getScriptContent(); String scriptFileName = buildScriptFileName(stepInstance); @@ -144,14 +195,16 @@ protected api_script_request getScriptRequest(StepInstanceDTO stepInstance) { AccountDTO accountInfo = getAccountBean(stepInstance.getAccountId(), stepInstance.getAccount(), stepInstance.getAppId()); - List agentList = GseRequestUtils.buildAgentList(jobIpSet, accountInfo.getAccount(), - accountInfo.getPassword()); + List agentList = GseRequestUtils.buildAgentList(targetAgentTaskMap.keySet(), + accountInfo.getAccount(), accountInfo.getPassword()); api_script_request request = GseRequestUtils.buildScriptRequest(agentList, scriptContent, scriptFileName, scriptFilePath, resolvedScriptParam, timeout); - request.setM_caller(buildTraceInfoMap()); + request.setM_caller(buildGSETraceInfo()); + return request; } + protected String buildScriptFileName(StepInstanceDTO stepInstance) { return this.scriptFileNamePrefix + ScriptTypeEnum.getExtByValue(stepInstance.getScriptType()); } @@ -159,7 +212,20 @@ protected String buildScriptFileName(StepInstanceDTO stepInstance) { /** * 构建shell脚本下发的请求 */ - private api_script_request buildShellScriptRequest(StepInstanceDTO stepInstance) { + private api_script_request buildShellScriptRequest() { + api_script_request request; + if (taskInstance.isPlanInstance()) { + // 执行方案脚本执行步骤，需要处理变量 + request = buildShellScriptRequestForPlan(); + } else { + // 快速执行脚本 + request = buildRequestWithoutAnyParam(stepInstance); + } + + return request; + } + + private api_script_request buildShellScriptRequestForPlan() { api_script_request request; boolean containsAnyImportedVariable = false; List importVariables = null; @@ -199,7 +265,7 @@ private String resolveScriptParamVariables(String scriptParam) { } String resolvedScriptParam = VariableValueResolver.resolve(scriptParam, - buildReferenceGlobalVarValueMap(stepInputVariables)); + buildStringGlobalVarKV(stepInputVariables)); resolvedScriptParam = resolvedScriptParam.replace("\n", " "); log.info("Origin script param:{}, resolved script param:{}", scriptParam, resolvedScriptParam); updateResolvedScriptParamIfNecessary(scriptParam, resolvedScriptParam); @@ -213,7 +279,6 @@ private void updateResolvedScriptParamIfNecessary(String originParam, String res } } - /** * 创建下发请求-不带任何全局参数 */ @@ -230,8 +295,8 @@ private api_script_request buildRequestWithoutAnyParam(StepInstanceDTO stepInsta AccountDTO accountInfo = getAccountBean(stepInstance.getAccountId(), stepInstance.getAccount(), stepInstance.getAppId()); - List agentList = GseRequestUtils.buildAgentList(jobIpSet, accountInfo.getAccount(), - accountInfo.getPassword()); + List agentList = GseRequestUtils.buildAgentList(targetAgentTaskMap.keySet(), + accountInfo.getAccount(), accountInfo.getPassword()); builder.addScriptTask(agentList, scriptFilePath, scriptFileName, resolvedScriptParam, timeout); return builder.build(); @@ -265,8 +330,8 @@ private api_script_request buildRequestWithConstParamOnly(StepInstanceDTO stepIn AccountDTO accountInfo = getAccountBean(stepInstance.getAccountId(), stepInstance.getAccount(), stepInstance.getAppId()); - List agentList = GseRequestUtils.buildAgentList(jobIpSet, accountInfo.getAccount(), - accountInfo.getPassword()); + List agentList = GseRequestUtils.buildAgentList(targetAgentTaskMap.keySet(), + accountInfo.getAccount(), accountInfo.getPassword()); builder.addScriptTask(agentList, scriptFilePath, wrapperScriptFileName, resolvedScriptParam, timeout); return builder.build(); } @@ -375,7 +440,6 @@ private String buildWrapperScriptWithConstParamOnly(String declareFileName, Stri * * @param stepInstance 步骤实例 * @param taskVariablesAnalyzeResult 参数 - * @return */ private api_script_request buildRequestWithChangeableParam(StepInstanceDTO stepInstance, TaskVariablesAnalyzeResult taskVariablesAnalyzeResult, @@ -386,8 +450,8 @@ private api_script_request buildRequestWithChangeableParam(StepInstanceDTO stepI AccountDTO accountInfo = getAccountBean(stepInstance.getAccountId(), stepInstance.getAccount(), stepInstance.getAppId()); - List agentList = GseRequestUtils.buildAgentList(jobIpSet, accountInfo.getAccount(), - accountInfo.getPassword()); + List agentList = GseRequestUtils.buildAgentList(targetAgentTaskMap.keySet(), + accountInfo.getAccount(), accountInfo.getPassword()); ScriptRequestBuilder builder = new ScriptRequestBuilder(); @@ -599,57 +663,52 @@ private String getNamespaceParamsOutputFilePath() { } @Override - public GseTaskExecuteResult stopGseTask() { - AccountDTO accountInfo = getAccountBean(stepInstance.getAccountId(), stepInstance.getAccount(), - stepInstance.getAppId()); - List agentList = GseRequestUtils.buildAgentList(jobIpSet, accountInfo.getAccount(), - accountInfo.getPassword()); - api_stop_task_request stopTaskRequest = new api_stop_task_request(); - GseTaskLogDTO gseTaskLog = gseTaskLogService.getGseTaskLog(stepInstanceId, executeCount); - if (gseTaskLog == null || StringUtils.isEmpty(gseTaskLog.getGseTaskId())) { - log.warn("Gse Task not send to gse server, not support stop"); - return new GseTaskExecuteResult(GseTaskExecuteResult.RESULT_CODE_STOP_FAILED, "Termination failed"); - } - stopTaskRequest.setStop_task_id(gseTaskLog.getGseTaskId()); - stopTaskRequest.setAgents(agentList); - stopTaskRequest.setType(StepExecuteTypeEnum.EXECUTE_SCRIPT.getValue()); - stopTaskRequest.setM_caller(buildTraceInfoMap()); - - GseTaskResponse gseTaskResponse = GseRequestUtils.sendForceStopTaskRequest(stepInstance.getId(), - stopTaskRequest); - if (GseTaskResponse.ERROR_CODE_SUCCESS != gseTaskResponse.getErrorCode()) { - log.info("sendForceStopTaskRequest response failed!"); - return new GseTaskExecuteResult(GseTaskExecuteResult.RESULT_CODE_STOP_FAILED, - "Termination failed， msg:" + gseTaskResponse.getErrorMessage()); - } else { - log.info("sendForceStopTaskRequest response success!"); - return new GseTaskExecuteResult(GseTaskExecuteResult.RESULT_CODE_STOP_SUCCESS, "Termination successfully"); - } - } - - @Override - void addExecutionResultHandleTask() { + protected final void addResultHandleTask() { ScriptResultHandleTask scriptResultHandleTask = - new ScriptResultHandleTask(taskInstance, stepInstance, taskVariablesAnalyzeResult, ipLogMap, gseTaskLog, - jobIpSet, requestId); - scriptResultHandleTask.initDependentService(taskInstanceService, gseTaskLogService, logService, - taskInstanceVariableService, stepInstanceVariableValueService, - taskManager, resultHandleTaskKeepaliveManager, exceptionStatusManager, taskEvictPolicyExecutor); + new ScriptResultHandleTask( + taskInstanceService, + gseTaskService, + logService, + taskInstanceVariableService, + stepInstanceVariableValueService, + taskExecuteMQEventDispatcher, + resultHandleTaskKeepaliveManager, + taskEvictPolicyExecutor, + scriptAgentTaskService, + stepInstanceService, + taskInstance, + stepInstance, + taskVariablesAnalyzeResult, + targetAgentTaskMap, + gseTask, + requestId); resultHandleManager.handleDeliveredTask(scriptResultHandleTask); } - @Override - public void resume() { - log.info("Resume script task from snapshot, stepInstanceId: {}", stepInstanceId); - } - - @Override - public void saveSnapshot() { - log.info("Save script task snapshot, stepInstanceId: {}", stepInstanceId); - } @Override - protected boolean checkHostExecutable() { - return !jobIpSet.isEmpty(); + protected final void handleStartGseTaskError(GseTaskResponse gseTaskResponse) { + long now = DateUtils.currentTimeMillis(); + updateGseTaskExecutionInfo(null, RunStatusEnum.FAIL, null, now, now - gseTask.getStartTime()); + + String errorMsg = "GSE Job failed:" + gseTaskResponse.getErrorMessage(); + int errorMsgLength = errorMsg.length(); + + List scriptLogs = new ArrayList<>(targetAgentTaskMap.size()); + for (AgentTaskDTO agentTask : targetAgentTaskMap.values()) { + // 日志输出 + ServiceScriptLogDTO scriptLog = logService.buildSystemScriptLog(agentTask.getHost(), errorMsg, + agentTask.getScriptLogOffset() + errorMsgLength, now); + scriptLogs.add(scriptLog); + + // AgentTask 结果更新 + agentTask.setGseTaskId(gseTask.getId()); + agentTask.setStartTime(gseTask.getStartTime()); + agentTask.setEndTime(now); + agentTask.setTotalTime(TaskCostCalculator.calculate(gseTask.getStartTime(), now, null)); + agentTask.setStatus(AgentTaskStatusEnum.SUBMIT_FAILED); + } + logService.batchWriteScriptLog(taskInstance.getCreateTime(), stepInstanceId, executeCount, batch, scriptLogs); + scriptAgentTaskService.batchUpdateAgentTasks(targetAgentTaskMap.values()); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/ScriptGseTaskStopCommand.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/ScriptGseTaskStopCommand.java new file mode 100644 index 0000000000..497fa07f98 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/ScriptGseTaskStopCommand.java @@ -0,0 +1,98 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.executor; + +import com.tencent.bk.gse.taskapi.api_agent; +import com.tencent.bk.gse.taskapi.api_stop_task_request; +import com.tencent.bk.job.execute.common.constants.RunStatusEnum; +import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; +import com.tencent.bk.job.execute.engine.gse.GseRequestUtils; +import com.tencent.bk.job.execute.engine.model.GseTaskResponse; +import com.tencent.bk.job.execute.model.AccountDTO; +import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.model.GseTaskDTO; +import com.tencent.bk.job.execute.model.StepInstanceDTO; +import com.tencent.bk.job.execute.model.TaskInstanceDTO; +import com.tencent.bk.job.execute.service.AccountService; +import com.tencent.bk.job.execute.service.AgentService; +import com.tencent.bk.job.execute.service.AgentTaskService; +import com.tencent.bk.job.execute.service.GseTaskService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.cloud.sleuth.Tracer; + +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +@Slf4j +public class ScriptGseTaskStopCommand extends AbstractGseTaskCommand { + + public ScriptGseTaskStopCommand(AgentService agentService, + AccountService accountService, + GseTaskService gseTaskService, + AgentTaskService agentTaskService, + Tracer tracer, + TaskInstanceDTO taskInstance, + StepInstanceDTO stepInstance, + GseTaskDTO gseTask) { + super(agentService, + accountService, + gseTaskService, + agentTaskService, + tracer, + taskInstance, + stepInstance, + gseTask); + } + + @Override + public void execute() { + log.info("Stop gse task, gseTask:" + gseTaskUniqueName); + List agentTasks = agentTaskService.listAgentTasksByGseTaskId(gseTask.getId()); + Set terminateAgentIds = agentTasks.stream() + .map(AgentTaskDTO::getAgentId) + .collect(Collectors.toSet()); + + AccountDTO accountInfo = getAccountBean(stepInstance.getAccountId(), stepInstance.getAccount(), + stepInstance.getAppId()); + List agentList = GseRequestUtils.buildAgentList(terminateAgentIds, accountInfo.getAccount(), + accountInfo.getPassword()); + api_stop_task_request stopTaskRequest = new api_stop_task_request(); + stopTaskRequest.setStop_task_id(gseTask.getGseTaskId()); + stopTaskRequest.setAgents(agentList); + stopTaskRequest.setType(StepExecuteTypeEnum.EXECUTE_SCRIPT.getValue()); + stopTaskRequest.setM_caller(buildGSETraceInfo()); + + GseTaskResponse gseTaskResponse = GseRequestUtils.sendForceStopTaskRequest(stepInstance.getId(), + stopTaskRequest); + if (GseTaskResponse.ERROR_CODE_SUCCESS != gseTaskResponse.getErrorCode()) { + log.error("Terminate gse task failed! gseTask: {}", gseTaskUniqueName); + } else { + log.info("Terminate gse task response success!"); + gseTask.setStatus(RunStatusEnum.STOPPING.getValue()); + gseTaskService.updateGseTask(gseTask); + } + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/GseRequestUtils.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/GseRequestUtils.java index d7518656da..bca1973d92 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/GseRequestUtils.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/GseRequestUtils.java @@ -45,7 +45,6 @@ import com.tencent.bk.gse.taskapi.api_task_detail_result; import com.tencent.bk.gse.taskapi.api_task_request; import com.tencent.bk.job.common.gse.constants.GseConstants; -import com.tencent.bk.job.common.model.dto.IpDTO; import com.tencent.bk.job.common.util.ApplicationContextRegister; import com.tencent.bk.job.common.util.ThreadUtils; import com.tencent.bk.job.execute.common.exception.ReadTimeoutException; @@ -63,7 +62,6 @@ import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -123,41 +121,21 @@ public static api_copy_fileinfoV2 buildCopyFileInfo(api_agent src, String srcPat return copyFileInfo; } - /** - * 创建 Agent 列表 - * - * @param userName 用户名 - * @param passwd 密码 - * @return Agent列表 - */ - public static List buildAgentList(Set jobIpSet, String userName, String passwd) { - List agentList = new ArrayList<>(); - for (String ip : jobIpSet) { - api_agent agent = buildAgent(ip, userName, passwd); - agentList.add(agent); - } - return agentList; - } - public static List buildAgentList(List ipList, String userName, String passwd) { + public static List buildAgentList(Collection agentIds, String userName, String password) { List agentList = new ArrayList<>(); - for (IpDTO ipDTO : ipList) { - String fullIp = ipDTO.getCloudAreaId() + ":" + ipDTO.getIp(); - api_agent agent = buildAgent(fullIp, userName, passwd); + for (String agentId : agentIds) { + api_agent agent = buildAgent(agentId, userName, password); agentList.add(agent); } return agentList; } - /** - * @param userName 用户名 - * @param cloudIp 云区域:IP - * @return Agent - */ - public static api_agent buildAgent(String cloudIp, String userName, String passwd) { + public static api_agent buildAgent(String agentId, String userName, String password) { + // 在gse api v1, agentId=云区域:IP int source = 0; String ip; - String[] ipArray = cloudIp.split(":"); + String[] ipArray = agentId.split(":"); if (ipArray.length > 1) { source = Integer.parseInt(ipArray[0]); ip = ipArray[1]; @@ -167,8 +145,8 @@ public static api_agent buildAgent(String cloudIp, String userName, String passw api_auth auth = new api_auth(); auth.setUser(userName); - if (passwd != null) { - auth.setPassword(passwd); + if (password != null) { + auth.setPassword(password); } else { auth.setPassword(""); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/model/GSEFileTaskResult.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/model/GSEFileTaskResult.java index f62461edd2..ed33b1c8a9 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/model/GSEFileTaskResult.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/model/GSEFileTaskResult.java @@ -44,6 +44,8 @@ public class GSEFileTaskResult { /** * 0:upload， 1:download + * + * @see FileDistModeEnum */ private Integer mode; @@ -103,6 +105,12 @@ public class GSEFileTaskResult { @JsonProperty("source_cloudid") private Long sourceCloudId; + /** + * 文件源主机AgentId + */ + @JsonProperty("source_agent_id") + private String sourceAgentId; + /** * 分发目标IP */ @@ -115,6 +123,12 @@ public class GSEFileTaskResult { @JsonProperty("dest_cloudid") private Long destCloudId; + /** + * 文件目标主机AgentId + */ + @JsonProperty("dest_agent_id") + private String destAgentId; + /** * 源文件目录 */ @@ -178,19 +192,26 @@ public class GSEFileTaskResult { */ private TaskType taskType; - - public String getSourceCloudIp() { - if (sourceCloudId == null || sourceIp == null) { - return null; + /** + * 文件源主机agentId + * 按照与GSE的约定，在没有bk_agent_id的场景下，使用{云区域:IP}作为agentId + */ + public String getSourceAgentId() { + if (sourceAgentId == null) { + sourceAgentId = sourceCloudId + ":" + sourceIp; } - return sourceCloudId + ":" + sourceIp; + return sourceAgentId; } - public String getDestCloudIp() { - if (destCloudId == null || destIp == null) { - return null; + /** + * 目标主机agentId + * 按照与GSE的约定，在没有bk_agent_id的场景下，使用{云区域:IP}作为agentId + */ + public String getDestAgentId() { + if (destAgentId == null) { + destAgentId = destCloudId + ":" + destIp; } - return destCloudId + ":" + destIp; + return destAgentId; } public boolean isDownloadMode() { @@ -227,25 +248,26 @@ public String getStandardDestFilePath() { } public String getTaskId() { - if (taskId != null) { - return taskId; + if (taskId == null) { + this.taskId = buildTaskId(mode, getSourceAgentId(), getStandardSourceFilePath(), getDestAgentId(), + getStandardDestFilePath()); } - if (isDownloadMode()) { - if (StringUtils.isNotEmpty(sourceIp)) { - this.taskId = concat(mode.toString(), getSourceCloudIp(), getStandardSourceFilePath(), - getDestCloudIp(), getStandardDestFilePath()); - } else { - // GSE BUG, 兼容处理 - this.taskId = concat(mode.toString(), "*", getStandardSourceFilePath(), - getDestCloudIp(), getStandardDestFilePath()); - } + return taskId; + } + + public static String buildTaskId(Integer mode, String sourceAgentId, String sourceFilePath, String destAgentId, + String destFilePath) { + String taskId; + if (FileDistModeEnum.getFileDistMode(mode) == FileDistModeEnum.DOWNLOAD) { + taskId = concat(mode.toString(), sourceAgentId, FilePathUtils.standardizedGSEFilePath(sourceFilePath), + destAgentId, destFilePath); } else { - this.taskId = concat(mode.toString(), getSourceCloudIp(), getStandardSourceFilePath()); + taskId = concat(mode.toString(), sourceAgentId, FilePathUtils.standardizedGSEFilePath(sourceFilePath)); } - return this.taskId; + return taskId; } - private String concat(String... strArgs) { + private static String concat(String... strArgs) { StringJoiner sj = new StringJoiner(":"); for (String strArg : strArgs) { if (StringUtils.isEmpty(strArg)) { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ConfirmStepEventHandler.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ConfirmStepEventHandler.java new file mode 100644 index 0000000000..21909481b6 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ConfirmStepEventHandler.java @@ -0,0 +1,181 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.listener; + +import com.tencent.bk.job.common.util.date.DateUtils; +import com.tencent.bk.job.execute.common.constants.RunStatusEnum; +import com.tencent.bk.job.execute.common.util.TaskCostCalculator; +import com.tencent.bk.job.execute.engine.consts.StepActionEnum; +import com.tencent.bk.job.execute.engine.listener.event.EventSource; +import com.tencent.bk.job.execute.engine.listener.event.JobEvent; +import com.tencent.bk.job.execute.engine.listener.event.StepEvent; +import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; +import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; +import com.tencent.bk.job.execute.model.StepInstanceDTO; +import com.tencent.bk.job.execute.model.TaskInstanceDTO; +import com.tencent.bk.job.execute.service.NotifyService; +import com.tencent.bk.job.execute.service.TaskInstanceService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * 人工确认步骤事件处理 + */ +@Component +@Slf4j +public class ConfirmStepEventHandler implements StepEventHandler { + + private final TaskInstanceService taskInstanceService; + private final TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher; + private final NotifyService notifyService; + + @Autowired + public ConfirmStepEventHandler(TaskInstanceService taskInstanceService, + TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, + NotifyService notifyService) { + this.taskInstanceService = taskInstanceService; + this.taskExecuteMQEventDispatcher = taskExecuteMQEventDispatcher; + this.notifyService = notifyService; + } + + @Override + public void handleEvent(StepEvent stepEvent, + StepInstanceDTO stepInstance) { + long stepInstanceId = stepEvent.getStepInstanceId(); + try { + StepActionEnum action = StepActionEnum.valueOf(stepEvent.getAction()); + if (action == null) { + log.error("Invalid step action: {}, ignore step event!", stepEvent.getAction()); + return; + } + + switch (action) { + case START: + executeConfirmStep(stepInstance); + break; + case CONFIRM_TERMINATE: + confirmStepTerminate(stepInstance); + break; + case CONFIRM_RESTART: + confirmStepRestart(stepInstance); + break; + case CONFIRM_CONTINUE: + confirmStepContinue(stepInstance); + break; + default: + log.error("Unhandled step event: {}", stepEvent); + } + } catch (Throwable e) { + String errorMsg = "Handling step event error,stepInstanceId:" + stepInstanceId; + log.error(errorMsg, e); + } + } + + private void confirmStepTerminate(StepInstanceDTO stepInstance) { + long stepInstanceId = stepInstance.getId(); + log.info("Confirm step terminate, stepInstanceId={}", stepInstanceId); + + TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(stepInstance.getTaskInstanceId()); + if (RunStatusEnum.WAITING_USER == stepInstance.getStatus()) { + Long endTime = DateUtils.currentTimeMillis(); + long taskTotalTime = TaskCostCalculator.calculate(taskInstance.getStartTime(), endTime, + taskInstance.getTotalTime()); + taskInstanceService.updateTaskExecutionInfo(taskInstance.getId(), RunStatusEnum.CONFIRM_TERMINATED, null, + null, endTime, taskTotalTime); + long stepTotalTime = TaskCostCalculator.calculate(stepInstance.getStartTime(), endTime, + stepInstance.getTotalTime()); + taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.CONFIRM_TERMINATED, null, + endTime, stepTotalTime); + } else { + log.warn("Unsupported step instance status for confirm step terminate action, stepInstanceId:{}, " + + "status:{}", stepInstanceId, stepInstance.getStatus()); + } + } + + private void confirmStepRestart(StepInstanceBaseDTO stepInstance) { + long stepInstanceId = stepInstance.getId(); + log.info("Confirm step restart, stepInstanceId={}", stepInstanceId); + + if (RunStatusEnum.CONFIRM_TERMINATED == stepInstance.getStatus()) { + executeConfirmStep(stepInstance); + } else { + log.warn("Unsupported step instance status for confirm-step-restart action, stepInstanceId:{}, status:{}" + , stepInstanceId, stepInstance.getStatus()); + } + } + + /** + * 人工确认步骤 + */ + private void executeConfirmStep(StepInstanceBaseDTO stepInstance) { + long stepInstanceId = stepInstance.getId(); + long taskInstanceId = stepInstance.getTaskInstanceId(); + + // 只有“未执行”和“确认终止”状态的，才可以重新执行人工确认步骤 + if (RunStatusEnum.BLANK == stepInstance.getStatus() + || RunStatusEnum.CONFIRM_TERMINATED == stepInstance.getStatus()) { + // 发送页面确认信息 + TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(taskInstanceId); + String stepOperator = stepInstance.getOperator(); + + if (StringUtils.isBlank(stepOperator)) { + log.info("The operator is empty, continue run step! stepInstanceId={}", stepInstanceId); + stepOperator = taskInstance.getOperator(); + stepInstance.setOperator(stepOperator); + } + taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.WAITING_USER, + System.currentTimeMillis(), null, null); + taskInstanceService.updateTaskStatus(taskInstanceId, RunStatusEnum.WAITING_USER.getValue()); + notifyService.asyncSendMQConfirmNotification(taskInstance, stepInstance); + } else { + log.warn("Unsupported step instance run status for executing confirm step, stepInstanceId={}, status={}", + stepInstanceId, stepInstance.getStatus()); + } + } + + private void confirmStepContinue(StepInstanceBaseDTO stepInstance) { + long taskInstanceId = stepInstance.getTaskInstanceId(); + long stepInstanceId = stepInstance.getId(); + log.info("Confirm step continue, stepInstanceId={}", stepInstanceId); + + RunStatusEnum stepStatus = stepInstance.getStatus(); + if (RunStatusEnum.WAITING_USER == stepStatus) { + taskInstanceService.updateTaskStatus(taskInstanceId, RunStatusEnum.RUNNING.getValue()); + long endTime = DateUtils.currentTimeMillis(); + long totalTime = TaskCostCalculator.calculate(stepInstance.getStartTime(), endTime, + stepInstance.getTotalTime()); + // 人工确认通过，该步骤状态标识为成功；终止成功的步骤保持状态不变 + taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.SUCCESS, null, endTime, + totalTime); + taskExecuteMQEventDispatcher.dispatchJobEvent( + JobEvent.refreshJob(taskInstanceId, EventSource.buildStepEventSource(stepInstanceId))); + } else { + log.warn("Unsupported step instance status for confirm-step-continue step action, stepInstanceId:{}, " + + "status:{}", stepInstanceId, stepInstance.getStatus()); + } + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java new file mode 100644 index 0000000000..cfbb83fb42 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java @@ -0,0 +1,822 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.listener; + +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.RollingModeEnum; +import com.tencent.bk.job.common.exception.NotImplementedException; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.common.util.date.DateUtils; +import com.tencent.bk.job.execute.common.constants.RunStatusEnum; +import com.tencent.bk.job.execute.common.util.TaskCostCalculator; +import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; +import com.tencent.bk.job.execute.engine.consts.StepActionEnum; +import com.tencent.bk.job.execute.engine.listener.event.EventSource; +import com.tencent.bk.job.execute.engine.listener.event.GseTaskEvent; +import com.tencent.bk.job.execute.engine.listener.event.JobEvent; +import com.tencent.bk.job.execute.engine.listener.event.StepEvent; +import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; +import com.tencent.bk.job.execute.engine.prepare.FilePrepareService; +import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.model.GseTaskDTO; +import com.tencent.bk.job.execute.model.RollingConfigDTO; +import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; +import com.tencent.bk.job.execute.model.StepInstanceDTO; +import com.tencent.bk.job.execute.model.StepInstanceRollingTaskDTO; +import com.tencent.bk.job.execute.model.TaskInstanceDTO; +import com.tencent.bk.job.execute.model.db.RollingHostsBatchDO; +import com.tencent.bk.job.execute.service.FileAgentTaskService; +import com.tencent.bk.job.execute.service.GseTaskService; +import com.tencent.bk.job.execute.service.RollingConfigService; +import com.tencent.bk.job.execute.service.ScriptAgentTaskService; +import com.tencent.bk.job.execute.service.StepInstanceRollingTaskService; +import com.tencent.bk.job.execute.service.StepInstanceService; +import com.tencent.bk.job.execute.service.TaskInstanceService; +import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; +import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +/** + * GSE 任务(脚本执行/文件分发)步骤事件处理 + */ +@Component +@Slf4j +public class GseStepEventHandler implements StepEventHandler { + + private final TaskInstanceService taskInstanceService; + private final StepInstanceService stepInstanceService; + private final TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher; + private final FilePrepareService filePrepareService; + private final GseTaskService gseTaskService; + private final RollingConfigService rollingConfigService; + private final StepInstanceRollingTaskService stepInstanceRollingTaskService; + private final ScriptAgentTaskService scriptAgentTaskService; + private final FileAgentTaskService fileAgentTaskService; + + @Autowired + public GseStepEventHandler(TaskInstanceService taskInstanceService, + StepInstanceService stepInstanceService, + TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, + FilePrepareService filePrepareService, + GseTaskService gseTaskService, + RollingConfigService rollingConfigService, + StepInstanceRollingTaskService stepInstanceRollingTaskService, + ScriptAgentTaskService scriptAgentTaskService, + FileAgentTaskService fileAgentTaskService) { + this.taskInstanceService = taskInstanceService; + this.stepInstanceService = stepInstanceService; + this.taskExecuteMQEventDispatcher = taskExecuteMQEventDispatcher; + this.filePrepareService = filePrepareService; + this.gseTaskService = gseTaskService; + this.rollingConfigService = rollingConfigService; + this.stepInstanceRollingTaskService = stepInstanceRollingTaskService; + this.scriptAgentTaskService = scriptAgentTaskService; + this.fileAgentTaskService = fileAgentTaskService; + } + + @Override + public void handleEvent(StepEvent stepEvent, + StepInstanceDTO stepInstance) { + long stepInstanceId = stepEvent.getStepInstanceId(); + try { + StepActionEnum action = StepActionEnum.valueOf(stepEvent.getAction()); + + switch (action) { + case START: + startStep(stepEvent, stepInstance); + break; + case STOP: + stopStep(stepInstance); + break; + case SKIP: + skipStep(stepInstance); + break; + case NEXT_STEP: + nextStep(stepInstance); + break; + case RETRY_ALL: + retryStepAll(stepInstance); + break; + case RETRY_FAIL: + retryStepFail(stepInstance); + break; + case IGNORE_ERROR: + ignoreError(stepInstance); + break; + case REFRESH: + refreshStep(stepEvent, stepInstance); + break; + case CONTINUE_FILE_PUSH: + continueGseFileStep(stepInstance); + break; + case CLEAR: + clearStep(stepInstance); + break; + default: + log.error("Unhandled step event: {}", stepEvent); + } + } catch (Throwable e) { + String errorMsg = "Handling step event error,stepInstanceId:" + stepInstanceId; + log.error(errorMsg, e); + } + } + + private void startStep(StepEvent stepEvent, StepInstanceDTO stepInstance) { + long stepInstanceId = stepInstance.getId(); + boolean isRollingStep = stepInstance.isRollingStep(); + if (isRollingStep) { + Integer batch = stepEvent.getBatch(); + if (batch == null) { + log.error("Empty batch for rolling step. Start step fail"); + return; + } + stepInstance.setBatch(batch); + log.info("Start rolling step, stepInstanceId={}, batch: {}", stepInstanceId, batch); + } else { + log.info("Start step, stepInstanceId={}", stepInstanceId); + } + + RunStatusEnum stepStatus = stepInstance.getStatus(); + // 只有当步骤状态为“未执行”、“滚动等待”, "等待用户"时可以启动步骤 + if (RunStatusEnum.BLANK == stepStatus + || RunStatusEnum.ROLLING_WAITING == stepStatus + || RunStatusEnum.WAITING_USER == stepStatus) { + + RollingConfigDTO rollingConfig = null; + if (isRollingStep) { + rollingConfig = rollingConfigService.getRollingConfig(stepInstance.getRollingConfigId()); + log.info("Rolling config: {}", rollingConfig); + // 更新滚动进度 + stepInstanceService.updateStepCurrentBatch(stepInstanceId, stepInstance.getBatch()); + // 初始化步骤滚动任务 + saveInitialStepInstanceRollingTask(stepInstance); + } + + Long gseTaskId = saveInitialGseTask(stepInstance); + saveGseAgentTasksForStartStep(gseTaskId, stepInstance, rollingConfig); + + taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.RUNNING, + stepInstance.getStartTime() == null ? DateUtils.currentTimeMillis() : null, null, null); + if (isRollingStep) { + stepInstanceRollingTaskService.updateRollingTask(stepInstanceId, stepInstance.getExecuteCount(), + stepInstance.getBatch(), RunStatusEnum.RUNNING, System.currentTimeMillis(), null, null); + } + + startGseTask(stepInstance, gseTaskId); + } else { + log.warn("Unsupported step instance run status for starting step, stepInstanceId={}, status={}", + stepInstanceId, stepStatus); + } + } + + /** + * 启动GSE任务 + * + * @param stepInstance 步骤实例 + * @param gseTaskId Gse任务ID + */ + private void startGseTask(StepInstanceDTO stepInstance, Long gseTaskId) { + if (stepInstance.isScriptStep()) { + taskExecuteMQEventDispatcher.dispatchGseTaskEvent(GseTaskEvent.startGseTask( + stepInstance.getId(), stepInstance.getExecuteCount(), stepInstance.getBatch(), gseTaskId, null)); + } else if (stepInstance.isFileStep()) { + if (filePrepareService.needToPrepareSourceFilesForGseTask(stepInstance)) { + filePrepareService.prepareFileForGseTask(stepInstance); + } else { + taskExecuteMQEventDispatcher.dispatchGseTaskEvent(GseTaskEvent.startGseTask( + stepInstance.getId(), stepInstance.getExecuteCount(), stepInstance.getBatch(), gseTaskId, + null)); + } + } + } + + /** + * 初始化的GSE任务 + * + * @param stepInstance 步骤实例 + * @return GSE 任务ID + */ + private Long saveInitialGseTask(StepInstanceDTO stepInstance) { + GseTaskDTO gseTask = new GseTaskDTO(stepInstance.getId(), stepInstance.getExecuteCount(), + stepInstance.getBatch()); + gseTask.setStatus(RunStatusEnum.WAITING_USER.getValue()); + + return gseTaskService.saveGseTask(gseTask); + } + + /** + * 启动步骤的时候保存 GSE Agent 任务 + * + * @param gseTaskId GSE任务ID + * @param stepInstance 步骤实例 + * @param rollingConfig 滚动配置 + */ + private void saveGseAgentTasksForStartStep(Long gseTaskId, + StepInstanceDTO stepInstance, + RollingConfigDTO rollingConfig) { + long stepInstanceId = stepInstance.getId(); + int executeCount = stepInstance.getExecuteCount(); + int batch = stepInstance.getBatch(); + + if (stepInstance.isRollingStep()) { + // 滚动步骤 + saveGseAgentTasksForStartRollingStep(gseTaskId, stepInstance, rollingConfig); + } else { + // 普通步骤，启动的时候需要初始化所有AgentTask + List agentTasks = new ArrayList<>( + buildGseAgentTasks(stepInstanceId, executeCount, executeCount, batch, + gseTaskId, stepInstance.getTargetServers().getIpList(), AgentTaskStatusEnum.WAITING)); + saveAgentTasks(stepInstance, agentTasks); + } + } + + /** + * 启动滚动执行步骤的时候保存 GSE Agent 任务 + * + * @param gseTaskId GSE任务ID + * @param stepInstance 步骤实例 + * @param rollingConfig 滚动配置 + */ + private void saveGseAgentTasksForStartRollingStep(Long gseTaskId, + StepInstanceDTO stepInstance, + RollingConfigDTO rollingConfig) { + long stepInstanceId = stepInstance.getId(); + int executeCount = stepInstance.getExecuteCount(); + int batch = stepInstance.getBatch(); + if (stepInstance.isFirstRollingBatch()) { + // 如果是第一批次的执行，需要提前初始化所有批次的agent任务（作业详情查询主机任务列表需要) + List agentTasks = new ArrayList<>(); + if (rollingConfig.isBatchRollingStep(stepInstanceId)) { + List serverBatchList = + rollingConfig.getConfigDetail().getHostsBatchList(); + serverBatchList.forEach(serverBatch -> { + agentTasks.addAll( + buildGseAgentTasks( + stepInstanceId, + executeCount, + serverBatch.getBatch() == 1 ? executeCount : null, + serverBatch.getBatch(), + serverBatch.getBatch() == 1 ? gseTaskId : 0, + serverBatch.getHosts(), + AgentTaskStatusEnum.WAITING) + ); + }); + saveAgentTasks(stepInstance, agentTasks); + } else { + // 暂时不支持，滚动执行二期需求 + log.warn("All rolling step is not supported!"); + throw new NotImplementedException("All rolling step is not supported", + ErrorCode.NOT_SUPPORT_FEATURE); + } + } else { + // 滚动执行步骤除了第一批次，后续的批次仅更新 AgentTask 的 actualExecuteCount、gse_task_id + if (stepInstance.isScriptStep()) { + scriptAgentTaskService.updateAgentTaskFields(stepInstanceId, executeCount, batch, executeCount, + gseTaskId); + } else if (stepInstance.isFileStep()) { + fileAgentTaskService.updateAgentTaskFields(stepInstanceId, executeCount, batch, executeCount, + gseTaskId); + } + } + } + + private List buildGseAgentTasks(long stepInstanceId, + int executeCount, + Integer actualExecuteCount, + int batch, + Long gseTaskId, + List hosts, + AgentTaskStatusEnum status) { + return hosts.stream() + .map(host -> buildGseAgentTask(stepInstanceId, executeCount, actualExecuteCount, + batch, gseTaskId, host, status)) + .collect(Collectors.toList()); + } + + protected AgentTaskDTO buildGseAgentTask(long stepInstanceId, + int executeCount, + Integer actualExecuteCount, + int batch, + Long gseTaskId, + HostDTO host, + AgentTaskStatusEnum status) { + AgentTaskDTO agentTask = new AgentTaskDTO(); + agentTask.setStepInstanceId(stepInstanceId); + agentTask.setExecuteCount(executeCount); + agentTask.setActualExecuteCount(actualExecuteCount); + agentTask.setBatch(batch); + agentTask.setGseTaskId(gseTaskId); + agentTask.setStatus(status); + agentTask.setFileTaskMode(FileTaskModeEnum.DOWNLOAD); + agentTask.setHostId(host.getHostId()); + if (StringUtils.isEmpty(host.getAgentId())) { + // 兼容老的Agent,没有AgentId,需要使用{bk_cloud_id:ip}的格式作为agentId传给GSE + agentTask.setAgentId(host.toCloudIp()); + } else { + agentTask.setAgentId(host.getAgentId()); + } + return agentTask; + } + + /** + * 保存初始化的步骤实例上的滚动任务 + * + * @param stepInstance 步骤实例 + */ + private void saveInitialStepInstanceRollingTask(StepInstanceDTO stepInstance) { + StepInstanceRollingTaskDTO stepInstanceRollingTask = new StepInstanceRollingTaskDTO(); + stepInstanceRollingTask.setStepInstanceId(stepInstance.getId()); + stepInstanceRollingTask.setBatch(stepInstance.getBatch()); + stepInstanceRollingTask.setExecuteCount(stepInstance.getExecuteCount()); + stepInstanceRollingTask.setStatus(RunStatusEnum.RUNNING); + stepInstanceRollingTask.setStartTime(System.currentTimeMillis()); + stepInstanceRollingTaskService.saveRollingTask(stepInstanceRollingTask); + } + + private void nextStep(StepInstanceDTO stepInstance) { + log.info("Next step, stepInstanceId={}", stepInstance.getId()); + + long taskInstanceId = stepInstance.getTaskInstanceId(); + long stepInstanceId = stepInstance.getId(); + RunStatusEnum stepStatus = stepInstance.getStatus(); + + if (RunStatusEnum.STOP_SUCCESS == stepStatus) { + taskInstanceService.updateTaskStatus(taskInstanceId, RunStatusEnum.RUNNING.getValue()); + long endTime = DateUtils.currentTimeMillis(); + long totalTime = TaskCostCalculator.calculate(stepInstance.getStartTime(), endTime, + stepInstance.getTotalTime()); + // 终止成功，进入下一步，该步骤设置为“跳过” + taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.SKIPPED, null, endTime, + totalTime); + taskExecuteMQEventDispatcher.dispatchJobEvent( + JobEvent.refreshJob(taskInstanceId, EventSource.buildStepEventSource(stepInstanceId))); + } else { + log.warn("Unsupported step instance status for next step action, stepInstanceId:{}, status:{}", + stepInstanceId, stepInstance.getStatus()); + } + } + + private void ignoreError(StepInstanceDTO stepInstance) { + log.info("Ignore step error, stepInstanceId={}", stepInstance.getId()); + + if (stepInstance.getStatus() != RunStatusEnum.FAIL) { + log.warn("Current step status does not support ignore error operation! stepInstanceId:{}, status:{}", + stepInstance.getId(), stepInstance.getStatus()); + return; + } + + long stepInstanceId = stepInstance.getId(); + boolean isRollingStep = stepInstance.isRollingStep(); + if (isRollingStep) { + log.info("Retry-fail for rolling step, stepInstanceId={}, batch: {}", stepInstanceId, + stepInstance.getBatch()); + } else { + log.info("Retry-fail for step, stepInstanceId={}", stepInstanceId); + } + + taskInstanceService.updateStepStatus(stepInstance.getId(), RunStatusEnum.IGNORE_ERROR.getValue()); + taskInstanceService.resetTaskExecuteInfoForRetry(stepInstance.getTaskInstanceId()); + if (isRollingStep) { + StepInstanceRollingTaskDTO stepInstanceRollingTask = + stepInstanceRollingTaskService.queryRollingTask(stepInstanceId, stepInstance.getExecuteCount(), + stepInstance.getBatch()); + if (stepInstanceRollingTask != null) { + finishRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), + RunStatusEnum.IGNORE_ERROR); + } + } + + taskExecuteMQEventDispatcher.dispatchJobEvent( + JobEvent.refreshJob(stepInstance.getTaskInstanceId(), + EventSource.buildStepEventSource(stepInstance.getId()))); + } + + + private void skipStep(StepInstanceDTO stepInstance) { + RunStatusEnum stepStatus = stepInstance.getStatus(); + long stepInstanceId = stepInstance.getId(); + long taskInstanceId = stepInstance.getTaskInstanceId(); + + TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(stepInstance.getTaskInstanceId()); + if (!(taskInstance.getCurrentStepInstanceId() == stepInstanceId)) { + log.warn("Only current running step is support for skipping, stepInstanceId={}", stepInstanceId); + return; + } + + log.info("Skip step, stepInstanceId={}", stepInstanceId); + + // 只有当步骤状态为'终止中'时可以跳过步骤 + if (RunStatusEnum.STOPPING == stepStatus) { + long now = DateUtils.currentTimeMillis(); + taskInstanceService.updateStepStartTimeIfNull(stepInstanceId, now); + taskInstanceService.updateStepStatus(stepInstanceId, RunStatusEnum.SKIPPED.getValue()); + taskInstanceService.updateStepEndTime(stepInstanceId, now); + + taskInstanceService.updateTaskStatus(taskInstanceId, RunStatusEnum.RUNNING.getValue()); + taskExecuteMQEventDispatcher.dispatchJobEvent( + JobEvent.refreshJob(taskInstanceId, EventSource.buildStepEventSource(stepInstanceId))); + } else { + log.warn("Unsupported step instance run status for skipping step, stepInstanceId={}, status={}", + stepInstanceId, stepStatus); + } + } + + private void stopStep(StepInstanceDTO stepInstance) { + long stepInstanceId = stepInstance.getId(); + log.info("Force stop step, stepInstanceId={}", stepInstanceId); + + RunStatusEnum stepStatus = stepInstance.getStatus(); + if (stepStatus == RunStatusEnum.WAITING_USER) { + log.info("Step status is WAITING_USER, set step status stop_success directly!"); + // 等待用户的步骤可以直接结束 + long endTime = DateUtils.currentTimeMillis(); + taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.STOP_SUCCESS, + null, endTime, TaskCostCalculator.calculate(stepInstance.getStartTime(), endTime, + stepInstance.getTotalTime())); + taskExecuteMQEventDispatcher.dispatchJobEvent(JobEvent.refreshJob(stepInstance.getTaskInstanceId(), + EventSource.buildStepEventSource(stepInstanceId))); + } else if (stepStatus == RunStatusEnum.RUNNING) { + // 正在运行中的任务无法立即结束，需要等待任务调度引擎检测到停止状态;这里只需要处理设置步骤状态即可 + taskInstanceService.updateStepStatus(stepInstanceId, RunStatusEnum.STOPPING.getValue()); + } + } + + /** + * 第三方文件源文件拉取完成后继续GSE文件分发 + * + * @param stepInstance 步骤实例 + */ + private void continueGseFileStep(StepInstanceDTO stepInstance) { + log.info("Continue file push step, stepInstanceId={}", stepInstance.getId()); + + GseTaskDTO gseTask = + gseTaskService.getGseTask(stepInstance.getId(), stepInstance.getExecuteCount(), stepInstance.getBatch()); + taskExecuteMQEventDispatcher.dispatchGseTaskEvent(GseTaskEvent.startGseTask( + gseTask.getStepInstanceId(), gseTask.getExecuteCount(), gseTask.getBatch(), gseTask.getId(), null)); + } + + /** + * 重新执行步骤失败的任务 + */ + private void retryStepFail(StepInstanceDTO stepInstance) { + long stepInstanceId = stepInstance.getId(); + boolean isRollingStep = stepInstance.isRollingStep(); + if (isRollingStep) { + log.info("Retry-fail for rolling step, stepInstanceId={}, batch: {}", stepInstanceId, + stepInstance.getBatch()); + } else { + log.info("Retry-fail for step, stepInstanceId={}", stepInstanceId); + } + + RunStatusEnum stepStatus = stepInstance.getStatus(); + if (isStepSupportRetry(stepStatus)) { + + resetExecutionInfoForRetry(stepInstance); + + if (isRollingStep) { + // 初始化步骤滚动任务 + saveInitialStepInstanceRollingTask(stepInstance); + } + + Long gseTaskId = saveInitialGseTask(stepInstance); + saveAgentTasksForRetryFail(stepInstance, stepInstance.getExecuteCount(), stepInstance.getBatch(), + gseTaskId); + + startGseTask(stepInstance, gseTaskId); + } else { + log.warn("Unsupported step instance run status for retry step, stepInstanceId={}, status={}", + stepInstanceId, stepStatus); + } + } + + private boolean isStepSupportRetry(RunStatusEnum stepStatus) { + return RunStatusEnum.FAIL == stepStatus || RunStatusEnum.ABNORMAL_STATE == stepStatus + || RunStatusEnum.STOP_SUCCESS == stepStatus; + } + + private void saveAgentTasksForRetryFail(StepInstanceBaseDTO stepInstance, int executeCount, Integer batch, + Long gseTaskId) { + List retryAgentTasks = listTargetAgentTasks(stepInstance, executeCount - 1); + + for (AgentTaskDTO retryAgentTask : retryAgentTasks) { + retryAgentTask.setExecuteCount(executeCount); + if (batch != null && retryAgentTask.getBatch() != batch) { + continue; + } + // 只有失败的目标主机才需要参与重试 + if (!AgentTaskStatusEnum.isSuccess(retryAgentTask.getStatus())) { + retryAgentTask.setActualExecuteCount(executeCount); + retryAgentTask.resetTaskInitialStatus(); + retryAgentTask.setGseTaskId(gseTaskId); + } + } + + saveAgentTasks(stepInstance, retryAgentTasks); + } + + + private void saveAgentTasksForRetryAll(StepInstanceBaseDTO stepInstance, int executeCount, Integer batch, + Long gseTaskId) { + List retryAgentTasks = listTargetAgentTasks(stepInstance, executeCount - 1); + + for (AgentTaskDTO retryAgentTask : retryAgentTasks) { + retryAgentTask.setExecuteCount(executeCount); + if (batch != null && retryAgentTask.getBatch() != batch) { + continue; + } + retryAgentTask.setActualExecuteCount(executeCount); + retryAgentTask.resetTaskInitialStatus(); + retryAgentTask.setGseTaskId(gseTaskId); + } + + saveAgentTasks(stepInstance, retryAgentTasks); + } + + private List listTargetAgentTasks(StepInstanceBaseDTO stepInstance, int executeCount) { + List agentTasks = Collections.emptyList(); + if (stepInstance.isScriptStep()) { + agentTasks = scriptAgentTaskService.listAgentTasks(stepInstance.getId(), executeCount, null); + } else if (stepInstance.isFileStep()) { + agentTasks = fileAgentTaskService.listAgentTasks(stepInstance.getId(), executeCount, null, + FileTaskModeEnum.DOWNLOAD); + } + return agentTasks; + } + + private void saveAgentTasks(StepInstanceBaseDTO stepInstance, List agentTasks) { + if (CollectionUtils.isNotEmpty(agentTasks)) { + if (stepInstance.isScriptStep()) { + scriptAgentTaskService.batchSaveAgentTasks(agentTasks); + } else if (stepInstance.isFileStep()) { + fileAgentTaskService.batchSaveAgentTasks(agentTasks); + } + } + } + + /** + * 从头执行步骤 + */ + private void retryStepAll(StepInstanceDTO stepInstance) { + + long stepInstanceId = stepInstance.getId(); + boolean isRollingStep = stepInstance.isRollingStep(); + if (isRollingStep) { + log.info("Retry-all for rolling step, stepInstanceId={}, batch: {}", stepInstanceId, + stepInstance.getBatch()); + } else { + log.info("Retry-all for step, stepInstanceId={}", stepInstanceId); + } + + RunStatusEnum stepStatus = stepInstance.getStatus(); + if (isStepSupportRetry(stepStatus)) { + + resetExecutionInfoForRetry(stepInstance); + + if (isRollingStep) { + // 初始化步骤滚动任务 + saveInitialStepInstanceRollingTask(stepInstance); + } + + Long gseTaskId = saveInitialGseTask(stepInstance); + saveAgentTasksForRetryAll(stepInstance, stepInstance.getExecuteCount(), stepInstance.getBatch(), + gseTaskId); + + startGseTask(stepInstance, gseTaskId); + } else { + log.warn("Unsupported step instance run status for retry step, stepInstanceId={}, status={}", + stepInstanceId, stepStatus); + } + } + + /** + * 清理执行完的步骤 + */ + private void clearStep(StepInstanceDTO stepInstance) { + log.info("Clear step, stepInstanceId={}", stepInstance.getId()); + + int executeType = stepInstance.getExecuteType(); + // 当前仅有文件分发类步骤需要清理中间文件 + if (TaskStepTypeEnum.FILE.getValue() == executeType) { + filePrepareService.clearPreparedTmpFile(stepInstance.getId()); + } + } + + /** + * 重置作业、步骤执行状态，包括结束时间、任务状态、任务耗时 + * + * @param stepInstance 步骤实例 + */ + private void resetExecutionInfoForRetry(StepInstanceDTO stepInstance) { + long stepInstanceId = stepInstance.getId(); + long taskInstanceId = stepInstance.getTaskInstanceId(); + + taskInstanceService.resetStepExecuteInfoForRetry(stepInstanceId); + taskInstanceService.resetTaskExecuteInfoForRetry(taskInstanceId); + } + + private void refreshStep(StepEvent stepEvent, StepInstanceDTO stepInstance) { + + long stepInstanceId = stepInstance.getId(); + EventSource eventSource = stepEvent.getSource(); + + GseTaskDTO gseTask = gseTaskService.getGseTask(eventSource.getGseTaskId()); + + RunStatusEnum gseTaskStatus = RunStatusEnum.valueOf(gseTask.getStatus()); + log.info("Refresh step according to gse task status, stepInstanceId: {}, gseTaskStatus: {}", + stepInstance.getId(), gseTaskStatus.name()); + + switch (gseTaskStatus) { + case SUCCESS: + onSuccess(stepInstance); + break; + case FAIL: + onFail(stepInstance); + break; + case STOP_SUCCESS: + onStopSuccess(stepInstance); + break; + case ABNORMAL_STATE: + onAbnormalState(stepInstance); + break; + default: + log.error("Refresh step fail because of unexpected gse task status. stepInstanceId: {}, " + + "gseTaskStatus: {}", + stepInstanceId, gseTaskStatus.getValue()); + setAbnormalStatusForStep(stepInstance); + break; + } + + // 更新作业状态 + taskExecuteMQEventDispatcher.dispatchJobEvent( + JobEvent.refreshJob(stepInstance.getTaskInstanceId(), EventSource.buildStepEventSource(stepInstanceId))); + } + + private void onSuccess(StepInstanceDTO stepInstance) { + long stepInstanceId = stepInstance.getId(); + long endTime = System.currentTimeMillis(); + long startTime = stepInstance.getStartTime(); + long totalTime = endTime - startTime; + + if (stepInstance.isRollingStep()) { + RollingConfigDTO rollingConfig = + rollingConfigService.getRollingConfig(stepInstance.getRollingConfigId()); + finishRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), + RunStatusEnum.SUCCESS); + int totalBatch = rollingConfig.getConfigDetail().getTotalBatch(); + boolean isLastBatch = totalBatch == stepInstance.getBatch(); + if (isLastBatch) { + taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.SUCCESS, + startTime, endTime, totalTime); + // 步骤执行成功后清理产生的临时文件 + clearStep(stepInstance); + } else { + taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.ROLLING_WAITING, + startTime, endTime, totalTime); + } + } else { + taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.SUCCESS, + startTime, endTime, totalTime); + // 步骤执行成功后清理产生的临时文件 + clearStep(stepInstance); + } + } + + private void onFail(StepInstanceDTO stepInstance) { + long stepInstanceId = stepInstance.getId(); + long endTime = System.currentTimeMillis(); + long startTime = stepInstance.getStartTime(); + long totalTime = endTime - startTime; + if (stepInstance.isIgnoreError()) { + log.info("Ignore error for step: {}", stepInstanceId); + taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.IGNORE_ERROR, + startTime, endTime, totalTime); + if (stepInstance.isRollingStep()) { + finishRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), + RunStatusEnum.IGNORE_ERROR); + } + return; + } + + if (stepInstance.isRollingStep()) { + RollingConfigDTO rollingConfig = + rollingConfigService.getRollingConfig(stepInstance.getRollingConfigId()); + RollingModeEnum rollingMode = RollingModeEnum.valOf(rollingConfig.getConfigDetail().getMode()); + switch (rollingMode) { + case IGNORE_ERROR: + log.info("Ignore error for rolling step, rollingMode: {}", rollingMode); + finishRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), + RunStatusEnum.IGNORE_ERROR); + taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.IGNORE_ERROR, + startTime, endTime, totalTime); + break; + case PAUSE_IF_FAIL: + case MANUAL: + finishRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), + RunStatusEnum.FAIL); + taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.FAIL, + startTime, endTime, totalTime); + break; + default: + log.error("Invalid rolling mode: {}", rollingMode); + } + } else { + taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.FAIL, + startTime, endTime, totalTime); + } + } + + private void onStopSuccess(StepInstanceDTO stepInstance) { + long stepInstanceId = stepInstance.getId(); + long endTime = System.currentTimeMillis(); + long startTime = stepInstance.getStartTime(); + long totalTime = endTime - startTime; + RunStatusEnum stepStatus = stepInstance.getStatus(); + + if (stepStatus == RunStatusEnum.STOPPING || stepStatus == RunStatusEnum.RUNNING) { + taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.STOP_SUCCESS, + startTime, endTime, totalTime); + if (stepInstance.isRollingStep()) { + finishRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), + RunStatusEnum.STOP_SUCCESS); + } + } else { + log.error("Refresh step fail, stepInstanceId: {}, stepStatus: {}, gseTaskStatus: {}", + stepInstanceId, stepStatus, RunStatusEnum.STOP_SUCCESS.getValue()); + } + } + + private void onAbnormalState(StepInstanceDTO stepInstance) { + setAbnormalStatusForStep(stepInstance); + if (stepInstance.isRollingStep()) { + finishRollingTask(stepInstance.getId(), stepInstance.getExecuteCount(), stepInstance.getBatch(), + RunStatusEnum.ABNORMAL_STATE); + } + } + + private void setAbnormalStatusForStep(StepInstanceDTO stepInstance) { + long endTime = System.currentTimeMillis(); + if (!RunStatusEnum.isFinishedStatus(stepInstance.getStatus())) { + long totalTime = TaskCostCalculator.calculate(stepInstance.getStartTime(), endTime, + stepInstance.getTotalTime()); + taskInstanceService.updateStepExecutionInfo( + stepInstance.getId(), + RunStatusEnum.ABNORMAL_STATE, + null, + endTime, + totalTime + ); + } else { + log.info( + "StepInstance {} already enter a final state:{}", + stepInstance.getId(), + stepInstance.getStatus() + ); + } + } + + private void finishRollingTask(long stepInstanceId, int executeCount, int batch, RunStatusEnum status) { + StepInstanceRollingTaskDTO rollingTask = + stepInstanceRollingTaskService.queryRollingTask(stepInstanceId, executeCount, batch); + if (rollingTask == null) { + log.error("Rolling task is not exist, skip update! stepInstanceId: {}, executeCount: {}, batch: {}", + stepInstanceId, executeCount, batch); + return; + } + long now = System.currentTimeMillis(); + long startTime = rollingTask.getStartTime() != null ? rollingTask.getStartTime() : now; + + stepInstanceRollingTaskService.updateRollingTask(stepInstanceId, executeCount, + batch, status, startTime, now, now - startTime); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseTaskListener.java similarity index 56% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepListener.java rename to src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseTaskListener.java index 25af70f996..150c77210c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepListener.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseTaskListener.java @@ -26,12 +26,13 @@ import com.tencent.bk.job.execute.common.exception.MessageHandleException; import com.tencent.bk.job.execute.common.exception.MessageHandlerUnavailableException; -import com.tencent.bk.job.execute.engine.GseTaskManager; -import com.tencent.bk.job.execute.engine.consts.GseStepActionEnum; -import com.tencent.bk.job.execute.engine.exception.ExceptionStatusManager; +import com.tencent.bk.job.execute.engine.consts.GseTaskActionEnum; +import com.tencent.bk.job.execute.engine.executor.GseTaskManager; +import com.tencent.bk.job.execute.engine.listener.event.GseTaskEvent; import com.tencent.bk.job.execute.engine.message.GseTaskProcessor; -import com.tencent.bk.job.execute.engine.model.StepControlMessage; +import com.tencent.bk.job.execute.model.GseTaskDTO; import com.tencent.bk.job.execute.monitor.metrics.GseTasksExceptionCounter; +import com.tencent.bk.job.execute.service.GseTaskService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cloud.stream.annotation.EnableBinding; @@ -40,64 +41,56 @@ import org.springframework.stereotype.Component; /** - * 执行引擎流程处理-GSE任务 + * 执行引擎事件处理-GSE任务 */ @Component @EnableBinding({GseTaskProcessor.class}) @Slf4j -public class GseStepListener { +public class GseTaskListener { private final GseTaskManager gseTaskManager; - private final ExceptionStatusManager exceptionStatusManager; - /** - * 任务执行异常Counter - */ + private final GseTaskService gseTaskService; private final GseTasksExceptionCounter gseTasksExceptionCounter; @Autowired - public GseStepListener( - GseTaskManager gseTaskManager, - ExceptionStatusManager exceptionStatusManager, - GseTasksExceptionCounter gseTasksExceptionCounter - ) { + public GseTaskListener(GseTaskManager gseTaskManager, + GseTaskService gseTaskService, + GseTasksExceptionCounter gseTasksExceptionCounter) { this.gseTaskManager = gseTaskManager; - this.exceptionStatusManager = exceptionStatusManager; + this.gseTaskService = gseTaskService; this.gseTasksExceptionCounter = gseTasksExceptionCounter; } + /** + * 处理GSE任务相关的事件 + * + * @param gseTaskEvent GSE任务事件 + */ @StreamListener(GseTaskProcessor.INPUT) - public void handleMessage(@Payload StepControlMessage gseStepControlMessage) { - log.info("Receive gse step control message, stepInstanceId={}, action={}, requestId={}, msgSendTime={}", - gseStepControlMessage.getStepInstanceId(), - gseStepControlMessage.getAction(), gseStepControlMessage.getRequestId(), gseStepControlMessage.getTime()); - long stepInstanceId = gseStepControlMessage.getStepInstanceId(); - String requestId = gseStepControlMessage.getRequestId(); + public void handleEvent(@Payload GseTaskEvent gseTaskEvent) { + log.info("Handel gse task event: {}", gseTaskEvent); + GseTaskDTO gseTask = gseTaskService.getGseTask(gseTaskEvent.getGseTaskId()); + String requestId = gseTaskEvent.getRequestId(); try { - int action = gseStepControlMessage.getAction(); - if (GseStepActionEnum.START.getValue() == action) { - gseTaskManager.startStep(stepInstanceId, gseStepControlMessage.getRequestId()); - } else if (GseStepActionEnum.STOP.getValue() == action) { - gseTaskManager.stopStep(stepInstanceId, requestId); - } else if (GseStepActionEnum.RETRY_FAIL.getValue() == action) { - gseTaskManager.retryFail(stepInstanceId, requestId); - } else if (GseStepActionEnum.RETRY_ALL.getValue() == action) { - gseTaskManager.retryAll(stepInstanceId, requestId); + int action = gseTaskEvent.getAction(); + if (GseTaskActionEnum.START.getValue() == action) { + gseTaskManager.startTask(gseTask, requestId); + } else if (GseTaskActionEnum.STOP.getValue() == action) { + gseTaskManager.stopTask(gseTask); } else { - log.error("Error gse step control action:{}", action); + log.error("Error gse task action:{}", action); } } catch (Throwable e) { - String errorMsg = "Handling gse step control message error,stepInstanceId:" + stepInstanceId; + String errorMsg = "Handling gse task event error, event:" + gseTaskEvent; log.error(errorMsg, e); - handleException(stepInstanceId, e); + handleException(e); } } - private void handleException(long stepInstanceId, Throwable e) throws MessageHandleException { + private void handleException(Throwable e) throws MessageHandleException { // 服务关闭，消息被拒绝，重新入队列 if (e instanceof MessageHandlerUnavailableException) { throw (MessageHandlerUnavailableException) e; } gseTasksExceptionCounter.increment(); - // 任务状态应当置为异常状态 - exceptionStatusManager.setAbnormalStatusForStep(stepInstanceId); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/JobListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/JobListener.java new file mode 100644 index 0000000000..493bf1e8aa --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/JobListener.java @@ -0,0 +1,373 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.listener; + +import com.google.common.collect.Lists; +import com.tencent.bk.job.common.constant.RollingModeEnum; +import com.tencent.bk.job.common.util.date.DateUtils; +import com.tencent.bk.job.execute.common.constants.RunStatusEnum; +import com.tencent.bk.job.execute.common.util.TaskCostCalculator; +import com.tencent.bk.job.execute.engine.consts.JobActionEnum; +import com.tencent.bk.job.execute.engine.listener.event.JobEvent; +import com.tencent.bk.job.execute.engine.listener.event.StepEvent; +import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; +import com.tencent.bk.job.execute.engine.message.TaskProcessor; +import com.tencent.bk.job.execute.engine.model.JobCallbackDTO; +import com.tencent.bk.job.execute.model.RollingConfigDTO; +import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; +import com.tencent.bk.job.execute.model.TaskInstanceDTO; +import com.tencent.bk.job.execute.model.db.RollingConfigDetailDO; +import com.tencent.bk.job.execute.service.NotifyService; +import com.tencent.bk.job.execute.service.RollingConfigService; +import com.tencent.bk.job.execute.service.StepInstanceService; +import com.tencent.bk.job.execute.service.TaskInstanceService; +import com.tencent.bk.job.execute.statistics.StatisticsService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cloud.stream.annotation.EnableBinding; +import org.springframework.cloud.stream.annotation.StreamListener; +import org.springframework.stereotype.Component; + +import java.util.Collection; +import java.util.List; + +/** + * 执行引擎事件处理-作业 + */ +@Component +@EnableBinding({TaskProcessor.class}) +@Slf4j +public class JobListener { + + private final TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher; + private final StatisticsService statisticsService; + private final TaskInstanceService taskInstanceService; + private final StepInstanceService stepInstanceService; + private final RollingConfigService rollingConfigService; + private final NotifyService notifyService; + + @Autowired + public JobListener(TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, + StatisticsService statisticsService, + TaskInstanceService taskInstanceService, + StepInstanceService stepInstanceService, + RollingConfigService rollingConfigService, + NotifyService notifyService) { + this.taskExecuteMQEventDispatcher = taskExecuteMQEventDispatcher; + this.statisticsService = statisticsService; + this.taskInstanceService = taskInstanceService; + this.stepInstanceService = stepInstanceService; + this.rollingConfigService = rollingConfigService; + this.notifyService = notifyService; + } + + + /** + * 处理作业执行相关的事件 + * + * @param jobEvent 作业执行相关的事件 + */ + @StreamListener(TaskProcessor.INPUT) + public void handleEvent(JobEvent jobEvent) { + log.info("Handle job event, event: {}", jobEvent); + long jobInstanceId = jobEvent.getJobInstanceId(); + JobActionEnum action = JobActionEnum.valueOf(jobEvent.getAction()); + try { + TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(jobInstanceId); + switch (action) { + case START: + startJob(taskInstance); + break; + case STOP: + stopJob(taskInstance); + break; + case RESTART: + restartJob(taskInstance); + break; + case REFRESH: + refreshJob(taskInstance); + break; + default: + log.error("Invalid job action: {}", action); + } + } catch (Throwable e) { + String errorMsg = "Handle job event error, jobInstanceId=" + jobInstanceId; + log.error(errorMsg, e); + } + } + + /** + * 启动作业 + * + * @param taskInstance 作业实例 + */ + private void startJob(TaskInstanceDTO taskInstance) { + long jobInstanceId = taskInstance.getId(); + // 首先验证作业的状态，只有状态为“未执行”的作业可以启动 + if (RunStatusEnum.BLANK == taskInstance.getStatus()) { + StepInstanceBaseDTO stepInstance = taskInstanceService.getFirstStepInstance(jobInstanceId); + taskInstanceService.updateTaskExecutionInfo(jobInstanceId, RunStatusEnum.RUNNING, stepInstance.getId(), + DateUtils.currentTimeMillis(), null, null); + startStep(stepInstance); + + // 触发任务开始统计分析 + statisticsService.updateStartJobStatistics(taskInstance); + } else { + log.error("Unsupported job instance run status for starting job, jobInstanceId={}, status={}", + jobInstanceId, taskInstance.getStatus()); + } + } + + /** + * 强制终止作业 + * + * @param taskInstance 作业实例 + */ + private void stopJob(TaskInstanceDTO taskInstance) { + long jobInstanceId = taskInstance.getId(); + RunStatusEnum taskStatus = taskInstance.getStatus(); + + if (RunStatusEnum.RUNNING == taskStatus || RunStatusEnum.WAITING_USER == taskStatus) { + taskInstanceService.updateTaskStatus(jobInstanceId, RunStatusEnum.STOPPING.getValue()); + long currentStepInstanceId = taskInstance.getCurrentStepInstanceId(); + taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.stopStep(currentStepInstanceId)); + } else { + log.warn("Unsupported job instance run status for stop task, jobInstanceId={}, status={}", + jobInstanceId, taskInstance.getStatus()); + } + } + + /** + * 重头执行作业 + * + * @param taskInstance 作业实例 + */ + private void restartJob(TaskInstanceDTO taskInstance) { + long jobInstanceId = taskInstance.getId(); + RunStatusEnum taskStatus = taskInstance.getStatus(); + // 验证作业状态，只有“执行失败”和“等待用户”的作业可以重头执行 + if (RunStatusEnum.WAITING_USER == taskStatus || RunStatusEnum.FAIL == taskStatus) { + + // 重置作业状态 + taskInstanceService.resetTaskStatus(jobInstanceId); + taskInstanceService.addStepInstanceExecuteCount(jobInstanceId); + + // 重置作业下步骤的状态、开始时间和结束时间等。 + List stepInstanceIdList = taskInstanceService.getTaskStepIdList(jobInstanceId); + for (long stepInstanceId : stepInstanceIdList) { + taskInstanceService.updateStepStatus(stepInstanceId, RunStatusEnum.BLANK.getValue()); + taskInstanceService.resetStepStatus(stepInstanceId); + } + + taskExecuteMQEventDispatcher.dispatchJobEvent(JobEvent.startJob(jobInstanceId)); + } else { + log.warn("Unsupported job instance run status for restart task, jobInstanceId={}, status={}", + jobInstanceId, taskInstance.getStatus()); + } + } + + /** + * 作业状态流转 + * + * @param taskInstance 作业实例 + */ + private void refreshJob(TaskInstanceDTO taskInstance) { + long jobInstanceId = taskInstance.getId(); + RunStatusEnum taskStatus = taskInstance.getStatus(); + + long currentStepInstanceId = taskInstance.getCurrentStepInstanceId(); + StepInstanceBaseDTO currentStepInstance = taskInstanceService.getBaseStepInstance(currentStepInstanceId); + RunStatusEnum stepStatus = currentStepInstance.getStatus(); + + // 验证作业状态，只有'正在执行'、'强制终止中'的作业可以刷新状态进入下一步或者结束 + if (RunStatusEnum.STOPPING == taskStatus) { + // 非正在执行的步骤可以直接终止 + if (RunStatusEnum.RUNNING != stepStatus) { + finishJob(taskInstance, currentStepInstance, RunStatusEnum.STOP_SUCCESS); + } else { + log.error("Unsupported job instance run status for refresh task, jobInstanceId={}, taskStatus={}, " + + "stepStatus: {}", jobInstanceId, taskStatus, stepStatus); + } + } else if (RunStatusEnum.RUNNING == taskStatus) { + // 步骤状态为成功、跳过、设为忽略错误、滚动等待，可以进入下一步 + if (RunStatusEnum.SUCCESS == stepStatus + || RunStatusEnum.SKIPPED == stepStatus + || RunStatusEnum.IGNORE_ERROR == stepStatus + || RunStatusEnum.ROLLING_WAITING == stepStatus) { + nextStep(taskInstance, currentStepInstance); + } else if (RunStatusEnum.FAIL == stepStatus || RunStatusEnum.ABNORMAL_STATE == stepStatus) { + // 步骤失败，任务结束 + finishJob(taskInstance, currentStepInstance, stepStatus); + } else { + log.warn("Unsupported job instance run status for refresh task, jobInstanceId={}, status={}", + jobInstanceId, taskInstance.getStatus()); + } + } else { + log.warn("Unsupported job instance run status for refresh task, jobInstanceId={}, status={}", + jobInstanceId, taskInstance.getStatus()); + } + } + + private void nextStep(TaskInstanceDTO taskInstance, StepInstanceBaseDTO currentStepInstance) { + if (currentStepInstance.isRollingStep()) { + RollingConfigDTO taskInstanceRollingConfig = + rollingConfigService.getRollingConfig(currentStepInstance.getRollingConfigId()); + RollingConfigDetailDO rollingConfig = taskInstanceRollingConfig.getConfigDetail(); + StepInstanceBaseDTO nextStepInstance = getNextStepInstance(taskInstance, currentStepInstance, + rollingConfig); + if (nextStepInstance == null) { + finishJob(taskInstance, currentStepInstance, RunStatusEnum.SUCCESS); + } else { + if (rollingConfig.getMode().equals(RollingModeEnum.MANUAL.getValue()) + && rollingConfig.isFirstRollingStep(nextStepInstance.getId())) { + log.info("Manual mode for rolling step[{}], pause and wait for user confirmation", + nextStepInstance.getId()); + taskInstanceService.updateStepStatus(nextStepInstance.getId(), + RunStatusEnum.WAITING_USER.getValue()); + taskInstanceService.updateTaskStatus(taskInstance.getId(), RunStatusEnum.WAITING_USER.getValue()); + } else { + // 执行下一步骤 + startStep(nextStepInstance); + } + } + } else { + StepInstanceBaseDTO nextStepInstance = getNextStepInstance(taskInstance, currentStepInstance, null); + if (nextStepInstance == null) { + finishJob(taskInstance, currentStepInstance, RunStatusEnum.SUCCESS); + } else { + // 执行下一步骤 + startStep(nextStepInstance); + } + } + + } + + private void finishJob(TaskInstanceDTO taskInstance, + StepInstanceBaseDTO stepInstance, + RunStatusEnum jobStatus) { + long jobInstanceId = taskInstance.getId(); + long stepInstanceId = stepInstance.getId(); + Long endTime = DateUtils.currentTimeMillis(); + long totalTime = TaskCostCalculator.calculate(taskInstance.getStartTime(), endTime, + taskInstance.getTotalTime()); + taskInstance.setEndTime(endTime); + taskInstance.setTotalTime(totalTime); + taskInstance.setStatus(jobStatus); + taskInstanceService.updateTaskExecutionInfo(jobInstanceId, jobStatus, null, null, endTime, totalTime); + + // 作业执行结果消息通知 + if (RunStatusEnum.SUCCESS == jobStatus || RunStatusEnum.IGNORE_ERROR == jobStatus) { + notifyService.asyncSendMQSuccessTaskNotification(taskInstance, stepInstance); + } else { + notifyService.asyncSendMQFailTaskNotification(taskInstance, stepInstance); + } + + // 触发作业结束统计分析 + statisticsService.updateEndJobStatistics(taskInstance); + + // 作业执行完成回调 + callback(taskInstance, jobInstanceId, jobStatus.getValue(), stepInstanceId, stepInstance.getStatus()); + } + + /** + * 获取下一个执行的步骤实例 + * + * @param taskInstance 作业实例 + * @param currentStepInstance 当前步骤实例 + * @param rollingConfig 滚动配置，仅当当前步骤为滚动步骤时才需要传入 + * @return 步骤实例;如果当前步骤已经是最后一个步骤，那么返回null + */ + private StepInstanceBaseDTO getNextStepInstance(TaskInstanceDTO taskInstance, + StepInstanceBaseDTO currentStepInstance, + RollingConfigDetailDO rollingConfig) { + StepInstanceBaseDTO nextStepInstance = null; + if (currentStepInstance.isRollingStep()) { + int currentBatch = currentStepInstance.getBatch(); + List includeStepInstanceIdList = rollingConfig.getIncludeStepInstanceIdList(); + boolean isLastRollingStep = rollingConfig.isLastRollingStep(currentStepInstance.getId()); + boolean isLastBatch = rollingConfig.getTotalBatch() == currentBatch; + + // 最后一个滚动步骤和滚动批次，那么该滚动任务执行结束，进入下一个步骤 + if (isLastRollingStep && isLastBatch) { + nextStepInstance = stepInstanceService.getNextStepInstance(taskInstance.getId(), + currentStepInstance.getStepOrder()); + } else { + Long nextRollingStepInstanceId; + if (isLastRollingStep) { + nextRollingStepInstanceId = includeStepInstanceIdList.get(0); + } else { + nextRollingStepInstanceId = getNextStepInstanceId(includeStepInstanceIdList, + currentStepInstance.getId()); + } + if (nextRollingStepInstanceId != null) { + nextStepInstance = taskInstanceService.getBaseStepInstance(nextRollingStepInstanceId); + } + } + } else { + nextStepInstance = stepInstanceService.getNextStepInstance(taskInstance.getId(), + currentStepInstance.getStepOrder()); + } + return nextStepInstance; + } + + private Long getNextStepInstanceId(List stepInstanceIdList, long currentStepInstanceId) { + int currentStepIndex = stepInstanceIdList.indexOf(currentStepInstanceId); + // 当前步骤为最后一个步骤 + if (currentStepIndex == stepInstanceIdList.size() - 1) { + return null; + } + return stepInstanceIdList.get(currentStepIndex + 1); + } + + private void startStep(StepInstanceBaseDTO stepInstance) { + taskInstanceService.updateTaskCurrentStepId(stepInstance.getTaskInstanceId(), stepInstance.getId()); + if (stepInstance.isRollingStep()) { + taskInstanceService.updateStepStatus(stepInstance.getId(), RunStatusEnum.ROLLING_WAITING.getValue()); + taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.startStep(stepInstance.getId(), + stepInstance.getBatch() + 1)); + } else { + taskInstanceService.updateStepStatus(stepInstance.getId(), RunStatusEnum.BLANK.getValue()); + taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.startStep(stepInstance.getId(), null)); + } + } + + private void callback(TaskInstanceDTO taskInstance, long jobInstanceId, int taskStatus, long currentStepId, + RunStatusEnum stepStatus) { + if (taskInstance.getCallbackUrl() != null) { + JobCallbackDTO callback = new JobCallbackDTO(); + callback.setId(jobInstanceId); + callback.setStatus(taskStatus); + callback.setCallbackUrl(taskInstance.getCallbackUrl()); + Collection stepInstanceList = Lists.newArrayList(); + JobCallbackDTO.StepInstanceStatus stepInstance = new JobCallbackDTO.StepInstanceStatus(); + stepInstance.setId(currentStepId); + stepInstance.setStatus(stepStatus.getValue()); + stepInstanceList.add(stepInstance); + callback.setStepInstances(stepInstanceList); + taskExecuteMQEventDispatcher.sendCallback(callback); + } + } + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ResultHandleResumeListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ResultHandleResumeListener.java index 46c966eef8..d6d06d6bc1 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ResultHandleResumeListener.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ResultHandleResumeListener.java @@ -26,15 +26,13 @@ import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.config.StorageSystemConfig; -import com.tencent.bk.job.execute.engine.TaskExecuteControlMsgSender; import com.tencent.bk.job.execute.engine.consts.FileDirTypeConf; -import com.tencent.bk.job.execute.engine.consts.IpStatus; import com.tencent.bk.job.execute.engine.evict.TaskEvictPolicyExecutor; -import com.tencent.bk.job.execute.engine.exception.ExceptionStatusManager; +import com.tencent.bk.job.execute.engine.listener.event.ResultHandleTaskResumeEvent; +import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; import com.tencent.bk.job.execute.engine.message.TaskResultHandleResumeProcessor; import com.tencent.bk.job.execute.engine.model.FileDest; import com.tencent.bk.job.execute.engine.model.JobFile; -import com.tencent.bk.job.execute.engine.model.StepControlMessage; import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; import com.tencent.bk.job.execute.engine.model.TaskVariablesAnalyzeResult; import com.tencent.bk.job.execute.engine.result.FileResultHandleTask; @@ -44,18 +42,20 @@ import com.tencent.bk.job.execute.engine.util.FilePathUtils; import com.tencent.bk.job.execute.engine.util.JobSrcFileUtils; import com.tencent.bk.job.execute.engine.util.NFSUtils; -import com.tencent.bk.job.execute.model.GseTaskIpLogDTO; -import com.tencent.bk.job.execute.model.GseTaskLogDTO; +import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.model.GseTaskDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.service.AgentService; -import com.tencent.bk.job.execute.service.GseTaskLogService; +import com.tencent.bk.job.execute.service.FileAgentTaskService; +import com.tencent.bk.job.execute.service.GseTaskService; import com.tencent.bk.job.execute.service.LogService; +import com.tencent.bk.job.execute.service.ScriptAgentTaskService; +import com.tencent.bk.job.execute.service.StepInstanceService; import com.tencent.bk.job.execute.service.StepInstanceVariableValueService; import com.tencent.bk.job.execute.service.TaskInstanceService; import com.tencent.bk.job.execute.service.TaskInstanceVariableService; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cloud.stream.annotation.EnableBinding; @@ -67,10 +67,9 @@ import java.util.Map; import java.util.Set; import java.util.UUID; -import java.util.stream.Collectors; /** - * 执行引擎流程处理-任务结果处理-恢复 + * 执行引擎事件处理-任务恢复 */ @Component @EnableBinding({TaskResultHandleResumeProcessor.class}) @@ -82,7 +81,7 @@ public class ResultHandleResumeListener { private final TaskInstanceVariableService taskInstanceVariableService; - private final GseTaskLogService gseTaskLogService; + private final GseTaskService gseTaskService; private final StorageSystemConfig storageSystemConfig; @@ -92,40 +91,47 @@ public class ResultHandleResumeListener { private final StepInstanceVariableValueService stepInstanceVariableValueService; - private final TaskExecuteControlMsgSender taskExecuteControlMsgSender; + private final TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher; private final ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager; - private final ExceptionStatusManager exceptionStatusManager; - private final TaskEvictPolicyExecutor taskEvictPolicyExecutor; + private final ScriptAgentTaskService scriptAgentTaskService; + + private final FileAgentTaskService fileAgentTaskService; + + private final StepInstanceService stepInstanceService; + @Autowired public ResultHandleResumeListener(TaskInstanceService taskInstanceService, ResultHandleManager resultHandleManager, TaskInstanceVariableService taskInstanceVariableService, - GseTaskLogService gseTaskLogService, + GseTaskService gseTaskService, StorageSystemConfig storageSystemConfig, AgentService agentService, LogService logService, StepInstanceVariableValueService stepInstanceVariableValueService, - TaskExecuteControlMsgSender taskExecuteControlMsgSender, + TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager, - ExceptionStatusManager exceptionStatusManager, - TaskEvictPolicyExecutor taskEvictPolicyExecutor) { + TaskEvictPolicyExecutor taskEvictPolicyExecutor, + ScriptAgentTaskService scriptAgentTaskService, + FileAgentTaskService fileAgentTaskService, + StepInstanceService stepInstanceService) { this.taskInstanceService = taskInstanceService; this.resultHandleManager = resultHandleManager; this.taskInstanceVariableService = taskInstanceVariableService; - this.gseTaskLogService = gseTaskLogService; + this.gseTaskService = gseTaskService; this.storageSystemConfig = storageSystemConfig; this.agentService = agentService; this.logService = logService; - this.stepInstanceVariableValueService = stepInstanceVariableValueService; - this.taskExecuteControlMsgSender = taskExecuteControlMsgSender; + this.taskExecuteMQEventDispatcher = taskExecuteMQEventDispatcher; this.resultHandleTaskKeepaliveManager = resultHandleTaskKeepaliveManager; - this.exceptionStatusManager = exceptionStatusManager; this.taskEvictPolicyExecutor = taskEvictPolicyExecutor; + this.scriptAgentTaskService = scriptAgentTaskService; + this.fileAgentTaskService = fileAgentTaskService; + this.stepInstanceService = stepInstanceService; } @@ -133,71 +139,33 @@ public ResultHandleResumeListener(TaskInstanceService taskInstanceService, * 恢复被中断的作业结果处理任务 */ @StreamListener(TaskResultHandleResumeProcessor.INPUT) - public void handleMessage(StepControlMessage stepControlMessage) { - log.info("Receive result handle task resume control message, action: {}, stepInstanceId: {}, executeCount: {}, requestId: {}, msgSendTime={}", - stepControlMessage.getAction(), stepControlMessage.getStepInstanceId(), - stepControlMessage.getExecuteCount(), - stepControlMessage.getRequestId(), stepControlMessage.getTime()); - long stepInstanceId = stepControlMessage.getStepInstanceId(); - int executeCount = stepControlMessage.getExecuteCount(); - String requestId = StringUtils.isNotEmpty(stepControlMessage.getRequestId()) ? stepControlMessage.getRequestId() + public void handleEvent(ResultHandleTaskResumeEvent event) { + log.info("Receive gse task result handle task resume event: {}", event); + GseTaskDTO gseTask = gseTaskService.getGseTask(event.getGseTaskId()); + long stepInstanceId = gseTask.getStepInstanceId(); + String requestId = StringUtils.isNotEmpty(event.getRequestId()) ? event.getRequestId() : UUID.randomUUID().toString(); + try { StepInstanceDTO stepInstance = taskInstanceService.getStepInstanceDetail(stepInstanceId); TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(stepInstance.getTaskInstanceId()); - GseTaskLogDTO gseTaskLog = gseTaskLogService.getGseTaskLog(stepInstanceId, executeCount); - if (!checkIsTaskResumeable(stepInstance, gseTaskLog)) { + if (!checkIsTaskResumeable(stepInstance, gseTask)) { log.warn("Task can not resume, stepStatus: {}, gseTaskStatus: {}", - stepInstance.getStatus(), gseTaskLog.getStatus()); + stepInstance.getStatus(), gseTask.getStatus()); return; } - Map ipLogMap = new HashMap<>(); - List gseTaskIpLogs = gseTaskLogService.getIpLog(stepInstanceId, executeCount, false); - if (CollectionUtils.isNotEmpty(gseTaskIpLogs)) { - gseTaskIpLogs.stream().filter(gseTaskIpLog -> - IpStatus.LAST_SUCCESS.getValue() != gseTaskIpLog.getStatus()) - .forEach(gseTaskIpLog -> ipLogMap.put(gseTaskIpLog.getCloudAreaAndIp(), gseTaskIpLog)); - } - - List taskVariables = taskInstanceVariableService.getByTaskInstanceId(stepInstance.getTaskInstanceId()); TaskVariablesAnalyzeResult taskVariablesAnalyzeResult = new TaskVariablesAnalyzeResult(taskVariables); if (stepInstance.isScriptStep()) { - ScriptResultHandleTask scriptResultHandleTask = new ScriptResultHandleTask(taskInstance, stepInstance, - taskVariablesAnalyzeResult, ipLogMap, gseTaskLog, ipLogMap.keySet(), - requestId); - scriptResultHandleTask.initDependentService(taskInstanceService, gseTaskLogService, logService, - taskInstanceVariableService, stepInstanceVariableValueService, taskExecuteControlMsgSender, - resultHandleTaskKeepaliveManager, exceptionStatusManager, taskEvictPolicyExecutor); - resultHandleManager.handleDeliveredTask(scriptResultHandleTask); + resumeScriptTask(taskInstance, stepInstance, taskVariablesAnalyzeResult, gseTask, requestId); } else if (stepInstance.isFileStep()) { - Set sendFiles = JobSrcFileUtils.parseSendFileList(stepInstance, - agentService.getLocalAgentBindIp(), - storageSystemConfig.getJobStorageRootPath()); - String targetDir = FilePathUtils.standardizedDirPath(stepInstance.getResolvedFileTargetPath()); - Map srcAndDestMap = JobSrcFileUtils.buildSourceDestPathMapping( - sendFiles, targetDir, stepInstance.getFileTargetName()); - Map sourceDestPathMap = buildSourceDestPathMap(srcAndDestMap); - // 初始化显示名称映射Map - Map sourceFileDisplayMap = JobSrcFileUtils.buildSourceFileDisplayMapping(sendFiles, - NFSUtils.getFileDir(storageSystemConfig.getJobStorageRootPath(), FileDirTypeConf.UPLOAD_FILE_DIR)); - - Set targetIps = gseTaskIpLogs.stream().filter(GseTaskIpLogDTO::isTargetServer) - .map(GseTaskIpLogDTO::getCloudAreaAndIp).collect(Collectors.toSet()); - FileResultHandleTask fileResultHandleTask = new FileResultHandleTask(taskInstance, stepInstance, - taskVariablesAnalyzeResult, ipLogMap, gseTaskLog, targetIps, sendFiles, - storageSystemConfig.getJobStorageRootPath(), sourceDestPathMap, sourceFileDisplayMap, - requestId); - fileResultHandleTask.initDependentService(taskInstanceService, gseTaskLogService, logService, - taskInstanceVariableService, stepInstanceVariableValueService, taskExecuteControlMsgSender, - resultHandleTaskKeepaliveManager, exceptionStatusManager, taskEvictPolicyExecutor); - resultHandleManager.handleDeliveredTask(fileResultHandleTask); + resumeFileTask(taskInstance, stepInstance, taskVariablesAnalyzeResult, gseTask, requestId); } else { - log.warn("Not support resume step type! stepType: {}", stepInstance.getExecuteType()); + log.error("Not support resume step type! stepType: {}", stepInstance.getExecuteType()); } } catch (Exception e) { String errorMsg = "Handling task control message error,stepInstanceId=" + stepInstanceId; @@ -205,19 +173,98 @@ public void handleMessage(StepControlMessage stepControlMessage) { } } + private void resumeScriptTask(TaskInstanceDTO taskInstance, + StepInstanceDTO stepInstance, + TaskVariablesAnalyzeResult taskVariablesAnalyzeResult, + GseTaskDTO gseTask, + String requestId) { + Map agentTaskMap = new HashMap<>(); + List agentTasks = scriptAgentTaskService.listAgentTasksByGseTaskId(gseTask.getId()); + agentTasks.forEach(agentTask -> agentTaskMap.put(agentTask.getAgentId(), agentTask)); + + ScriptResultHandleTask scriptResultHandleTask = new ScriptResultHandleTask( + taskInstanceService, + gseTaskService, + logService, + taskInstanceVariableService, + stepInstanceVariableValueService, + taskExecuteMQEventDispatcher, + resultHandleTaskKeepaliveManager, + taskEvictPolicyExecutor, + scriptAgentTaskService, + stepInstanceService, + taskInstance, + stepInstance, + taskVariablesAnalyzeResult, + agentTaskMap, + gseTask, + requestId); + resultHandleManager.handleDeliveredTask(scriptResultHandleTask); + } + + private void resumeFileTask(TaskInstanceDTO taskInstance, + StepInstanceDTO stepInstance, + TaskVariablesAnalyzeResult taskVariablesAnalyzeResult, + GseTaskDTO gseTask, + String requestId) { + Set sendFiles = JobSrcFileUtils.parseSendFileList(stepInstance, + agentService.getLocalAgentHost(), + storageSystemConfig.getJobStorageRootPath()); + String targetDir = FilePathUtils.standardizedDirPath(stepInstance.getResolvedFileTargetPath()); + Map srcAndDestMap = JobSrcFileUtils.buildSourceDestPathMapping( + sendFiles, targetDir, stepInstance.getFileTargetName()); + Map sourceDestPathMap = buildSourceDestPathMap(srcAndDestMap); + // 初始化显示名称映射Map + Map sourceFileDisplayMap = JobSrcFileUtils.buildSourceFileDisplayMapping(sendFiles, + NFSUtils.getFileDir(storageSystemConfig.getJobStorageRootPath(), FileDirTypeConf.UPLOAD_FILE_DIR)); + + Map sourceAgentTaskMap = new HashMap<>(); + Map targetAgentTaskMap = new HashMap<>(); + List agentTasks = fileAgentTaskService.listAgentTasksByGseTaskId(gseTask.getId()); + agentTasks.forEach(agentTask -> { + if (agentTask.isTarget()) { + targetAgentTaskMap.put(agentTask.getAgentId(), agentTask); + } else { + sourceAgentTaskMap.put(agentTask.getAgentId(), agentTask); + } + }); + + FileResultHandleTask fileResultHandleTask = new FileResultHandleTask( + taskInstanceService, + gseTaskService, + logService, + taskInstanceVariableService, + stepInstanceVariableValueService, + taskExecuteMQEventDispatcher, + resultHandleTaskKeepaliveManager, + taskEvictPolicyExecutor, + fileAgentTaskService, + stepInstanceService, + taskInstance, + stepInstance, + taskVariablesAnalyzeResult, + targetAgentTaskMap, + sourceAgentTaskMap, + gseTask, + sendFiles, + storageSystemConfig.getJobStorageRootPath(), + sourceDestPathMap, + sourceFileDisplayMap, + requestId); + resultHandleManager.handleDeliveredTask(fileResultHandleTask); + } + private Map buildSourceDestPathMap(Map srcAndDestMap) { Map sourceDestPathMap = new HashMap<>(); - srcAndDestMap.forEach((fileKey, dest) -> { - sourceDestPathMap.put(fileKey, dest.getDestPath()); - }); + srcAndDestMap.forEach((fileKey, dest) -> sourceDestPathMap.put(fileKey, dest.getDestPath())); return sourceDestPathMap; } - private boolean checkIsTaskResumeable(StepInstanceDTO stepInstance, GseTaskLogDTO gseTaskLog) { - RunStatusEnum stepStatus = RunStatusEnum.valueOf(stepInstance.getStatus()); - RunStatusEnum gseTaskStatus = RunStatusEnum.valueOf(gseTaskLog.getStatus()); - return (stepStatus == RunStatusEnum.WAITING || stepStatus == RunStatusEnum.RUNNING - || stepStatus == RunStatusEnum.STOPPING) && (gseTaskStatus == RunStatusEnum.WAITING + private boolean checkIsTaskResumeable(StepInstanceDTO stepInstance, GseTaskDTO gseTask) { + RunStatusEnum stepStatus = stepInstance.getStatus(); + RunStatusEnum gseTaskStatus = RunStatusEnum.valueOf(gseTask.getStatus()); + return (stepStatus == RunStatusEnum.WAITING_USER || stepStatus == RunStatusEnum.RUNNING + || stepStatus == RunStatusEnum.STOPPING) && (gseTaskStatus == RunStatusEnum.WAITING_USER || gseTaskStatus == RunStatusEnum.RUNNING || gseTaskStatus == RunStatusEnum.STOPPING); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/TaskControlMessage.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/StepEventHandler.java similarity index 81% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/TaskControlMessage.java rename to src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/StepEventHandler.java index 22889142b5..7cc313ba41 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/TaskControlMessage.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/StepEventHandler.java @@ -22,19 +22,15 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.execute.engine.model; +package com.tencent.bk.job.execute.engine.listener; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; +import com.tencent.bk.job.execute.engine.listener.event.StepEvent; +import com.tencent.bk.job.execute.model.StepInstanceDTO; -import java.time.LocalDateTime; +/** + * 步骤事件处理 + */ +public interface StepEventHandler { -@Data -@NoArgsConstructor -@AllArgsConstructor -public class TaskControlMessage { - private int action; - private long taskInstanceId; - private LocalDateTime time; + void handleEvent(StepEvent stepEvent, StepInstanceDTO stepInstance); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/StepListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/StepListener.java index a407584b82..98bd40084b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/StepListener.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/StepListener.java @@ -24,380 +24,72 @@ package com.tencent.bk.job.execute.engine.listener; -import com.tencent.bk.job.common.util.date.DateUtils; -import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; -import com.tencent.bk.job.execute.common.util.TaskCostCalculator; -import com.tencent.bk.job.execute.engine.TaskExecuteControlMsgSender; +import com.tencent.bk.job.execute.engine.listener.event.StepEvent; import com.tencent.bk.job.execute.engine.message.StepProcessor; -import com.tencent.bk.job.execute.engine.model.StepControlMessage; -import com.tencent.bk.job.execute.engine.prepare.FilePrepareService; -import com.tencent.bk.job.execute.model.NotifyDTO; -import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; -import com.tencent.bk.job.execute.model.TaskInstanceDTO; -import com.tencent.bk.job.execute.model.TaskNotifyDTO; import com.tencent.bk.job.execute.service.TaskInstanceService; -import com.tencent.bk.job.manage.common.consts.notify.ExecuteStatusEnum; -import com.tencent.bk.job.manage.common.consts.notify.ResourceTypeEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cloud.stream.annotation.EnableBinding; import org.springframework.cloud.stream.annotation.StreamListener; import org.springframework.stereotype.Component; -import static com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum.EXECUTE_SCRIPT; -import static com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum.EXECUTE_SQL; -import static com.tencent.bk.job.execute.engine.consts.StepActionEnum.CLEAR; -import static com.tencent.bk.job.execute.engine.consts.StepActionEnum.CONFIRM_CONTINUE; -import static com.tencent.bk.job.execute.engine.consts.StepActionEnum.CONFIRM_RESTART; -import static com.tencent.bk.job.execute.engine.consts.StepActionEnum.CONFIRM_TERMINATE; -import static com.tencent.bk.job.execute.engine.consts.StepActionEnum.CONTINUE_FILE_PUSH; -import static com.tencent.bk.job.execute.engine.consts.StepActionEnum.IGNORE_ERROR; -import static com.tencent.bk.job.execute.engine.consts.StepActionEnum.NEXT_STEP; -import static com.tencent.bk.job.execute.engine.consts.StepActionEnum.RETRY_ALL; -import static com.tencent.bk.job.execute.engine.consts.StepActionEnum.RETRY_FAIL; -import static com.tencent.bk.job.execute.engine.consts.StepActionEnum.SKIP; -import static com.tencent.bk.job.execute.engine.consts.StepActionEnum.START; -import static com.tencent.bk.job.execute.engine.consts.StepActionEnum.STOP; - /** - * 执行引擎流程处理-步骤 + * 执行引擎事件处理-步骤 */ @Component @EnableBinding({StepProcessor.class}) @Slf4j public class StepListener { private final TaskInstanceService taskInstanceService; - private final TaskExecuteControlMsgSender taskControlMsgSender; - private final FilePrepareService filePrepareService; + private final GseStepEventHandler gseStepEventHandler; + private final ConfirmStepEventHandler confirmStepEventHandler; @Autowired public StepListener(TaskInstanceService taskInstanceService, - TaskExecuteControlMsgSender taskControlMsgSender, - FilePrepareService filePrepareService) { + GseStepEventHandler gseStepEventHandler, + ConfirmStepEventHandler confirmStepEventHandler) { this.taskInstanceService = taskInstanceService; - this.taskControlMsgSender = taskControlMsgSender; - this.filePrepareService = filePrepareService; + this.gseStepEventHandler = gseStepEventHandler; + this.confirmStepEventHandler = confirmStepEventHandler; } /** - * 处理步骤控制相关消息，包含：预启动步骤、启动步骤、重新执行步骤和跳过步骤 + * 处理步骤执行相关的事件 + * + * @param stepEvent 步骤执行相关的事件 */ @StreamListener(StepProcessor.INPUT) - public void handleMessage(StepControlMessage stepControlMessage) { - log.info("Receive step control message, stepInstanceId={}, action={}, msgSendTime={}", - stepControlMessage.getStepInstanceId(), - stepControlMessage.getAction(), stepControlMessage.getTime()); - long stepInstanceId = stepControlMessage.getStepInstanceId(); + public void handleEvent(StepEvent stepEvent) { + log.info("Handle step event: {}", stepEvent); + long stepInstanceId = stepEvent.getStepInstanceId(); try { - int action = stepControlMessage.getAction(); - StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(stepInstanceId); - if (START.getValue() == action) { - log.info("Start step, stepInstanceId={}", stepInstanceId); - startStep(stepInstance); - } else if (SKIP.getValue() == action) { - TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(stepInstance.getTaskInstanceId()); - if (taskInstance.getCurrentStepId() == stepInstanceId) { - log.info("Skip step, stepInstanceId={}", stepInstanceId); - skipStep(stepInstance); - } else { - log.warn("Only current running step is support for skipping, stepInstanceId={}", stepInstanceId); - } - } else if (RETRY_FAIL.getValue() == action) { - log.info("Retry step fail, stepInstanceId={}", stepInstanceId); - retryStepFail(stepInstance); - } else if (RETRY_ALL.getValue() == action) { - log.info("Retry step all, stepInstanceId={}", stepInstanceId); - retryStepAll(stepInstance); - } else if (STOP.getValue() == action) { - log.info("Force stop step, stepInstanceId={}", stepInstanceId); - stopStep(stepInstance); - } else if (IGNORE_ERROR.getValue() == action) { - log.info("Ignore step error, stepInstanceId={}", stepInstanceId); - ignoreError(stepInstance); - } else if (NEXT_STEP.getValue() == action) { - log.info("Next step, stepInstanceId={}", stepInstanceId); - nextStep(stepInstance); - } else if (CONFIRM_TERMINATE.getValue() == action) { - log.info("Confirm step terminate, stepInstanceId={}", stepInstanceId); - confirmStepTerminate(stepInstance); - } else if (CONFIRM_RESTART.getValue() == action) { - log.info("Confirm step restart, stepInstanceId={}", stepInstanceId); - confirmStepRestart(stepInstance); - } else if (CONFIRM_CONTINUE.getValue() == action) { - log.info("Confirm step continue, stepInstanceId={}", stepInstanceId); - confirmStepContinue(stepInstance); - } else if (CONTINUE_FILE_PUSH.getValue() == action) { - log.info("continue file push step, stepInstanceId={}", stepInstanceId); - continueGseFileStep(stepInstance); - } else if (CLEAR.getValue() == action) { - log.info("clear step, stepInstanceId={}", stepInstanceId); - clearStep(stepInstance); - } else { - log.warn("Error step control action:{}", action); - } - } catch (Exception e) { - String errorMsg = "Handling step control message error,stepInstanceId:" + stepInstanceId; - log.warn(errorMsg, e); - } - - } - - private void confirmStepTerminate(StepInstanceBaseDTO stepInstance) { - long stepInstanceId = stepInstance.getId(); - TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(stepInstance.getTaskInstanceId()); - if (RunStatusEnum.WAITING.getValue().equals(stepInstance.getStatus())) { - Long endTime = DateUtils.currentTimeMillis(); - long taskTotalTime = TaskCostCalculator.calculate(taskInstance.getStartTime(), endTime, - taskInstance.getTotalTime()); - taskInstanceService.updateTaskExecutionInfo(taskInstance.getId(), RunStatusEnum.CONFIRM_TERMINATED, null, - null, endTime, taskTotalTime); - long stepTotalTime = TaskCostCalculator.calculate(stepInstance.getStartTime(), endTime, - stepInstance.getTotalTime()); - taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.CONFIRM_TERMINATED, null, - endTime, stepTotalTime); - } else { - log.warn("Unsupported step instance status for confirm step terminate action, stepInstanceId:{}, " + - "status:{}", stepInstanceId, stepInstance.getStatus()); - } - } - - private void confirmStepRestart(StepInstanceBaseDTO stepInstance) { - long stepInstanceId = stepInstance.getId(); - if (RunStatusEnum.CONFIRM_TERMINATED.getValue().equals(stepInstance.getStatus())) { - executeConfirmStep(stepInstance); - } else { - log.warn("Unsupported step instance status for confirm-step-restart action, stepInstanceId:{}, status:{}" - , stepInstanceId, stepInstance.getStatus()); - } - } - - private void nextStep(StepInstanceBaseDTO stepInstance) { - long taskInstanceId = stepInstance.getTaskInstanceId(); - long stepInstanceId = stepInstance.getId(); - int stepStatus = stepInstance.getStatus(); - - if (RunStatusEnum.STOP_SUCCESS.getValue() == stepStatus) { - taskInstanceService.updateTaskStatus(taskInstanceId, RunStatusEnum.RUNNING.getValue()); - long endTime = DateUtils.currentTimeMillis(); - long totalTime = TaskCostCalculator.calculate(stepInstance.getStartTime(), endTime, - stepInstance.getTotalTime()); - // 终止成功，进入下一步，该步骤设置为“跳过” - taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.SKIPPED, null, endTime, - totalTime); - taskControlMsgSender.refreshTask(taskInstanceId); - } else { - log.warn("Unsupported step instance status for next step action, stepInstanceId:{}, status:{}", - stepInstanceId, stepInstance.getStatus()); - } - } - - private void confirmStepContinue(StepInstanceBaseDTO stepInstance) { - long taskInstanceId = stepInstance.getTaskInstanceId(); - long stepInstanceId = stepInstance.getId(); - int stepStatus = stepInstance.getStatus(); - - if (RunStatusEnum.WAITING.getValue() == stepStatus) { - taskInstanceService.updateTaskStatus(taskInstanceId, RunStatusEnum.RUNNING.getValue()); - long endTime = DateUtils.currentTimeMillis(); - long totalTime = TaskCostCalculator.calculate(stepInstance.getStartTime(), endTime, - stepInstance.getTotalTime()); - // 人工确认通过，该步骤状态标识为成功；终止成功的步骤保持状态不变 - taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.SUCCESS, null, endTime, - totalTime); - taskControlMsgSender.refreshTask(taskInstanceId); - } else { - log.warn("Unsupported step instance status for confirm-step-continue step action, stepInstanceId:{}, " + - "status:{}", stepInstanceId, stepInstance.getStatus()); - } - } - - private void ignoreError(StepInstanceBaseDTO stepInstance) { - if (!stepInstance.getStatus().equals(RunStatusEnum.FAIL.getValue())) { - log.warn("Current step status does not support ignore error operation! stepInstanceId:{}, status:{}", - stepInstance.getId(), stepInstance.getStatus()); - return; + StepInstanceDTO stepInstance = taskInstanceService.getStepInstanceDetail(stepInstanceId); + dispatchEvent(stepEvent, stepInstance); + } catch (Throwable e) { + String errorMsg = "Handling step event error,stepInstanceId:" + stepInstanceId; + log.error(errorMsg, e); } - taskInstanceService.updateStepStatus(stepInstance.getId(), RunStatusEnum.IGNORE_ERROR.getValue()); - taskInstanceService.resetTaskExecuteInfoForResume(stepInstance.getTaskInstanceId()); - taskControlMsgSender.refreshTask(stepInstance.getTaskInstanceId()); - } - - private void startStep(StepInstanceBaseDTO stepInstance) { - int stepStatus = stepInstance.getStatus(); - long stepInstanceId = stepInstance.getId(); - long taskInstanceId = stepInstance.getTaskInstanceId(); - - // 只有当步骤状态为'等待用户'和'未执行'时可以启动步骤 - if (RunStatusEnum.BLANK.getValue() == stepStatus || RunStatusEnum.WAITING.getValue() == stepStatus) { - taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.RUNNING, - DateUtils.currentTimeMillis(), null, null); - taskInstanceService.updateTaskStatus(taskInstanceId, RunStatusEnum.RUNNING.getValue()); - - int stepType = stepInstance.getExecuteType(); - if (EXECUTE_SCRIPT.getValue() == stepType || StepExecuteTypeEnum.EXECUTE_SQL.getValue() == stepType) { - taskControlMsgSender.startGseStep(stepInstanceId); - } else if (TaskStepTypeEnum.FILE.getValue() == stepType) { - filePrepareService.prepareFileForGseTask(stepInstanceId); - } else if (TaskStepTypeEnum.APPROVAL.getValue() == stepType) { - executeConfirmStep(stepInstance); - } else { - log.warn("Unsupported step type, skip it! stepInstanceId={}, stepType={}", stepInstanceId, stepType); - taskInstanceService.updateTaskStatus(taskInstanceId, RunStatusEnum.SKIPPED.getValue()); - taskControlMsgSender.refreshTask(taskInstanceId); - } - } else { - log.warn("Unsupported step instance run status for starting step, stepInstanceId={}, status={}", - stepInstanceId, stepStatus); - } } - private void skipStep(StepInstanceBaseDTO stepInstance) { - int stepStatus = stepInstance.getStatus(); - long stepInstanceId = stepInstance.getId(); - long taskInstanceId = stepInstance.getTaskInstanceId(); - - // 只有当步骤状态为'终止中'时可以跳过步骤 - if (RunStatusEnum.STOPPING.getValue() == stepStatus) { - long now = DateUtils.currentTimeMillis(); - taskInstanceService.updateStepStartTimeIfNull(stepInstanceId, now); - taskInstanceService.updateStepStatus(stepInstanceId, RunStatusEnum.SKIPPED.getValue()); - taskInstanceService.updateStepEndTime(stepInstanceId, now); + private void dispatchEvent(StepEvent stepEvent, StepInstanceDTO stepInstance) { + StepExecuteTypeEnum stepType = StepExecuteTypeEnum.valueOf(stepInstance.getExecuteType()); - taskInstanceService.updateTaskStatus(taskInstanceId, RunStatusEnum.RUNNING.getValue()); - taskControlMsgSender.refreshTask(taskInstanceId); - } else { - log.warn("Unsupported step instance run status for skipping step, stepInstanceId={}, status={}", - stepInstanceId, stepStatus); + switch (stepType) { + case EXECUTE_SCRIPT: + case SEND_FILE: + case EXECUTE_SQL: + gseStepEventHandler.handleEvent(stepEvent, stepInstance); + break; + case MANUAL_CONFIRM: + confirmStepEventHandler.handleEvent(stepEvent, stepInstance); + break; + default: + log.error("Unhandled step event: {}", stepEvent); } } - private void stopStep(StepInstanceBaseDTO stepInstance) { - long stepInstanceId = stepInstance.getId(); - long taskInstanceId = stepInstance.getTaskInstanceId(); - - int executeType = stepInstance.getExecuteType(); - if (TaskStepTypeEnum.SCRIPT.getValue() == executeType || TaskStepTypeEnum.FILE.getValue() == executeType - || EXECUTE_SQL.getValue().equals(executeType)) { - taskInstanceService.updateTaskStatus(taskInstanceId, RunStatusEnum.STOPPING.getValue()); - } else { - log.warn("Not gse step type, can not stop! stepInstanceId={}, stepType={}", stepInstanceId, executeType); - } - } - /** - * 第三方文件源文件拉取完成后继续GSE文件分发 - * - * @param stepInstance 步骤实例 - */ - private void continueGseFileStep(StepInstanceBaseDTO stepInstance) { - taskControlMsgSender.startGseStep(stepInstance.getId()); - } - - /** - * 重新执行步骤失败的任务 - */ - private void retryStepFail(StepInstanceBaseDTO stepInstance) { - resetStatusForRetry(stepInstance); - filePrepareService.retryPrepareFile(stepInstance.getId()); - taskControlMsgSender.retryGseStepFail(stepInstance.getId()); - } - - /** - * 从头执行步骤 - */ - private void retryStepAll(StepInstanceBaseDTO stepInstance) { - resetStatusForRetry(stepInstance); - filePrepareService.retryPrepareFile(stepInstance.getId()); - taskControlMsgSender.retryGseStepAll(stepInstance.getId()); - } - - /** - * 清理执行完的步骤 - */ - private void clearStep(StepInstanceBaseDTO stepInstance) { - int executeType = stepInstance.getExecuteType(); - // 当前仅有文件分发类步骤需要清理中间文件 - if (TaskStepTypeEnum.FILE.getValue() == executeType) { - filePrepareService.clearPreparedTmpFile(stepInstance.getId()); - } - } - - private void resetStatusForRetry(StepInstanceBaseDTO stepInstance) { - long stepInstanceId = stepInstance.getId(); - long taskInstanceId = stepInstance.getTaskInstanceId(); - - taskInstanceService.resetStepExecuteInfoForRetry(stepInstanceId); - taskInstanceService.resetTaskExecuteInfoForResume(taskInstanceId); - } - - /** - * 人工确认步骤 - */ - private void executeConfirmStep(StepInstanceBaseDTO stepInstance) { - long stepInstanceId = stepInstance.getId(); - long taskInstanceId = stepInstance.getTaskInstanceId(); - - // 只有“未执行”和“确认终止”状态的，才可以重新执行人工确认步骤 - if (RunStatusEnum.BLANK.getValue().equals(stepInstance.getStatus()) - || RunStatusEnum.CONFIRM_TERMINATED.getValue().equals(stepInstance.getStatus())) { - // 发送页面确认信息 - TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(taskInstanceId); - String stepOperator = stepInstance.getOperator(); - - if (StringUtils.isBlank(stepOperator)) { - log.info("The operator is empty, continue run step! stepInstanceId={}", stepInstanceId); - stepOperator = taskInstance.getOperator(); - stepInstance.setOperator(stepOperator); - } - taskInstanceService.updateStepStatus(stepInstanceId, RunStatusEnum.WAITING.getValue()); - taskInstanceService.updateTaskStatus(taskInstanceId, RunStatusEnum.WAITING.getValue()); - asyncNotifyConfirm(taskInstance, stepInstance); - } else { - log.warn("Unsupported step instance run status for executing confirm step, stepInstanceId={}, status={}", - stepInstanceId, stepInstance.getStatus()); - } - } - - private void asyncNotifyConfirm(TaskInstanceDTO taskInstance, StepInstanceBaseDTO stepInstance) { - StepInstanceDTO stepInstanceDetail = taskInstanceService.getStepInstanceDetail(stepInstance.getId()); - if (stepInstanceDetail == null) { - log.warn("StepInstance is not exist, stepInstanceId: {}", stepInstance.getId()); - return; - } - TaskNotifyDTO taskNotifyDTO = buildCommonTaskNotification(taskInstance, stepInstance); - taskNotifyDTO.setResourceExecuteStatus(ExecuteStatusEnum.READY.getStatus()); - taskNotifyDTO.setStepName(stepInstance.getName()); - taskNotifyDTO.setConfirmMessage(stepInstanceDetail.getConfirmMessage()); - NotifyDTO notifyDTO = new NotifyDTO(); - notifyDTO.setReceiverUsers(stepInstanceDetail.getConfirmUsers()); - notifyDTO.setReceiverRoles(stepInstanceDetail.getConfirmRoles()); - notifyDTO.setChannels(stepInstanceDetail.getNotifyChannels()); - notifyDTO.setTriggerUser(stepInstance.getOperator()); - taskNotifyDTO.setNotifyDTO(notifyDTO); - taskNotifyDTO.setResourceId(String.valueOf(taskInstance.getTaskId())); - taskNotifyDTO.setResourceType(ResourceTypeEnum.JOB.getType()); - taskNotifyDTO.setOperator(stepInstance.getOperator()); - taskControlMsgSender.asyncSendNotifyMsg(taskNotifyDTO); - } - - private TaskNotifyDTO buildCommonTaskNotification(TaskInstanceDTO taskInstance, StepInstanceBaseDTO stepInstance) { - TaskNotifyDTO taskNotifyDTO = new TaskNotifyDTO(); - taskNotifyDTO.setAppId(taskInstance.getAppId()); - String operator = getOperator(taskInstance.getOperator(), stepInstance.getOperator()); - taskNotifyDTO.setStartupMode(taskInstance.getStartupMode()); - taskNotifyDTO.setOperator(operator); - taskNotifyDTO.setTaskInstanceId(taskInstance.getId()); - taskNotifyDTO.setTaskInstanceName(taskInstance.getName()); - taskNotifyDTO.setTaskId(taskInstance.getTaskId()); - return taskNotifyDTO; - } - - private String getOperator(String taskOperator, String stepOperator) { - return StringUtils.isNotEmpty(stepOperator) ? stepOperator : taskOperator; - } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/TaskListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/TaskListener.java deleted file mode 100644 index f9c71f378a..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/TaskListener.java +++ /dev/null @@ -1,364 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.engine.listener; - -import com.google.common.collect.Lists; -import com.tencent.bk.job.common.util.date.DateUtils; -import com.tencent.bk.job.execute.common.constants.RunStatusEnum; -import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; -import com.tencent.bk.job.execute.common.util.TaskCostCalculator; -import com.tencent.bk.job.execute.engine.TaskExecuteControlMsgSender; -import com.tencent.bk.job.execute.engine.consts.JobActionEnum; -import com.tencent.bk.job.execute.engine.message.TaskProcessor; -import com.tencent.bk.job.execute.engine.model.JobCallbackDTO; -import com.tencent.bk.job.execute.engine.model.TaskControlMessage; -import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; -import com.tencent.bk.job.execute.model.TaskInstanceDTO; -import com.tencent.bk.job.execute.model.TaskNotifyDTO; -import com.tencent.bk.job.execute.service.TaskInstanceService; -import com.tencent.bk.job.execute.statistics.StatisticsService; -import com.tencent.bk.job.manage.common.consts.notify.ExecuteStatusEnum; -import com.tencent.bk.job.manage.common.consts.notify.ResourceTypeEnum; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.StreamListener; -import org.springframework.stereotype.Component; - -import java.util.Collection; -import java.util.List; - -/** - * 执行引擎流程处理-作业 - */ -@Component -@EnableBinding({TaskProcessor.class}) -@Slf4j -public class TaskListener { - - private final TaskExecuteControlMsgSender taskExecuteControlMsgSender; - private final StatisticsService statisticsService; - private final TaskInstanceService taskInstanceService; - - @Autowired - public TaskListener(TaskExecuteControlMsgSender taskExecuteControlMsgSender, - StatisticsService statisticsService, TaskInstanceService taskInstanceService) { - this.taskExecuteControlMsgSender = taskExecuteControlMsgSender; - this.statisticsService = statisticsService; - this.taskInstanceService = taskInstanceService; - } - - - /** - * 处理和作业相关的控制消息：启动作业、停止作业、重启作业、忽略错误和作业状态刷新 - */ - @StreamListener(TaskProcessor.INPUT) - public void handleMessage(TaskControlMessage taskControlMessage) { - log.info("Receive task control message, taskInstanceId={}, action={}, msgSendTime={}", - taskControlMessage.getTaskInstanceId(), - taskControlMessage.getAction(), taskControlMessage.getTime()); - long taskInstanceId = taskControlMessage.getTaskInstanceId(); - int action = taskControlMessage.getAction(); - try { - TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(taskInstanceId); - if (JobActionEnum.START.getValue() == action) { - log.info("Start task, taskInstanceId={}", taskInstanceId); - startJob(taskInstance); - } else if (JobActionEnum.STOP.getValue() == action) { - log.info("Stop task, taskInstanceId={}", taskInstanceId); - stopJob(taskInstance); - } else if (JobActionEnum.RESTART.getValue() == action) { - log.info("Restart task, taskInstanceId={}", taskInstanceId); - restartJob(taskInstance); - } else if (JobActionEnum.REFRESH.getValue() == action) { - log.info("Refresh task, taskInstanceId={}", taskInstanceId); - refreshJob(taskInstance); - } else { - log.warn("Error task control action:{}", action); - } - } catch (Exception e) { - String errorMsg = "Handling task control message error,taskInstanceId=" + taskInstanceId; - log.error(errorMsg, e); - } - } - - /* - * 启动作业 - */ - private void startJob(TaskInstanceDTO taskInstance) { - long taskInstanceId = taskInstance.getId(); - // 首先验证作业的状态，只有状态为“未执行”的作业可以启动 - if (RunStatusEnum.BLANK.getValue().equals(taskInstance.getStatus())) { - long firstStepId = taskInstanceService.getTaskStepIdList(taskInstanceId).get(0); - taskInstanceService.updateTaskExecutionInfo(taskInstanceId, RunStatusEnum.RUNNING, firstStepId, - DateUtils.currentTimeMillis(), null, null); - taskExecuteControlMsgSender.startStep(firstStepId); - // 触发任务开始统计分析 - statisticsService.updateStartJobStatistics(taskInstance); - } else { - log.warn("Unsupported task instance run status for starting task, taskInstanceId={}, status={}", - taskInstanceId, taskInstance.getStatus()); - } - } - - /* - * 强制终止作业 - */ - private void stopJob(TaskInstanceDTO taskInstance) { - long taskInstanceId = taskInstance.getId(); - int taskStatus = taskInstance.getStatus(); - - if (RunStatusEnum.RUNNING.getValue() == taskStatus) { - taskInstanceService.updateTaskStatus(taskInstanceId, RunStatusEnum.STOPPING.getValue()); - } else { - log.warn("Unsupported task instance run status for stop task, taskInstanceId={}, status={}", - taskInstanceId, taskInstance.getStatus()); - } - } - - /* - * 重头执行作业 - */ - private void restartJob(TaskInstanceDTO taskInstance) { - long taskInstanceId = taskInstance.getId(); - int taskStatus = taskInstance.getStatus(); - // 验证作业状态，只有“执行失败”和“等待用户”的作业可以重头执行 - if (RunStatusEnum.WAITING.getValue() == taskStatus || RunStatusEnum.FAIL.getValue() == taskStatus) { - - // 重置作业状态 - taskInstanceService.resetTaskStatus(taskInstanceId); - taskInstanceService.addTaskExecuteCount(taskInstanceId); - - // 重置作业下步骤的状态、开始时间和结束时间等。 - List stepInstanceIdList = taskInstanceService.getTaskStepIdList(taskInstanceId); - for (long stepInstanceId : stepInstanceIdList) { - taskInstanceService.updateStepStatus(stepInstanceId, RunStatusEnum.BLANK.getValue()); - taskInstanceService.resetStepStatus(stepInstanceId); - } - - taskExecuteControlMsgSender.startTask(taskInstanceId); - } else { - log.warn("Unsupported task instance run status for restart task, taskInstanceId={}, status={}", - taskInstanceId, taskInstance.getStatus()); - } - } - - /* - * 刷新作业流程控制 - */ - private void refreshJob(TaskInstanceDTO taskInstance) { - long taskInstanceId = taskInstance.getId(); - int taskStatus = taskInstance.getStatus(); - - long currentStepId = taskInstance.getCurrentStepId(); - StepInstanceBaseDTO currentStep = taskInstanceService.getBaseStepInstance(currentStepId); - int stepStatus = currentStep.getStatus(); - - // 验证作业状态，只有'正在执行'、'强制终止中'的作业可以刷新状态进入下一步或者结束 - if (RunStatusEnum.STOPPING.getValue() == taskStatus) { - if (RunStatusEnum.STOP_SUCCESS.getValue() == stepStatus - || RunStatusEnum.SUCCESS.getValue() == stepStatus - || RunStatusEnum.FAIL.getValue() == stepStatus) { - Long endTime = DateUtils.currentTimeMillis(); - long totalTime = TaskCostCalculator.calculate(taskInstance.getStartTime(), endTime, - taskInstance.getTotalTime()); - taskInstance.setEndTime(endTime); - taskInstance.setTotalTime(totalTime); - taskInstance.setStatus(RunStatusEnum.STOP_SUCCESS.getValue()); - taskInstanceService.updateTaskExecutionInfo(taskInstanceId, RunStatusEnum.STOP_SUCCESS, null, null, - endTime, totalTime); - - int status = stepStatus; - if (RunStatusEnum.SUCCESS.getValue() == stepStatus) { - status = RunStatusEnum.STOP_SUCCESS.getValue(); - asyncNotifySuccess(taskInstance, currentStep); - // 触发任务结束统计分析 - statisticsService.updateEndJobStatistics(taskInstance); - } else if (RunStatusEnum.FAIL.getValue() == stepStatus) { - asyncNotifyFail(taskInstance, currentStep); - // 触发任务结束统计分析 - statisticsService.updateEndJobStatistics(taskInstance); - } else { - taskInstanceService.updateTaskStatus(taskInstanceId, stepStatus); - } - callback(taskInstance, taskInstanceId, status, currentStepId, stepStatus); - } else { - log.warn("Unsupported task instance run status for refresh task, taskInstanceId={}, status={}", - taskInstanceId, taskInstance.getStatus()); - } - - } else if (RunStatusEnum.RUNNING.getValue() == taskStatus) { - // 步骤执行成功、跳过、设为忽略错误、终止成功，可以进入下一步 - if (RunStatusEnum.SUCCESS.getValue() == stepStatus - || RunStatusEnum.SKIPPED.getValue() == stepStatus - || RunStatusEnum.IGNORE_ERROR.getValue() == stepStatus) { - List taskStepIdList = taskInstanceService.getTaskStepIdList(taskInstanceId); - long nextStepId = getNextStepId(taskStepIdList, currentStepId); - // 当前步骤执行成功或者用户手动跳过时，均应该进入下一步 - if (nextStepId == currentStepId) { // 当前执行步骤为最后一步 - Long endTime = DateUtils.currentTimeMillis(); - long totalTime = TaskCostCalculator.calculate(taskInstance.getStartTime(), endTime, - taskInstance.getTotalTime()); - taskInstanceService.updateTaskExecutionInfo(taskInstanceId, RunStatusEnum.SUCCESS, null, null, - endTime, totalTime); - taskInstance.setEndTime(endTime); - taskInstance.setTotalTime(totalTime); - taskInstance.setStatus(RunStatusEnum.SUCCESS.getValue()); - asyncNotifySuccess(taskInstance, currentStep); - callback(taskInstance, taskInstanceId, RunStatusEnum.SUCCESS.getValue(), currentStepId, stepStatus); - // 触发任务结束统计分析 - statisticsService.updateEndJobStatistics(taskInstance); - } else { // 进入下一步 - taskInstanceService.updateTaskCurrentStepId(taskInstanceId, nextStepId); - taskExecuteControlMsgSender.startStep(nextStepId); - } - // 步骤执行成功后清理产生的临时文件 - taskExecuteControlMsgSender.clearStep(currentStepId); - } else if (RunStatusEnum.FAIL.getValue() == stepStatus) { - if (currentStep.isIgnoreError()) { - taskInstanceService.updateStepStatus(currentStepId, RunStatusEnum.IGNORE_ERROR.getValue()); - goToNextStep(taskInstance, currentStep); - return; - } - // 步骤失败，任务结束 - Long endTime = DateUtils.currentTimeMillis(); - long totalTime = TaskCostCalculator.calculate(taskInstance.getStartTime(), endTime, - taskInstance.getTotalTime()); - taskInstance.setEndTime(endTime); - taskInstance.setTotalTime(totalTime); - taskInstance.setStatus(RunStatusEnum.FAIL.getValue()); - taskInstanceService.updateTaskExecutionInfo(taskInstanceId, RunStatusEnum.FAIL, null, null, endTime, - totalTime); - asyncNotifyFail(taskInstance, currentStep); - callback(taskInstance, taskInstanceId, RunStatusEnum.FAIL.getValue(), currentStepId, stepStatus); - // 触发任务结束统计分析 - statisticsService.updateEndJobStatistics(taskInstance); - } else { - log.warn("Unsupported task instance run status for refresh task, taskInstanceId={}, status={}", - taskInstanceId, taskInstance.getStatus()); - } - } else { - log.warn("Unsupported task instance run status for refresh task, taskInstanceId={}, status={}", - taskInstanceId, taskInstance.getStatus()); - } - } - - private void goToNextStep(TaskInstanceDTO taskInstance, StepInstanceBaseDTO currentStep) { - long taskInstanceId = taskInstance.getId(); - long currentStepId = currentStep.getId(); - List taskStepIdList = taskInstanceService.getTaskStepIdList(taskInstanceId); - long nextStepId = getNextStepId(taskStepIdList, currentStepId); - // 当前步骤执行成功或者用户手动跳过时，均应该进入下一步 - if (nextStepId == currentStepId) { // 当前执行步骤为最后一步 - Long endTime = DateUtils.currentTimeMillis(); - long totalTime = TaskCostCalculator.calculate(taskInstance.getStartTime(), endTime, - taskInstance.getTotalTime()); - taskInstanceService.updateTaskExecutionInfo(taskInstanceId, RunStatusEnum.SUCCESS, null, null, endTime, - totalTime); - - asyncNotifySuccess(taskInstance, currentStep); - callback(taskInstance, taskInstanceId, RunStatusEnum.SUCCESS.getValue(), currentStepId, - currentStep.getStatus()); - } else { // 进入下一步 - taskInstanceService.updateTaskCurrentStepId(taskInstanceId, nextStepId); - - taskExecuteControlMsgSender.startStep(nextStepId); - } - } - - private void callback(TaskInstanceDTO taskInstance, long taskInstanceId, int taskStatus, long currentStepId, - int stepStatus) { - if (taskInstance.getCallbackUrl() != null) { - JobCallbackDTO dto = new JobCallbackDTO(); - dto.setId(taskInstanceId); - dto.setStatus(taskStatus); - dto.setCallbackUrl(taskInstance.getCallbackUrl()); - Collection instances = Lists.newArrayList(); - JobCallbackDTO.StepInstanceStatus e = new JobCallbackDTO.StepInstanceStatus(); - e.setId(currentStepId); - e.setStatus(stepStatus); - instances.add(e); - dto.setStepInstances(instances); - taskExecuteControlMsgSender.sendCallback(dto); - } - } - - public long getNextStepId(List stepIdList, long currentStepId) { - int currentStepIndex = stepIdList.indexOf(currentStepId); - int nextStepIndex = Math.min(currentStepIndex + 1, stepIdList.size() - 1); - return stepIdList.get(nextStepIndex); - } - - private void setResourceInfo(TaskInstanceDTO taskInstance, StepInstanceBaseDTO stepInstance, - TaskNotifyDTO taskNotifyDTO) { - Long taskPlanId = taskInstance.getTaskId(); - taskNotifyDTO.setResourceId(String.valueOf(taskPlanId)); - if (taskPlanId == -1L) { - if (stepInstance.getExecuteType().equals(StepExecuteTypeEnum.EXECUTE_SCRIPT.getValue())) { - taskNotifyDTO.setResourceType(ResourceTypeEnum.SCRIPT.getType()); - } else if (stepInstance.getExecuteType().equals(StepExecuteTypeEnum.SEND_FILE.getValue())) { - taskNotifyDTO.setResourceType(ResourceTypeEnum.FILE.getType()); - } else { - log.warn("notify resourceType not supported yet:{}, use Job", stepInstance.getExecuteType()); - taskNotifyDTO.setResourceType(ResourceTypeEnum.JOB.getType()); - } - } else { - taskNotifyDTO.setResourceType(ResourceTypeEnum.JOB.getType()); - } - } - - private void asyncNotifyFail(TaskInstanceDTO taskInstance, StepInstanceBaseDTO stepInstance) { - TaskNotifyDTO taskNotifyDTO = buildCommonTaskNotification(taskInstance, stepInstance); - taskNotifyDTO.setResourceExecuteStatus(ExecuteStatusEnum.FAIL.getStatus()); - taskNotifyDTO.setStepName(stepInstance.getName()); - setResourceInfo(taskInstance, stepInstance, taskNotifyDTO); - taskExecuteControlMsgSender.asyncSendNotifyMsg(taskNotifyDTO); - } - - private void asyncNotifySuccess(TaskInstanceDTO taskInstance, StepInstanceBaseDTO stepInstance) { - TaskNotifyDTO taskNotifyDTO = buildCommonTaskNotification(taskInstance, stepInstance); - taskNotifyDTO.setResourceExecuteStatus(ExecuteStatusEnum.SUCCESS.getStatus()); - taskNotifyDTO.setCost(taskInstance.getTotalTime()); - setResourceInfo(taskInstance, stepInstance, taskNotifyDTO); - taskExecuteControlMsgSender.asyncSendNotifyMsg(taskNotifyDTO); - } - - private TaskNotifyDTO buildCommonTaskNotification(TaskInstanceDTO taskInstance, StepInstanceBaseDTO stepInstance) { - TaskNotifyDTO taskNotifyDTO = new TaskNotifyDTO(); - taskNotifyDTO.setAppId(taskInstance.getAppId()); - String operator = getOperator(taskInstance.getOperator(), stepInstance.getOperator()); - taskNotifyDTO.setOperator(operator); - taskNotifyDTO.setTaskInstanceId(taskInstance.getId()); - taskNotifyDTO.setStartupMode(taskInstance.getStartupMode()); - taskNotifyDTO.setTaskId(taskInstance.getTaskId()); - taskNotifyDTO.setTaskInstanceName(taskInstance.getName()); - return taskNotifyDTO; - } - - private String getOperator(String taskOperator, String stepOperator) { - return StringUtils.isNotEmpty(stepOperator) ? stepOperator : taskOperator; - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ResultGroupBaseDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/Event.java similarity index 81% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ResultGroupBaseDTO.java rename to src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/Event.java index 34faabb1d0..89ed821450 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ResultGroupBaseDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/Event.java @@ -22,25 +22,25 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.execute.model; +package com.tencent.bk.job.execute.engine.listener.event; +import com.fasterxml.jackson.annotation.JsonInclude; import lombok.Data; +import java.time.LocalDateTime; + /** - * 执行结果分组 + * 执行引擎调度事件基础类 */ @Data -public class ResultGroupBaseDTO { - /** - * 执行结果 - */ - private Integer resultType; +@JsonInclude(JsonInclude.Include.NON_NULL) +public class Event { /** - * 用户脚本输出的结果分类tag + * 事件源 */ - private String tag; + protected EventSource source; /** - * 分组下的Agent任务数 + * 事件发生时间 */ - private int agentTaskCount; + protected LocalDateTime time; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/EventSource.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/EventSource.java new file mode 100644 index 0000000000..475c3fb06e --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/EventSource.java @@ -0,0 +1,101 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.listener.event; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.execute.engine.consts.EventSourceTypeEnum; +import lombok.Data; + +/** + * 执行引擎事件源 + */ +@Data +@JsonInclude(JsonInclude.Include.NON_NULL) +public class EventSource { + /** + * Job 执行引擎任务调度事件源类型 + */ + private EventSourceTypeEnum sourceType; + + /** + * 作业实例ID + */ + private Long jobInstanceId; + + /** + * 步骤实例ID + */ + private Long stepInstanceId; + + /** + * 步骤执行次数 + */ + private Integer executeCount; + + /** + * 滚动执行批次 + */ + private Integer batch; + + /** + * GSE 任务ID + */ + private Long gseTaskId; + + /** + * 构造事件源 - 步骤 + * + * @param stepInstanceId 步骤实例ID + * @return 执行引擎事件源 + */ + public static EventSource buildStepEventSource(long stepInstanceId) { + EventSource eventSource = new EventSource(); + eventSource.setStepInstanceId(stepInstanceId); + eventSource.setSourceType(EventSourceTypeEnum.STEP); + return eventSource; + } + + /** + * 构造事件源 - GSE 任务 + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 步骤执行次数 + * @param batch 滚动执行批次 + * @param gseTaskId GSE 任务ID + * @return 执行引擎事件源 + */ + public static EventSource buildGseTaskEventSource(Long stepInstanceId, + Integer executeCount, + Integer batch, + Long gseTaskId) { + EventSource eventSource = new EventSource(); + eventSource.setStepInstanceId(stepInstanceId); + eventSource.setExecuteCount(executeCount); + eventSource.setBatch(batch); + eventSource.setGseTaskId(gseTaskId); + eventSource.setSourceType(EventSourceTypeEnum.GSE_TASK); + return eventSource; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/GseTaskEvent.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/GseTaskEvent.java new file mode 100644 index 0000000000..dbd4494e00 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/GseTaskEvent.java @@ -0,0 +1,141 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.listener.event; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.execute.engine.consts.GseTaskActionEnum; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.apache.commons.lang3.StringUtils; + +import java.time.LocalDateTime; +import java.util.StringJoiner; +import java.util.UUID; + +/** + * 执行引擎-GSE任务事件 + */ +@Getter +@Setter +@NoArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) +public class GseTaskEvent extends Event { + /** + * GSE任务操作 + * + * @see com.tencent.bk.job.execute.engine.consts.GseTaskActionEnum + */ + private int action; + /** + * 步骤实例ID + */ + private Long stepInstanceId; + /** + * 执行次数 + */ + private Integer executeCount; + /** + * 滚动批次 + */ + private Integer batch; + /** + * GSE任务ID + */ + private Long gseTaskId; + /** + * 请求ID,防止重复下发任务 + */ + private String requestId; + + /** + * 构造启动 GSE 任务 事件 + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @param batch 滚动批次 + * @param gseTaskId GSE任务ID + * @param requestId 请求ID,防止重复下发任务 + * @return 事件 + */ + public static GseTaskEvent startGseTask(Long stepInstanceId, + Integer executeCount, + Integer batch, + Long gseTaskId, + String requestId) { + GseTaskEvent gseTaskEvent = buildGseTaskEvent(stepInstanceId, executeCount, batch, gseTaskId); + if (StringUtils.isNotEmpty(requestId)) { + gseTaskEvent.setRequestId(requestId); + } else { + gseTaskEvent.setRequestId(UUID.randomUUID().toString()); + } + gseTaskEvent.setAction(GseTaskActionEnum.START.getValue()); + return gseTaskEvent; + } + + /** + * 构造停止 GSE 任务 事件 + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @param batch 滚动批次 + * @param gseTaskId GSE任务ID + * @return 事件 + */ + public static GseTaskEvent stopGseTask(Long stepInstanceId, + Integer executeCount, + Integer batch, + Long gseTaskId) { + GseTaskEvent gseTaskEvent = buildGseTaskEvent(stepInstanceId, executeCount, batch, gseTaskId); + gseTaskEvent.setAction(GseTaskActionEnum.STOP.getValue()); + return gseTaskEvent; + } + + private static GseTaskEvent buildGseTaskEvent(Long stepInstanceId, + Integer executeCount, + Integer batch, + Long gseTaskId) { + GseTaskEvent gseTaskEvent = new GseTaskEvent(); + gseTaskEvent.setGseTaskId(gseTaskId); + gseTaskEvent.setStepInstanceId(stepInstanceId); + gseTaskEvent.setExecuteCount(executeCount); + gseTaskEvent.setBatch(batch); + gseTaskEvent.setTime(LocalDateTime.now()); + + return gseTaskEvent; + } + + @Override + public String toString() { + return new StringJoiner(", ", GseTaskEvent.class.getSimpleName() + "[", "]") + .add("action=" + action) + .add("actionDesc=" + GseTaskActionEnum.valueOf(action)) + .add("gseTaskId=" + gseTaskId) + .add("requestId='" + requestId + "'") + .add("source=" + source) + .add("time=" + time) + .toString(); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/JobEvent.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/JobEvent.java new file mode 100644 index 0000000000..00f822e46a --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/JobEvent.java @@ -0,0 +1,125 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.listener.event; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.execute.engine.consts.JobActionEnum; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.time.LocalDateTime; +import java.util.StringJoiner; + +/** + * 执行引擎-Job事件 + */ +@Getter +@Setter +@NoArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) +public class JobEvent extends Event { + /** + * 作业操作 + * + * @see com.tencent.bk.job.execute.engine.consts.JobActionEnum + */ + private int action; + /** + * 作业实例ID + */ + @JsonProperty("jobInstanceId") + private Long jobInstanceId; + + /** + * 构造启动作业事件 + * + * @param jobInstanceId 作业实例ID + * @return 事件 + */ + public static JobEvent startJob(long jobInstanceId) { + JobEvent jobEvent = new JobEvent(); + jobEvent.setJobInstanceId(jobInstanceId); + jobEvent.setAction(JobActionEnum.START.getValue()); + jobEvent.setTime(LocalDateTime.now()); + return jobEvent; + } + + /** + * 构造停止作业事件 + * + * @param jobInstanceId 作业实例ID + * @return 事件 + */ + public static JobEvent stopJob(long jobInstanceId) { + JobEvent jobEvent = new JobEvent(); + jobEvent.setJobInstanceId(jobInstanceId); + jobEvent.setAction(JobActionEnum.STOP.getValue()); + jobEvent.setTime(LocalDateTime.now()); + return jobEvent; + } + + /** + * 构造重新执行作业事件 + * + * @param jobInstanceId 作业实例ID + * @return 事件 + */ + public static JobEvent restartJob(long jobInstanceId) { + JobEvent jobEvent = new JobEvent(); + jobEvent.setJobInstanceId(jobInstanceId); + jobEvent.setAction(JobActionEnum.RESTART.getValue()); + jobEvent.setTime(LocalDateTime.now()); + return jobEvent; + } + + /** + * 构造刷新作业事件 + * + * @param jobInstanceId 作业实例ID + * @param eventSource 事件源 + * @return 事件 + */ + public static JobEvent refreshJob(long jobInstanceId, EventSource eventSource) { + JobEvent jobEvent = new JobEvent(); + jobEvent.setJobInstanceId(jobInstanceId); + jobEvent.setSource(eventSource); + jobEvent.setAction(JobActionEnum.REFRESH.getValue()); + jobEvent.setTime(LocalDateTime.now()); + return jobEvent; + } + + @Override + public String toString() { + return new StringJoiner(", ", JobEvent.class.getSimpleName() + "[", "]") + .add("action=" + action) + .add("actionDesc=" + JobActionEnum.valueOf(action)) + .add("jobInstanceId=" + jobInstanceId) + .add("eventSource=" + source) + .add("time=" + time) + .toString(); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/ResultHandleTaskResumeEvent.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/ResultHandleTaskResumeEvent.java new file mode 100644 index 0000000000..8ec4909866 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/ResultHandleTaskResumeEvent.java @@ -0,0 +1,89 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.listener.event; + +import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; + +import java.time.LocalDateTime; + +/** + * 执行引擎-GSE任务结果处理任务恢复事件 + */ +@Getter +@Setter +@ToString +@NoArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) +public class ResultHandleTaskResumeEvent extends Event { + /** + * 步骤实例ID + */ + private Long stepInstanceId; + /** + * 执行次数 + */ + private Integer executeCount; + /** + * 滚动批次 + */ + private Integer batch; + /** + * GSE 任务ID + */ + private Long gseTaskId; + /** + * 请求ID,防止重复下发任务 + */ + private String requestId; + + /** + * 构造恢复任务事件 + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @param batch 滚动批次 + * @param gseTaskId GSE 任务ID + * @param requestId 请求ID,防止重复下发任务 + * @return 事件 + */ + public static ResultHandleTaskResumeEvent resume(Long stepInstanceId, + Integer executeCount, + Integer batch, + Long gseTaskId, + String requestId) { + ResultHandleTaskResumeEvent event = new ResultHandleTaskResumeEvent(); + event.setStepInstanceId(stepInstanceId); + event.setExecuteCount(executeCount); + event.setBatch(batch); + event.setGseTaskId(gseTaskId); + event.setRequestId(requestId); + event.setTime(LocalDateTime.now()); + return event; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/StepEvent.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/StepEvent.java new file mode 100644 index 0000000000..6b326b176e --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/StepEvent.java @@ -0,0 +1,241 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.listener.event; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.execute.engine.consts.StepActionEnum; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.time.LocalDateTime; +import java.util.StringJoiner; + +/** + * 执行引擎-步骤事件 + */ +@Getter +@Setter +@NoArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) +public class StepEvent extends Event { + /** + * 步骤操作 + * + * @see com.tencent.bk.job.execute.engine.consts.StepActionEnum + */ + private int action; + /** + * 步骤实例ID + */ + private long stepInstanceId; + /** + * 滚动执行批次 + */ + private Integer batch; + + /** + * 构造`忽略错误`事件 + * + * @param stepInstanceId 步骤实例ID + * @return 事件 + */ + public static StepEvent ignoreError(long stepInstanceId) { + StepEvent stepEvent = new StepEvent(); + stepEvent.setStepInstanceId(stepInstanceId); + stepEvent.setAction(StepActionEnum.IGNORE_ERROR.getValue()); + stepEvent.setTime(LocalDateTime.now()); + return stepEvent; + } + + /** + * 构造`下一步`事件 + * + * @param stepInstanceId 步骤实例ID + * @return 事件 + */ + public static StepEvent nextStep(long stepInstanceId) { + StepEvent stepEvent = new StepEvent(); + stepEvent.setStepInstanceId(stepInstanceId); + stepEvent.setAction(StepActionEnum.NEXT_STEP.getValue()); + stepEvent.setTime(LocalDateTime.now()); + return stepEvent; + } + + /** + * 构造`人工确认-重新发起确认`事件 + * + * @param stepInstanceId 步骤实例ID + * @return 事件 + */ + public static StepEvent confirmStepContinue(long stepInstanceId) { + StepEvent stepEvent = new StepEvent(); + stepEvent.setStepInstanceId(stepInstanceId); + stepEvent.setAction(StepActionEnum.CONFIRM_CONTINUE.getValue()); + stepEvent.setTime(LocalDateTime.now()); + return stepEvent; + } + + /** + * 构造`人工确认-终止流程`事件 + * + * @param stepInstanceId 步骤实例ID + * @return 事件 + */ + public static StepEvent confirmStepTerminate(long stepInstanceId) { + StepEvent stepEvent = new StepEvent(); + stepEvent.setStepInstanceId(stepInstanceId); + stepEvent.setAction(StepActionEnum.CONFIRM_TERMINATE.getValue()); + stepEvent.setTime(LocalDateTime.now()); + return stepEvent; + } + + /** + * 构造`人工确认-重新发起确认`事件 + * + * @param stepInstanceId 步骤实例ID + * @return 事件 + */ + public static StepEvent confirmStepRestart(long stepInstanceId) { + StepEvent stepEvent = new StepEvent(); + stepEvent.setStepInstanceId(stepInstanceId); + stepEvent.setAction(StepActionEnum.CONFIRM_RESTART.getValue()); + stepEvent.setTime(LocalDateTime.now()); + return stepEvent; + } + + /** + * 构造`启动步骤`事件 + * + * @param stepInstanceId 步骤实例ID + * @param batch 滚动执行批次 + * @return 事件 + */ + public static StepEvent startStep(long stepInstanceId, Integer batch) { + StepEvent stepEvent = new StepEvent(); + stepEvent.setStepInstanceId(stepInstanceId); + stepEvent.setBatch(batch); + stepEvent.setAction(StepActionEnum.START.getValue()); + stepEvent.setTime(LocalDateTime.now()); + return stepEvent; + } + + /** + * 构造`跳过步骤`事件 + * + * @param stepInstanceId 步骤实例ID + * @return 事件 + */ + public static StepEvent skipStep(long stepInstanceId) { + StepEvent stepEvent = new StepEvent(); + stepEvent.setStepInstanceId(stepInstanceId); + stepEvent.setAction(StepActionEnum.SKIP.getValue()); + stepEvent.setTime(LocalDateTime.now()); + return stepEvent; + } + + /** + * 构造`停止步骤`事件 + * + * @param stepInstanceId 步骤实例ID + * @return 事件 + */ + public static StepEvent stopStep(long stepInstanceId) { + StepEvent stepEvent = new StepEvent(); + stepEvent.setStepInstanceId(stepInstanceId); + stepEvent.setAction(StepActionEnum.STOP.getValue()); + stepEvent.setTime(LocalDateTime.now()); + return stepEvent; + } + + /** + * 构造`失败IP重试`事件 + * + * @param stepInstanceId 步骤实例ID + * @return 事件 + */ + public static StepEvent retryStepFail(long stepInstanceId) { + StepEvent stepEvent = new StepEvent(); + stepEvent.setStepInstanceId(stepInstanceId); + stepEvent.setAction(StepActionEnum.RETRY_FAIL.getValue()); + stepEvent.setTime(LocalDateTime.now()); + return stepEvent; + } + + /** + * 构造`全部重试`事件 + * + * @param stepInstanceId 步骤实例ID + * @return 事件 + */ + public static StepEvent retryStepAll(long stepInstanceId) { + StepEvent stepEvent = new StepEvent(); + stepEvent.setStepInstanceId(stepInstanceId); + stepEvent.setAction(StepActionEnum.RETRY_ALL.getValue()); + stepEvent.setTime(LocalDateTime.now()); + return stepEvent; + } + + /** + * 构造`第三方文件源文件拉取完成后继续GSE分发`事件 + * + * @param stepInstanceId 步骤实例ID + * @return 事件 + */ + public static StepEvent continueGseFileStep(long stepInstanceId) { + StepEvent stepEvent = new StepEvent(); + stepEvent.setStepInstanceId(stepInstanceId); + stepEvent.setAction(StepActionEnum.CONTINUE_FILE_PUSH.getValue()); + stepEvent.setTime(LocalDateTime.now()); + return stepEvent; + } + + /** + * 构造`刷新步骤状态`事件 + * + * @param stepInstanceId 步骤实例ID + * @return 事件 + */ + public static StepEvent refreshStep(long stepInstanceId, EventSource eventSource) { + StepEvent stepEvent = new StepEvent(); + stepEvent.setStepInstanceId(stepInstanceId); + stepEvent.setAction(StepActionEnum.REFRESH.getValue()); + stepEvent.setSource(eventSource); + stepEvent.setTime(LocalDateTime.now()); + return stepEvent; + } + + @Override + public String toString() { + return new StringJoiner(", ", StepEvent.class.getSimpleName() + "[", "]") + .add("action=" + action) + .add("actionDesc=" + StepActionEnum.valueOf(action)) + .add("stepInstanceId=" + stepInstanceId) + .add("batch=" + batch) + .add("source=" + source) + .add("time=" + time) + .toString(); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/TaskExecuteMQEventDispatcher.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/TaskExecuteMQEventDispatcher.java new file mode 100644 index 0000000000..dbf64ed009 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/TaskExecuteMQEventDispatcher.java @@ -0,0 +1,154 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.listener.event; + +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.execute.engine.message.CallbackProcessor; +import com.tencent.bk.job.execute.engine.message.GseTaskProcessor; +import com.tencent.bk.job.execute.engine.message.NotifyMsgProcessor; +import com.tencent.bk.job.execute.engine.message.StepProcessor; +import com.tencent.bk.job.execute.engine.message.TaskProcessor; +import com.tencent.bk.job.execute.engine.message.TaskResultHandleResumeProcessor; +import com.tencent.bk.job.execute.engine.model.JobCallbackDTO; +import com.tencent.bk.job.execute.model.TaskNotifyDTO; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.messaging.MessageChannel; +import org.springframework.messaging.support.MessageBuilder; +import org.springframework.stereotype.Component; + +/** + * 作业执行MQ事件分发 + */ +@Component +@Slf4j +public class TaskExecuteMQEventDispatcher { + /** + * 消息通道-作业 + */ + private final MessageChannel taskOutput; + /** + * 消息通道-步骤 + */ + private final MessageChannel stepOutput; + /** + * 消息通道-GSE任务 + */ + private final MessageChannel gseTaskOutput; + + /** + * 消息通道-消息通知 + */ + private final MessageChannel notifyMsgOutput; + /** + * 消息通道-作业执行完成回调 + */ + private final MessageChannel callbackOutput; + + /** + * 消息通道-作业执行完成回调 + */ + private final MessageChannel resultHandleTaskResumeOutput; + + @Autowired + public TaskExecuteMQEventDispatcher(@Qualifier(TaskProcessor.OUTPUT) MessageChannel taskOutput, + @Qualifier(StepProcessor.OUTPUT) MessageChannel stepOutput, + @Qualifier(GseTaskProcessor.OUTPUT) MessageChannel gseTaskOutput, + @Qualifier(NotifyMsgProcessor.OUTPUT) MessageChannel notifyMsgOutput, + @Qualifier(CallbackProcessor.OUTPUT) MessageChannel callbackOutput, + @Qualifier(TaskResultHandleResumeProcessor.OUTPUT) MessageChannel resultHandleTaskResumeOutput) { + this.taskOutput = taskOutput; + this.stepOutput = stepOutput; + this.gseTaskOutput = gseTaskOutput; + this.notifyMsgOutput = notifyMsgOutput; + this.callbackOutput = callbackOutput; + this.resultHandleTaskResumeOutput = resultHandleTaskResumeOutput; + } + + /** + * 分发作业事件 + * + * @param jobEvent 作业事件 + */ + public void dispatchJobEvent(JobEvent jobEvent) { + log.info("Begin to dispatch job event, event: {}", jobEvent); + taskOutput.send(MessageBuilder.withPayload(jobEvent).build()); + log.info("Dispatch job event successfully, event: {}", jobEvent); + } + + /** + * 分发步骤事件 + * + * @param stepEvent 步骤事件 + */ + public void dispatchStepEvent(StepEvent stepEvent) { + log.info("Begin to dispatch step event, event: {}", stepEvent); + stepOutput.send(MessageBuilder.withPayload(stepEvent).build()); + log.info("Dispatch step event successfully, event: {}", stepEvent); + } + + /** + * 分发GSE任务事件 + * + * @param gseTaskEvent GSE任务事件 + */ + public void dispatchGseTaskEvent(GseTaskEvent gseTaskEvent) { + log.info("Begin to dispatch gse task event, event: {}", gseTaskEvent); + gseTaskOutput.send(MessageBuilder.withPayload(gseTaskEvent).build()); + log.info("Dispatch gse task event successfully, event: {}", gseTaskEvent); + } + + /** + * 分发结果处理任务恢复事件 + * + * @param event 结果处理任务恢复事件 + */ + public void dispatchResultHandleTaskResumeEvent(ResultHandleTaskResumeEvent event) { + log.info("Begin to dispatch gse task result handle resume event, event: {}", event); + resultHandleTaskResumeOutput.send(MessageBuilder.withPayload(event).build()); + log.info("Dispatch gse task result handle resume event successfully, event: {}", event); + } + + /** + * 异步发送消息通知事件 + * + * @param notification 消息内容 + */ + public void asyncSendNotifyMsg(TaskNotifyDTO notification) { + log.info("Async send notification event:{}", JsonUtils.toJson(notification)); + notifyMsgOutput.send(MessageBuilder.withPayload(JsonUtils.toJson(notification)).build()); + } + + /** + * 发送回调信息事件 + * + * @param jobCallback 回调内容 + */ + public void sendCallback(JobCallbackDTO jobCallback) { + log.info("Async invoke callback url, callback:{}", JsonUtils.toJson(jobCallback)); + callbackOutput.send(MessageBuilder.withPayload(JsonUtils.toJson(jobCallback)).build()); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/JobFile.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/JobFile.java index b4b7992b9b..db1e06fba4 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/JobFile.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/JobFile.java @@ -24,31 +24,27 @@ package com.tencent.bk.job.execute.engine.model; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.engine.util.FilePathUtils; +import com.tencent.bk.job.manage.common.consts.task.TaskFileTypeEnum; import lombok.Data; import lombok.ToString; import org.apache.commons.lang3.StringUtils; -import java.util.Objects; - /** - * 分发的单个文件信息（一个IP+一个Path），含用户输入的原始路径、解析后的真实路径、用于展示的路径等信息 + * 分发的单个文件信息，含用户输入的原始路径、解析后的真实路径、用于展示的路径等信息 */ @Data @ToString(exclude = {"password"}) public class JobFile { /** - * 是否本地文件 - */ - private boolean localUploadFile; - /** - * 是否包含敏感信息 + * 文件类型 */ - private boolean sensitive; + private TaskFileTypeEnum fileType; /** - * 文件源主机ip + * 源文件主机 */ - private String cloudAreaIdAndIp; + private HostDTO host; /** * 文件路径(用户输入) */ @@ -97,43 +93,55 @@ public class JobFile { private String uniqueKey; /** - * @param cloudAreaIdAndIp 云区域:IP - * @param dir 目录名称 - * @param fileName 文件名 - */ - public JobFile(String cloudAreaIdAndIp, String dir, String fileName) { - this.cloudAreaIdAndIp = cloudAreaIdAndIp; - this.dir = dir; - this.fileName = fileName; - } - - /** - * @param isLocalUploadFile 是否本地文件 - * @param cloudAreaIdAndIp 云区域:IP - * @param filePath 文件路径 - * @param dir 目录名称 - * @param fileName 文件名 - * @param account 源文件账号 - * @param password 源文件密码 - * @param displayFilePath 要展示的文件路径 - */ - public JobFile(boolean isLocalUploadFile, String cloudAreaIdAndIp, String filePath, String dir, - String fileName, String account, String password, String displayFilePath) { - this.localUploadFile = isLocalUploadFile; - this.cloudAreaIdAndIp = cloudAreaIdAndIp; + * @param fileType 文件类型 + * @param host 源文件主机 + * @param filePath 文件路径 + * @param dir 目录名称 + * @param fileName 文件名 + * @param account 源文件账号 + * @param password 源文件密码 + * @param displayFilePath 要展示的文件路径 + */ + public JobFile(TaskFileTypeEnum fileType, + HostDTO host, + String filePath, + String dir, + String fileName, + String account, + String password, + String displayFilePath) { + this.fileType = fileType; + this.host = host; this.filePath = filePath; this.dir = dir; this.fileName = fileName; this.account = account; this.password = password; this.displayFilePath = displayFilePath; - this.sensitive = this.localUploadFile; } - public JobFile(boolean isLocalUploadFile, String cloudAreaIdAndIp, String filePath, String displayFilePath, - String dir, String fileName, Long appId, Long accountId, String accountAlias) { - this.localUploadFile = isLocalUploadFile; - this.cloudAreaIdAndIp = cloudAreaIdAndIp; + /** + * @param fileType 文件类型 + * @param host 源文件主机 + * @param filePath 文件路径 + * @param displayFilePath 要展示的文件路径 + * @param dir 目录名称 + * @param fileName 文件名 + * @param appId 业务ID + * @param accountId 账号ID + * @param accountAlias 账号别名 + */ + public JobFile(TaskFileTypeEnum fileType, + HostDTO host, + String filePath, + String displayFilePath, + String dir, + String fileName, + Long appId, + Long accountId, + String accountAlias) { + this.fileType = fileType; + this.host = host; this.filePath = filePath; this.displayFilePath = displayFilePath; this.dir = dir; @@ -141,7 +149,6 @@ public JobFile(boolean isLocalUploadFile, String cloudAreaIdAndIp, String filePa this.appId = appId; this.accountId = accountId; this.accountAlias = accountAlias; - this.sensitive = this.localUploadFile; } @Override @@ -157,45 +164,31 @@ public boolean equals(Object obj) { } JobFile target = (JobFile) obj; - if (this.cloudAreaIdAndIp == null || target.cloudAreaIdAndIp == null - || !this.cloudAreaIdAndIp.equals(target.cloudAreaIdAndIp)) { - return false; - } - if (this.dir == null || target.dir == null || !this.dir.equals(target.dir)) { - return false; - } - if (this.fileName == null || target.fileName == null) { - return false; - } - return this.fileName.equals(target.fileName); + return this.getUniqueKey().equals(target.getUniqueKey()); } @Override public int hashCode() { - return Objects.hash(cloudAreaIdAndIp, dir, fileName, account); - } - - public String getDisplayFilePath() { - if (!StringUtils.isEmpty(this.displayFilePath)) { - return this.displayFilePath; - } - if (localUploadFile) { - this.displayFilePath = this.fileName; - } else { - this.displayFilePath = this.filePath; - } - return this.displayFilePath; + return getUniqueKey().hashCode(); } - public String getFileUniqueKey() { + /** + * 获取文件的唯一KEY，用于去重等操作 + * + * @return 文件KEY + */ + public String getUniqueKey() { if (!StringUtils.isEmpty(this.uniqueKey)) { return this.uniqueKey; } - if (localUploadFile) { - this.uniqueKey = fileName; - } else { - this.uniqueKey = getCloudAreaIdAndIp() + ":" + getStandardFilePath(); + StringBuilder sb = new StringBuilder(); + sb.append(fileType.name()).append(":"); + if (fileType == TaskFileTypeEnum.SERVER) { + // 远程文件分发，需要源主机信息才能唯一确定一个源文件 + sb.append(host.getUniqueKey()).append(":"); } + sb.append(getStandardFilePath()); + this.uniqueKey = sb.toString(); return this.uniqueKey; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/LogPullProgress.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/LogPullProgress.java index 7d6a850c91..6551bd2e27 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/LogPullProgress.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/LogPullProgress.java @@ -31,7 +31,13 @@ */ @Data public class LogPullProgress { - private String ip; + /** + * bk_agent_id + */ + private String agentId; + /** + * 任务编号 + */ private int mid; /** * 日志偏移 - 字节 diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/TaskVariablesAnalyzeResult.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/TaskVariablesAnalyzeResult.java index dc93e7f9aa..9b0d92fa8e 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/TaskVariablesAnalyzeResult.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/TaskVariablesAnalyzeResult.java @@ -32,8 +32,8 @@ import java.util.ArrayList; import java.util.List; -/* - *全局变量分析结果 +/** + * 全局变量分析结果 */ @Getter @ToString @@ -43,10 +43,10 @@ public class TaskVariablesAnalyzeResult { private boolean existOnlyConstVar = true; //作业中是否仅存在常量 private boolean existNamespaceVar = false; //是否存在命名空间变量 private boolean existChangeableGlobalVar = false; //是否存在可赋值全局变量 - private List allVarNames = new ArrayList<>(); // 所有变量列表 - private List constVarNames = new ArrayList<>(); // 常量列表 - private List namespaceVarNames = new ArrayList<>(); // 命名空间变量列表 - private List changeableGlobalVarNames = new ArrayList<>(); // 赋值可变变量列表 + private final List allVarNames = new ArrayList<>(); // 所有变量列表 + private final List constVarNames = new ArrayList<>(); // 常量列表 + private final List namespaceVarNames = new ArrayList<>(); // 命名空间变量列表 + private final List changeableGlobalVarNames = new ArrayList<>(); // 赋值可变变量列表 public TaskVariablesAnalyzeResult(List taskVars) { if (CollectionUtils.isEmpty(taskVars)) { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareControlTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareControlTask.java index f194d96a05..88a4373c2d 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareControlTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareControlTask.java @@ -108,10 +108,10 @@ private boolean needToStop(StepInstanceDTO stepInstance) { // 刷新步骤状态 stepInstance = taskInstanceService.getStepInstanceDetail(stepInstance.getId()); // 如果任务处于“终止中”状态，触发任务终止 - if (taskInstance.getStatus().equals(RunStatusEnum.STOPPING.getValue())) { + if (taskInstance.getStatus() == RunStatusEnum.STOPPING) { // 已经发送过停止命令的就不再重复发送了 - return !RunStatusEnum.STOPPING.getValue().equals(stepInstance.getStatus()) - && !RunStatusEnum.STOP_SUCCESS.getValue().equals(stepInstance.getStatus()); + return !(RunStatusEnum.STOPPING == stepInstance.getStatus() + || RunStatusEnum.STOP_SUCCESS == stepInstance.getStatus()); } return false; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareService.java index c1025482a8..a60fa685c7 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareService.java @@ -24,13 +24,15 @@ package com.tencent.bk.job.execute.engine.prepare; +import com.tencent.bk.job.execute.model.StepInstanceDTO; + public interface FilePrepareService { /** * 为后续的分发阶段准备本地/第三方源文件 * - * @param stepInstanceId 步骤实例Id + * @param stepInstance 步骤实例 */ - void prepareFileForGseTask(long stepInstanceId); + void prepareFileForGseTask(StepInstanceDTO stepInstance); /** * 停止正在进行的准备文件过程 @@ -52,4 +54,11 @@ public interface FilePrepareService { * @param stepInstanceId 步骤实例Id */ void clearPreparedTmpFile(long stepInstanceId); + + /** + * 是否需要准备源文件 + * + * @param stepInstance 步骤实例 + */ + boolean needToPrepareSourceFilesForGseTask(StepInstanceDTO stepInstance); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareServiceImpl.java index d84214ed88..d0c5c817a2 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareServiceImpl.java @@ -25,7 +25,10 @@ package com.tencent.bk.job.execute.engine.prepare; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; -import com.tencent.bk.job.execute.engine.TaskExecuteControlMsgSender; +import com.tencent.bk.job.execute.engine.listener.event.EventSource; +import com.tencent.bk.job.execute.engine.listener.event.JobEvent; +import com.tencent.bk.job.execute.engine.listener.event.StepEvent; +import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; import com.tencent.bk.job.execute.engine.prepare.local.LocalFilePrepareService; import com.tencent.bk.job.execute.engine.prepare.local.LocalFilePrepareTaskResultHandler; import com.tencent.bk.job.execute.engine.prepare.third.ThirdFilePrepareService; @@ -35,6 +38,7 @@ import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.service.TaskInstanceService; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; import org.slf4j.helpers.FormattingTuple; import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; @@ -56,20 +60,19 @@ public class FilePrepareServiceImpl implements FilePrepareService { private final LocalFilePrepareService localFilePrepareService; private final ThirdFilePrepareService thirdFilePrepareService; private final TaskInstanceService taskInstanceService; - private final TaskExecuteControlMsgSender taskControlMsgSender; + private final TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher; private final ResultHandleManager resultHandleManager; @Autowired - public FilePrepareServiceImpl( - LocalFilePrepareService localFilePrepareService, - ThirdFilePrepareService thirdFilePrepareService, - TaskInstanceService taskInstanceService, - TaskExecuteControlMsgSender taskControlMsgSender, - ResultHandleManager resultHandleManager) { + public FilePrepareServiceImpl(LocalFilePrepareService localFilePrepareService, + ThirdFilePrepareService thirdFilePrepareService, + TaskInstanceService taskInstanceService, + TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, + ResultHandleManager resultHandleManager) { this.localFilePrepareService = localFilePrepareService; this.thirdFilePrepareService = thirdFilePrepareService; this.taskInstanceService = taskInstanceService; - this.taskControlMsgSender = taskControlMsgSender; + this.taskExecuteMQEventDispatcher = taskExecuteMQEventDispatcher; this.resultHandleManager = resultHandleManager; } @@ -84,24 +87,6 @@ public void clearPreparedTmpFile(long stepInstanceId) { thirdFilePrepareService.clearPreparedTmpFile(stepInstanceId); } - private boolean hasLocalFile(List fileSourceList) { - for (FileSourceDTO fileSourceDTO : fileSourceList) { - if (fileSourceDTO.isLocalUpload()) { - return true; - } - } - return false; - } - - private boolean hasThirdFile(List fileSourceList) { - for (FileSourceDTO fileSourceDTO : fileSourceList) { - if (fileSourceDTO.getFileSourceId() != null && fileSourceDTO.getFileSourceId() > 0) { - return true; - } - } - return false; - } - private void startPrepareLocalFileTask(long stepInstanceId, List fileSourceList, List resultList, @@ -167,12 +152,11 @@ public void onFailed(JobTaskContext taskContext) { } @Override - public void prepareFileForGseTask(long stepInstanceId) { - StepInstanceDTO stepInstance = taskInstanceService.getStepInstanceDetail(stepInstanceId); + public void prepareFileForGseTask(StepInstanceDTO stepInstance) { + log.info("Begin to prepare source files for step, stepInstanceId: {}", stepInstance.getId()); List fileSourceList = stepInstance.getFileSourceList(); - if (fileSourceList == null) { - log.warn("stepInstanceId={},fileSourceList is null", stepInstanceId); - taskControlMsgSender.startGseStep(stepInstance.getId()); + if (CollectionUtils.isEmpty(fileSourceList)) { + log.error("FileSource is empty, stepInstanceId: {}", stepInstance.getId()); return; } int taskCount = 0; @@ -182,15 +166,14 @@ public void prepareFileForGseTask(long stepInstanceId) { if (hasThirdFile) taskCount += 1; if (taskCount == 0) { // 没有需要准备文件的本地文件/第三方源文件 - taskControlMsgSender.startGseStep(stepInstance.getId()); + log.error("FileSource no need to prepare, stepInstanceId: {}", stepInstance.getId()); return; } - log.debug("stepInstanceId={},prepareTaskCount={}", stepInstanceId, taskCount); CountDownLatch latch = new CountDownLatch(taskCount); final List resultList = Collections.synchronizedList(new ArrayList<>(taskCount)); if (hasLocalFile) { // 启动异步准备本地文件任务 - startPrepareLocalFileTask(stepInstanceId, fileSourceList, resultList, latch); + startPrepareLocalFileTask(stepInstance.getId(), fileSourceList, resultList, latch); } if (hasThirdFile) { // 启动异步准备第三方源文件任务 @@ -232,6 +215,24 @@ public void onException(StepInstanceDTO stepInstance, Throwable t) { resultHandleManager.handleDeliveredTask(filePrepareControlTask); } + private boolean hasLocalFile(List fileSourceList) { + for (FileSourceDTO fileSourceDTO : fileSourceList) { + if (fileSourceDTO.isLocalUpload()) { + return true; + } + } + return false; + } + + private boolean hasThirdFile(List fileSourceList) { + for (FileSourceDTO fileSourceDTO : fileSourceList) { + if (fileSourceDTO.getFileSourceId() != null && fileSourceDTO.getFileSourceId() > 0) { + return true; + } + } + return false; + } + @Override public void stopPrepareFile(long stepInstanceId) { localFilePrepareService.stopPrepareLocalFilesAsync(stepInstanceId); @@ -272,7 +273,7 @@ private void handleFinalTaskResult(List resultList, StepI private void onSuccess(StepInstanceDTO stepInstance, FilePrepareTaskResult finalResult) { if (!finalResult.getTaskContext().isForRetry()) { // 直接进行下一步 - taskControlMsgSender.continueGseFileStep(stepInstance.getId()); + taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.continueGseFileStep(stepInstance.getId())); } } @@ -280,14 +281,25 @@ private void onStopped(StepInstanceDTO stepInstance, FilePrepareTaskResult final // 步骤状态变更 taskInstanceService.updateStepStatus(stepInstance.getId(), RunStatusEnum.STOP_SUCCESS.getValue()); // 任务状态变更 - taskControlMsgSender.refreshTask(stepInstance.getTaskInstanceId()); - // 强制终止成功后就不再下发GSE Task了 - // taskControlMsgSender.continueGseFileStep(stepInstance.getId()); + taskExecuteMQEventDispatcher.dispatchJobEvent( + JobEvent.refreshJob(stepInstance.getTaskInstanceId(), + EventSource.buildStepEventSource(stepInstance.getId()))); } private void onFailed(StepInstanceDTO stepInstance, FilePrepareTaskResult finalResult) { // 文件源文件下载失败 taskInstanceService.updateStepStatus(stepInstance.getId(), RunStatusEnum.FAIL.getValue()); - taskControlMsgSender.refreshTask(stepInstance.getTaskInstanceId()); + taskExecuteMQEventDispatcher.dispatchJobEvent( + JobEvent.refreshJob(stepInstance.getTaskInstanceId(), + EventSource.buildStepEventSource(stepInstance.getId()))); + } + + @Override + public boolean needToPrepareSourceFilesForGseTask(StepInstanceDTO stepInstance) { + List fileSourceList = stepInstance.getFileSourceList(); + if (fileSourceList == null) { + return false; + } + return hasLocalFile(fileSourceList) || hasThirdFile(fileSourceList); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareService.java index 6e3e701300..0db5ebc860 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareService.java @@ -26,12 +26,13 @@ import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.model.InternalResponse; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.util.ThreadUtils; import com.tencent.bk.job.common.util.file.PathUtil; import com.tencent.bk.job.execute.client.FileSourceTaskResourceClient; import com.tencent.bk.job.execute.dao.FileSourceTaskLogDAO; -import com.tencent.bk.job.execute.engine.TaskExecuteControlMsgSender; +import com.tencent.bk.job.execute.engine.listener.event.GseTaskEvent; +import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; import com.tencent.bk.job.execute.engine.prepare.JobTaskContext; import com.tencent.bk.job.execute.engine.result.ResultHandleManager; import com.tencent.bk.job.execute.model.FileDetailDTO; @@ -40,6 +41,7 @@ import com.tencent.bk.job.execute.model.ServersDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.service.AccountService; +import com.tencent.bk.job.execute.service.HostService; import com.tencent.bk.job.execute.service.LogService; import com.tencent.bk.job.execute.service.TaskInstanceService; import com.tencent.bk.job.file_gateway.consts.TaskStatusEnum; @@ -71,23 +73,28 @@ public class ThirdFilePrepareService { private final TaskInstanceService taskInstanceService; private final FileSourceTaskLogDAO fileSourceTaskLogDAO; private final AccountService accountService; + private final HostService hostService; private final LogService logService; - private final TaskExecuteControlMsgSender taskControlMsgSender; + private final TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher; private final Map taskMap = new ConcurrentHashMap<>(); @Autowired public ThirdFilePrepareService(ResultHandleManager resultHandleManager, FileSourceTaskResourceClient fileSourceTaskResource, TaskInstanceService taskInstanceService, - FileSourceTaskLogDAO fileSourceTaskLogDAO, AccountService accountService, - LogService logService, TaskExecuteControlMsgSender taskControlMsgSender) { + FileSourceTaskLogDAO fileSourceTaskLogDAO, + AccountService accountService, + HostService hostService, + LogService logService, + TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher) { this.resultHandleManager = resultHandleManager; this.fileSourceTaskResource = fileSourceTaskResource; this.taskInstanceService = taskInstanceService; this.fileSourceTaskLogDAO = fileSourceTaskLogDAO; this.accountService = accountService; + this.hostService = hostService; this.logService = logService; - this.taskControlMsgSender = taskControlMsgSender; + this.taskExecuteMQEventDispatcher = taskExecuteMQEventDispatcher; } private void setTaskInfoIntoThirdFileSource(TaskInfoDTO taskInfoDTO, FileSourceDTO fileSourceDTO) { @@ -95,10 +102,10 @@ private void setTaskInfoIntoThirdFileSource(TaskInfoDTO taskInfoDTO, FileSourceD if (fileSourceDTO.getServers() == null) { fileSourceDTO.setServers(new ServersDTO()); } - List ipDTOList = new ArrayList<>(); - ipDTOList.add(new IpDTO(taskInfoDTO.getCloudId(), taskInfoDTO.getIp())); - fileSourceDTO.getServers().setStaticIpList(ipDTOList); - fileSourceDTO.getServers().setIpList(ipDTOList); + List hostDTOList = new ArrayList<>(); + hostDTOList.add(new HostDTO(taskInfoDTO.getCloudId(), taskInfoDTO.getIp())); + fileSourceDTO.getServers().setStaticIpList(hostDTOList); + fileSourceDTO.getServers().setIpList(hostDTOList); fileSourceDTO.setFileSourceTaskId(fileSourceTaskId); fileSourceDTO.getFiles().forEach(fileDetailDTO -> { // 第二次处理，加上文件源名称的文件路径 @@ -209,7 +216,10 @@ public ThirdFilePrepareTask prepareThirdFileAsync( List thirdFileSourceList = thirdFileSource.getLeft(); List fileSourceTaskList = thirdFileSource.getRight(); if (thirdFileSourceList == null || thirdFileSourceList.isEmpty()) { - taskControlMsgSender.startGseStep(stepInstance.getId()); + // TODO-Rolling + taskExecuteMQEventDispatcher.dispatchGseTaskEvent( + GseTaskEvent.startGseTask(stepInstance.getId(), stepInstance.getExecuteCount(), + stepInstance.getBatch(), null, null)); return null; } log.debug("Start FileSourceBatchTask: {}", fileSourceTaskList); @@ -299,7 +309,7 @@ private ThirdFilePrepareTask asyncWatchThirdFilePulling( new RecordableThirdFilePrepareTaskResultHandler(stepInstance.getId(), resultHandler) ); batchResultHandleTask.initDependentService(fileSourceTaskResource, taskInstanceService, accountService, - logService, taskControlMsgSender, fileSourceTaskLogDAO); + hostService, logService, taskExecuteMQEventDispatcher, fileSourceTaskLogDAO); resultHandleManager.handleDeliveredTask(batchResultHandleTask); return batchResultHandleTask; } @@ -312,7 +322,8 @@ private BatchTaskInfoDTO startFileSourceDownloadTask(String username, Long appId req.setStepInstanceId(stepInstanceId); req.setExecuteCount(executeCount); req.setFileSourceTaskList(fileSourceTaskList); - InternalResponse resp = fileSourceTaskResource.startFileSourceBatchDownloadTask(username, req); + InternalResponse resp = fileSourceTaskResource.startFileSourceBatchDownloadTask(username, + req); log.debug("resp={}", resp); if (resp.isSuccess()) { return resp.getData(); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareTask.java index 0fc95d3a1d..b3cb71806b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareTask.java @@ -25,12 +25,15 @@ package com.tencent.bk.job.execute.engine.prepare.third; import com.tencent.bk.job.common.model.InternalResponse; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.execute.client.FileSourceTaskResourceClient; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.dao.FileSourceTaskLogDAO; -import com.tencent.bk.job.execute.engine.TaskExecuteControlMsgSender; +import com.tencent.bk.job.execute.engine.listener.event.EventSource; +import com.tencent.bk.job.execute.engine.listener.event.JobEvent; +import com.tencent.bk.job.execute.engine.listener.event.StepEvent; +import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; import com.tencent.bk.job.execute.engine.prepare.JobTaskContext; import com.tencent.bk.job.execute.engine.result.ContinuousScheduledTask; import com.tencent.bk.job.execute.engine.result.ScheduleStrategy; @@ -42,13 +45,15 @@ import com.tencent.bk.job.execute.model.ServersDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.service.AccountService; +import com.tencent.bk.job.execute.service.HostService; import com.tencent.bk.job.execute.service.LogService; import com.tencent.bk.job.execute.service.TaskInstanceService; import com.tencent.bk.job.file_gateway.consts.TaskStatusEnum; import com.tencent.bk.job.file_gateway.model.req.inner.StopBatchTaskReq; import com.tencent.bk.job.file_gateway.model.resp.inner.BatchTaskStatusDTO; import com.tencent.bk.job.file_gateway.model.resp.inner.FileSourceTaskStatusDTO; -import com.tencent.bk.job.logsvr.model.service.ServiceIpLogDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceHostLogDTO; +import com.tencent.bk.job.manage.model.inner.ServiceHostDTO; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.slf4j.helpers.FormattingTuple; @@ -81,8 +86,9 @@ public class ThirdFilePrepareTask implements ContinuousScheduledTask, JobTaskCon private FileSourceTaskResourceClient fileSourceTaskResource; private TaskInstanceService taskInstanceService; private AccountService accountService; + private HostService hostService; private LogService logService; - private TaskExecuteControlMsgSender taskControlMsgSender; + private TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher; private FileSourceTaskLogDAO fileSourceTaskLogDAO; private ThirdFilePrepareTaskResultHandler resultHandler; private int pullTimes = 0; @@ -111,15 +117,17 @@ public void initDependentService( FileSourceTaskResourceClient fileSourceTaskResource, TaskInstanceService taskInstanceService, AccountService accountService, + HostService hostService, LogService logService, - TaskExecuteControlMsgSender taskControlMsgSender, + TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, FileSourceTaskLogDAO fileSourceTaskLogDAO ) { this.fileSourceTaskResource = fileSourceTaskResource; this.taskInstanceService = taskInstanceService; this.accountService = accountService; + this.hostService = hostService; this.logService = logService; - this.taskControlMsgSender = taskControlMsgSender; + this.taskExecuteMQEventDispatcher = taskExecuteMQEventDispatcher; this.fileSourceTaskLogDAO = fileSourceTaskLogDAO; } @@ -171,7 +179,7 @@ public BatchTaskStatusDTO getFileSourceBatchTaskResults(StepInstanceDTO stepInst int maxLogSize = 0; // 写日志 for (FileSourceTaskStatusDTO fileSourceTaskStatusDTO : fileSourceTaskStatusInfoList) { - List logList = fileSourceTaskStatusDTO.getLogList(); + List logList = fileSourceTaskStatusDTO.getLogList(); if (logList != null && !logList.isEmpty()) { writeLogs(stepInstance, logList); if (logList.size() > maxLogSize) { @@ -212,6 +220,16 @@ public BatchTaskStatusDTO getFileSourceBatchTaskResults(StepInstanceDTO stepInst return batchTaskStatusDTO; } + private void fillHostInfo(HostDTO hostDTO, ServiceHostDTO serviceHostDTO) { + if (hostDTO == null || serviceHostDTO == null) { + return; + } + hostDTO.setHostId(serviceHostDTO.getHostId()); + hostDTO.setBkCloudId(serviceHostDTO.getCloudAreaId()); + hostDTO.setIp(serviceHostDTO.getIp()); + hostDTO.setAgentId(serviceHostDTO.getFinalAgentId()); + } + private void handleFileSourceTaskResult( StepInstanceDTO stepInstance, List fileSourceList, @@ -227,7 +245,7 @@ private void handleFileSourceTaskResult( List resultList = batchTaskStatusDTO.getFileSourceTaskStatusInfoList(); if (resultList.isEmpty()) { // 直接进行下一步 - taskControlMsgSender.continueGseFileStep(stepInstance.getId()); + taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.continueGseFileStep(stepInstance.getId())); } else { // 需要处理业务 boolean allSuccess = true; @@ -256,9 +274,7 @@ private void handleFileSourceTaskResult( } if (allSuccess) { Map map = new HashMap<>(); - resultList.forEach(result -> { - map.put(result.getTaskId(), result); - }); + resultList.forEach(result -> map.put(result.getTaskId(), result)); //添加服务器文件信息 for (FileSourceDTO fileSourceDTO : fileSourceList) { String fileSourceTaskId = fileSourceDTO.getFileSourceTaskId(); @@ -270,19 +286,31 @@ private void handleFileSourceTaskResult( //业务无root账号，报错提示 log.warn("No root account in appId={}, plz config one", stepInstance.getAppId()); taskInstanceService.updateStepStatus(stepInstance.getId(), RunStatusEnum.FAIL.getValue()); - taskControlMsgSender.refreshTask(stepInstance.getTaskInstanceId()); + taskExecuteMQEventDispatcher.dispatchJobEvent( + JobEvent.refreshJob(stepInstance.getTaskInstanceId(), + EventSource.buildStepEventSource(stepInstance.getId()))); return; } fileSourceDTO.setAccountId(accountDTO.getId()); fileSourceDTO.setLocalUpload(false); ServersDTO servers = new ServersDTO(); - IpDTO ipDTO = new IpDTO(fileSourceTaskStatusDTO.getCloudId(), fileSourceTaskStatusDTO.getIp()); - List ipDTOList = Collections.singletonList(ipDTO); - servers.addStaticIps(ipDTOList); + HostDTO hostDTO = new HostDTO(fileSourceTaskStatusDTO.getCloudId(), + fileSourceTaskStatusDTO.getIp()); + ServiceHostDTO serviceHostDTO = hostService.getHost(hostDTO); + if (serviceHostDTO == null) { + log.warn( + "cannot find file-worker host info by {}, " + + "plz check whether file-worker gse agent is installed", + hostDTO + ); + } + fillHostInfo(hostDTO, serviceHostDTO); + List hostDTOList = Collections.singletonList(hostDTO); + servers.addStaticIps(hostDTOList); if (servers.getIpList() == null) { - servers.setIpList(ipDTOList); + servers.setIpList(hostDTOList); } else { - servers.getIpList().addAll(ipDTOList); + servers.getIpList().addAll(hostDTOList); // 去重 servers.setIpList(new ArrayList<>(new HashSet<>(servers.getIpList()))); } @@ -310,10 +338,11 @@ private void handleFileSourceTaskResult( } } - private void writeLogs(StepInstanceDTO stepInstance, List logDTOList) { - for (ServiceIpLogDTO serviceLogDTO : logDTOList) { + private void writeLogs(StepInstanceDTO stepInstance, List logDTOList) { + for (ServiceHostLogDTO serviceLogDTO : logDTOList) { logService.writeFileLogWithTimestamp(stepInstance.getCreateTime(), stepInstance.getId(), - stepInstance.getExecuteCount(), serviceLogDTO.getIp(), serviceLogDTO, System.currentTimeMillis()); + stepInstance.getExecuteCount(), stepInstance.getBatch(), + HostDTO.fromHostId(serviceLogDTO.getHostId()), serviceLogDTO, System.currentTimeMillis()); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/AbstractResultHandleTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/AbstractResultHandleTask.java index 88b98b1f32..68296c2e50 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/AbstractResultHandleTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/AbstractResultHandleTask.java @@ -25,29 +25,33 @@ package com.tencent.bk.job.execute.engine.result; import com.tencent.bk.job.common.constant.JobConstants; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.redis.util.LockUtils; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; -import com.tencent.bk.job.execute.engine.TaskExecuteControlMsgSender; -import com.tencent.bk.job.execute.engine.consts.IpStatus; +import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; import com.tencent.bk.job.execute.engine.evict.TaskEvictPolicyExecutor; -import com.tencent.bk.job.execute.engine.exception.ExceptionStatusManager; +import com.tencent.bk.job.execute.engine.listener.event.EventSource; +import com.tencent.bk.job.execute.engine.listener.event.GseTaskEvent; +import com.tencent.bk.job.execute.engine.listener.event.ResultHandleTaskResumeEvent; +import com.tencent.bk.job.execute.engine.listener.event.StepEvent; +import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; import com.tencent.bk.job.execute.engine.model.GseLog; import com.tencent.bk.job.execute.engine.model.GseLogBatchPullResult; import com.tencent.bk.job.execute.engine.model.GseTaskExecuteResult; import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; import com.tencent.bk.job.execute.engine.model.TaskVariablesAnalyzeResult; import com.tencent.bk.job.execute.engine.result.ha.ResultHandleTaskKeepaliveManager; -import com.tencent.bk.job.execute.engine.util.IpHelper; -import com.tencent.bk.job.execute.model.GseTaskIpLogDTO; -import com.tencent.bk.job.execute.model.GseTaskLogDTO; +import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.model.GseTaskDTO; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; -import com.tencent.bk.job.execute.service.GseTaskLogService; +import com.tencent.bk.job.execute.service.AgentTaskService; +import com.tencent.bk.job.execute.service.GseTaskService; import com.tencent.bk.job.execute.service.LogService; +import com.tencent.bk.job.execute.service.StepInstanceService; import com.tencent.bk.job.execute.service.StepInstanceVariableValueService; import com.tencent.bk.job.execute.service.TaskInstanceService; import com.tencent.bk.job.execute.service.TaskInstanceVariableService; @@ -55,7 +59,6 @@ import org.apache.commons.collections4.CollectionUtils; import org.springframework.util.StopWatch; -import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -89,13 +92,14 @@ public abstract class AbstractResultHandleTask implements ContinuousScheduled // ---------------- dependent service -------------------- protected LogService logService; protected TaskInstanceService taskInstanceService; - protected GseTaskLogService gseTaskLogService; + protected GseTaskService gseTaskService; protected TaskInstanceVariableService taskInstanceVariableService; protected StepInstanceVariableValueService stepInstanceVariableValueService; - protected TaskExecuteControlMsgSender taskManager; + protected TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher; protected ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager; - protected ExceptionStatusManager exceptionStatusManager; protected TaskEvictPolicyExecutor taskEvictPolicyExecutor; + protected AgentTaskService agentTaskService; + protected StepInstanceService stepInstanceService; /** * 任务请求的requestId，用于防止重复下发任务 */ @@ -123,11 +127,11 @@ public abstract class AbstractResultHandleTask implements ContinuousScheduled /** * GSE 任务执行结果 */ - protected GseTaskLogDTO gseTaskLog; + protected GseTaskDTO gseTask; /** - * GSE 主机任务执行结果 + * GSE 主机任务执行结果，Map */ - protected Map ipLogMap; + protected Map targetAgentTasks; /** * 全局参数分析结果 */ @@ -139,29 +143,29 @@ public abstract class AbstractResultHandleTask implements ContinuousScheduled /** * 任务包含的所有目标服务器 */ - protected Set targetIpSet = new HashSet<>(); + protected Set targetAgentIds = new HashSet<>(); /** - * 未开始任务的服务器 + * 未开始任务的目标服务器 */ - protected Set notStartedIpSet = new HashSet<>(); + protected Set notStartedTargetAgentIds = new HashSet<>(); /** - * 正在执行任务的服务器 + * 正在执行任务的目标服务器 */ - protected Set runningIpSet = new HashSet<>(); + protected Set runningTargetAgentIds = new HashSet<>(); // ---------------- analysed task execution result for server -------------------- /** * 已经分析结果完成的目标服务器 */ - protected Set analyseFinishedIpSet = new HashSet<>(); + protected Set analyseFinishedTargetAgentIds = new HashSet<>(); /** - * 执行成功的服务器 + * 执行成功的目标服务器 */ - protected Set successIpSet = new HashSet<>(); + protected Set successTargetAgentIds = new HashSet<>(); /** - * 不合法的服务器 + * Agent ID 与 host 映射关系 */ - protected Set invalidIpSet = new HashSet<>(); + protected Map agentIdHostMap; /** * 任务成功被终止 */ @@ -200,24 +204,32 @@ public abstract class AbstractResultHandleTask implements ContinuousScheduled // ---------------- task lifecycle properties -------------------- - /** - * Constructor - * - * @param taskInstance 作业实例 - * @param stepInstance 步骤实例 - * @param taskVariablesAnalyzeResult 变量信息 - * @param ipLogMap GSE任务-主机-信息 - * @param gseTaskLog GSE任务整体信息 - * @param targetIps 目标服务器IP - * @param requestId 请求ID,防止重复执行 - */ - protected AbstractResultHandleTask(TaskInstanceDTO taskInstance, + protected AbstractResultHandleTask(TaskInstanceService taskInstanceService, + GseTaskService gseTaskService, + LogService logService, + TaskInstanceVariableService taskInstanceVariableService, + StepInstanceVariableValueService stepInstanceVariableValueService, + TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, + ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager, + TaskEvictPolicyExecutor taskEvictPolicyExecutor, + AgentTaskService agentTaskService, + StepInstanceService stepInstanceService, + TaskInstanceDTO taskInstance, StepInstanceDTO stepInstance, TaskVariablesAnalyzeResult taskVariablesAnalyzeResult, - Map ipLogMap, - GseTaskLogDTO gseTaskLog, - Set targetIps, + Map targetAgentTasks, + GseTaskDTO gseTask, String requestId) { + this.taskInstanceService = taskInstanceService; + this.gseTaskService = gseTaskService; + this.logService = logService; + this.taskInstanceVariableService = taskInstanceVariableService; + this.stepInstanceVariableValueService = stepInstanceVariableValueService; + this.taskExecuteMQEventDispatcher = taskExecuteMQEventDispatcher; + this.resultHandleTaskKeepaliveManager = resultHandleTaskKeepaliveManager; + this.taskEvictPolicyExecutor = taskEvictPolicyExecutor; + this.agentTaskService = agentTaskService; + this.stepInstanceService = stepInstanceService; this.requestId = requestId; this.taskInstance = taskInstance; this.taskInstanceId = taskInstance.getId(); @@ -225,44 +237,26 @@ protected AbstractResultHandleTask(TaskInstanceDTO taskInstance, this.appId = stepInstance.getAppId(); this.stepInstanceId = stepInstance.getId(); this.taskVariablesAnalyzeResult = taskVariablesAnalyzeResult; - this.ipLogMap = ipLogMap; - this.gseTaskLog = gseTaskLog; - this.targetIpSet.addAll(targetIps); - this.notStartedIpSet.addAll(targetIps); - if (CollectionUtils.isNotEmpty(stepInstance.getInvalidIps())) { - this.invalidIpSet.addAll(stepInstance.getInvalidIps()); - } - - List taskVariables = taskVariablesAnalyzeResult.getTaskVars(); - if (taskVariables != null && !taskVariables.isEmpty()) { - taskVariables.forEach(var -> initialVariables.put(var.getName(), var)); + this.targetAgentTasks = targetAgentTasks; + this.gseTask = gseTask; + + targetAgentTasks.values().forEach(agentTask -> { + this.targetAgentIds.add(agentTask.getAgentId()); + }); + this.notStartedTargetAgentIds.addAll(targetAgentIds); + + this.agentIdHostMap = stepInstanceService.computeStepHosts(stepInstance, + host -> host.getAgentId() != null ? host.getAgentId() : host.toCloudIp()); + + // 如果是执行方案，需要初始化全局变量 + if (taskInstance.isPlanInstance()) { + List taskVariables = taskVariablesAnalyzeResult.getTaskVars(); + if (taskVariables != null && !taskVariables.isEmpty()) { + taskVariables.forEach(var -> initialVariables.put(var.getName(), var)); + } } } - /** - * 初始化依赖的服务 - */ - public void initDependentService(TaskInstanceService taskInstanceService, - GseTaskLogService gseTaskLogService, - LogService logService, - TaskInstanceVariableService taskInstanceVariableService, - StepInstanceVariableValueService stepInstanceVariableValueService, - TaskExecuteControlMsgSender taskManager, - ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager, - ExceptionStatusManager exceptionStatusManager, - TaskEvictPolicyExecutor taskEvictPolicyExecutor - ) { - this.taskInstanceService = taskInstanceService; - this.gseTaskLogService = gseTaskLogService; - this.logService = logService; - this.taskInstanceVariableService = taskInstanceVariableService; - this.stepInstanceVariableValueService = stepInstanceVariableValueService; - this.taskManager = taskManager; - this.resultHandleTaskKeepaliveManager = resultHandleTaskKeepaliveManager; - this.exceptionStatusManager = exceptionStatusManager; - this.taskEvictPolicyExecutor = taskEvictPolicyExecutor; - } - /** * 检查是否应当驱逐当前任务，若应当驱逐则驱逐并更新任务状态 * @@ -291,7 +285,7 @@ private boolean checkAndEvictTaskIfNeed(StopWatch watch) { * @return 是否应当继续后续流程 */ private boolean pullGSELogsAndCheckPullResult(StopWatch watch) { - log.info("[{}]: Start pull log, times: {}", stepInstanceId, pullLogTimes.addAndGet(1)); + log.info("[{}]: Start pull gse task result, times: {}", stepInstanceId, pullLogTimes.addAndGet(1)); GseLogBatchPullResult gseLogBatchPullResult; int batch = 0; do { @@ -300,7 +294,7 @@ private boolean pullGSELogsAndCheckPullResult(StopWatch watch) { watch.start("pull-task-result-batch-" + batch); // 分批拉取GSE任务执行结果 - gseLogBatchPullResult = pullGseTaskLogInBatches(); + gseLogBatchPullResult = pullGseTaskResultInBatches(); // 拉取结果校验 if (!checkPullResult(gseLogBatchPullResult)) { @@ -315,10 +309,10 @@ private boolean pullGSELogsAndCheckPullResult(StopWatch watch) { try { watch.start("analyse-task-result-batch-" + batch); - this.executeResult = analyseGseTaskLog(gseLog); + this.executeResult = analyseGseTaskResult(gseLog); watch.stop(); } catch (Throwable e) { - log.error("[" + stepInstanceId + "]: analyse gse task log result error.", e); + log.error("[" + stepInstanceId + "]: analyse gse task result error.", e); throw e; } } while (!gseLogBatchPullResult.isLastBatch()); @@ -327,6 +321,7 @@ private boolean pullGSELogsAndCheckPullResult(StopWatch watch) { public void execute() { StopWatch watch = new StopWatch("Result-Handle-Task-" + stepInstanceId); + String lockKey = buildGseTaskLockKey(gseTask); try { if (!checkTaskActiveAndSetRunningStatus()) { return; @@ -336,8 +331,8 @@ public void execute() { } watch.start("get-lock"); - if (!LockUtils.tryGetReentrantLock("job:result:handle:" + stepInstanceId, requestId, 30000L)) { - log.info("Fail to get result handle lock, stepInstanceId: {}", stepInstanceId); + if (!LockUtils.tryGetReentrantLock(lockKey, requestId, 30000L)) { + log.error("Fail to get result handle lock, lockKey: {}", lockKey); this.executeResult = GseTaskExecuteResult.DISCARDED; return; } @@ -367,7 +362,7 @@ public void execute() { handleExecuteResult(this.executeResult); } finally { this.isRunning = false; - LockUtils.releaseDistributedLock("job:result:handle:", String.valueOf(stepInstanceId), requestId); + LockUtils.releaseDistributedLock(lockKey, requestId); if (watch.isRunning()) { watch.stop(); } @@ -378,6 +373,10 @@ public void execute() { } } + private String buildGseTaskLockKey(GseTaskDTO gseTask) { + return "job:result:handle:" + gseTask.getId(); + } + private boolean checkTaskActiveAndSetRunningStatus() { if (!isActive) { log.info("Task is inactive, stepInstanceId: {}", stepInstanceId); @@ -396,11 +395,12 @@ private void terminateGseTaskIfDetectTaskStatusIsStopping() { if (!isGseTaskTerminating) { this.taskInstance = taskInstanceService.getTaskInstance(taskInstanceId); // 如果任务处于“终止中”状态，触发任务终止 - if (this.taskInstance.getStatus().equals(RunStatusEnum.STOPPING.getValue())) { + if (this.taskInstance.getStatus() == RunStatusEnum.STOPPING) { log.info("Task instance status is stopping, stop executing the step! taskInstanceId:{}, " + "stepInstanceId:{}", taskInstance.getId(), stepInstance.getId()); - taskManager.stopGseStep(stepInstanceId); + taskExecuteMQEventDispatcher.dispatchGseTaskEvent(GseTaskEvent.stopGseTask( + gseTask.getStepInstanceId(), gseTask.getExecuteCount(), gseTask.getBatch(), gseTask.getId())); this.isGseTaskTerminating = true; log.info("Send stop gse step control action successfully!"); } @@ -409,19 +409,20 @@ private void terminateGseTaskIfDetectTaskStatusIsStopping() { private boolean shouldSkipStep() { StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(stepInstanceId); - return stepInstance == null || RunStatusEnum.SKIPPED.getValue().equals(stepInstance.getStatus()); + return stepInstance == null || RunStatusEnum.SKIPPED == stepInstance.getStatus(); } private void saveStatusWhenSkip() { - List notFinishedIpLogs = - ipLogMap.values().stream().filter(not(GseTaskIpLogDTO::isFinished)).collect(Collectors.toList()); - if (CollectionUtils.isNotEmpty(notFinishedIpLogs)) { - notFinishedIpLogs.forEach(ipLog -> { - ipLog.setStatus(IpStatus.UNKNOWN.getValue()); - ipLog.setEndTime(System.currentTimeMillis()); + List notFinishedGseAgentTasks = + targetAgentTasks.values().stream().filter(not(AgentTaskDTO::isFinished)).collect(Collectors.toList()); + if (CollectionUtils.isNotEmpty(notFinishedGseAgentTasks)) { + notFinishedGseAgentTasks.forEach(agentTask -> { + agentTask.setStatus(AgentTaskStatusEnum.UNKNOWN); + agentTask.setEndTime(System.currentTimeMillis()); + }); } - gseTaskLogService.batchSaveIpLog(notFinishedIpLogs); + agentTaskService.batchUpdateAgentTasks(notFinishedGseAgentTasks); } /* @@ -432,7 +433,7 @@ private boolean determineTaskAbnormal(GseLog gseLog) { } private boolean checkTaskTimeout() { - long startTime = gseTaskLog.getStartTime(); + long startTime = gseTask.getStartTime(); long runDuration = System.currentTimeMillis() - startTime; boolean isTimeout = false; // 作业执行超时,但是GSE并没有按照预期结束作业;job最大容忍5min，然后判定任务异常 @@ -442,7 +443,7 @@ private boolean checkTaskTimeout() { log.warn("[{}]: Task execution timeout! runDuration: {}ms, timeout: {}s", stepInstanceId, runDuration, stepInstance.getTimeout()); this.executeResult = GseTaskExecuteResult.FAILED; - saveFailInfoForUnfinishedIpTask(IpStatus.LOG_ERROR.getValue(), + saveFailInfoForUnfinishedAgentTask(AgentTaskStatusEnum.LOG_ERROR, "Task execution may be abnormal or timeout."); handleExecuteResult(GseTaskExecuteResult.FAILED); isTimeout = true; @@ -464,7 +465,7 @@ private boolean checkEmptyGseResult(GseLog gseLog) { if (currentTimeMillis - latestPullGseLogSuccessTimeMillis >= GSE_TASK_EMPTY_RESULT_MAX_TOLERATION_MILLS) { log.warn("[{}]: Execution result log always empty!", stepInstanceId); this.executeResult = GseTaskExecuteResult.FAILED; - saveFailInfoForUnfinishedIpTask(IpStatus.LOG_ERROR.getValue(), "Execution result log always empty."); + saveFailInfoForUnfinishedAgentTask(AgentTaskStatusEnum.LOG_ERROR, "Execution result log always empty."); handleExecuteResult(GseTaskExecuteResult.FAILED); isAbnormal = true; } @@ -476,9 +477,10 @@ private boolean checkEmptyGseResult(GseLog gseLog) { private boolean checkPullResult(GseLogBatchPullResult gseLogBatchPullResult) { if (!gseLogBatchPullResult.isSuccess()) { - log.error("[{}] Pull gse log error, errorMsg: {}", stepInstanceId, gseLogBatchPullResult.getErrorMsg()); + log.error("[{}] Pull gse task result error, errorMsg: {}", stepInstanceId, + gseLogBatchPullResult.getErrorMsg()); this.executeResult = GseTaskExecuteResult.FAILED; - saveFailInfoForUnfinishedIpTask(IpStatus.LOG_ERROR.getValue(), gseLogBatchPullResult.getErrorMsg()); + saveFailInfoForUnfinishedAgentTask(AgentTaskStatusEnum.LOG_ERROR, gseLogBatchPullResult.getErrorMsg()); handleExecuteResult(GseTaskExecuteResult.FAILED); return false; } @@ -487,28 +489,28 @@ private boolean checkPullResult(GseLogBatchPullResult gseLogBatchPullResult) /** - * 更新IP统计状态集合，设置任务起止时间 + * 设置目标gent任务结束状态 * - * @param cloudIp IP + * @param agentId agentId * @param startTime 起始时间 * @param endTime 终止时间 - * @param ipLog 日志 + * @param agentTask 日志 */ - protected void dealIPFinish(String cloudIp, Long startTime, Long endTime, GseTaskIpLogDTO ipLog) { - log.info("[{}]: Deal ip finished| ip={}| startTime:{}, endTime:{}, ipLog:{}", - stepInstanceId, cloudIp, startTime, endTime, JsonUtils.toJsonWithoutSkippedFields(ipLog)); + protected void dealTargetAgentFinish(String agentId, Long startTime, Long endTime, AgentTaskDTO agentTask) { + log.info("[{}]: Deal target agent finished| agentId={}| startTime:{}, endTime:{}, agentTask:{}", + stepInstanceId, agentId, startTime, endTime, JsonUtils.toJsonWithoutSkippedFields(agentTask)); - notStartedIpSet.remove(cloudIp); - runningIpSet.remove(cloudIp); - analyseFinishedIpSet.add(cloudIp); + notStartedTargetAgentIds.remove(agentId); + runningTargetAgentIds.remove(agentId); + analyseFinishedTargetAgentIds.add(agentId); if (endTime - startTime <= 0) { - ipLog.setTotalTime(100L); + agentTask.setTotalTime(100L); } else { - ipLog.setTotalTime(endTime - startTime); + agentTask.setTotalTime(endTime - startTime); } - ipLog.setStartTime(startTime); - ipLog.setEndTime(endTime); + agentTask.setStartTime(startTime); + agentTask.setEndTime(endTime); } /** @@ -523,102 +525,65 @@ private void handleExecuteResult(GseTaskExecuteResult result) { return; } - // 处理GSE任务执行结果,并更新任务步骤状态 + // 处理GSE任务执行结果 log.info("Handle execute result, stepInstanceId:{}, executeResult:{}", stepInstanceId, gseTaskExecuteResult); - long startTime = this.gseTaskLog.getStartTime(); + long startTime = this.gseTask.getStartTime(); long endTime = DateUtils.currentTimeMillis(); long gseTotalTime = endTime - startTime; - long stepTotalTime = endTime - stepInstance.getStartTime(); - - int targetIpNum = this.targetIpSet.size(); - int allSuccessIPNum = this.successIpSet.size(); - int invalidIpNum = this.invalidIpSet == null ? 0 : this.invalidIpSet.size(); - int successTargetIpNum = successIpSet.size(); - int failTargetIpNum = targetIpNum - successTargetIpNum; - boolean isSuccess = CollectionUtils.isEmpty(this.invalidIpSet) && allSuccessIPNum == targetIpNum; + int targetAgentNum = this.targetAgentIds.size(); + int allSuccessAgentNum = this.successTargetAgentIds.size(); + boolean isSuccess = !stepInstance.hasInvalidHost() && allSuccessAgentNum == targetAgentNum; - saveGseTaskLogExecutionInfo(result, isSuccess, endTime, gseTotalTime); - - if (gseTaskExecuteResult == GseTaskExecuteResult.RESULT_CODE_STOP_SUCCESS) { - int stepStatus = taskInstanceService.getBaseStepInstance(stepInstanceId).getStatus(); - if (stepStatus == RunStatusEnum.STOPPING.getValue() || stepStatus == RunStatusEnum.RUNNING.getValue()) { - taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.STOP_SUCCESS, - startTime, endTime, stepTotalTime, targetIpNum + invalidIpNum, - successTargetIpNum, failTargetIpNum + invalidIpNum); - taskManager.refreshTask(stepInstance.getTaskInstanceId()); - } - } else { - int stepStatus = taskInstanceService.getBaseStepInstance(stepInstanceId).getStatus(); - if (gseTaskExecuteResult == GseTaskExecuteResult.RESULT_CODE_EXCEPTION) { - taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.ABNORMAL_STATE, - startTime, endTime, stepTotalTime, targetIpNum + invalidIpNum, - successTargetIpNum, failTargetIpNum + invalidIpNum); - exceptionStatusManager.setAbnormalStatusForStep(stepInstanceId); - } else if (gseTaskExecuteResult == GseTaskExecuteResult.RESULT_CODE_FAILED) { - taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.FAIL, - startTime, endTime, stepTotalTime, targetIpNum + invalidIpNum, - successTargetIpNum, failTargetIpNum + invalidIpNum); - } else if (stepStatus == RunStatusEnum.STOPPING.getValue()) { - taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.STOP_SUCCESS, - startTime, endTime, stepTotalTime, targetIpNum + invalidIpNum, - successTargetIpNum, failTargetIpNum + invalidIpNum); - } else if (gseTaskExecuteResult == GseTaskExecuteResult.RESULT_CODE_SUCCESS) { - taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.SUCCESS, - startTime, endTime, stepTotalTime, targetIpNum + invalidIpNum, - successTargetIpNum, failTargetIpNum + invalidIpNum); - } - taskManager.refreshTask(stepInstance.getTaskInstanceId()); - } + updateGseTaskExecutionInfo(result, isSuccess, endTime, gseTotalTime); + taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.refreshStep(stepInstanceId, + EventSource.buildGseTaskEventSource(stepInstanceId, stepInstance.getExecuteCount(), + stepInstance.getBatch(), gseTask.getId()))); } - private void saveGseTaskLogExecutionInfo(GseTaskExecuteResult result, boolean isSuccess, long endTime, - long totalTime) { + private void updateGseTaskExecutionInfo(GseTaskExecuteResult result, boolean isSuccess, long endTime, + long totalTime) { if (GseTaskExecuteResult.RESULT_CODE_STOP_SUCCESS == result.getResultCode()) { - gseTaskLog.setStatus(RunStatusEnum.STOP_SUCCESS.getValue()); + gseTask.setStatus(RunStatusEnum.STOP_SUCCESS.getValue()); } else { - gseTaskLog.setStatus(isSuccess ? RunStatusEnum.SUCCESS.getValue() : RunStatusEnum.FAIL.getValue()); + gseTask.setStatus(isSuccess ? RunStatusEnum.SUCCESS.getValue() : RunStatusEnum.FAIL.getValue()); } - gseTaskLog.setEndTime(endTime); - gseTaskLog.setTotalTime(totalTime); - gseTaskLogService.saveGseTaskLog(gseTaskLog); - } - - protected void batchSaveChangedIpLogs() { - List changedIpLogs = - this.ipLogMap.values().stream().filter(GseTaskIpLogDTO::isChanged).collect(Collectors.toList()); - gseTaskLogService.batchSaveIpLog(changedIpLogs); - changedIpLogs.forEach(ipLog -> ipLog.setChanged(false)); + gseTask.setEndTime(endTime); + gseTask.setTotalTime(totalTime); + gseTaskService.updateGseTask(gseTask); } - protected void saveFailInfoForUnfinishedIpTask(int errorType, String errorMsg) { - log.info("[{}]: Deal unfinished ip result| noStartJobIPSet={}| runningJobIPSet={}", - stepInstanceId, this.notStartedIpSet, this.runningIpSet); - Set unfinishedIPSet = new HashSet<>(); - unfinishedIPSet.addAll(notStartedIpSet); - unfinishedIPSet.addAll(this.runningIpSet); - long startTime = (this.gseTaskLog != null && this.gseTaskLog.getStartTime() != null) ? - this.gseTaskLog.getStartTime() : System.currentTimeMillis(); - batchSaveFailIpLog(unfinishedIPSet, startTime, System.currentTimeMillis(), errorType); + /** + * 批量更新AgentTask并重置changed标志 + * + * @param agentTasks agent任务列表 + */ + protected void batchSaveChangedGseAgentTasks(Collection agentTasks) { + if (CollectionUtils.isNotEmpty(agentTasks)) { + List changedGseAgentTasks = + agentTasks.stream().filter(AgentTaskDTO::isChanged).collect(Collectors.toList()); + agentTaskService.batchUpdateAgentTasks(changedGseAgentTasks); + changedGseAgentTasks.forEach(agentTask -> agentTask.setChanged(false)); + } } - private void batchSaveFailIpLog(Collection ipSet, long startTime, long endTime, int status) { - List ipLogList = new ArrayList<>(); - for (String ip : ipSet) { - GseTaskIpLogDTO ipLog = new GseTaskIpLogDTO(); - ipLog.setStepInstanceId(stepInstanceId); - ipLog.setExecuteCount(stepInstance.getExecuteCount()); - IpDTO ipDto = IpHelper.transform(ip); - ipLog.setCloudAreaAndIp(IpHelper.compose(ipDto)); - ipLog.setDisplayIp(ipDto.getIp()); - ipLog.setStartTime(startTime); - ipLog.setEndTime(endTime); - ipLog.setStatus(status); - ipLogList.add(ipLog); + protected void saveFailInfoForUnfinishedAgentTask(AgentTaskStatusEnum status, String errorMsg) { + log.info("[{}]: Deal unfinished agent result| noStartJobAgentIds : {}| runningJobAgentIds : {}", + stepInstanceId, notStartedTargetAgentIds, runningTargetAgentIds); + Set unfinishedAgentIds = new HashSet<>(); + unfinishedAgentIds.addAll(notStartedTargetAgentIds); + unfinishedAgentIds.addAll(runningTargetAgentIds); + long startTime = (gseTask != null && gseTask.getStartTime() != null) ? + gseTask.getStartTime() : System.currentTimeMillis(); + for (String agentId : unfinishedAgentIds) { + AgentTaskDTO agentTask = targetAgentTasks.get(agentId); + agentTask.setStartTime(startTime); + agentTask.setEndTime(System.currentTimeMillis()); + agentTask.setStatus(status); } - gseTaskLogService.batchSaveIpLog(ipLogList); + batchSaveChangedGseAgentTasks(targetAgentTasks.values()); } @Override @@ -637,7 +602,10 @@ private void tryStopImmediately() { if (!this.isRunning) { log.info("ResultHandleTask-onStop start, stepInstanceId: {}", stepInstanceId); resultHandleTaskKeepaliveManager.stopKeepaliveInfoTask(getTaskId()); - taskManager.resumeGseStep(stepInstanceId, stepInstance.getExecuteCount(), requestId); + taskExecuteMQEventDispatcher.dispatchResultHandleTaskResumeEvent( + ResultHandleTaskResumeEvent.resume(gseTask.getStepInstanceId(), + gseTask.getExecuteCount(), gseTask.getBatch(), gseTask.getId(), requestId)); + this.isStopped = true; StopTaskCounter.getInstance().decrement(getTaskId()); log.info("ResultHandleTask-onStop end, stepInstanceId: {}", stepInstanceId); @@ -681,7 +649,7 @@ protected final GseTaskExecuteResult getExecuteResult() { * * @return 日志 */ - abstract GseLogBatchPullResult pullGseTaskLogInBatches(); + abstract GseLogBatchPullResult pullGseTaskResultInBatches(); /** * 解析GSE日志并获取结果 @@ -689,5 +657,5 @@ protected final GseTaskExecuteResult getExecuteResult() { * @param gseLog GSE日志 * @return 任务执行结果 */ - abstract GseTaskExecuteResult analyseGseTaskLog(GseLog gseLog); + abstract GseTaskExecuteResult analyseGseTaskResult(GseLog gseLog); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/FileResultHandleTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/FileResultHandleTask.java index 28f88a2509..370bb26fde 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/FileResultHandleTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/FileResultHandleTask.java @@ -26,34 +26,43 @@ import com.google.common.collect.Sets; import com.tencent.bk.gse.taskapi.api_map_rsp; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.util.ip.IpUtils; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.execute.common.constants.FileDistModeEnum; import com.tencent.bk.job.execute.common.constants.FileDistStatusEnum; +import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; import com.tencent.bk.job.execute.engine.consts.FileDirTypeConf; import com.tencent.bk.job.execute.engine.consts.GSECode; -import com.tencent.bk.job.execute.engine.consts.GseConstants; -import com.tencent.bk.job.execute.engine.consts.IpStatus; +import com.tencent.bk.job.execute.engine.evict.TaskEvictPolicyExecutor; import com.tencent.bk.job.execute.engine.gse.GseRequestUtils; import com.tencent.bk.job.execute.engine.gse.model.CopyFileRsp; import com.tencent.bk.job.execute.engine.gse.model.GSEFileTaskResult; +import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; import com.tencent.bk.job.execute.engine.model.FileTaskLog; import com.tencent.bk.job.execute.engine.model.GseLog; import com.tencent.bk.job.execute.engine.model.GseLogBatchPullResult; import com.tencent.bk.job.execute.engine.model.GseTaskExecuteResult; import com.tencent.bk.job.execute.engine.model.JobFile; import com.tencent.bk.job.execute.engine.model.TaskVariablesAnalyzeResult; +import com.tencent.bk.job.execute.engine.result.ha.ResultHandleTaskKeepaliveManager; import com.tencent.bk.job.execute.engine.util.FilePathUtils; +import com.tencent.bk.job.execute.engine.util.GseUtils; import com.tencent.bk.job.execute.engine.util.NFSUtils; -import com.tencent.bk.job.execute.engine.util.Utils; import com.tencent.bk.job.execute.engine.util.WindowsHelper; -import com.tencent.bk.job.execute.model.GseTaskIpLogDTO; -import com.tencent.bk.job.execute.model.GseTaskLogDTO; +import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.model.GseTaskDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; +import com.tencent.bk.job.execute.service.FileAgentTaskService; +import com.tencent.bk.job.execute.service.GseTaskService; +import com.tencent.bk.job.execute.service.LogService; +import com.tencent.bk.job.execute.service.StepInstanceService; +import com.tencent.bk.job.execute.service.StepInstanceVariableValueService; +import com.tencent.bk.job.execute.service.TaskInstanceService; +import com.tencent.bk.job.execute.service.TaskInstanceVariableService; import com.tencent.bk.job.logsvr.model.service.ServiceFileTaskLogDTO; -import com.tencent.bk.job.logsvr.model.service.ServiceIpLogDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceHostLogDTO; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -61,11 +70,11 @@ import org.springframework.util.StopWatch; import java.text.DecimalFormat; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; -import java.util.StringJoiner; /** * 文件任务执行结果处理 @@ -74,6 +83,10 @@ public class FileResultHandleTask extends AbstractResultHandleTask { private static final String FILE_TASK_MODE_DOWNLOAD = "download"; private static final String FILE_TASK_MODE_UPLOAD = "upload"; + /** + * GSE 源 Agent 任务, Map + */ + private final Map sourceAgentTasks; /** * 待分发文件，文件传输的源文件 */ @@ -81,33 +94,33 @@ public class FileResultHandleTask extends AbstractResultHandleTask /** * 任务包含的源服务器 */ - private final Set fileSourceIPSet = new HashSet<>(); + private final Set sourceAgentIds = new HashSet<>(); /** * 已经分析结果完成的源服务器 */ - protected Set analyseFinishedSourceIpSet = new HashSet<>(); + protected Set analyseFinishedSourceAgentIds = new HashSet<>(); /** - * 成功的文件下载任务，key: ip, value: 成功的文件任务名称 + * 成功的文件下载任务，key: agentId, value: 成功的文件任务名称 */ private final Map> successDownloadFileMap = new HashMap<>(); /** - * 已经结束的文件下载任务，key: ip, value: 已完成文件任务名称 + * 已经结束的文件下载任务，key: agentId, value: 已完成文件任务名称 */ private final Map> finishedDownloadFileMap = new HashMap<>(); /** - * 下载文件任务数，key: ip, value: 主机对应的文件任务数 + * 下载文件任务数，key: agentId, value: 主机对应的文件任务数 */ private final Map fileDownloadTaskNumMap = new HashMap<>(); /** - * 成功的文件上传任务，key: ip, value: 成功的文件任务名称 + * 成功的文件上传任务，key: agentId, value: 成功的文件任务名称 */ private final Map> successUploadFileMap = new HashMap<>(); /** - * 已经结束的文件上传任务，key: ip, value: 已完成文件任务名称 + * 已经结束的文件上传任务，key: agentId, value: 已完成文件任务名称 */ private final Map> finishedUploadFileMap = new HashMap<>(); /** - * 上传文件任务数，key: ip, value: 主机对应的文件任务数 + * 上传文件任务数，key: agentId, value: 主机对应的文件任务数 */ private final Map fileUploadTaskNumMap = new HashMap<>(); /** @@ -138,11 +151,11 @@ public class FileResultHandleTask extends AbstractResultHandleTask /** * 未开始任务的文件源服务器 */ - private final Set notStartedFileSourceIpSet = new HashSet<>(); + private final Set notStartedFileSourceAgentIds = new HashSet<>(); /** * 正在执行任务的文件源服务器 */ - private final Set runningFileSourceIpSet = new HashSet<>(); + private final Set runningFileSourceAgentIds = new HashSet<>(); /** * 文件任务执行结果处理调度策略 */ @@ -152,18 +165,44 @@ public class FileResultHandleTask extends AbstractResultHandleTask */ private String taskInfo; - public FileResultHandleTask(TaskInstanceDTO taskInstance, + public FileResultHandleTask(TaskInstanceService taskInstanceService, + GseTaskService gseTaskService, + LogService logService, + TaskInstanceVariableService taskInstanceVariableService, + StepInstanceVariableValueService stepInstanceVariableValueService, + TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, + ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager, + TaskEvictPolicyExecutor taskEvictPolicyExecutor, + FileAgentTaskService fileAgentTaskService, + StepInstanceService stepInstanceService, + TaskInstanceDTO taskInstance, StepInstanceDTO stepInstance, TaskVariablesAnalyzeResult taskVariablesAnalyzeResult, - Map ipLogMap, - GseTaskLogDTO gseTaskLog, - Set targetIps, + Map targetAgentTasks, + Map sourceAgentTasks, + GseTaskDTO gseTask, Set sendFiles, String storageRootPath, Map sourceDestPathMap, Map sourceFileDisplayMap, String requestId) { - super(taskInstance, stepInstance, taskVariablesAnalyzeResult, ipLogMap, gseTaskLog, targetIps, requestId); + super(taskInstanceService, + gseTaskService, + logService, + taskInstanceVariableService, + stepInstanceVariableValueService, + taskExecuteMQEventDispatcher, + resultHandleTaskKeepaliveManager, + taskEvictPolicyExecutor, + fileAgentTaskService, + stepInstanceService, + taskInstance, + stepInstance, + taskVariablesAnalyzeResult, + targetAgentTasks, + gseTask, + requestId); + this.sourceAgentTasks = sourceAgentTasks; this.sendFiles = sendFiles; this.localUploadDir = NFSUtils.getFileDir(storageRootPath, FileDirTypeConf.UPLOAD_FILE_DIR); if (sourceDestPathMap != null) { @@ -173,12 +212,12 @@ public FileResultHandleTask(TaskInstanceDTO taskInstance, } this.sourceFileDisplayMap = sourceFileDisplayMap; initFileTaskNumMap(); - initSourceServerIp(); + initSourceAgentIds(); initFileSourceIntIpMapping(); - log.info("InitFileResultHandleTask|stepInstanceId: {}|sendFiles: {}|fileSourceIpSet: {}|targetIpSet: {}|" + - "fileUploadTaskNumMap: {}|fileDownloadTaskNumMap: {}", - stepInstance.getId(), sendFiles, fileSourceIPSet, targetIpSet, fileUploadTaskNumMap, + log.info("InitFileResultHandleTask|stepInstanceId: {}|sendFiles: {}|fileSourceAgentIds: {}|targetAgentIds: {}|" + + "fileUploadTaskNumMap: {}|fileDownloadTaskNumMap: {}", + stepInstance.getId(), sendFiles, sourceAgentIds, targetAgentIds, fileUploadTaskNumMap, fileDownloadTaskNumMap); } @@ -187,34 +226,28 @@ public FileResultHandleTask(TaskInstanceDTO taskInstance, * 初始化文件任务计数器 */ private void initFileTaskNumMap() { - for (String ip : this.targetIpSet) { - this.fileDownloadTaskNumMap.put(ip, this.sendFiles.size()); + for (String agentId : this.targetAgentIds) { + this.fileDownloadTaskNumMap.put(agentId, this.sendFiles.size()); } for (JobFile sendFile : this.sendFiles) { - String ip = sendFile.getCloudAreaIdAndIp(); - this.fileUploadTaskNumMap.put(ip, this.fileUploadTaskNumMap.get(ip) == null ? 1 : - (this.fileUploadTaskNumMap.get(ip) + 1)); + String agentId = sendFile.getHost().getAgentId(); + this.fileUploadTaskNumMap.put(agentId, this.fileUploadTaskNumMap.get(agentId) == null ? 1 : + (this.fileUploadTaskNumMap.get(agentId) + 1)); } } - /* - * 初始化文件源服务器IP - */ - private void initSourceServerIp() { - if (this.sendFiles != null) { - for (JobFile sendFile : this.sendFiles) { - String fileSourceCloudIp = sendFile.getCloudAreaIdAndIp(); - this.fileSourceIPSet.add(fileSourceCloudIp); - this.notStartedFileSourceIpSet.add(fileSourceCloudIp); - } - } + private void initSourceAgentIds() { + sourceAgentTasks.values().forEach(agentTask -> { + this.notStartedFileSourceAgentIds.add(agentTask.getAgentId()); + this.sourceAgentIds.add(agentTask.getAgentId()); + }); } private void initFileSourceIntIpMapping() { // gse bug，使用int保存ip会溢出，所以必须根据当前的源ip反向推出key中源ip if (this.intSourceIpMapping.isEmpty()) { - for (String cloudIp : this.fileSourceIPSet) { + for (String cloudIp : this.sourceAgentIds) { this.intSourceIpMapping.put( String.valueOf((int) IpUtils.getStringIpToLong(cloudIp.split(":")[1])), cloudIp); @@ -223,19 +256,19 @@ private void initFileSourceIntIpMapping() { } @Override - GseLogBatchPullResult pullGseTaskLogInBatches() { + GseLogBatchPullResult pullGseTaskResultInBatches() { api_map_rsp gseLog; - if (CollectionUtils.isNotEmpty(this.analyseFinishedSourceIpSet) - || CollectionUtils.isNotEmpty(this.analyseFinishedIpSet)) { - Set notFinishedIps = new HashSet<>(); - notFinishedIps.addAll(notStartedFileSourceIpSet); - notFinishedIps.addAll(runningFileSourceIpSet); - notFinishedIps.addAll(notStartedIpSet); - notFinishedIps.addAll(runningIpSet); - gseLog = GseRequestUtils.pullCopyFileTaskLog(this.stepInstanceId, this.gseTaskLog.getGseTaskId(), - notFinishedIps); + if (CollectionUtils.isNotEmpty(this.analyseFinishedSourceAgentIds) + || CollectionUtils.isNotEmpty(this.analyseFinishedTargetAgentIds)) { + Set notFinishedAgentIds = new HashSet<>(); + notFinishedAgentIds.addAll(notStartedFileSourceAgentIds); + notFinishedAgentIds.addAll(runningFileSourceAgentIds); + notFinishedAgentIds.addAll(notStartedTargetAgentIds); + notFinishedAgentIds.addAll(runningTargetAgentIds); + gseLog = GseRequestUtils.pullCopyFileTaskLog(this.stepInstanceId, this.gseTask.getGseTaskId(), + notFinishedAgentIds); } else { - gseLog = GseRequestUtils.pullCopyFileTaskLog(this.stepInstanceId, this.gseTaskLog.getGseTaskId()); + gseLog = GseRequestUtils.pullCopyFileTaskLog(this.stepInstanceId, this.gseTask.getGseTaskId()); } GseLogBatchPullResult pullResult; if (gseLog != null) { @@ -248,10 +281,10 @@ GseLogBatchPullResult pullGseTaskLogInBatches() { } @Override - GseTaskExecuteResult analyseGseTaskLog(GseLog taskDetail) { + GseTaskExecuteResult analyseGseTaskResult(GseLog taskDetail) { Set> ipResults = taskDetail.getGseLog().getResult().entrySet(); - // 执行日志, Map - Map executionLogs = new HashMap<>(); + // 执行日志, Map + Map executionLogs = new HashMap<>(); StopWatch watch = new StopWatch("analyse-gse-file-task"); watch.start("analyse"); @@ -266,17 +299,17 @@ GseTaskExecuteResult analyseGseTaskLog(GseLog taskDetail) { continue; } - boolean isDownloadLog = fileTaskResult.isDownloadMode(); - String cloudIp = isDownloadLog ? - fileTaskResult.getDestCloudIp() : fileTaskResult.getSourceCloudIp(); + boolean isDownloadResult = fileTaskResult.isDownloadMode(); + String agentId = isDownloadResult ? + fileTaskResult.getDestAgentId() : fileTaskResult.getSourceAgentId(); - if (isDownloadLog && this.targetIpSet.contains(cloudIp)) { - this.runningIpSet.add(cloudIp); - } else if (!isDownloadLog && this.fileSourceIPSet.contains(cloudIp)) { - this.runningFileSourceIpSet.add(cloudIp); + if (isDownloadResult && this.targetAgentIds.contains(agentId)) { + this.runningTargetAgentIds.add(agentId); + } else if (!isDownloadResult && this.sourceAgentIds.contains(agentId)) { + this.runningFileSourceAgentIds.add(agentId); } - analyseFileResult(cloudIp, copyFileRsp, executionLogs, isDownloadLog); + analyseFileResult(agentId, copyFileRsp, executionLogs, isDownloadResult); } watch.stop(); @@ -286,21 +319,22 @@ GseTaskExecuteResult analyseGseTaskLog(GseLog taskDetail) { watch.stop(); // 保存任务执行结果 - watch.start("saveIpLogs"); - batchSaveChangedIpLogs(); + watch.start("saveAgentTasks"); + batchSaveChangedGseAgentTasks(targetAgentTasks.values()); + batchSaveChangedGseAgentTasks(sourceAgentTasks.values()); watch.stop(); - log.info("Analyse gse task log [{}] -> runningTargetIpSet={}, " + - "notStartedTargetIpSet={}, runningFileSourceIpSet={}, notStartedFileSourceIpSet={}, " + - "analyseFinishedTargetIpSet={}, analyseFinishedSourceIpSet={}, finishedDownloadFileMap={}, " + + log.info("Analyse gse task log [{}] -> runningTargetAgentIds={}, " + + "notStartedTargetAgentIds={}, runningFileSourceAgentIds={}, notStartedFileSourceAgentIds={}, " + + "analyseFinishedTargetAgentIds={}, analyseFinishedSourceAgentIds={}, finishedDownloadFileMap={}, " + "successDownloadFileMap={}, finishedUploadFileMap={}, successUploadFileMap={}", this.stepInstanceId, - this.runningIpSet, - this.notStartedIpSet, - this.runningFileSourceIpSet, - this.notStartedFileSourceIpSet, - this.analyseFinishedIpSet, - this.analyseFinishedSourceIpSet, + this.runningTargetAgentIds, + this.notStartedTargetAgentIds, + this.runningFileSourceAgentIds, + this.notStartedFileSourceAgentIds, + this.analyseFinishedTargetAgentIds, + this.analyseFinishedSourceAgentIds, this.finishedDownloadFileMap, this.successDownloadFileMap, this.finishedUploadFileMap, @@ -313,60 +347,69 @@ GseTaskExecuteResult analyseGseTaskLog(GseLog taskDetail) { return analyseExecuteResult(); } - private void analyseFileResult(String cloudIp, CopyFileRsp copyFileRsp, Map executionLogs, - boolean isDownloadLog) { - GseTaskIpLogDTO ipLog = this.ipLogMap.get(cloudIp); - if (ipLog.getStartTime() == null) { - ipLog.setStartTime(System.currentTimeMillis()); + private void analyseFileResult(String agentId, CopyFileRsp copyFileRsp, + Map executionLogs, + boolean isDownloadResult) { + AgentTaskDTO agentTask = getAgentTask(isDownloadResult, agentId); + if (agentTask.getStartTime() == null) { + agentTask.setStartTime(System.currentTimeMillis()); } - ipLog.setErrCode(copyFileRsp.getFinalErrorCode()); + agentTask.setErrorCode(copyFileRsp.getFinalErrorCode()); GSECode.AtomicErrorCode errorCode = GSECode.AtomicErrorCode.getErrorCode(copyFileRsp.getFinalErrorCode()); GSEFileTaskResult fileTaskResult = copyFileRsp.getGseFileTaskResult(); switch (errorCode) { case RUNNING: parseExecutionLog(copyFileRsp, executionLogs); - ipLog.setStatus(IpStatus.RUNNING.getValue()); - if (isDownloadLog) { - this.notStartedIpSet.remove(cloudIp); + agentTask.setStatus(AgentTaskStatusEnum.RUNNING); + if (isDownloadResult) { + this.notStartedTargetAgentIds.remove(agentId); } break; case FINISHED: parseExecutionLog(copyFileRsp, executionLogs); if (fileTaskResult.getProcess() == 100) { - if (isDownloadLog) { + if (isDownloadResult) { addFinishedFile(true, true, - fileTaskResult.getDestCloudIp(), fileTaskResult.getTaskId()); + fileTaskResult.getDestAgentId(), fileTaskResult.getTaskId()); } else { addFinishedFile(true, false, - fileTaskResult.getSourceCloudIp(), fileTaskResult.getTaskId()); + fileTaskResult.getSourceAgentId(), fileTaskResult.getTaskId()); } - // 分析日志，更新successIpSet、notStartedIpSet等状态集合 - analyseIpResult(errorCode.getValue(), cloudIp, fileTaskResult.getStartTime(), - fileTaskResult.getEndTime(), isDownloadLog); + // 分析日志，更新successAgentIds、notStartedAgentIds等状态集合 + analyseAgentTaskResult(errorCode.getValue(), agentId, fileTaskResult.getStartTime(), + fileTaskResult.getEndTime(), isDownloadResult); } else { - ipLog.setStatus(IpStatus.RUNNING.getValue()); - this.notStartedIpSet.remove(cloudIp); + agentTask.setStatus(AgentTaskStatusEnum.RUNNING); + this.notStartedTargetAgentIds.remove(agentId); } break; case TERMINATE: parseExecutionLog(copyFileRsp, executionLogs); - if (isDownloadLog) { + if (isDownloadResult) { addFinishedFile(false, true, - fileTaskResult.getDestCloudIp(), fileTaskResult.getTaskId()); + fileTaskResult.getDestAgentId(), fileTaskResult.getTaskId()); } else { addFinishedFile(false, false, - fileTaskResult.getSourceCloudIp(), fileTaskResult.getTaskId()); + fileTaskResult.getSourceAgentId(), fileTaskResult.getTaskId()); } - analyseIpResult(errorCode.getValue(), cloudIp, fileTaskResult.getStartTime(), - fileTaskResult.getEndTime(), isDownloadLog); + analyseAgentTaskResult(errorCode.getValue(), agentId, fileTaskResult.getStartTime(), + fileTaskResult.getEndTime(), isDownloadResult); this.isTerminatedSuccess = true; break; default: - dealIpTaskFail(copyFileRsp, executionLogs, isDownloadLog); + dealIpTaskFail(copyFileRsp, executionLogs, isDownloadResult); break; } } + private AgentTaskDTO getAgentTask(boolean isDownloadResult, String agentId) { + if (isDownloadResult) { + return targetAgentTasks.get(agentId); + } else { + return sourceAgentTasks.get(agentId); + } + } + /* * 分析执行结果 * @@ -375,12 +418,12 @@ private void analyseFileResult(String cloudIp, CopyFileRsp copyFileRsp, Map 15_000L) { - int targetIPNum = this.targetIpSet.size(); - int successTargetIpNum = this.successIpSet.size(); - boolean isSuccess = this.invalidIpSet.isEmpty() && successTargetIpNum == targetIPNum; + int targetIPNum = this.targetAgentIds.size(); + int successTargetIpNum = this.successTargetAgentIds.size(); + boolean isSuccess = !stepInstance.hasInvalidHost() && successTargetIpNum == targetIPNum; rst = isSuccess ? GseTaskExecuteResult.SUCCESS : GseTaskExecuteResult.FAILED; log.info("[{}] AnalyseExecuteResult-> Result: finished. Download tasks are finished, " + "but upload tasks are not finished after 15 seconds. Ignore upload tasks", @@ -425,23 +468,23 @@ private boolean shouldAnalyse(GSEFileTaskResult fileTaskResult) { return false; } boolean isDownloadMode = fileTaskResult.isDownloadMode(); - String cloudIp = isDownloadMode ? - fileTaskResult.getDestCloudIp() : fileTaskResult.getSourceCloudIp(); + String agentId = isDownloadMode ? + fileTaskResult.getDestAgentId() : fileTaskResult.getSourceAgentId(); boolean shouldAnalyse = true; if (isDownloadMode) { - if (this.analyseFinishedIpSet.contains(cloudIp) // 该ip已经日志分析结束，不要再分析 + if (this.analyseFinishedTargetAgentIds.contains(agentId) // 该ip已经日志分析结束，不要再分析 // 该文件下载任务已结束 - || (this.finishedDownloadFileMap.get(cloudIp) != null - && this.finishedDownloadFileMap.get(cloudIp).contains(fileTaskResult.getTaskId())) + || (this.finishedDownloadFileMap.get(agentId) != null + && this.finishedDownloadFileMap.get(agentId).contains(fileTaskResult.getTaskId())) // 非目标IP - || !this.fileDownloadTaskNumMap.containsKey(cloudIp)) { + || !this.fileDownloadTaskNumMap.containsKey(agentId)) { shouldAnalyse = false; } } else { - if ((this.finishedUploadFileMap.get(cloudIp) != null - && this.finishedUploadFileMap.get(cloudIp).contains(fileTaskResult.getTaskId())) + if ((this.finishedUploadFileMap.get(agentId) != null + && this.finishedUploadFileMap.get(agentId).contains(fileTaskResult.getTaskId())) // 非源IP - || !this.fileUploadTaskNumMap.containsKey(cloudIp)) { + || !this.fileUploadTaskNumMap.containsKey(agentId)) { shouldAnalyse = false; } } @@ -496,13 +539,13 @@ private CopyFileRsp parseCopyFileRspFromResultKey(CopyFileRsp copyFileRsp, Strin if (fileTaskResult.getMode() == null) { fileTaskResult.setMode(parseFileTaskModeFromKey(isGSEAgentLog, taskProps).getValue()); } - IpDTO cloudIp = parseCloudIpFromKey(taskProps); + HostDTO cloudIp = parseCloudIpFromKey(taskProps); if (FileDistModeEnum.DOWNLOAD.getValue().equals(fileTaskResult.getMode())) { fileTaskResult.setDestIp(cloudIp.getIp()); - fileTaskResult.setDestCloudId(cloudIp.getCloudAreaId()); - IpDTO fileSourceCloudIp = parseFileSourceIpFromKey(isGSEAgentLog, cloudIp, taskProps); + fileTaskResult.setDestCloudId(cloudIp.getBkCloudId()); + HostDTO fileSourceCloudIp = parseFileSourceIpFromKey(isGSEAgentLog, cloudIp, taskProps); if (fileSourceCloudIp != null) { - fileTaskResult.setSourceCloudId(fileSourceCloudIp.getCloudAreaId()); + fileTaskResult.setSourceCloudId(fileSourceCloudIp.getBkCloudId()); fileTaskResult.setSourceIp(fileSourceCloudIp.getIp()); } // GSE BUG, 只有目标文件信息，没有源文件信息 @@ -512,7 +555,7 @@ private CopyFileRsp parseCopyFileRspFromResultKey(CopyFileRsp copyFileRsp, Strin fileTaskResult.setDestFileName(dirAndFileName.getRight()); } else { fileTaskResult.setSourceIp(cloudIp.getIp()); - fileTaskResult.setSourceCloudId(cloudIp.getCloudAreaId()); + fileTaskResult.setSourceCloudId(cloudIp.getBkCloudId()); String sourceFilePath = parseFilePathFromKey(taskProps); Pair dirAndFileName = FilePathUtils.parseDirAndFileName(sourceFilePath); fileTaskResult.setSrcDirPath(dirAndFileName.getLeft()); @@ -536,10 +579,10 @@ private String parseFilePathFromKey(String[] taskProps) { } - private IpDTO parseCloudIpFromKey(String[] taskProps) { + private HostDTO parseCloudIpFromKey(String[] taskProps) { String ip = taskProps[taskProps.length - 1]; long cloudAreaId = Long.parseLong(taskProps[taskProps.length - 2].trim()); - return new IpDTO(cloudAreaId, ip); + return new HostDTO(cloudAreaId, ip); } private FileDistModeEnum parseFileTaskModeFromKey(boolean isGseAgentLog, String[] taskProps) { @@ -548,12 +591,12 @@ private FileDistModeEnum parseFileTaskModeFromKey(boolean isGseAgentLog, String[ fileDistMode = taskProps[0].equals(FILE_TASK_MODE_DOWNLOAD) ? FileDistModeEnum.DOWNLOAD : FileDistModeEnum.UPLOAD; } else { - IpDTO cloudIp = parseCloudIpFromKey(taskProps); + HostDTO cloudIp = parseCloudIpFromKey(taskProps); // GSE Task Server 日志， 如果该key为目标服务器ip的日志，并且key的第一个字段如果为"-1"或者为源文件ip, 表示download日志 String fileSourceCloudIp = this.intSourceIpMapping.get(taskProps[0]); - if (this.targetIpSet.contains(cloudIp.convertToStrIp()) + if (this.targetAgentIds.contains(cloudIp.toCloudIp()) && (taskProps[0].equals("-1") - || (fileSourceCloudIp != null && this.fileSourceIPSet.contains(fileSourceCloudIp)))) { + || (fileSourceCloudIp != null && this.sourceAgentIds.contains(fileSourceCloudIp)))) { fileDistMode = FileDistModeEnum.DOWNLOAD; } else { fileDistMode = FileDistModeEnum.UPLOAD; @@ -562,14 +605,14 @@ private FileDistModeEnum parseFileTaskModeFromKey(boolean isGseAgentLog, String[ return fileDistMode; } - private IpDTO parseFileSourceIpFromKey(boolean isGseAgentLog, IpDTO cloudIp, String[] taskProps) { - IpDTO fileSourceIp = null; + private HostDTO parseFileSourceIpFromKey(boolean isGseAgentLog, HostDTO cloudIp, String[] taskProps) { + HostDTO fileSourceIp = null; if (isGseAgentLog) { if (FILE_TASK_MODE_DOWNLOAD.equalsIgnoreCase(taskProps[0])) { // GSE BUG, download日志无法获取到源IP的云区域 String fileSourceCloudIp = guessFileSourceCloudIp(taskProps[2]); if (StringUtils.isNotEmpty(fileSourceCloudIp)) { - fileSourceIp = IpDTO.fromCloudAreaIdAndIpStr(fileSourceCloudIp); + fileSourceIp = HostDTO.fromCloudIp(fileSourceCloudIp); } } else { fileSourceIp = cloudIp; @@ -579,9 +622,9 @@ private IpDTO parseFileSourceIpFromKey(boolean isGseAgentLog, IpDTO cloudIp, Str // GSE BUG, GSE Task Server 日志， key的第一个字段可能为-1 String fileSourceCloudIp = guessFileSourceCloudIp(taskProps[0]); if (StringUtils.isNotEmpty(fileSourceCloudIp)) { - fileSourceIp = IpDTO.fromCloudAreaIdAndIpStr(fileSourceCloudIp); + fileSourceIp = HostDTO.fromCloudIp(fileSourceCloudIp); } else { - if (this.fileSourceIPSet.contains(cloudIp.convertToStrIp())) { + if (this.sourceAgentIds.contains(cloudIp.toCloudIp())) { fileSourceIp = cloudIp; } } @@ -594,55 +637,35 @@ private String guessFileSourceCloudIp(String intIp) { return this.intSourceIpMapping.get(intIp); } - private void recordFinishedFileSourceIPSet( - CopyFileRsp copyFileRsp, - Map executionLogs, - Set affectIps - ) { - GSEFileTaskResult taskResult = copyFileRsp.getGseFileTaskResult(); - String destCloudIp = taskResult.getDestCloudIp(); - log.info("Target agent down, sourceIp is null"); - for (String fileSourceIp : this.fileSourceIPSet) { - boolean isAddSuccess = addFinishedFile(false, true, destCloudIp, - buildTaskId(taskResult.getMode(), fileSourceIp, taskResult.getStandardSourceFilePath(), - destCloudIp, taskResult.getStandardDestFilePath())); - if (isAddSuccess) { - addFileTaskLog(executionLogs, destCloudIp, - new ServiceFileTaskLogDTO(FileDistModeEnum.DOWNLOAD.getValue(), - destCloudIp, taskResult.getStandardDestFilePath(), fileSourceIp, fileSourceIp, - taskResult.getStandardSourceFilePath(), - taskResult.getStandardSourceFilePath() == null ? null : - sourceFileDisplayMap.get(taskResult.getStandardSourceFilePath()), - null, FileDistStatusEnum.FAILED.getValue(), - FileDistStatusEnum.FAILED.getName(), - null, null, copyFileRsp.getFinalErrorMsg())); - affectIps.add(fileSourceIp); - } - } - } - - private void recordFinishedFile( - CopyFileRsp copyFileRsp, - Map executionLogs, - Set affectIps - ) { + private void recordDownloadFileFail(CopyFileRsp copyFileRsp, + Map executionLogs) { GSEFileTaskResult taskResult = copyFileRsp.getGseFileTaskResult(); - String destCloudIp = taskResult.getDestCloudIp(); - String sourceCloudIp = taskResult.getSourceCloudIp(); - boolean isAddSuccess = addFinishedFile(false, true, destCloudIp, - buildTaskId(taskResult.getMode(), sourceCloudIp, taskResult.getStandardSourceFilePath(), - destCloudIp, taskResult.getStandardDestFilePath())); + String destAgentId = taskResult.getDestAgentId(); + String sourceAgentId = taskResult.getSourceAgentId(); + AgentTaskDTO destAgentTask = targetAgentTasks.get(destAgentId); + AgentTaskDTO sourceAgentTask = sourceAgentTasks.get(sourceAgentId); + boolean isAddSuccess = addFinishedFile(false, true, destAgentId, + GSEFileTaskResult.buildTaskId(taskResult.getMode(), sourceAgentId, taskResult.getStandardSourceFilePath(), + destAgentId, taskResult.getStandardDestFilePath())); if (isAddSuccess) { - addFileTaskLog(executionLogs, destCloudIp, - new ServiceFileTaskLogDTO(FileDistModeEnum.DOWNLOAD.getValue(), - destCloudIp, taskResult.getStandardDestFilePath(), taskResult.getSourceCloudIp(), - taskResult.getSourceCloudIp(), + addFileTaskLog(executionLogs, + new ServiceFileTaskLogDTO( + FileDistModeEnum.DOWNLOAD.getValue(), + destAgentTask.getHostId(), + agentIdHostMap.get(destAgentTask.getAgentId()).toCloudIp(), + taskResult.getStandardDestFilePath(), + sourceAgentTask.getHostId(), + agentIdHostMap.get(sourceAgentTask.getAgentId()).toCloudIp(), + agentIdHostMap.get(sourceAgentTask.getAgentId()).getDisplayIp(), taskResult.getStandardSourceFilePath(), taskResult.getStandardSourceFilePath() == null ? null : sourceFileDisplayMap.get(taskResult.getStandardSourceFilePath()), null, FileDistStatusEnum.FAILED.getValue(), - FileDistStatusEnum.FAILED.getName(), null, null, copyFileRsp.getFinalErrorMsg())); - affectIps.add(sourceCloudIp); + FileDistStatusEnum.FAILED.getName(), + null, + null, + copyFileRsp.getFinalErrorMsg()) + ); } } @@ -656,91 +679,46 @@ private long getEndTimeOrDefault(GSEFileTaskResult content) { content.getEndTime() : System.currentTimeMillis(); } - private void dealIpTaskFail( - CopyFileRsp copyFileRsp, - Map executionLogs, - boolean isDownloadLog - ) { + private void dealIpTaskFail(CopyFileRsp copyFileRsp, + Map executionLogs, + boolean isDownloadResult) { GSEFileTaskResult taskResult = copyFileRsp.getGseFileTaskResult(); GSEFileTaskResult content = copyFileRsp.getGseFileTaskResult(); long startTime = getStartTimeOrDefault(content); long endTime = getEndTimeOrDefault(content); - if (isDownloadLog) { - // 被该错误影响的目标ip - Set affectedTargetIps = new HashSet<>(); - String destCloudIp = taskResult.getDestCloudIp(); - String sourceCloudIp = taskResult.getSourceCloudIp(); - affectedTargetIps.add(destCloudIp); - if (copyFileRsp.getFinalErrorCode().equals(GSECode.AGENT_DOWN) && StringUtils.isEmpty(sourceCloudIp)) { - // GSE BUG, agent异常场景需要特殊处理，此时，返回的源IP可能是-1 - // GSE Server 返回的download失败日志，在多个源文件的情况下，会丢失源ip的信息，需要job补全 - recordFinishedFileSourceIPSet(copyFileRsp, executionLogs, affectedTargetIps); - } else { - recordFinishedFile(copyFileRsp, executionLogs, affectedTargetIps); - } - - String cloudIp = taskResult.getDestCloudIp(); - for (String affectIp : affectedTargetIps) { - if (affectIp.equals(cloudIp)) { - analyseIpResult(copyFileRsp.getFinalErrorCode(), affectIp, startTime, endTime, true); - } else { - analyseIpResult(0, affectIp, startTime, endTime, true); - } - } + if (isDownloadResult) { + String destAgentId = taskResult.getDestAgentId(); + recordDownloadFileFail(copyFileRsp, executionLogs); + analyseAgentTaskResult(copyFileRsp.getFinalErrorCode(), destAgentId, startTime, endTime, true); } else { // 上传源IP本身处理 - String sourceCloudIp = taskResult.getSourceCloudIp(); - analyseIpResult(copyFileRsp.getFinalErrorCode(), sourceCloudIp, startTime, endTime, false); + String sourceAgentId = taskResult.getSourceAgentId(); + analyseAgentTaskResult(copyFileRsp.getFinalErrorCode(), sourceAgentId, startTime, endTime, false); // 如果上传源失败，除了影响上传，还会影响到下载的目标IP Set affectedTargetIps = new HashSet<>(); dealUploadFail(copyFileRsp, executionLogs, affectedTargetIps); for (String affectedTargetIp : affectedTargetIps) { - analyseIpResult(0, affectedTargetIp, startTime, endTime, true); - } - } - } - - private String buildTaskId(Integer mode, String sourceIp, String sourceFilePath, String destIp, - String destFilePath) { - String taskId; - if (isDownloadLog(mode)) { - if (StringUtils.isNotEmpty(sourceIp)) { - taskId = concat(mode.toString(), sourceIp, FilePathUtils.standardizedGSEFilePath(sourceFilePath), - destIp, destFilePath); - } else { - // GSE BUG, 兼容处理 - taskId = concat(mode.toString(), "*", FilePathUtils.standardizedGSEFilePath(sourceFilePath), - destIp, destFilePath); + analyseAgentTaskResult(0, affectedTargetIp, startTime, endTime, true); } - } else { - taskId = concat(mode.toString(), sourceIp, FilePathUtils.standardizedGSEFilePath(sourceFilePath)); - } - return taskId; - } - - private String concat(String... strArgs) { - StringJoiner sj = new StringJoiner(":"); - for (String strArg : strArgs) { - sj.add(strArg); } - return sj.toString(); } /** * 根据copyFileRsp内容填充executionLogs与affectIps * - * @param copyFileRsp GSE响应内容 - * @param executionLogs 执行日志总Map - * @param affectedTargetIps 受影响的目标IP集合 + * @param copyFileRsp GSE响应内容 + * @param executionLogs 执行日志总Map + * @param affectedTargetAgentIds 受影响的目标Agent集合 */ - private void dealUploadFail(CopyFileRsp copyFileRsp, Map executionLogs, - Set affectedTargetIps) { + private void dealUploadFail(CopyFileRsp copyFileRsp, Map executionLogs, + Set affectedTargetAgentIds) { GSEFileTaskResult taskResult = copyFileRsp.getGseFileTaskResult(); - String sourceCloudIp = taskResult.getSourceCloudIp(); + String sourceAgentId = taskResult.getSourceAgentId(); + HostDTO sourceHost = agentIdHostMap.get(sourceAgentId); // 记录源IP单个文件上传任务的结束状态 - addFinishedFile(false, false, taskResult.getSourceCloudIp(), taskResult.getTaskId()); + addFinishedFile(false, false, taskResult.getSourceAgentId(), taskResult.getTaskId()); String sourceFilePath = taskResult.getStandardSourceFilePath(); String displayFilePath = sourceFileDisplayMap.get(sourceFilePath); @@ -749,138 +727,172 @@ private void dealUploadFail(CopyFileRsp copyFileRsp, Map= fileNum) { - // 每个文件都处理完了，才算IP完成执行 - if (isDownload && isTargetIp) { - ipLog.setStatus(IpStatus.SUCCESS.getValue()); - this.successIpSet.add(cloudIp); + // 每个文件都处理完了，才算任务完成执行 + agentTask.setStatus(AgentTaskStatusEnum.SUCCESS); + if (isDownload) { + this.successTargetAgentIds.add(agentId); } } else { - int ipStatus = IpStatus.FAILED.getValue(); + AgentTaskStatusEnum agentTaskStatus = AgentTaskStatusEnum.FAILED; if (errorCode != 0) { - ipStatus = Utils.getStatusByGseErrorCode(errorCode); - if (ipStatus < 0) { - ipStatus = IpStatus.FILE_ERROR_UNCLASSIFIED.getValue(); - } + agentTaskStatus = GseUtils.getStatusByGseErrorCode(errorCode); } - ipLog.setStatus(ipStatus); + agentTask.setStatus(agentTaskStatus); } } /** - * 分析日志，更新successIpSet、notStartedIpSet等状态集合，用于判定最终整体任务状态 + * 分析结果，更新successAgentIds、notStartedAgentIds等状态集合，用于判定最终整体任务状态 * - * @param errorCode GSE错误码 - * @param cloudIp IP - * @param startTime 任务起始时间 - * @param endTime 任务终止时间 - * @param isDownload 是否为下载过程 + * @param errorCode GSE错误码 + * @param agentId agentId + * @param startTime 任务起始时间 + * @param endTime 任务终止时间 + * @param isDownloadResult 是否为下载结果 */ - private void analyseIpResult(int errorCode, String cloudIp, long startTime, long endTime, boolean isDownload) { + private void analyseAgentTaskResult(int errorCode, String agentId, long startTime, long endTime, + boolean isDownloadResult) { int finishedNum; int fileNum; int successNum; - boolean isTargetIp = targetIpSet.contains(cloudIp); - boolean isSourceIp = fileSourceIPSet.contains(cloudIp); - if (isDownload && isTargetIp) { - finishedNum = this.finishedDownloadFileMap.get(cloudIp) == null ? 0 : - this.finishedDownloadFileMap.get(cloudIp).size(); - fileNum = this.fileDownloadTaskNumMap.get(cloudIp) == null ? 0 : this.fileDownloadTaskNumMap.get(cloudIp); - successNum = this.successDownloadFileMap.get(cloudIp) == null ? 0 : - this.successDownloadFileMap.get(cloudIp).size(); - } else if (isSourceIp) { - finishedNum = this.finishedUploadFileMap.get(cloudIp) == null ? 0 : - this.finishedUploadFileMap.get(cloudIp).size(); - successNum = this.successUploadFileMap.get(cloudIp) == null ? 0 : - this.successUploadFileMap.get(cloudIp).size(); - fileNum = this.fileUploadTaskNumMap.get(cloudIp) == null ? 0 : this.fileUploadTaskNumMap.get(cloudIp); + if (isDownloadResult) { + finishedNum = this.finishedDownloadFileMap.get(agentId) == null ? 0 : + this.finishedDownloadFileMap.get(agentId).size(); + fileNum = this.fileDownloadTaskNumMap.get(agentId) == null ? 0 : this.fileDownloadTaskNumMap.get(agentId); + successNum = this.successDownloadFileMap.get(agentId) == null ? 0 : + this.successDownloadFileMap.get(agentId).size(); } else { - return; + finishedNum = this.finishedUploadFileMap.get(agentId) == null ? 0 : + this.finishedUploadFileMap.get(agentId).size(); + successNum = this.successUploadFileMap.get(agentId) == null ? 0 : + this.successUploadFileMap.get(agentId).size(); + fileNum = this.fileUploadTaskNumMap.get(agentId) == null ? 0 : this.fileUploadTaskNumMap.get(agentId); } - GseTaskIpLogDTO ipLog = ipLogMap.get(cloudIp); + AgentTaskDTO agentTask = getAgentTask(isDownloadResult, agentId); if (finishedNum >= fileNum) { log.info("[{}] Ip analyse finished! ip: {}, finishedTaskNum: {}, expectedTaskNum: {}", - stepInstanceId, cloudIp, finishedNum, fileNum); - if (isDownload && isTargetIp) { - // 更新IP统计状态集合，为ipLog设置任务起止时间 - dealIPFinish(cloudIp, startTime, endTime, ipLog); - } - if (isSourceIp) { - // 更新IP统计状态集合 - dealUploadIpFinished(cloudIp); + stepInstanceId, agentId, finishedNum, fileNum); + // 更新AgentTask结果 + if (isDownloadResult) { + dealTargetAgentFinish(agentId, startTime, endTime, agentTask); + } else { + dealUploadAgentFinished(agentId, startTime, endTime, agentTask); } - updateFinishedIpStatusAndLog(errorCode, cloudIp, fileNum, successNum, isDownload, ipLog); + analyseAgentStatus(errorCode, agentId, fileNum, successNum, isDownloadResult, agentTask); } else { - ipLog.setStatus(IpStatus.RUNNING.getValue()); - this.notStartedIpSet.remove(cloudIp); + agentTask.setStatus(AgentTaskStatusEnum.RUNNING); + this.notStartedTargetAgentIds.remove(agentId); } } - private void dealUploadIpFinished(String sourceCloudIp) { - this.runningFileSourceIpSet.remove(sourceCloudIp); - this.notStartedFileSourceIpSet.remove(sourceCloudIp); - this.analyseFinishedSourceIpSet.add(sourceCloudIp); + /** + * 设置源agent任务结束状态 + * + * @param agentId agentId + * @param startTime 起始时间 + * @param endTime 终止时间 + * @param agentTask 日志 + */ + private void dealUploadAgentFinished(String agentId, Long startTime, Long endTime, AgentTaskDTO agentTask) { + log.info("[{}]: Deal source agent finished| agentId={}| startTime:{}, endTime:{}, agentTask:{}", + stepInstanceId, agentId, startTime, endTime, JsonUtils.toJsonWithoutSkippedFields(agentTask)); + + this.runningFileSourceAgentIds.remove(agentId); + this.notStartedFileSourceAgentIds.remove(agentId); + this.analyseFinishedSourceAgentIds.add(agentId); + if (endTime - startTime <= 0) { + agentTask.setTotalTime(100L); + } else { + agentTask.setTotalTime(endTime - startTime); + } + agentTask.setStartTime(startTime); + agentTask.setEndTime(endTime); } /* * 从执行结果生成执行日志 */ - private void parseExecutionLog(CopyFileRsp copyFileRsp, Map executionLogs) { + private void parseExecutionLog(CopyFileRsp copyFileRsp, Map executionLogs) { GSEFileTaskResult taskResult = copyFileRsp.getGseFileTaskResult(); if (null != taskResult) { Integer mode = taskResult.getMode(); - boolean isDownloadLog = isDownloadLog(mode); - String cloudIp = isDownloadLog ? taskResult.getDestCloudIp() : taskResult.getSourceCloudIp(); + boolean isDownloadResult = isDownloadResult(mode); GSECode.AtomicErrorCode errorCode = GSECode.AtomicErrorCode.getErrorCode(copyFileRsp.getFinalErrorCode()); String key = taskResult.getTaskId(); Integer process = processMap.computeIfAbsent(key, k -> -1); @@ -891,9 +903,9 @@ private void parseExecutionLog(CopyFileRsp copyFileRsp, Map 0) { // 兼容GSE不返回size的情况 - fileSize = GseConstants.tranByteReadable(taskResult.getSize()); + fileSize = GseUtils.tranByteReadable(taskResult.getSize()); logContent.append(" FileSize: ").append(fileSize); } - if (StringUtils.isNotEmpty(taskResult.getStatusDesc())) { - logContent.append(" State: ").append(taskResult.getStatusDesc()); - } if (taskResult.getSpeed() != null) { speed = formatSpeed(taskResult.getSpeed()) + " KB/s"; logContent.append(" Speed: ").append(speed); @@ -930,42 +939,74 @@ private void parseExecutionLog(CopyFileRsp copyFileRsp, Map executionLogs, String ip, - ServiceFileTaskLogDTO fileTaskLog) { - ServiceIpLogDTO ipExecutionLog = executionLogs.get(ip); - if (ipExecutionLog == null) { - ipExecutionLog = new ServiceIpLogDTO(); - ipExecutionLog.setStepInstanceId(stepInstanceId); - ipExecutionLog.setIp(ip); - ipExecutionLog.setExecuteCount(stepInstance.getExecuteCount()); - executionLogs.put(ip, ipExecutionLog); + + private void addFileTaskLog(Map hostLogs, ServiceFileTaskLogDTO fileTaskLog) { + boolean isDownloadResult = isDownloadResult(fileTaskLog.getMode()); + Long hostId = isDownloadResult ? fileTaskLog.getDestHostId() : fileTaskLog.getSrcHostId(); + String cloudIp = isDownloadResult ? fileTaskLog.getDestIp() : fileTaskLog.getSrcIp(); + ServiceHostLogDTO hostLog = hostLogs.get(hostId); + if (hostLog == null) { + hostLog = new ServiceHostLogDTO(); + hostLog.setStepInstanceId(stepInstanceId); + hostLog.setHostId(hostId); + hostLog.setIp(cloudIp); + hostLog.setBatch(stepInstance.getBatch()); + hostLog.setExecuteCount(stepInstance.getExecuteCount()); + hostLogs.put(hostId, hostLog); } - ipExecutionLog.addFileTaskLog(fileTaskLog); + hostLog.addFileTaskLog(fileTaskLog); } - private void writeFileTaskLogContent(Map executionLogs) { - executionLogs.forEach((ip, executionLog) -> - logService.writeFileLogWithTimestamp(taskInstance.getCreateTime(), stepInstanceId, - stepInstance.getExecuteCount(), ip, executionLog, System.currentTimeMillis())); + private void writeFileTaskLogContent(Map executionLogs) { + if (!executionLogs.isEmpty()) { + logService.writeFileLogs(taskInstance.getCreateTime(), new ArrayList<>(executionLogs.values())); + } } /** @@ -1020,26 +1059,26 @@ private void writeFileTaskLogContent(Map executionLogs) * * @param isSuccess 单个文件上传/下载任务是否成功 * @param isDownloadMode 是否为下载 - * @param cloudIp 含云区域ID的IP + * @param agentId bk_agent_id * @param taskId 单个文件任务唯一Key * @return 是否添加成功 */ - private boolean addFinishedFile(boolean isSuccess, boolean isDownloadMode, String cloudIp, String taskId) { + private boolean addFinishedFile(boolean isSuccess, boolean isDownloadMode, String agentId, String taskId) { if (isDownloadMode) { - return addFinishedFile(isSuccess, cloudIp, taskId, finishedDownloadFileMap, successDownloadFileMap); + return addFinishedFile(isSuccess, agentId, taskId, finishedDownloadFileMap, successDownloadFileMap); } else { - return addFinishedFile(isSuccess, cloudIp, taskId, finishedUploadFileMap, successUploadFileMap); + return addFinishedFile(isSuccess, agentId, taskId, finishedUploadFileMap, successUploadFileMap); } } - private boolean addFinishedFile(boolean isSuccess, String cloudIp, String taskId, + private boolean addFinishedFile(boolean isSuccess, String agentId, String taskId, Map> finishedFileMap, Map> successFileMap) { - Set finishedFileSet = finishedFileMap.computeIfAbsent(cloudIp, k -> Sets.newHashSet()); + Set finishedFileSet = finishedFileMap.computeIfAbsent(agentId, k -> Sets.newHashSet()); boolean isAdd = finishedFileSet.add(taskId); if (isSuccess) { - Set successFileSet = successFileMap.computeIfAbsent(cloudIp, k -> Sets.newHashSet()); + Set successFileSet = successFileMap.computeIfAbsent(agentId, k -> Sets.newHashSet()); successFileSet.add(taskId); } return isAdd; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ResultHandleManager.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ResultHandleManager.java index a08569b2ea..7f1d188fcf 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ResultHandleManager.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ResultHandleManager.java @@ -24,8 +24,6 @@ package com.tencent.bk.job.execute.engine.result; -import brave.ScopedSpan; -import brave.Tracing; import com.tencent.bk.job.execute.common.exception.MessageHandlerUnavailableException; import com.tencent.bk.job.execute.common.ha.DestroyOrder; import com.tencent.bk.job.execute.config.JobExecuteConfig; @@ -34,6 +32,8 @@ import com.tencent.bk.job.execute.monitor.metrics.ExecuteMonitor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cloud.sleuth.Span; +import org.springframework.cloud.sleuth.Tracer; import org.springframework.context.SmartLifecycle; import org.springframework.core.task.SimpleAsyncTaskExecutor; import org.springframework.stereotype.Component; @@ -58,9 +58,9 @@ @Slf4j public class ResultHandleManager implements SmartLifecycle { /** - * 日志调用链Tracing + * 日志调用链Tracer */ - private final Tracing tracing; + private final Tracer tracer; /** * 结果处理任务存活管理 */ @@ -137,10 +137,10 @@ public class ResultHandleManager implements SmartLifecycle { new LinkedBlockingQueue<>()); @Autowired - public ResultHandleManager(Tracing tracing, ExecuteMonitor counters, + public ResultHandleManager(Tracer tracer, ExecuteMonitor counters, ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager, ResultHandleTaskSampler resultHandleTaskSampler, JobExecuteConfig jobExecuteConfig) { - this.tracing = tracing; + this.tracer = tracer; this.counters = counters; this.resultHandleTaskKeepaliveManager = resultHandleTaskKeepaliveManager; this.resultHandleTaskSampler = resultHandleTaskSampler; @@ -156,7 +156,7 @@ public void handleDeliveredTask(ContinuousScheduledTask task) { resultHandleLimiter.acquire(); log.info("Handle delivered task: {}", task); ScheduledContinuousResultHandleTask scheduleTask = - new ScheduledContinuousResultHandleTask(resultHandleTaskSampler, tracing, task, this, + new ScheduledContinuousResultHandleTask(resultHandleTaskSampler, tracer, task, this, resultHandleTaskKeepaliveManager, resultHandleLimiter); synchronized (lifecycleMonitor) { if (!isActive()) { @@ -267,7 +267,7 @@ private void stopTasksGraceful() { stopTaskCounter.initCounter(scheduledTasks.keySet()); } for (ScheduledContinuousResultHandleTask task : scheduledTasks.values()) { - shutdownExecutorService.execute(new StopTask(task, tracing)); + shutdownExecutorService.execute(new StopTask(task, tracer)); } } try { @@ -365,18 +365,18 @@ public int getPhase() { private static final class StopTask implements Runnable { private final ScheduledContinuousResultHandleTask task; - private final Tracing tracing; + private final Tracer tracer1; - StopTask(ScheduledContinuousResultHandleTask task, Tracing tracing) { + StopTask(ScheduledContinuousResultHandleTask task, Tracer tracer1) { this.task = task; - this.tracing = tracing; + this.tracer1 = tracer1; } @Override public void run() { - ScopedSpan span = null; + Span span = null; try { - span = tracing.tracer().startScopedSpanWithParent("stop-task", task.getTraceContext()); + span = tracer1.nextSpan(task.getTraceContext()).name("stop-task"); log.info("Begin to stop task, task: {}", task.getResultHandleTask()); task.getResultHandleTask().stop(); log.info("Stop task successfully, task: {}", task.getResultHandleTask()); @@ -385,7 +385,7 @@ public void run() { log.warn(errorMsg, e); } finally { if (span != null) { - span.finish(); + span.end(); } } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScheduledContinuousResultHandleTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScheduledContinuousResultHandleTask.java index 0dcea51095..d1fb42871f 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScheduledContinuousResultHandleTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScheduledContinuousResultHandleTask.java @@ -24,14 +24,12 @@ package com.tencent.bk.job.execute.engine.result; -import brave.ScopedSpan; -import brave.Tracer; -import brave.Tracing; -import brave.propagation.TraceContext; import com.tencent.bk.job.execute.engine.result.ha.ResultHandleLimiter; import com.tencent.bk.job.execute.engine.result.ha.ResultHandleTaskKeepaliveManager; import com.tencent.bk.job.execute.monitor.ExecuteMetricNames; import lombok.extern.slf4j.Slf4j; +import org.springframework.cloud.sleuth.Span; +import org.springframework.cloud.sleuth.Tracer; import java.util.Objects; import java.util.StringJoiner; @@ -70,25 +68,25 @@ public class ScheduledContinuousResultHandleTask extends DelayedTask { /** * 调用链父上下文 */ - private TraceContext parent; + private final Span parent; /** * ScheduledContinuousQueuedTask Constructor * * @param sampler 采样器 - * @param tracing 日志调用链 + * @param tracer 日志调用链 * @param task 任务 * @param resultHandleManager resultHandleManager * @param resultHandleLimiter 限流 */ public ScheduledContinuousResultHandleTask(ResultHandleTaskSampler sampler, - Tracing tracing, ContinuousScheduledTask task, + Tracer tracer, ContinuousScheduledTask task, ResultHandleManager resultHandleManager, ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager, ResultHandleLimiter resultHandleLimiter) { this.sampler = sampler; - this.tracer = tracing.tracer(); - this.parent = tracing.currentTraceContext().get(); + this.tracer = tracer; + this.parent = tracer.currentSpan(); this.task = task; this.delayedTask = new DelayedTask(this.task, this.task.getScheduleStrategy().getDelay()); this.resultHandleManager = resultHandleManager; @@ -97,10 +95,23 @@ public ScheduledContinuousResultHandleTask(ResultHandleTaskSampler sampler, this.resultHandleLimiter = resultHandleLimiter; } + private Span getChildSpan() { + return this.tracer.nextSpan(parent).name("execute-task"); + } + @Override public void execute() { - ScopedSpan span = this.tracer.startScopedSpanWithParent("execute-task", - this.parent); + Span span = getChildSpan(); + try (Tracer.SpanInScope ignored = this.tracer.withSpan(span.start())) { + doExecute(); + } catch (Exception e) { + span.error(e); + } finally { + span.end(); + } + } + + public void doExecute() { // 任务是否执行完成 boolean isDone = false; // 任务是否是可执行的 @@ -131,7 +142,6 @@ public void execute() { isDone = true; } } catch (Exception | Error e) { - span.error(e); status = "error"; throw e; } finally { @@ -156,8 +166,6 @@ public void execute() { if (!isReScheduled) { resultHandleLimiter.release(); } - this.parent = span.context(); - span.finish(); } } @@ -201,7 +209,7 @@ public long getExpireTime() { return this.delayedTask.getExpireTime(); } - public TraceContext getTraceContext() { + public Span getTraceContext() { return this.parent; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScriptResultHandleTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScriptResultHandleTask.java index be4eac5c57..28a4de5425 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScriptResultHandleTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScriptResultHandleTask.java @@ -27,14 +27,17 @@ import com.tencent.bk.gse.taskapi.api_agent_task_rst; import com.tencent.bk.gse.taskapi.api_query_task_info_v2; import com.tencent.bk.gse.taskapi.api_task_detail_result; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.util.BatchUtil; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.execute.common.exception.ReadTimeoutException; import com.tencent.bk.job.execute.constants.VariableValueTypeEnum; +import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; import com.tencent.bk.job.execute.engine.consts.GSECode; -import com.tencent.bk.job.execute.engine.consts.IpStatus; +import com.tencent.bk.job.execute.engine.evict.TaskEvictPolicyExecutor; import com.tencent.bk.job.execute.engine.gse.GseRequestPrinter; import com.tencent.bk.job.execute.engine.gse.GseRequestUtils; +import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; import com.tencent.bk.job.execute.engine.model.GseLog; import com.tencent.bk.job.execute.engine.model.GseLogBatchPullResult; import com.tencent.bk.job.execute.engine.model.GseTaskExecuteResult; @@ -42,14 +45,22 @@ import com.tencent.bk.job.execute.engine.model.ScriptTaskLog; import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; import com.tencent.bk.job.execute.engine.model.TaskVariablesAnalyzeResult; -import com.tencent.bk.job.execute.engine.util.Utils; -import com.tencent.bk.job.execute.model.GseTaskIpLogDTO; -import com.tencent.bk.job.execute.model.GseTaskLogDTO; +import com.tencent.bk.job.execute.engine.result.ha.ResultHandleTaskKeepaliveManager; +import com.tencent.bk.job.execute.engine.util.GseUtils; +import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.model.GseTaskDTO; import com.tencent.bk.job.execute.model.HostVariableValuesDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.StepInstanceVariableValuesDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.VariableValueDTO; +import com.tencent.bk.job.execute.service.GseTaskService; +import com.tencent.bk.job.execute.service.LogService; +import com.tencent.bk.job.execute.service.ScriptAgentTaskService; +import com.tencent.bk.job.execute.service.StepInstanceService; +import com.tencent.bk.job.execute.service.StepInstanceVariableValueService; +import com.tencent.bk.job.execute.service.TaskInstanceService; +import com.tencent.bk.job.execute.service.TaskInstanceVariableService; import com.tencent.bk.job.logsvr.model.service.ServiceScriptLogDTO; import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; import lombok.extern.slf4j.Slf4j; @@ -57,8 +68,6 @@ import org.springframework.util.StopWatch; import java.nio.charset.StandardCharsets; -import java.time.ZoneId; -import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -68,6 +77,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; /** * 脚本任务执行结果处理 @@ -75,37 +85,37 @@ @Slf4j public class ScriptResultHandleTask extends AbstractResultHandleTask { /** - * GSE日志查询支持的每一批次的最大IP数目 + * GSE日志查询支持的每一批次的最大Agent数目 */ private static final int MAX_BATCH_SIZE = 1000; /** - * GSE日志查询支持的每一批次的最小IP数目 + * GSE日志查询支持的每一批次的最小Agent数目 */ private static final int MIN_BATCH_SIZE = 10; /** - * 每次批次的IP数目 - 优先选择列表 + * 每次批次的Agent数目 - 优先选择列表 */ private static final int[] BATCH_SIZE_PRIORITY_ARRAY = new int[]{MAX_BATCH_SIZE, 100, MIN_BATCH_SIZE}; /** - * 脚本任务执行日志进度 + * 脚本任务执行日志进度, Map */ - private Map logPullProgressMap = new HashMap<>(); + private final Map logPullProgressMap = new HashMap<>(); /** - * 保存命名空间参数，Map> + * 保存命名空间参数，Map> */ - private Map> namespaceParamValues = new HashMap<>(); + private final Map> namespaceParamValues = new HashMap<>(); /** - * 保存可变全局参数，Map> + * 保存可变全局参数，Map> */ - private Map> changeableGlobalParamValues = new HashMap<>(); + private final Map> changeableGlobalParamValues = new HashMap<>(); /** * 脚本任务结果处理调度策略 */ private volatile ScheduleStrategy scheduleStrategy; /** - * 目标IP分批 + * 目标Agent分批 */ - private List> pullIpBatches = new LinkedList<>(); + private List> pullAgentIdBatches = new LinkedList<>(); /** * 当前使用的批次大小 */ @@ -113,7 +123,7 @@ public class ScriptResultHandleTask extends AbstractResultHandleTask ipLogMap, - GseTaskLogDTO gseTaskLog, - Set targetIps, - String requestId - ) { - super(taskInstance, stepInstance, taskVariablesAnalyzeResult, ipLogMap, gseTaskLog, targetIps, requestId); - initLogPullProcess(ipLogMap.values()); + public ScriptResultHandleTask(TaskInstanceService taskInstanceService, + GseTaskService gseTaskService, + LogService logService, + TaskInstanceVariableService taskInstanceVariableService, + StepInstanceVariableValueService stepInstanceVariableValueService, + TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, + ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager, + TaskEvictPolicyExecutor taskEvictPolicyExecutor, + ScriptAgentTaskService scriptAgentTaskService, + StepInstanceService stepInstanceService, + TaskInstanceDTO taskInstance, + StepInstanceDTO stepInstance, + TaskVariablesAnalyzeResult taskVariablesAnalyzeResult, + Map agentTaskMap, + GseTaskDTO gseTask, + String requestId) { + super(taskInstanceService, + gseTaskService, + logService, + taskInstanceVariableService, + stepInstanceVariableValueService, + taskExecuteMQEventDispatcher, + resultHandleTaskKeepaliveManager, + taskEvictPolicyExecutor, + scriptAgentTaskService, + stepInstanceService, + taskInstance, + stepInstance, + taskVariablesAnalyzeResult, + agentTaskMap, + gseTask, + requestId); + initLogPullProcess(agentTaskMap.values()); } - private void initLogPullProcess(Collection ipLogs) { - ipLogs.forEach(ipLog -> { + private void initLogPullProcess(Collection agentTasks) { + agentTasks.forEach(agentTask -> { LogPullProgress process = new LogPullProgress(); - process.setIp(ipLog.getCloudAreaAndIp()); - process.setByteOffset(ipLog.getOffset()); + process.setAgentId(agentTask.getAgentId()); + process.setByteOffset(agentTask.getScriptLogOffset()); process.setMid(0); - logPullProgressMap.put(ipLog.getCloudAreaAndIp(), process); + logPullProgressMap.put(agentTask.getAgentId(), process); }); } @Override - GseLogBatchPullResult pullGseTaskLogInBatches() { - if (pullIpBatches.isEmpty()) { - Set queryIpSet = new HashSet<>(); - queryIpSet.addAll(notStartedIpSet); - queryIpSet.addAll(runningIpSet); - List queryIpList = new ArrayList<>(queryIpSet); - pullIpBatches = BatchUtil.buildBatchList(queryIpList, currentBatchSize); + GseLogBatchPullResult pullGseTaskResultInBatches() { + if (pullAgentIdBatches.isEmpty()) { + Set queryAgentIds = new HashSet<>(); + queryAgentIds.addAll(notStartedTargetAgentIds); + queryAgentIds.addAll(runningTargetAgentIds); + List queryAgentIdList = new ArrayList<>(queryAgentIds); + pullAgentIdBatches = BatchUtil.buildBatchList(queryAgentIdList, currentBatchSize); } - return tryPullGseLogWithRetry(); + return tryPullGseResultWithRetry(); } - private GseLogBatchPullResult tryPullGseLogWithRetry() { - List pullLogIps = pullIpBatches.get(pullIpBatchesIndex.get() - 1); + private GseLogBatchPullResult tryPullGseResultWithRetry() { + List pullLogAgentIds = pullAgentIdBatches.get(pullResultBatchesIndex.get() - 1); try { - api_task_detail_result detailRst = pullGseLog(pullLogIps); - boolean isLastBatch = pullIpBatchesIndex.get() == pullIpBatches.size(); + api_task_detail_result detailRst = pullGseTaskResult(pullLogAgentIds); + boolean isLastBatch = pullResultBatchesIndex.get() == pullAgentIdBatches.size(); GseLogBatchPullResult batchPullResult = new GseLogBatchPullResult<>(true, isLastBatch, new ScriptTaskLog(detailRst), null); if (isLastBatch) { resetBatch(); } else { - pullIpBatchesIndex.incrementAndGet(); + pullResultBatchesIndex.incrementAndGet(); } return batchPullResult; } catch (ReadTimeoutException e) { @@ -185,8 +216,8 @@ private GseLogBatchPullResult tryPullGseLogWithRetry() { if (isSuccess) { log.info("Reduce batch size and rebuild batch list successfully, currentBatchSize: {}, batches: {}. " + "Retry pull!", - this.currentBatchSize, this.pullIpBatches); - return tryPullGseLogWithRetry(); + this.currentBatchSize, this.pullAgentIdBatches); + return tryPullGseResultWithRetry(); } else { log.warn("Try pull gse log with min batch size, but fail!"); return new GseLogBatchPullResult<>(false, true, null, "Pull gse task log timeout"); @@ -195,12 +226,12 @@ private GseLogBatchPullResult tryPullGseLogWithRetry() { } private void resetBatch() { - pullIpBatchesIndex = new AtomicInteger(1); - pullIpBatches.clear(); + pullResultBatchesIndex = new AtomicInteger(1); + pullAgentIdBatches.clear(); } - private api_task_detail_result pullGseLog(List ips) { - api_query_task_info_v2 requestV2 = GseRequestUtils.buildScriptLogRequestV2(gseTaskLog.getGseTaskId(), ips, + private api_task_detail_result pullGseTaskResult(List agentIds) { + api_query_task_info_v2 requestV2 = GseRequestUtils.buildScriptLogRequestV2(gseTask.getGseTaskId(), agentIds, logPullProgressMap); return GseRequestUtils.getScriptTaskDetailRst(stepInstanceId, requestV2); } @@ -213,14 +244,15 @@ private boolean tryReduceBatchSizeAndRebuildBatchList() { } currentBatchSize = getBatchSizeLessThanCurrentSize(); List> newBatchList = new ArrayList<>(); - List leftIps = new ArrayList<>(); - if (pullIpBatchesIndex.get() > 1) { - List> pullFinishedBatchList = pullIpBatches.subList(0, pullIpBatchesIndex.get() - 1); + List leftAgentIds = new ArrayList<>(); + if (pullResultBatchesIndex.get() > 1) { + List> pullFinishedBatchList = pullAgentIdBatches.subList(0, pullResultBatchesIndex.get() - 1); newBatchList.addAll(pullFinishedBatchList); } - pullIpBatches.subList(pullIpBatchesIndex.get() - 1, pullIpBatches.size()).forEach(leftIps::addAll); - newBatchList.addAll(BatchUtil.buildBatchList(leftIps, currentBatchSize)); - pullIpBatches = newBatchList; + pullAgentIdBatches.subList(pullResultBatchesIndex.get() - 1, pullAgentIdBatches.size()) + .forEach(leftAgentIds::addAll); + newBatchList.addAll(BatchUtil.buildBatchList(leftAgentIds, currentBatchSize)); + pullAgentIdBatches = newBatchList; return true; } @@ -234,37 +266,36 @@ private int getBatchSizeLessThanCurrentSize() { } @Override - GseTaskExecuteResult analyseGseTaskLog(GseLog taskDetail) { + GseTaskExecuteResult analyseGseTaskResult(GseLog taskDetail) { long currentTime = DateUtils.currentTimeMillis(); // 当前时间 - Set analysedIpSet = new HashSet<>(); List scriptLogs = new ArrayList<>(); StopWatch watch = new StopWatch("analyse-gse-script-task"); watch.start("analyse"); - for (api_agent_task_rst ipResult : taskDetail.getGseLog().getResult()) { - log.info("[{}]: ipResult={}", stepInstanceId, GseRequestPrinter.printAgentTaskResult(ipResult)); + for (api_agent_task_rst agentTaskResult : taskDetail.getGseLog().getResult()) { + log.info("[{}]: agentTaskResult={}", stepInstanceId, + GseRequestPrinter.printAgentTaskResult(agentTaskResult)); /*为了解决shell上下文传参的问题，在下发用户脚本的时候，实际上下下发两个脚本。第一个脚本是用户脚本，第二个脚本 *是获取上下文参数的脚本。所以m_id=0的是用户脚本的执行日志，需要分析记录；m_id=1的，则是获取上下文参数 *输出的日志内容，不需要记录，仅需要从日志分析提取上下文参数*/ - boolean isUserScriptResult = ipResult.getAtomic_task_id() == 0; - String cloudIp = ipResult.getGse_composite_id() + ":" + ipResult.getIp(); + boolean isUserScriptResult = agentTaskResult.getAtomic_task_id() == 0; + String agentId = agentTaskResult.getGse_composite_id() + ":" + agentTaskResult.getIp(); - // 该ip已经日志分析结束，不要再分析 - if (this.analyseFinishedIpSet.contains(cloudIp)) { + // 该Agent已经日志分析结束，不要再分析 + if (this.analyseFinishedTargetAgentIds.contains(agentId)) { continue; } - GseTaskIpLogDTO ipLog = ipLogMap.get(cloudIp); - if (ipLog == null) { + AgentTaskDTO agentTask = targetAgentTasks.get(agentId); + if (agentTask == null) { continue; } - analysedIpSet.add(cloudIp); if (isUserScriptResult) { - addScriptLogsAndRefreshPullProgress(scriptLogs, ipResult, cloudIp, ipLog, currentTime); + addScriptLogsAndRefreshPullProgress(scriptLogs, agentTaskResult, agentId, agentTask, currentTime); } - analyseIpResult(ipResult, ipLog, cloudIp, isUserScriptResult, currentTime); + analyseAgentResult(agentTaskResult, agentTask, agentId, isUserScriptResult, currentTime); } watch.stop(); @@ -272,8 +303,8 @@ GseTaskExecuteResult analyseGseTaskLog(GseLog taskDetail saveScriptLogContent(scriptLogs); watch.stop(); - watch.start("saveIpLogs"); - batchSaveChangedIpLogs(); + watch.start("saveGseAgentTasks"); + batchSaveChangedGseAgentTasks(targetAgentTasks.values()); watch.stop(); GseTaskExecuteResult rst = analyseExecuteResult(); @@ -289,65 +320,62 @@ GseTaskExecuteResult analyseGseTaskLog(GseLog taskDetail return rst; } - private void addScriptLogsAndRefreshPullProgress(List logs, api_agent_task_rst ipResult, - String cloudIp, GseTaskIpLogDTO ipLog, long currentTime) { - if (GSECode.AtomicErrorCode.getErrorCode(ipResult.getBk_error_code()) == GSECode.AtomicErrorCode.ERROR) { - logs.add(logService.buildSystemScriptLog(cloudIp, ipResult.getBk_error_msg(), ipLog.getOffset(), + private void addScriptLogsAndRefreshPullProgress(List logs, api_agent_task_rst agentTaskResult, + String agentId, AgentTaskDTO agentTask, long currentTime) { + HostDTO host = agentIdHostMap.get(agentTask.getAgentId()); + if (GSECode.AtomicErrorCode.getErrorCode(agentTaskResult.getBk_error_code()) == GSECode.AtomicErrorCode.ERROR) { + logs.add(logService.buildSystemScriptLog(host, agentTaskResult.getBk_error_msg(), + agentTask.getScriptLogOffset(), currentTime)); } else { - String content = ipResult.getScreen(); + String content = agentTaskResult.getScreen(); if (StringUtils.isEmpty(content)) { return; } - int offset = ipLog.getOffset(); + int offset = agentTask.getScriptLogOffset(); if (StringUtils.isNotEmpty(content)) { int bytes = content.getBytes(StandardCharsets.UTF_8).length; offset += bytes; - ipLog.setOffset(offset); + agentTask.setScriptLogOffset(offset); } - logs.add(new ServiceScriptLogDTO(cloudIp, offset, ipResult.getScreen())); + logs.add(new ServiceScriptLogDTO(host, offset, agentTaskResult.getScreen())); } // 刷新日志拉取偏移量 - refreshPullLogProgress(ipResult.getScreen(), cloudIp, ipResult.getAtomic_task_id()); + refreshPullLogProgress(agentTaskResult.getScreen(), agentId, agentTaskResult.getAtomic_task_id()); } private void saveScriptLogContent(List logs) { - logService.batchWriteScriptLog(DateUtils.formatUnixTimestamp(taskInstance.getCreateTime(), ChronoUnit.MILLIS, - "yyyy_MM_dd", ZoneId.of("UTC")), stepInstanceId, stepInstance.getExecuteCount(), - logs); + logService.batchWriteScriptLog(taskInstance.getCreateTime(), stepInstanceId, stepInstance.getExecuteCount(), + stepInstance.getBatch(), logs); } - private void analyseIpResult(api_agent_task_rst ipResult, GseTaskIpLogDTO ipLog, String cloudIp, - boolean isUserScriptResult, long currentTime) { + private void analyseAgentResult(api_agent_task_rst agentResult, AgentTaskDTO agentTask, String agentId, + boolean isUserScriptResult, long currentTime) { boolean isShellScript = (stepInstance.getScriptType().equals(ScriptTypeEnum.SHELL.getValue())); - if (ipLog.getStartTime() == null) { - ipLog.setStartTime(currentTime); + if (agentTask.getStartTime() == null) { + agentTask.setStartTime(currentTime); } - ipLog.setErrCode(ipResult.getBk_error_code()); - if (GSECode.AtomicErrorCode.getErrorCode(ipResult.getBk_error_code()) == GSECode.AtomicErrorCode.ERROR) { + agentTask.setErrorCode(agentResult.getBk_error_code()); + if (GSECode.AtomicErrorCode.getErrorCode(agentResult.getBk_error_code()) == GSECode.AtomicErrorCode.ERROR) { // 脚本执行失败 - dealIPFinish(cloudIp, ipResult, ipLog); - int ipStatus = Utils.getStatusByGseErrorCode(ipResult.getBk_error_code()); - if (ipStatus < 0) { - ipStatus = IpStatus.FAILED.getValue(); - } - ipLog.setStatus(ipStatus); - } else if (GSECode.AtomicErrorCode.getErrorCode(ipResult.getBk_error_code()) + dealAgentFinish(agentId, agentResult, agentTask); + agentTask.setStatus(GseUtils.getStatusByGseErrorCode(agentResult.getBk_error_code())); + } else if (GSECode.AtomicErrorCode.getErrorCode(agentResult.getBk_error_code()) == GSECode.AtomicErrorCode.TERMINATE) { - dealIPFinish(cloudIp, ipResult, ipLog); - ipLog.setStatus(IpStatus.GSE_TASK_TERMINATE_SUCCESS.getValue()); + dealAgentFinish(agentId, agentResult, agentTask); + agentTask.setStatus(AgentTaskStatusEnum.GSE_TASK_TERMINATE_SUCCESS); this.isTerminatedSuccess = true; } else { // 分析GSE的返回状态 - GSECode.Status status = GSECode.Status.getStatus(ipResult.getStatus()); + GSECode.Status status = GSECode.Status.getStatus(agentResult.getStatus()); switch (status) { case UNSTART: // 0：原子任务已派发； case RUNNING: // 1：原子任务执行中； - notStartedIpSet.remove(cloudIp); - runningIpSet.add(cloudIp); - ipLog.setStatus(IpStatus.RUNNING.getValue()); + notStartedTargetAgentIds.remove(agentId); + runningTargetAgentIds.add(agentId); + agentTask.setStatus(AgentTaskStatusEnum.RUNNING); break; case SUCCESS: if (isShellScript && isUserScriptResult) { @@ -355,59 +383,60 @@ private void analyseIpResult(api_agent_task_rst ipResult, GseTaskIpLogDTO ipLog, && (taskVariablesAnalyzeResult.isExistChangeableGlobalVar() || taskVariablesAnalyzeResult.isExistNamespaceVar())) { //对于包含云参或者上下文参数的任务，下发任务的时候包含了2个任务；第一个是执行用户脚本；第二个获取参数的值 - ipLog.setStatus(IpStatus.RUNNING.getValue()); - notStartedIpSet.remove(cloudIp); - runningIpSet.add(cloudIp); - refreshPullLogProgress("", cloudIp, 1); + agentTask.setStatus(AgentTaskStatusEnum.RUNNING); + notStartedTargetAgentIds.remove(agentId); + runningTargetAgentIds.add(agentId); + refreshPullLogProgress("", agentId, 1); } else { //普通任务，拉取日志，设置为成功 - dealIPFinish(cloudIp, ipResult, ipLog); - ipLog.setStatus(IpStatus.SUCCESS.getValue()); - if (this.targetIpSet.contains(cloudIp)) { - successIpSet.add(cloudIp); + dealAgentFinish(agentId, agentResult, agentTask); + agentTask.setStatus(AgentTaskStatusEnum.SUCCESS); + if (this.targetAgentIds.contains(agentId)) { + successTargetAgentIds.add(agentId); } } } else { //获取输出参数的任务执行完成，需要分析日志 - dealIPFinish(cloudIp, ipResult, ipLog); - ipLog.setStatus(IpStatus.SUCCESS.getValue()); - if (this.targetIpSet.contains(cloudIp)) { - successIpSet.add(cloudIp); + dealAgentFinish(agentId, agentResult, agentTask); + agentTask.setStatus(AgentTaskStatusEnum.SUCCESS); + if (this.targetAgentIds.contains(agentId)) { + successTargetAgentIds.add(agentId); } - parseVariableValueFromResult(ipResult, cloudIp); + parseVariableValueFromResult(agentResult, agentId); } if (isUserScriptResult) { - ipLog.setTag(ipResult.getTag()); + agentTask.setTag(agentResult.getTag()); } break; case TIMEOUT: - dealIPFinish(cloudIp, ipResult, ipLog); - ipLog.setStatus(IpStatus.SCRIPT_TIMEOUT.getValue()); + dealAgentFinish(agentId, agentResult, agentTask); + agentTask.setStatus(AgentTaskStatusEnum.SCRIPT_TIMEOUT); break; case DISCARD: - dealIPFinish(cloudIp, ipResult, ipLog); - ipLog.setStatus(IpStatus.SCRIPT_TERMINATE.getValue()); + dealAgentFinish(agentId, agentResult, agentTask); + agentTask.setStatus(AgentTaskStatusEnum.SCRIPT_TERMINATE); break; default: - dealIPFinish(cloudIp, ipResult, ipLog); - int errCode = ipResult.getBk_error_code(); - int exitCode = getExitCode(ipResult.getExitcode()); + dealAgentFinish(agentId, agentResult, agentTask); + int errCode = agentResult.getBk_error_code(); + int exitCode = getExitCode(agentResult.getExitcode()); if (errCode == 0) { if (exitCode != 0) { - ipLog.setStatus(IpStatus.SCRIPT_NOT_ZERO_EXIT_CODE.getValue()); + agentTask.setStatus(AgentTaskStatusEnum.SCRIPT_NOT_ZERO_EXIT_CODE); } else { - ipLog.setStatus(IpStatus.SCRIPT_FAILED.getValue()); + agentTask.setStatus(AgentTaskStatusEnum.SCRIPT_FAILED); } - ipLog.setTag(ipResult.getTag()); + agentTask.setTag(agentResult.getTag()); } break; } } } - private void parseVariableValueFromResult(api_agent_task_rst ipResult, String ip) { - if (ipResult.getAtomic_task_id() == 1 && ipResult.getStatus() == GSECode.Status.SUCCESS.getValue()) { - String paramsContent = ipResult.getScreen(); + private void parseVariableValueFromResult(api_agent_task_rst agentTaskResult, String agentId) { + if (agentTaskResult.getAtomic_task_id() == 1 + && agentTaskResult.getStatus() == GSECode.Status.SUCCESS.getValue()) { + String paramsContent = agentTaskResult.getScreen(); if (!StringUtils.isEmpty(paramsContent)) { String[] varKeyAndValueStrArray = paramsContent.split("\n"); for (String varKeyAndValueStr : varKeyAndValueStrArray) { @@ -426,11 +455,11 @@ private void parseVariableValueFromResult(api_agent_task_rst ipResult, String ip continue; } if (taskVariablesAnalyzeResult.isNamespaceVar(paramName)) { - Map param2Values = namespaceParamValues.computeIfAbsent(ip, + Map param2Values = namespaceParamValues.computeIfAbsent(agentId, k -> new HashMap<>()); param2Values.put(paramName, parseShellEscapeValue(paramValue)); } else if (taskVariablesAnalyzeResult.isChangeableGlobalVar(paramName)) { - Map param2Values = changeableGlobalParamValues.computeIfAbsent(ip, + Map param2Values = changeableGlobalParamValues.computeIfAbsent(agentId, k -> new HashMap<>()); param2Values.put(paramName, parseShellEscapeValue(paramValue)); } @@ -498,9 +527,11 @@ private List buildNamespaceVariableValues() { for (Map.Entry> entry : namespaceParamValues.entrySet()) { HostVariableValuesDTO hostVariableValues = new HostVariableValuesDTO(); List paramValues = toVariableValuesList(entry.getValue()); - String ip = entry.getKey(); - hostVariableValues.setIp(ip); + AgentTaskDTO agentTask = targetAgentTasks.get(entry.getKey()); + hostVariableValues.setHostId(agentTask.getHostId()); + hostVariableValues.setAgentId(entry.getKey()); hostVariableValues.setValues(paramValues); + hostVariableValues.setIp(agentTask.getCloudIp()); hostVariableValuesList.add(hostVariableValues); } return hostVariableValuesList; @@ -518,9 +549,9 @@ private List toVariableValuesList(Map varNameV return variableValues; } - private void dealIPFinish(String ip, api_agent_task_rst ipResult, GseTaskIpLogDTO ipLog) { - dealIPFinish(ip, ipResult.getStart_time(), ipResult.getEnd_time(), ipLog); - ipLog.setExitCode(getExitCode(ipResult.getExitcode())); + private void dealAgentFinish(String agentId, api_agent_task_rst agentTaskResult, AgentTaskDTO agentTask) { + dealTargetAgentFinish(agentId, agentTaskResult.getStart_time(), agentTaskResult.getEnd_time(), agentTask); + agentTask.setExitCode(getExitCode(agentTaskResult.getExitcode())); } private int getExitCode(int exitCode) { @@ -535,19 +566,19 @@ private int getExitCode(int exitCode) { * 刷新拉取日志进度 * * @param context 日志内容 - * @param ip agent ip + * @param agentId bk_agent_id * @param mid gse任务的m_id */ - private void refreshPullLogProgress(String context, String ip, int mid) { + private void refreshPullLogProgress(String context, String agentId, int mid) { int increase = 0; if (null != context) { increase = context.getBytes(StandardCharsets.UTF_8).length; } - LogPullProgress progress = logPullProgressMap.get(ip); + LogPullProgress progress = logPullProgressMap.get(agentId); if (null == progress) { progress = new LogPullProgress(); - logPullProgressMap.put(ip, progress); + logPullProgressMap.put(agentId, progress); } int prevMid = progress.getMid(); if (prevMid != mid) { @@ -556,7 +587,7 @@ private void refreshPullLogProgress(String context, String ip, int mid) { } else { progress.setByteOffset(progress.getByteOffset() + increase); } - progress.setIp(ip); + progress.setAgentId(agentId); progress.setMid(mid); } @@ -566,10 +597,10 @@ private void refreshPullLogProgress(String context, String ip, int mid) { */ private GseTaskExecuteResult analyseExecuteResult() { GseTaskExecuteResult rst; - if (this.notStartedIpSet.isEmpty() && this.runningIpSet.isEmpty()) { - int targetIPNum = this.targetIpSet.size(); - int successTargetIpNum = this.successIpSet.size(); - boolean isSuccess = this.invalidIpSet.isEmpty() && successTargetIpNum == targetIPNum; + if (this.notStartedTargetAgentIds.isEmpty() && this.runningTargetAgentIds.isEmpty()) { + int targetAgentNum = this.targetAgentIds.size(); + int successTargetAgentNum = this.successTargetAgentIds.size(); + boolean isSuccess = !stepInstance.hasInvalidHost() && successTargetAgentNum == targetAgentNum; if (isSuccess) { rst = GseTaskExecuteResult.SUCCESS; } else { @@ -586,31 +617,23 @@ private GseTaskExecuteResult analyseExecuteResult() { } @Override - protected void saveFailInfoForUnfinishedIpTask(int errorType, String errorMsg) { - super.saveFailInfoForUnfinishedIpTask(errorType, errorMsg); + protected void saveFailInfoForUnfinishedAgentTask(AgentTaskStatusEnum status, String errorMsg) { + super.saveFailInfoForUnfinishedAgentTask(status, errorMsg); long endTime = System.currentTimeMillis(); - Set unfinishedIPSet = new HashSet<>(); - unfinishedIPSet.addAll(notStartedIpSet); - unfinishedIPSet.addAll(this.runningIpSet); + Set unfinishedAgentIds = new HashSet<>(); + unfinishedAgentIds.addAll(notStartedTargetAgentIds); + unfinishedAgentIds.addAll(runningTargetAgentIds); if (StringUtils.isNotEmpty(errorMsg)) { - logService.batchWriteJobSystemScriptLog(taskInstance.getCreateTime(), stepInstanceId, - stepInstance.getExecuteCount(), buildIpAndLogOffsetMap(unfinishedIPSet), errorMsg, endTime); + List scriptLogs = unfinishedAgentIds.stream().map(agentId -> { + AgentTaskDTO agentTask = targetAgentTasks.get(agentId); + return logService.buildSystemScriptLog(agentTask.getHost(), errorMsg, agentTask.getScriptLogOffset(), + endTime); + }).collect(Collectors.toList()); + logService.batchWriteScriptLog(taskInstance.getCreateTime(), stepInstanceId, stepInstance.getExecuteCount(), + stepInstance.getBatch(), scriptLogs); } } - private Map buildIpAndLogOffsetMap(Collection ips) { - Map ipAndLogOffsetMap = new HashMap<>(); - ips.forEach(ip -> { - GseTaskIpLogDTO ipLog = ipLogMap.get(ip); - if (ipLog != null) { - ipAndLogOffsetMap.put(ip, ipLog.getOffset()); - } else { - ipAndLogOffsetMap.put(ip, 0); - } - }); - return ipAndLogOffsetMap; - } - @Override public boolean isFinished() { return !getExecuteResult().getResultCode().equals(GseTaskExecuteResult.RESULT_CODE_RUNNING); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ha/NotAliveResultHandleTaskDetector.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ha/NotAliveResultHandleTaskDetector.java index 71a090fc9c..f0dbc96a9c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ha/NotAliveResultHandleTaskDetector.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ha/NotAliveResultHandleTaskDetector.java @@ -25,7 +25,7 @@ package com.tencent.bk.job.execute.engine.result.ha; import com.tencent.bk.job.common.redis.util.LockUtils; -import com.tencent.bk.job.execute.engine.TaskExecuteControlMsgSender; +import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; import com.tencent.bk.job.execute.monitor.metrics.ExecuteMonitor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; @@ -44,15 +44,15 @@ @EnableScheduling public class NotAliveResultHandleTaskDetector { private final ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager; - private final TaskExecuteControlMsgSender taskExecuteControlMsgSender; + private final TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher; private final ExecuteMonitor executeMonitor; private final String requestId = UUID.randomUUID().toString(); public NotAliveResultHandleTaskDetector(ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager, - TaskExecuteControlMsgSender taskExecuteControlMsgSender, + TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, ExecuteMonitor executeMonitor) { this.resultHandleTaskKeepaliveManager = resultHandleTaskKeepaliveManager; - this.taskExecuteControlMsgSender = taskExecuteControlMsgSender; + this.taskExecuteMQEventDispatcher = taskExecuteMQEventDispatcher; this.executeMonitor = executeMonitor; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/ExponentIncrementRollingExprPart.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/ExponentIncrementRollingExprPart.java new file mode 100644 index 0000000000..31b92657b6 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/ExponentIncrementRollingExprPart.java @@ -0,0 +1,90 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.rolling; + +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.common.exception.RollingExprParseException; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * 滚动执行子表达式-指数递增 + */ +@Getter +@Setter +@ToString +@Slf4j +public class ExponentIncrementRollingExprPart extends RollingExprPart { + /** + * 滚动表示式匹配正则 + */ + private static final String EXPR_REGEX = "^\\*(\\d+)$"; + /** + * 每一批次递增的指数 + */ + private int exponent; + + @Override + public RollingExprPart parseExpr(String expr) throws RollingExprParseException { + Pattern pattern = Pattern.compile(EXPR_REGEX); + Matcher matcher = pattern.matcher(expr); + if (matcher.find()) { + ExponentIncrementRollingExprPart rollingExprPart = new ExponentIncrementRollingExprPart(); + rollingExprPart.setExpr(expr); + + int exponent = Integer.parseInt(matcher.group(1)); + if (exponent <= 0) { + log.warn("Invalid rolling expr part : {}", expr); + throw new RollingExprParseException(); + } + rollingExprPart.setExponent(exponent); + + return rollingExprPart; + } else { + return null; + } + } + + @Override + public List compute(RollingServerBatchContext context) throws RollingExprParseException { + List candidateServers = context.getRemainedServers(); + // 上一批次的主机 + RollingServerBatch preServerBatch = context.getServerBatches().get(context.getServerBatches().size() - 1); + if (preServerBatch == null) { + log.warn("Invalid rolling expr part : {}, no preServerBatch", getExpr()); + throw new RollingExprParseException(); + } + + int currentBatchSize = preServerBatch.getServers().size() * exponent; + return new ArrayList<>(candidateServers.subList(0, Math.min(currentBatchSize, candidateServers.size()))); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/PercentRollingExprPart.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/PercentRollingExprPart.java new file mode 100644 index 0000000000..3095e72468 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/PercentRollingExprPart.java @@ -0,0 +1,79 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.rolling; + +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.common.exception.RollingExprParseException; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * 滚动执行子表达式-按照服务器百分比解析 + */ +@Getter +@Setter +@ToString +@Slf4j +public class PercentRollingExprPart extends RollingExprPart { + private static final String PERCENT_EXPR_REGEX = "^(\\d+)%$"; + private int percent; + + @Override + public RollingExprPart parseExpr(String expr) throws RollingExprParseException { + Pattern pattern = Pattern.compile(PERCENT_EXPR_REGEX); + Matcher matcher = pattern.matcher(expr); + if (matcher.find()) { + PercentRollingExprPart rollingExprPart = new PercentRollingExprPart(); + rollingExprPart.setExpr(expr); + + int percent = Integer.parseInt(matcher.group(1)); + if (percent <= 0 || percent > 100) { + log.warn("Invalid rolling expr part : {}", expr); + throw new RollingExprParseException(); + } + rollingExprPart.setPercent(percent); + + return rollingExprPart; + } else { + return null; + } + } + + @Override + public List compute(RollingServerBatchContext context) throws RollingExprParseException { + List candidateServers = context.getRemainedServers(); + int total = context.getTotalServersSize(); + // 批次大小，需要向上取整 + int batchSize = (total * percent + 100 - 1) / 100; + return new ArrayList<>(candidateServers.subList(0, Math.min(batchSize, candidateServers.size()))); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/PlusIncrementRollingExprPart.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/PlusIncrementRollingExprPart.java new file mode 100644 index 0000000000..30b66afc13 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/PlusIncrementRollingExprPart.java @@ -0,0 +1,90 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.rolling; + +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.common.exception.RollingExprParseException; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * 滚动执行子表达式-加法递增 + */ +@Getter +@Setter +@ToString +@Slf4j +public class PlusIncrementRollingExprPart extends RollingExprPart { + /** + * 滚动表示式匹配正则 + */ + private static final String EXPR_REGEX = "^\\+(\\d+)$"; + /** + * 每一批次递增的主机数量 + */ + private int addend; + + @Override + public RollingExprPart parseExpr(String expr) throws RollingExprParseException { + Pattern pattern = Pattern.compile(EXPR_REGEX); + Matcher matcher = pattern.matcher(expr); + if (matcher.find()) { + PlusIncrementRollingExprPart rollingExprPart = new PlusIncrementRollingExprPart(); + rollingExprPart.setExpr(expr); + + int addend = Integer.parseInt(matcher.group(1)); + if (addend <= 0) { + log.warn("Invalid rolling expr part : {}", expr); + throw new RollingExprParseException(); + } + rollingExprPart.setAddend(addend); + + return rollingExprPart; + } else { + return null; + } + } + + @Override + public List compute(RollingServerBatchContext context) throws RollingExprParseException { + List candidateServers = context.getRemainedServers(); + // 上一批次的主机 + RollingServerBatch preServerBatch = context.getServerBatches().get(context.getServerBatches().size() - 1); + if (preServerBatch == null) { + log.warn("Invalid rolling expr part : {}, no preServerBatch", getExpr()); + throw new RollingExprParseException(); + } + + int currentBatchSize = preServerBatch.getServers().size() + addend; + return new ArrayList<>(candidateServers.subList(0, Math.min(currentBatchSize, candidateServers.size()))); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/QuantityRollingExprPart.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/QuantityRollingExprPart.java new file mode 100644 index 0000000000..8f7a3295cd --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/QuantityRollingExprPart.java @@ -0,0 +1,77 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.rolling; + +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.common.exception.RollingExprParseException; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * 滚动执行子表达式-按照服务器数量解析 + */ +@Getter +@Setter +@ToString +@Slf4j +public class QuantityRollingExprPart extends RollingExprPart { + private static final String QUANTITY_EXPR_REGEX = "^(\\d+)$"; + private int quantity; + + @Override + public RollingExprPart parseExpr(String expr) throws RollingExprParseException { + + Pattern pattern = Pattern.compile(QUANTITY_EXPR_REGEX); + Matcher matcher = pattern.matcher(expr); + if (matcher.find()) { + QuantityRollingExprPart rollingExprPart = new QuantityRollingExprPart(); + rollingExprPart.setExpr(expr); + + int quantity = Integer.parseInt(matcher.group(1)); + if (quantity <= 0) { + log.warn("Invalid rolling expr part : {}", expr); + throw new RollingExprParseException(); + } + rollingExprPart.setQuantity(quantity); + + return rollingExprPart; + } else { + return null; + } + } + + @Override + public List compute(RollingServerBatchContext context) throws RollingExprParseException { + List candidateServers = context.getRemainedServers(); + return new ArrayList<>(candidateServers.subList(0, Math.min(this.quantity, candidateServers.size()))); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingBatchServersResolver.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingBatchServersResolver.java new file mode 100644 index 0000000000..c9c082f395 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingBatchServersResolver.java @@ -0,0 +1,93 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.rolling; + +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.FailedPreconditionException; +import com.tencent.bk.job.common.model.dto.HostDTO; +import lombok.extern.slf4j.Slf4j; + +import java.util.List; + +/** + * 服务器滚动分批Resolver + */ +@Slf4j +public class RollingBatchServersResolver { + + /** + * 允许的最大滚动批次 + */ + public static final int MAX_ALLOWED_ROLLING_BATCH_SIZE = 500; + + /** + * 滚动策略表达式 + */ + private final String rollingExpr; + + /** + * 滚动表达式解析上下文 + */ + private final RollingServerBatchContext context; + + /** + * Constructor + * + * @param servers 滚动执行的主机 + * @param rollingExpr 滚动表达式 + */ + public RollingBatchServersResolver(List servers, String rollingExpr) { + this.context = new RollingServerBatchContext(servers); + this.rollingExpr = rollingExpr; + } + + /** + * 解析滚动表达式，滚动分批 + * + * @return 服务器分批情况 + */ + public List resolve() { + RollingExpr rollingExpr = new RollingExpr(this.rollingExpr); + while (context.hasRemainedServer()) { + context.increaseBatchCount(); + RollingExprPart rollingExprPart = rollingExpr.nextRollingExprPart(context.getBatchCount()); + List serversOnBatch = rollingExprPart.compute(context); + context.removeResolvedServers(serversOnBatch); + RollingServerBatch rollingServerBatch = new RollingServerBatch(); + rollingServerBatch.setBatch(context.getBatchCount()); + rollingServerBatch.setServers(serversOnBatch); + rollingServerBatch.setRollingExprPart(rollingExprPart); + context.addServerBatch(rollingServerBatch); + } + + if (context.getServerBatches().size() > MAX_ALLOWED_ROLLING_BATCH_SIZE) { + log.warn("Batch {} size greater than {}", context.getServerBatches().size(), MAX_ALLOWED_ROLLING_BATCH_SIZE); + throw new FailedPreconditionException(ErrorCode.EXCEED_MAX_ALLOWED_BATCH_SIZE, + new Integer[]{MAX_ALLOWED_ROLLING_BATCH_SIZE}); + } + return context.getServerBatches(); + } + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingExpr.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingExpr.java new file mode 100644 index 0000000000..bf93d5f69f --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingExpr.java @@ -0,0 +1,106 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.rolling; + +import com.tencent.bk.job.execute.common.exception.RollingExprParseException; +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.List; + +/** + * 滚动策略表达式 + */ +@Slf4j +public class RollingExpr { + /** + * 滚动策略子表达式解析器 + */ + private static final List ROLLING_EXPR_PART_PARSER = new ArrayList<>(); + + static { + ROLLING_EXPR_PART_PARSER.add(new PercentRollingExprPart()); + ROLLING_EXPR_PART_PARSER.add(new QuantityRollingExprPart()); + ROLLING_EXPR_PART_PARSER.add(new PlusIncrementRollingExprPart()); + ROLLING_EXPR_PART_PARSER.add(new ExponentIncrementRollingExprPart()); + } + + /** + * 滚动策略表达式 + */ + private final String rollingExpr; + /** + * 解析之后的滚动策略子表达式列表 + */ + private final List rollingExprParts; + + /** + * Constructor + * + * @param rollingExprStr 滚动策略表达式 + */ + public RollingExpr(String rollingExprStr) { + this.rollingExpr = rollingExprStr; + this.rollingExprParts = parseRollingExpr(rollingExprStr); + } + + private List parseRollingExpr(String rollingExprStr) { + String[] rollingExprParts = rollingExprStr.split(" "); + List rollingExprPartList = new ArrayList<>(); + for (String exprPart : rollingExprParts) { + String rollingExprPartStr = exprPart.trim(); + RollingExprPart rollingExprPart = parseRollingExprPart(rollingExprPartStr); + rollingExprPartList.add(rollingExprPart); + } + return rollingExprPartList; + } + + private RollingExprPart parseRollingExprPart(String rollingExprPartStr) { + for (RollingExprPart rollingExprPart : ROLLING_EXPR_PART_PARSER) { + RollingExprPart parseResult = rollingExprPart.parseExpr(rollingExprPartStr); + if (parseResult != null) { + return parseResult; + } + } + log.warn("Invalid rolling expr, no supported expr resolver!"); + throw new RollingExprParseException(); + } + + /** + * 获取滚动策略表达式 + */ + public String getRollingExpr() { + return rollingExpr; + } + + /** + * 根据滚动执行批次获取滚动策略子表达式 + * + * @param batch 滚动执行批次 + */ + public RollingExprPart nextRollingExprPart(int batch) { + return this.rollingExprParts.get(Math.min(this.rollingExprParts.size(), batch) - 1); + } +} diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BizSetAttr.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingExprPart.java similarity index 66% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BizSetAttr.java rename to src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingExprPart.java index f8d9890d90..e79c5d8ef9 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BizSetAttr.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingExprPart.java @@ -22,62 +22,42 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.common.cc.model.bizset; +package com.tencent.bk.job.execute.engine.rolling; -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Builder; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.common.exception.RollingExprParseException; import lombok.Getter; import lombok.Setter; import lombok.ToString; +import java.util.List; + /** - * CMDB接口请求实体类，定义业务集字段 + * 滚动策略表达式部分 */ -@Builder -@Setter @Getter +@Setter @ToString -public class BizSetAttr { - - /** - * ID - */ - @JsonProperty("bk_biz_set_id") - private long id; - - /** - * 名称 - */ - @JsonProperty("bk_biz_set_name") - private String name; - - /** - * 描述 - */ - @JsonProperty("bk_biz_set_desc") - private String desc; - - /** - * 运维人员 - */ - @JsonProperty("bk_biz_maintainer") - private String maintainer; - +public abstract class RollingExprPart { /** - * 供应商 + * 滚动表达式部分 */ - @JsonProperty("bk_supplier_account") - private String supplierAccount; + private String expr; /** - * 时区 + * 解析滚动表达式 + * + * @param expr 滚动表达式 + * @return 子表达式 */ - @JsonProperty("time_zone") - private String timeZone; + public abstract RollingExprPart parseExpr(String expr) throws RollingExprParseException; /** - * 语言 + * 根据滚动子表达式以及分批计算上下文，计算当前批次的主机 + * + * @param context 解析上下文 + * @return 当前批次的主机 */ - private String language; + public abstract List compute(RollingServerBatchContext context) throws RollingExprParseException; } diff --git a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ScriptLogQueryRequest.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingServerBatch.java similarity index 84% rename from src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ScriptLogQueryRequest.java rename to src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingServerBatch.java index 53cf24e499..b89ee6974f 100644 --- a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ScriptLogQueryRequest.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingServerBatch.java @@ -22,15 +22,19 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.logsvr.model.service; +package com.tencent.bk.job.execute.engine.rolling; +import com.tencent.bk.job.common.model.dto.HostDTO; import lombok.Data; import java.util.List; +/** + * 滚动执行服务器分批 + */ @Data -public class ScriptLogQueryRequest { - private Long stepInstanceId; - private Integer executeCount; - private List ips; +public class RollingServerBatch { + private RollingExprPart rollingExprPart; + private int batch; + private List servers; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingServerBatchContext.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingServerBatchContext.java new file mode 100644 index 0000000000..8d47563e07 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingServerBatchContext.java @@ -0,0 +1,83 @@ +package com.tencent.bk.job.execute.engine.rolling; + +import com.tencent.bk.job.common.model.dto.HostDTO; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * 主机计算滚动分批上下文 + */ +@Getter +@Setter +@ToString +public class RollingServerBatchContext { + /** + * 需要分批的服务器 + */ + private List servers; + /** + * 未分批的服务器 + */ + private List remainedServers; + /** + * 需要分批的服务器数量 + */ + private int totalServersSize; + /** + * 分批数量 + */ + private int batchCount; + /** + * 分批结果 + */ + private List serverBatches; + + /** + * Constructor + * + * @param servers 所有参与滚动的主机 + */ + public RollingServerBatchContext(List servers) { + this.servers = servers; + this.remainedServers = new ArrayList<>(this.servers); + this.totalServersSize = servers.size(); + this.serverBatches = new ArrayList<>(); + } + + /** + * 增加已经解析的分批数量 + */ + public void increaseBatchCount() { + this.batchCount++; + } + + /** + * 是否还有剩余服务器没有被分批 + */ + public boolean hasRemainedServer() { + return this.remainedServers.size() > 0; + } + + /** + * 移除已经解析过的主机 + * + * @param resolvedServers 已经解析(分批)过的主机 + */ + public void removeResolvedServers(Collection resolvedServers) { + this.remainedServers.removeAll(resolvedServers); + } + + /** + * 新增一批滚动主机 + * + * @param rollingServerBatch 一个滚动批次的主机 + */ + public void addServerBatch(RollingServerBatch rollingServerBatch) { + this.serverBatches.add(rollingServerBatch); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/GseConstants.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/GseUtils.java similarity index 71% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/GseConstants.java rename to src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/GseUtils.java index b47f96c33b..d74ace741a 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/GseConstants.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/GseUtils.java @@ -22,20 +22,25 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.execute.engine.consts; +package com.tencent.bk.job.execute.engine.util; +import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; +import com.tencent.bk.job.execute.engine.consts.Consts; -public class GseConstants { +/** + * 处理 GSE 工具类 + */ +public class GseUtils { /** * 字节转换的进制 */ - private static int SIZE_UNIT = 1024; + private static final int SIZE_UNIT = 1024; /** * 将字节转换为 human readable format * - * @param size - * @return + * @param size 大小(单位Byte) + * @return 便于阅读的大小表示 */ public static String tranByteReadable(double size) { if (size < SIZE_UNIT) { @@ -57,11 +62,23 @@ public static String tranByteReadable(double size) { /** * 转换单位 - * - * @param size - * @return */ private static double changeUnit(double size) { return (double) Math.round(size / SIZE_UNIT * 1000) / 1000; } + + /** + * 根据gse ErrorCode返回 Status + */ + public static AgentTaskStatusEnum getStatusByGseErrorCode(int gseErrorCode) { + AgentTaskStatusEnum status = Consts.GSE_ERROR_CODE_2_STATUS_MAP.get(gseErrorCode); + if (status == null) { + if (gseErrorCode > 0) { + status = AgentTaskStatusEnum.UNKNOWN_ERROR; + } else { + status = AgentTaskStatusEnum.SUCCESS; + } + } + return status; + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/IpHelper.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/IpHelper.java index 5ee5cce024..28b40876f0 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/IpHelper.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/IpHelper.java @@ -24,7 +24,7 @@ package com.tencent.bk.job.execute.engine.util; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import org.apache.commons.lang3.math.NumberUtils; /** @@ -47,19 +47,19 @@ public static String fix1To0(String sourceAndIp) { return compose(transform(sourceAndIp)); } - public static IpDTO transform(String sourceAndIp) { - IpDTO ipDTO = null; + public static HostDTO transform(String sourceAndIp) { + HostDTO hostDTO = null; if (sourceAndIp != null) { String[] split = sourceAndIp.split(SI_SPLIT); if (split.length == 2) { long source = optLong(split[0].trim(), DEFAULT_CLOUD_ID); if (source == 1) source = DEFAULT_CLOUD_ID; - ipDTO = new IpDTO(source, split[1].trim()); + hostDTO = new HostDTO(source, split[1].trim()); } else { - ipDTO = new IpDTO(DEFAULT_CLOUD_ID, sourceAndIp.trim()); + hostDTO = new HostDTO(DEFAULT_CLOUD_ID, sourceAndIp.trim()); } } - return ipDTO; + return hostDTO; } private static long optLong(String str, long defaultValue) { @@ -72,11 +72,11 @@ private static long optLong(String str, long defaultValue) { return defaultValue; } - public static String compose(IpDTO ipDTO) { - if (ipDTO == null) { + public static String compose(HostDTO hostDTO) { + if (hostDTO == null) { return null; } - return compose(ipDTO.getCloudAreaId(), ipDTO.getIp()); + return compose(hostDTO.getBkCloudId(), hostDTO.getIp()); } public static String compose(Long cloudId, String ip) { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/JobSrcFileUtils.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/JobSrcFileUtils.java index eb5299b13c..e574506d10 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/JobSrcFileUtils.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/JobSrcFileUtils.java @@ -25,7 +25,7 @@ package com.tencent.bk.job.execute.engine.util; import com.google.common.collect.Sets; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.util.file.PathUtil; import com.tencent.bk.job.common.util.function.LambdasUtil; import com.tencent.bk.job.execute.engine.consts.FileDirTypeConf; @@ -65,8 +65,8 @@ public static Map buildSourceDestPathMapping(Set srcF long currentTime = System.currentTimeMillis(); for (JobFile srcFile : srcFiles) { // 本地文件的源ip是本机ip，展开源文件IP地址宏采用"0.0.0.0" - String destDirPath = MacroUtil.resolveFileSrcIpMacro(standardTargetDir, srcFile.isLocalUploadFile() ? - "0_0.0.0.0" : srcFile.getCloudAreaIdAndIp()); + String destDirPath = MacroUtil.resolveFileSrcIpMacro(standardTargetDir, + srcFile.getFileType() == TaskFileTypeEnum.LOCAL ? "0_0.0.0.0" : srcFile.getHost().toCloudIp()); destDirPath = MacroUtil.resolveDate(destDirPath, currentTime); addSourceDestPathMapping(sourceDestPathMap, srcFile, destDirPath, targetFileName); } @@ -77,7 +77,7 @@ private static void addSourceDestPathMapping(Map sourceDestPat JobFile sourceFile, String destDirPath, String destName) { - sourceDestPathMap.put(sourceFile.getFileUniqueKey(), buildFileDest(sourceFile, destDirPath, destName)); + sourceDestPathMap.put(sourceFile.getUniqueKey(), buildFileDest(sourceFile, destDirPath, destName)); } private static FileDest buildFileDest(JobFile sourceFile, String destDirPath, String destName) { @@ -104,11 +104,11 @@ private static boolean isServerOrThirdFileSource(FileSourceDTO fileSource) { * 从步骤解析源文件，处理服务器文件、本地文件、第三方源文件的差异，统一为IP+Path信息 * * @param stepInstance 步骤 - * @param localServerIp job server ip + * @param localHost job local host * @param jobStorageRootDir job共享存储根目录 * @return 多个要分发的源文件信息集合 */ - public static Set parseSendFileList(StepInstanceDTO stepInstance, String localServerIp, + public static Set parseSendFileList(StepInstanceDTO stepInstance, HostDTO localHost, String jobStorageRootDir) { Set sendFiles = Sets.newHashSet(); for (FileSourceDTO fileSource : stepInstance.getFileSourceList()) { @@ -124,25 +124,26 @@ public static Set parseSendFileList(StepInstanceDTO stepInstance, Strin Long accountId = fileSource.getAccountId(); String accountAlias = fileSource.getAccountAlias(); // 远程文件 - List ipList = fileSource.getServers().getIpList(); - Set invalidIpSet = stepInstance.getInvalidIps(); + List sourceHosts = fileSource.getServers().getIpList(); + Set invalidHosts = stepInstance.getInvalidHosts(); for (FileDetailDTO file : files) { String filePath = StringUtils.isNotEmpty(file.getResolvedFilePath()) ? file.getResolvedFilePath() : file.getFilePath(); Pair fileNameAndPath = FilePathUtils.parseDirAndFileName(filePath); String dir = fileNameAndPath.getLeft(); String fileName = fileNameAndPath.getRight(); - Predicate predicate = LambdasUtil.not(ip -> invalidIpSet.contains(ip.convertToStrIp())); - for (IpDTO ipDTO : ipList) { - if (predicate.test(ipDTO)) { + Predicate predicate = LambdasUtil.not(invalidHosts::contains); + for (HostDTO sourceHost : sourceHosts) { + if (predicate.test(sourceHost)) { // 第三方源文件的displayName不同 if (isThirdFile) { - sendFiles.add(new JobFile(false, ipDTO.convertToStrIp(), filePath, - file.getThirdFilePathWithFileSourceName(), dir, fileName, - stepInstance.getAppId(), accountId, accountAlias)); + sendFiles.add(new JobFile(TaskFileTypeEnum.FILE_SOURCE, sourceHost, + file.getThirdFilePathWithFileSourceName(), + file.getThirdFilePathWithFileSourceName(), + dir, fileName, stepInstance.getAppId(), accountId, accountAlias)); } else { - sendFiles.add(new JobFile(false, ipDTO.convertToStrIp(), filePath, - filePath, dir, fileName, stepInstance.getAppId(), accountId, accountAlias)); + sendFiles.add(new JobFile(TaskFileTypeEnum.SERVER, sourceHost, filePath, filePath, dir, + fileName, stepInstance.getAppId(), accountId, accountAlias)); } } } @@ -157,15 +158,15 @@ public static Set parseSendFileList(StepInstanceDTO stepInstance, Strin String fileName = fileNameAndPath.getRight(); ServersDTO servers = fileSource.getServers(); if (servers != null && servers.getIpList() != null && !servers.getIpList().isEmpty()) { - List ipList = servers.getIpList(); - for (IpDTO ipDTO : ipList) { - sendFiles.add(new JobFile(fileSource.isLocalUpload(), ipDTO.convertToStrIp(), - file.getFilePath(), dir, fileName, "root", null, + List ipList = servers.getIpList(); + for (HostDTO hostDTO : ipList) { + sendFiles.add(new JobFile(TaskFileTypeEnum.LOCAL, hostDTO, file.getFilePath(), dir, + fileName, "root", null, FilePathUtils.parseDirAndFileName(file.getFilePath()).getRight())); } } else { - sendFiles.add(new JobFile(fileSource.isLocalUpload(), IpHelper.fix1To0(localServerIp), - file.getFilePath(), dir, fileName, "root", null, + sendFiles.add(new JobFile(TaskFileTypeEnum.LOCAL, localHost, file.getFilePath(), dir, + fileName, "root", null, FilePathUtils.parseDirAndFileName(file.getFilePath()).getRight())); } } @@ -176,10 +177,10 @@ public static Set parseSendFileList(StepInstanceDTO stepInstance, Strin String dir = NFSUtils.getFileDir(jobStorageRootDir, FileDirTypeConf.UPLOAD_FILE_DIR) + fileNameAndPath.getLeft(); String fileName = fileNameAndPath.getRight(); - List ipList = fileSource.getServers().getIpList(); - for (IpDTO ipDTO : ipList) { - sendFiles.add(new JobFile(fileSource.isLocalUpload(), ipDTO.convertToStrIp(), - file.getFilePath(), dir, fileName, "root", null, + List ipList = fileSource.getServers().getIpList(); + for (HostDTO hostDTO : ipList) { + sendFiles.add(new JobFile(TaskFileTypeEnum.BASE64_FILE, hostDTO, file.getFilePath(), dir, + fileName, "root", null, FilePathUtils.parseDirAndFileName(file.getFilePath()).getRight())); } } @@ -200,7 +201,7 @@ public static Map buildSourceFileDisplayMapping(Set sou sourceFiles.forEach(sourceFile -> { Pair pair = FilePathUtils.parseDirAndFileName(sourceFile.getFilePath()); String standardPath = FilePathUtils.standardizedDirPath(pair.getLeft()) + pair.getRight(); - if (sourceFile.isLocalUploadFile() && !standardPath.startsWith(localUploadDir)) { + if (sourceFile.getFileType() == TaskFileTypeEnum.LOCAL && !standardPath.startsWith(localUploadDir)) { sourceFileDisplayMap.put(PathUtil.joinFilePath(localUploadDir, standardPath), sourceFile.getDisplayFilePath()); } else { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/JobLastHostsVariableResolver.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/JobLastHostsVariableResolver.java index 63468a219c..6bcad90fc0 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/JobLastHostsVariableResolver.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/JobLastHostsVariableResolver.java @@ -24,13 +24,15 @@ package com.tencent.bk.job.execute.engine.variable; -import com.tencent.bk.job.common.model.dto.IpDTO; -import com.tencent.bk.job.execute.engine.consts.IpStatus; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.engine.consts.JobBuildInVariables; -import com.tencent.bk.job.execute.model.GseTaskIpLogDTO; +import com.tencent.bk.job.execute.model.AgentTaskDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; -import com.tencent.bk.job.execute.service.GseTaskLogService; +import com.tencent.bk.job.execute.service.FileAgentTaskService; +import com.tencent.bk.job.execute.service.ScriptAgentTaskService; import com.tencent.bk.job.execute.service.TaskInstanceService; +import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; +import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -41,6 +43,8 @@ import java.util.Set; import java.util.stream.Collectors; +import static com.tencent.bk.job.common.util.function.LambdasUtil.not; + /** * 任务前置步骤目标主机-变量解析器 */ @@ -48,14 +52,17 @@ @Slf4j public class JobLastHostsVariableResolver implements VariableResolver { private final TaskInstanceService taskInstanceService; - private final GseTaskLogService gseTaskLogService; + private final ScriptAgentTaskService scriptAgentTaskService; + private final FileAgentTaskService fileAgentTaskService; private final Set BUILD_IN_VARIABLES = new HashSet<>(); @Autowired public JobLastHostsVariableResolver(TaskInstanceService taskInstanceService, - GseTaskLogService gseTaskLogService) { + ScriptAgentTaskService scriptAgentTaskService, + FileAgentTaskService fileAgentTaskService) { this.taskInstanceService = taskInstanceService; - this.gseTaskLogService = gseTaskLogService; + this.scriptAgentTaskService = scriptAgentTaskService; + this.fileAgentTaskService = fileAgentTaskService; init(); } @@ -76,41 +83,73 @@ public String resolve(VariableResolveContext context, String variableName) { StepInstanceDTO preStepInstance = taskInstanceService.getPreExecutableStepInstance(taskInstanceId, stepInstanceId); if (preStepInstance == null) { - log.info("Resolve value from latest executable step instance, but no pre step exist! taskInstanceId: {}, stepInstanceId:{}", + log.info("Resolve value from latest executable step instance, but no pre step exist! taskInstanceId: {}, " + + "stepInstanceId:{}", taskInstanceId, stepInstanceId); return null; } - Set hosts = null; - if (JobBuildInVariables.JOB_LAST_ALL.equals(variableName)) { - hosts = extractAllHosts(preStepInstance); - } else if (JobBuildInVariables.JOB_LAST_SUCCESS.equals(variableName)) { - List ipLogs = gseTaskLogService.getIpLog(preStepInstance.getId(), - preStepInstance.getExecuteCount(), true); - if (CollectionUtils.isNotEmpty(ipLogs)) { - hosts = - ipLogs.stream().filter(ipLog -> (ipLog.getStatus() == IpStatus.SUCCESS.getValue() - || ipLog.getStatus() == IpStatus.LAST_SUCCESS.getValue())) - .map(ipLog -> new IpDTO(ipLog.getCloudAreaId(), ipLog.getIp())).collect(Collectors.toSet()); + Set hosts = null; + switch (variableName) { + case JobBuildInVariables.JOB_LAST_ALL: + hosts = extractAllHosts(preStepInstance); + break; + case JobBuildInVariables.JOB_LAST_SUCCESS: { + List agentTasks = listAgentTasks(preStepInstance.getStepType(), preStepInstance.getId(), + preStepInstance.getExecuteCount()); + if (CollectionUtils.isNotEmpty(agentTasks)) { + hosts = agentTasks.stream() + .filter(AgentTaskDTO::isSuccess) + .map(AgentTaskDTO::getHost) + .collect(Collectors.toSet()); + } + break; } - } else if (JobBuildInVariables.JOB_LAST_FAIL.equals(variableName)) { - List ipLogs = gseTaskLogService.getIpLog(preStepInstance.getId(), - preStepInstance.getExecuteCount(), true); - if (CollectionUtils.isNotEmpty(ipLogs)) { - hosts = - ipLogs.stream().filter(ipLog -> (ipLog.getStatus() != IpStatus.SUCCESS.getValue() - && ipLog.getStatus() != IpStatus.LAST_SUCCESS.getValue())) - .map(ipLog -> new IpDTO(ipLog.getCloudAreaId(), ipLog.getIp())).collect(Collectors.toSet()); + case JobBuildInVariables.JOB_LAST_FAIL: { + List agentTasks = listAgentTasks(preStepInstance.getStepType(), preStepInstance.getId(), + preStepInstance.getExecuteCount()); + if (CollectionUtils.isNotEmpty(agentTasks)) { + hosts = agentTasks.stream() + .filter(not(AgentTaskDTO::isSuccess)) + .map(AgentTaskDTO::getHost) + .collect(Collectors.toSet()); + } + break; } } + String value = VariableResolveUtils.formatHosts(hosts); log.info("Resolve value from latest executable step instance, variableName: {}, value: {}", variableName, value); return value; } +// private List fillIpInfo(StepInstanceDTO stepInstance, List hosts) { +// Map stepHosts = stepInstanceService.computeStepHosts(stepInstance); +// hosts.forEach(host -> { +// if (StringUtils.isEmpty(host.getIp())) { +// HostDTO stepHost = stepHosts.get(host.getHostId()); +// if (stepHost != null) { +// host.setBkCloudId(stepHost.getBkCloudId()); +// host.setIp(stepHost.getIp()); +// } +// } +// }); +// } + + private List listAgentTasks(TaskStepTypeEnum stepType, long stepInstanceId, int executeCount) { + List agentTasks = null; + if (stepType == TaskStepTypeEnum.SCRIPT) { + agentTasks = scriptAgentTaskService.listAgentTasks(stepInstanceId, executeCount, null); + } else if (stepType == TaskStepTypeEnum.FILE) { + agentTasks = fileAgentTaskService.listAgentTasks(stepInstanceId, executeCount, null, + FileTaskModeEnum.DOWNLOAD); + } + return agentTasks; + } + - private Set extractAllHosts(StepInstanceDTO stepInstance) { - Set hosts = new HashSet<>(); + private Set extractAllHosts(StepInstanceDTO stepInstance) { + Set hosts = new HashSet<>(); if (CollectionUtils.isNotEmpty(stepInstance.getTargetServers().getIpList())) { hosts.addAll(stepInstance.getTargetServers().getIpList()); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/VariableResolveUtils.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/VariableResolveUtils.java index 925b80c821..06009afad0 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/VariableResolveUtils.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/VariableResolveUtils.java @@ -24,7 +24,7 @@ package com.tencent.bk.job.execute.engine.variable; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import org.apache.commons.collections4.CollectionUtils; import java.util.Collection; @@ -40,12 +40,12 @@ public class VariableResolveUtils { * @param hosts 主机列表 * @return 主机值 */ - public static String formatHosts(Collection hosts) { + public static String formatHosts(Collection hosts) { if (CollectionUtils.isEmpty(hosts)) { return null; } StringJoiner joiner = new StringJoiner(","); - hosts.forEach(host -> joiner.add(host.convertToStrIp())); + hosts.forEach(host -> joiner.add(host.toCloudIp())); return joiner.toString(); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/gse/GseApiExecutor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/gse/GseApiExecutor.java index 93631c834c..324ff84c34 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/gse/GseApiExecutor.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/gse/GseApiExecutor.java @@ -24,7 +24,12 @@ package com.tencent.bk.job.execute.gse; -import com.tencent.bk.gse.taskapi.*; +import com.tencent.bk.gse.taskapi.api_agent; +import com.tencent.bk.gse.taskapi.api_map_rsp; +import com.tencent.bk.gse.taskapi.api_process_base_info; +import com.tencent.bk.gse.taskapi.api_process_extra_info; +import com.tencent.bk.gse.taskapi.api_process_req; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.engine.gse.GseRequestUtils; import com.tencent.bk.job.execute.engine.model.GseTaskResponse; import com.tencent.bk.job.execute.gse.model.GseProcessInfoDTO; @@ -32,6 +37,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; @Slf4j public class GseApiExecutor { @@ -45,7 +51,8 @@ public GseApiExecutor(String uid) { public GseTaskResponse processOperateProcess(List processInfoList, int reqType) { List procReqList = new ArrayList<>(); for (GseProcessInfoDTO processInfo : processInfoList) { - List agentList = GseRequestUtils.buildAgentList(processInfo.getIpList(), + List agentList = GseRequestUtils.buildAgentList( + processInfo.getIpList().stream().map(HostDTO::toCloudIp).collect(Collectors.toList()), processInfo.getUserName(), null); api_process_base_info processBaseInfo = createProcessBaseInfo(processInfo); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/gse/model/GseProcessInfoDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/gse/model/GseProcessInfoDTO.java index 160510635d..7924a03cdf 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/gse/model/GseProcessInfoDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/gse/model/GseProcessInfoDTO.java @@ -24,7 +24,7 @@ package com.tencent.bk.job.execute.gse.model; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import lombok.Getter; import lombok.Setter; import lombok.ToString; @@ -35,7 +35,7 @@ @Setter @ToString public class GseProcessInfoDTO { - private List ipList; + private List ipList; private String userName; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskDTO.java new file mode 100644 index 0000000000..b2fff0a631 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskDTO.java @@ -0,0 +1,278 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model; + +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; +import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import org.apache.commons.lang3.StringUtils; + +/** + * GSE Agent 任务 + */ +@Getter +@Setter +@ToString +@NoArgsConstructor +public class AgentTaskDTO { + /** + * 步骤实例ID + */ + private long stepInstanceId; + /** + * 步骤执行次数 + */ + private int executeCount; + /** + * Agent 任务对应的实际的步骤执行次数（重试场景，可能Agent任务并没有实际被执行) + */ + private Integer actualExecuteCount; + /** + * 滚动执行批次 + */ + private int batch; + /** + * GSE 任务ID + */ + private Long gseTaskId; + /** + * 主机ID + */ + private Long hostId; + /** + * Agent ID + */ + private String agentId; + /** + * 服务器云区域+IP + */ + @Deprecated + @CompatibleImplementation(name = "rolling_execute", explain = "兼容字段，后续AgentTask仅包含hostId,不再存储具体的IP数据", + version = "3.7.x") + private String cloudIp; + /** + * 展示给用户的服务器IP + */ + @Deprecated + @CompatibleImplementation(name = "rolling_execute", explain = "兼容字段，后续AgentTask仅包含hostId,不再存储具体的IP数据", + version = "3.7.x") + private String displayIp; + /** + * 任务状态 + */ + private AgentTaskStatusEnum status; + /** + * 任务开始时间 + */ + private Long startTime; + /** + * 任务结束时间 + */ + private Long endTime; + /** + * 耗时，毫秒 + */ + private Long totalTime; + /** + * GSE返回错误码 + */ + private int errorCode; + /** + * 脚本任务-执行程序退出码， 0 脚本执行成功，非 0 脚本执行失败 + */ + private Integer exitCode; + /** + * 脚本任务-用户自定义执行结果分组 + */ + private String tag = ""; + /** + * 脚本任务-日志偏移量。Job 从 GSE 根据 scriptLogOffset 增量拉取执行日志 + */ + private int scriptLogOffset; + /** + * 脚本任务-执行日志 + */ + private String scriptLogContent; + /** + * 文件任务类型 + */ + private FileTaskModeEnum fileTaskMode; + /** + * 结果是否发生变化 + */ + private volatile boolean changed; + + public AgentTaskDTO(long stepInstanceId, int executeCount, int batch, Long hostId, String agentId) { + this.stepInstanceId = stepInstanceId; + this.executeCount = executeCount; + this.batch = batch; + this.hostId = hostId; + this.agentId = agentId; + } + + public AgentTaskDTO(long stepInstanceId, int executeCount, int batch, FileTaskModeEnum fileTaskMode, + Long hostId, String agentId) { + this.stepInstanceId = stepInstanceId; + this.executeCount = executeCount; + this.batch = batch; + this.fileTaskMode = fileTaskMode; + this.hostId = hostId; + this.agentId = agentId; + } + + public AgentTaskDTO(AgentTaskDTO agentTask) { + this.stepInstanceId = agentTask.getStepInstanceId(); + this.executeCount = agentTask.getExecuteCount(); + this.actualExecuteCount = agentTask.getActualExecuteCount(); + this.batch = agentTask.getBatch(); + this.fileTaskMode = agentTask.getFileTaskMode(); + this.hostId = agentTask.getHostId(); + this.agentId = agentTask.getAgentId(); + this.cloudIp = agentTask.getCloudIp(); + this.displayIp = agentTask.getDisplayIp(); + this.status = agentTask.getStatus(); + this.startTime = agentTask.getStartTime(); + this.endTime = agentTask.getEndTime(); + this.totalTime = agentTask.getTotalTime(); + this.errorCode = agentTask.getErrorCode(); + this.exitCode = agentTask.getExitCode(); + this.tag = agentTask.getTag(); + this.scriptLogOffset = agentTask.getScriptLogOffset(); + this.scriptLogContent = agentTask.getScriptLogContent(); + this.fileTaskMode = agentTask.getFileTaskMode(); + this.gseTaskId = agentTask.getGseTaskId(); + this.changed = agentTask.isChanged(); + } + + public void setStatus(AgentTaskStatusEnum status) { + this.changed = true; + this.status = status; + } + + public void setStartTime(Long startTime) { + this.changed = true; + this.startTime = startTime; + } + + public void setEndTime(Long endTime) { + this.changed = true; + this.endTime = endTime; + } + + public void setTotalTime(Long totalTime) { + this.changed = true; + this.totalTime = totalTime; + } + + public void setErrorCode(int errorCode) { + this.changed = true; + this.errorCode = errorCode; + } + + public void setExitCode(Integer exitCode) { + this.changed = true; + this.exitCode = exitCode; + } + + public void setScriptLogOffset(int scriptLogOffset) { + this.changed = true; + this.scriptLogOffset = scriptLogOffset; + } + + /** + * 任务是否结束 + * + * @return 任务是否结束 + */ + public boolean isFinished() { + return status != AgentTaskStatusEnum.WAITING && status != AgentTaskStatusEnum.RUNNING; + } + + /** + * 计算任务运行时间 + */ + public void calculateTotalTime() { + if (this.endTime != null && this.startTime != null && this.endTime > this.startTime) { + this.totalTime = this.endTime - this.startTime; + } + } + + /** + * 重置任务状态数据 + */ + public void resetTaskInitialStatus() { + this.status = AgentTaskStatusEnum.WAITING; + this.startTime = null; + this.endTime = null; + this.totalTime = null; + + this.errorCode = 0; + this.scriptLogOffset = 0; + this.exitCode = null; + this.gseTaskId = null; + } + + /** + * 是否任务执行目标Agent + */ + public boolean isTarget() { + // 非文件分发文件源主机，目前来说都是目标主机 + return !(fileTaskMode != null && fileTaskMode == FileTaskModeEnum.UPLOAD); + } + + public HostDTO getHost() { + HostDTO host = null; + if (hostId != null) { + host = new HostDTO(); + host.setHostId(hostId); + host.setAgentId(agentId); + } else if (StringUtils.isNotEmpty(cloudIp)) { + host = HostDTO.fromCloudIp(cloudIp); + } + + return host; + } + + /** + * 任务是否执行成功 + */ + public boolean isSuccess() { + return AgentTaskStatusEnum.isSuccess(status); + } + + public String getAgentId() { + if (StringUtils.isNotBlank(agentId)) { + return agentId; + } else { + // 兼容没有agentId的历史数据 + return getCloudIp(); + } + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskDetailDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskDetailDTO.java new file mode 100644 index 0000000000..e183776964 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskDetailDTO.java @@ -0,0 +1,88 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model; + +import com.tencent.bk.job.common.model.dto.HostDTO; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import org.apache.commons.lang3.StringUtils; + +/** + * GSE Agent 任务详情，包含主机的详细信息 + */ +@Getter +@Setter +@ToString +@NoArgsConstructor +public class AgentTaskDetailDTO extends AgentTaskDTO { + /** + * 服务器IP,不包含云区域 + */ + private String ip; + /** + * 云区域ID + */ + private Long bkCloudId; + /** + * 云区域名称 + */ + private String bkCloudName; + /** + * 展示给用户的IP + */ + private String displayIp; + + public AgentTaskDetailDTO(AgentTaskDTO agentTask) { + super(agentTask); + if (StringUtils.isNotEmpty(agentTask.getCloudIp())) { + HostDTO host = HostDTO.fromCloudIp(agentTask.getAgentId()); + this.ip = host.getIp(); + this.bkCloudId = host.getBkCloudId(); + } + } + + public String getDisplayIp() { + if (StringUtils.isNotEmpty(displayIp)) { + return displayIp; + } else { + return ip; + } + } + + public HostDTO getHost() { + HostDTO host = new HostDTO(); + host.setHostId(getHostId()); + host.setIp(getIp()); + host.setBkCloudId(getBkCloudId()); + host.setBkCloudName(getBkCloudName()); + host.setDisplayIp(getDisplayIp()); + host.setAgentId(getAgentId()); + return host; + } + + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ExecutionResultGroupDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskResultGroupBaseDTO.java similarity index 63% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ExecutionResultGroupDTO.java rename to src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskResultGroupBaseDTO.java index 50fa63001f..cc54c02624 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ExecutionResultGroupDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskResultGroupBaseDTO.java @@ -24,46 +24,50 @@ package com.tencent.bk.job.execute.model; +import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; import lombok.Data; import lombok.NoArgsConstructor; -import java.util.List; import java.util.Objects; /** - * 执行结果分组 + * Agent任务执行结果分组 */ @Data @NoArgsConstructor -public class ExecutionResultGroupDTO { +public class AgentTaskResultGroupBaseDTO implements Comparable { /** - * 执行结果分类 + * 任务状态 + * + * @see AgentTaskStatusEnum */ - private Integer resultType; + private Integer status; /** - * 用户通过job_success/job_fail自定义的结果分类tag + * 用户脚本输出的自定义分组tag */ private String tag; /** - * Agent任务执行情况 + * Agent任务总数 */ - private List agentTaskExecutionDetail; - /** - * 结果分组下的agent任务数 - */ - private int agentTaskSize; + private int totalAgentTasks; - public ExecutionResultGroupDTO(Integer resultType, String tag) { - this.resultType = resultType; + public AgentTaskResultGroupBaseDTO(Integer status, String tag) { + this.status = status; this.tag = tag; } + public AgentTaskResultGroupBaseDTO(AgentTaskResultGroupBaseDTO baseResultGroup) { + this.status = baseResultGroup.getStatus(); + this.tag = baseResultGroup.getTag(); + this.totalAgentTasks = baseResultGroup.getTotalAgentTasks(); + } + @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - ExecutionResultGroupDTO that = (ExecutionResultGroupDTO) o; - return resultType.equals(that.resultType) && + AgentTaskResultGroupBaseDTO that = (AgentTaskResultGroupBaseDTO) o; + return status.equals(that.status) && tagEquals(tag, that.tag); } @@ -76,9 +80,20 @@ private boolean tagEquals(String thisTag, String thatTag) { @Override public int hashCode() { if (this.tag == null) { - return Objects.hash(resultType, ""); + return Objects.hash(status, ""); } else { - return Objects.hash(resultType, tag); + return Objects.hash(status, tag); + } + } + + @Override + public int compareTo(AgentTaskResultGroupBaseDTO that) { + int result = status.compareTo(that.status); + if (result != 0) { + return result; } + String tag1 = this.tag == null ? "" : this.tag; + String tag2 = that.tag == null ? "" : that.tag; + return tag1.compareTo(tag2); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskResultGroupDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskResultGroupDTO.java index c8a00f40b0..491947df7b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskResultGroupDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskResultGroupDTO.java @@ -24,31 +24,31 @@ package com.tencent.bk.job.execute.model; -import com.tencent.bk.job.common.model.dto.IpDTO; -import com.tencent.bk.job.execute.engine.consts.IpStatus; -import lombok.Data; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; import java.util.List; /** * Agent任务执行结果分组 */ -@Data -public class AgentTaskResultGroupDTO { +@Getter +@Setter +@ToString +@NoArgsConstructor +public class AgentTaskResultGroupDTO extends AgentTaskResultGroupBaseDTO { /** - * 结果类型 + * Agent任务 */ - private IpStatus resultType; - /** - * 用户脚本输出的tag - */ - private String tag; - /** - * 分组下的ip数目 - */ - private Integer count; - /** - * ip列表 - */ - private List ipList; + private List agentTasks; + + public AgentTaskResultGroupDTO(Integer status, String tag) { + super(status, tag); + } + + public AgentTaskResultGroupDTO(AgentTaskResultGroupBaseDTO baseResultGroup) { + super(baseResultGroup); + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/DangerousRecordDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/DangerousRecordDTO.java index 64050ac681..6c62320e33 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/DangerousRecordDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/DangerousRecordDTO.java @@ -117,9 +117,7 @@ public DangerousRecordVO toDangerousRecordVO() { recordVO.setScopeType(resourceScope.getType().getValue()); recordVO.setScopeId(resourceScope.getId()); - // TODO:发布后去除 recordVO.setAppName(appName); - recordVO.setScopeName(appName); recordVO.setCreateTime(createTime); if (checkResult != null && CollectionUtils.isNotEmpty(checkResult.getResults())) { recordVO.setCheckResultItems(checkResult.getResults().stream().map(scriptCheckItem -> { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/DynamicServerGroupDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/DynamicServerGroupDTO.java index 4b6728b418..60b7850d52 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/DynamicServerGroupDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/DynamicServerGroupDTO.java @@ -24,7 +24,7 @@ package com.tencent.bk.job.execute.model; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import lombok.Data; import lombok.NoArgsConstructor; @@ -45,7 +45,7 @@ public class DynamicServerGroupDTO implements Cloneable { /** * 动态分组对应的静态IP */ - private List ipList; + private List ipList; public DynamicServerGroupDTO(String groupId) { this.groupId = groupId; @@ -55,7 +55,7 @@ public DynamicServerGroupDTO clone() { DynamicServerGroupDTO cloneServerGroup = new DynamicServerGroupDTO(); cloneServerGroup.setGroupId(groupId); if (ipList != null) { - List cloneIpList = new ArrayList<>(ipList.size()); + List cloneIpList = new ArrayList<>(ipList.size()); ipList.forEach(ip -> cloneIpList.add(ip.clone())); cloneServerGroup.setIpList(cloneIpList); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/DynamicServerTopoNodeDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/DynamicServerTopoNodeDTO.java index 2a7eb84c8b..8cce29ba31 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/DynamicServerTopoNodeDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/DynamicServerTopoNodeDTO.java @@ -24,7 +24,7 @@ package com.tencent.bk.job.execute.model; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import lombok.Data; import lombok.NoArgsConstructor; @@ -48,7 +48,7 @@ public class DynamicServerTopoNodeDTO implements Cloneable { /** * 分布式拓扑节点对应的静态IP */ - private List ipList; + private List ipList; public DynamicServerTopoNodeDTO(long topoNodeId, String nodeType) { this.topoNodeId = topoNodeId; @@ -74,7 +74,7 @@ public DynamicServerTopoNodeDTO clone() { cloneTopoNode.setNodeType(nodeType); cloneTopoNode.setTopoNodeId(topoNodeId); if (ipList != null) { - List cloneIpList = new ArrayList<>(ipList.size()); + List cloneIpList = new ArrayList<>(ipList.size()); ipList.forEach(ip -> cloneIpList.add(ip.clone())); cloneTopoNode.setIpList(cloneIpList); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FastTaskDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FastTaskDTO.java new file mode 100644 index 0000000000..ce1c2fce37 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FastTaskDTO.java @@ -0,0 +1,57 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model; + +import lombok.Builder; +import lombok.Data; + +/** + * 快速执行任务 + */ +@Data +@Builder +public class FastTaskDTO { + /** + * 作业实例 + */ + private TaskInstanceDTO taskInstance; + /** + * 步骤实例 + */ + private StepInstanceDTO stepInstance; + /** + * 滚动配置 + */ + private StepRollingConfigDTO rollingConfig; + + /** + * 是否滚动执行 + * + * @return 是否滚动执行 + */ + public boolean isRollingEnabled() { + return this.rollingConfig != null; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/GseTaskDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/GseTaskDTO.java new file mode 100644 index 0000000000..8e73890d68 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/GseTaskDTO.java @@ -0,0 +1,82 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model; + +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * GSE 任务 + */ +@Data +@NoArgsConstructor +public class GseTaskDTO { + /** + * GSE任务ID + */ + private Long id; + /** + * 步骤实例ID + */ + private Long stepInstanceId; + /** + * 步骤执行次数 + */ + private Integer executeCount; + /** + * 滚动执行批次 + */ + private int batch; + /** + * 任务开始时间 + */ + private Long startTime; + /** + * 任务结束时间 + */ + private Long endTime; + /** + * 任务耗时，单位毫秒 + */ + private Long totalTime; + /** + * 任务状态 + */ + private Integer status = 1; + /** + * GSE 任务ID + */ + private String gseTaskId; + + public GseTaskDTO(Long stepInstanceId, Integer executeCount, int batch) { + this.stepInstanceId = stepInstanceId; + this.executeCount = executeCount; + this.batch = batch; + } + + public String getTaskUniqueName() { + return "GseTask:" + id + ":" + stepInstanceId + ":" + executeCount + ":" + batch; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/GseTaskIpLogDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/GseTaskIpLogDTO.java deleted file mode 100644 index 8c5d77507a..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/GseTaskIpLogDTO.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.model; - -import com.tencent.bk.job.execute.engine.consts.IpStatus; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.ToString; - -/** - * gse 任务IPLog - */ -@Getter -@Setter -@ToString -@NoArgsConstructor -public class GseTaskIpLogDTO { - - private long stepInstanceId; - - private int executeCount = 0; - - /** - * 服务器IP,包含云区域 - */ - private String cloudAreaAndIp; - - /** - * 服务器IP,不包含云区域 - */ - private String ip; - /** - * 是否目标服务器 - */ - private boolean isTargetServer = true; - /** - * 是否文件源服务器 - */ - private boolean isSourceServer = false; - - /** - * 云区域ID - */ - private Long cloudAreaId; - - private String displayIp; - - /** - * 状态： 0.系统错误、1.Agent 异常、3.上次已成功、5.等待执行、7.正在执行、9.执行成功、11.执行失败、13.任务超时、15.任务日志错误、101.脚本执行失败、102.脚本执行超时、202.拷贝文件失败 - */ - private int status = -1; - - private Long startTime; - - private Long endTime; - - /** - * 耗时，毫秒 - */ - private Long totalTime; - - /** - * GSE返回错误码 - */ - private int errCode; - - /** - * 执行程序退出码， 0 脚本执行成功，非 0 脚本执行失败 - */ - private Integer exitCode; - - /** - * IP 结果标签 - */ - private String tag = ""; - /** - * 当前日志偏移量 - */ - private int offset; - - private String logContent; - - /** - * 结果是否发生变化 - */ - private volatile boolean changed; - - public void setStatus(int status) { - this.changed = this.status != status; - this.status = status; - } - - public void setStartTime(Long startTime) { - this.changed = true; - this.startTime = startTime; - } - - public void setEndTime(Long endTime) { - this.changed = true; - this.endTime = endTime; - } - - public void setTotalTime(Long totalTime) { - this.changed = true; - this.totalTime = totalTime; - } - - public void setErrCode(int errCode) { - this.changed = true; - this.errCode = errCode; - } - - public void setExitCode(Integer exitCode) { - this.changed = true; - this.exitCode = exitCode; - } - - public void setOffset(int offset) { - this.changed = this.offset != offset; - this.offset = offset; - } - - /** - * 任务是否结束 - * - * @return 任务是否结束 - */ - public boolean isFinished() { - return status != IpStatus.WAITING.getValue() && status != IpStatus.RUNNING.getValue(); - } - - public ExecutionResultGroupDTO getExecutionResultGroup() { - return new ExecutionResultGroupDTO(status, tag); - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/HostVariableValuesDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/HostVariableValuesDTO.java index a1867f33c2..59076784a9 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/HostVariableValuesDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/HostVariableValuesDTO.java @@ -24,12 +24,31 @@ package com.tencent.bk.job.execute.model; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; import lombok.Data; import java.util.List; +/** + * 作业-全局变量-命名空间-主机变量 + */ @Data public class HostVariableValuesDTO { + /** + * 主机IP + */ + @CompatibleImplementation(explain = "由于ip不再唯一，使用hostId/agentId替代该参数", version = "3.7.x") private String ip; + /** + * 主机ID + */ + private Long hostId; + /** + * bk_agent_id + */ + private String agentId; + /** + * 变量值 + */ private List values; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/OperationLogDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/OperationLogDTO.java index e3d597639e..c280311af6 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/OperationLogDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/OperationLogDTO.java @@ -25,6 +25,7 @@ package com.tencent.bk.job.execute.model; import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.common.annotation.PersistenceObject; import com.tencent.bk.job.execute.constants.UserOperationEnum; import lombok.Data; @@ -43,10 +44,12 @@ public class OperationLogDTO { */ @Data @JsonInclude(JsonInclude.Include.NON_NULL) + @PersistenceObject public static class OperationDetail { private Long taskInstanceId; private Long stepInstanceId; private Integer executeCount; + private Integer batch; private String stepName; private Integer startupMode; private String appCode; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/RollingConfigDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/RollingConfigDTO.java new file mode 100644 index 0000000000..c01a5febc5 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/RollingConfigDTO.java @@ -0,0 +1,64 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model; + +import com.tencent.bk.job.execute.model.db.RollingConfigDetailDO; +import lombok.Data; + +/** + * 作业滚动配置实例 + */ +@Data +public class RollingConfigDTO { + /** + * id + */ + private Long id; + + /** + * 执行作业实例id + */ + private Long taskInstanceId; + + /** + * 滚动配置名称 + */ + private String configName; + + /** + * 滚动配置 + */ + private RollingConfigDetailDO configDetail; + + /** + * 步骤是否分批执行 + * + * @param stepInstanceId 步骤实例ID + */ + public boolean isBatchRollingStep(long stepInstanceId) { + return configDetail != null && configDetail.getStepRollingConfigs().get(stepInstanceId).isBatch(); + } +} + diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/RollingStepBatchTaskDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/RollingStepBatchTaskDTO.java new file mode 100644 index 0000000000..3c5d8e04af --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/RollingStepBatchTaskDTO.java @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +@ApiModel("步骤滚动任务") +@Data +public class RollingStepBatchTaskDTO { + @ApiModelProperty("滚动批次") + private Integer batch; + + /** + * 任务状态 + * + * @see com.tencent.bk.job.execute.common.constants.RunStatusEnum + */ + @ApiModelProperty("任务状态,值同步骤状态一致。1-等待执行，2-正在执行，3-执行成功，4-执行失败，5-跳过，6-忽略错误，7-等待用户" + + "，8-手动结束，9-状态异常，10-强制终止中，11-强制终止成功，12-强制终止失败，13-确认终止，14-被丢弃，15-滚动等待") + private Integer status; + + @ApiModelProperty("是否正在滚动中") + private boolean running; +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ScriptIpLogContent.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ScriptHostLogContent.java similarity index 81% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ScriptIpLogContent.java rename to src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ScriptHostLogContent.java index 0f600c124c..8751750724 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ScriptIpLogContent.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ScriptHostLogContent.java @@ -24,6 +24,7 @@ package com.tencent.bk.job.execute.model; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; import lombok.Data; import lombok.NoArgsConstructor; @@ -32,7 +33,7 @@ */ @Data @NoArgsConstructor -public class ScriptIpLogContent { +public class ScriptHostLogContent { /** * 步骤实例ID */ @@ -41,9 +42,14 @@ public class ScriptIpLogContent { * 执行次数 */ private int executeCount; + /** + * 主机ID + */ + private Long hostId; /** * 目标IP */ + @CompatibleImplementation(name = "rolling_execute", explain = "兼容字段，后续使用hostId替换", version = "3.7.x") private String ip; /** * 日志内容 @@ -54,9 +60,11 @@ public class ScriptIpLogContent { */ private boolean finished; - public ScriptIpLogContent(long stepInstanceId, int executeCount, String ip, String content, boolean finished) { + public ScriptHostLogContent(long stepInstanceId, int executeCount, Long hostId, + String ip, String content, boolean finished) { this.stepInstanceId = stepInstanceId; this.executeCount = executeCount; + this.hostId = hostId; this.ip = ip; this.content = content; this.finished = finished; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ServersDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ServersDTO.java index 3191f9c47f..5ec40badb9 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ServersDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ServersDTO.java @@ -24,13 +24,19 @@ package com.tencent.bk.job.execute.model; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.annotation.PersistenceObject; +import com.tencent.bk.job.common.model.dto.HostDTO; import lombok.Data; import org.apache.commons.collections4.CollectionUtils; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.StringJoiner; @Data +@PersistenceObject public class ServersDTO implements Cloneable { /** * 如果目标服务器是通过全局变量-主机列表定义的，variable 表示变量 name @@ -39,7 +45,7 @@ public class ServersDTO implements Cloneable { /** * 用户选择的服务器ip列表（静态） */ - private List staticIpList; + private List staticIpList; /** * 服务器动态分组列表 @@ -54,12 +60,12 @@ public class ServersDTO implements Cloneable { /** * 服务器（动态分组、静态ip、动态topo节点）等对应的所有ip的集合 */ - private List ipList; + private List ipList; /** * 非法服务器 */ - private List invalidIpList; + private List invalidIpList; /** * 非法动态分组 @@ -88,7 +94,7 @@ public String buildIpListStr() { return null; } StringJoiner sj = new StringJoiner(","); - ipList.forEach(ipDTO -> sj.add(ipDTO.getCloudAreaId() + ":" + ipDTO.getIp())); + ipList.forEach(ipDTO -> sj.add(ipDTO.getBkCloudId() + ":" + ipDTO.getIp())); return sj.toString(); } @@ -96,7 +102,7 @@ public ServersDTO clone() { ServersDTO cloneServersDTO = new ServersDTO(); cloneServersDTO.setVariable(variable); if (staticIpList != null) { - List cloneStaticIpList = new ArrayList<>(staticIpList.size()); + List cloneStaticIpList = new ArrayList<>(staticIpList.size()); staticIpList.forEach(staticIp -> cloneStaticIpList.add(staticIp.clone())); cloneServersDTO.setStaticIpList(cloneStaticIpList); } @@ -109,12 +115,12 @@ public ServersDTO clone() { cloneServersDTO.setTopoNodes(topoNodes); } if (ipList != null) { - List cloneIpList = new ArrayList<>(ipList.size()); + List cloneIpList = new ArrayList<>(ipList.size()); ipList.forEach(ip -> cloneIpList.add(ip.clone())); cloneServersDTO.setIpList(cloneIpList); } if (invalidIpList != null) { - List cloneIpList = new ArrayList<>(invalidIpList.size()); + List cloneIpList = new ArrayList<>(invalidIpList.size()); invalidIpList.forEach(ip -> cloneIpList.add(ip.clone())); cloneServersDTO.setInvalidIpList(cloneIpList); } @@ -172,7 +178,7 @@ public ServersDTO merge(ServersDTO servers) { return this; } - public void addStaticIps(Collection ips) { + public void addStaticIps(Collection ips) { if (staticIpList == null) { staticIpList = new ArrayList<>(); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepExecutionDetailDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepExecutionDetailDTO.java index 4c3eab07eb..98e18a010a 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepExecutionDetailDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepExecutionDetailDTO.java @@ -24,7 +24,12 @@ package com.tencent.bk.job.execute.model; +import com.tencent.bk.job.execute.common.constants.RunStatusEnum; +import com.tencent.bk.job.execute.common.constants.StepRunModeEnum; +import com.tencent.bk.job.execute.common.util.TaskCostCalculator; +import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; import lombok.Data; +import lombok.NoArgsConstructor; import java.util.List; @@ -32,11 +37,16 @@ * 步骤执行详情 */ @Data +@NoArgsConstructor public class StepExecutionDetailDTO { + /** + * 步骤实例 + */ + private StepInstanceBaseDTO stepInstance; /** * 任务执行分组状态 */ - List resultGroups; + List resultGroups; /** * 步骤实例ID */ @@ -45,6 +55,10 @@ public class StepExecutionDetailDTO { * 执行次数 */ private Integer executeCount; + /** + * 滚动任务-最新批次 + */ + private Integer latestBatch; /** * 步骤执行是否结束 */ @@ -68,7 +82,7 @@ public class StepExecutionDetailDTO { /** * 执行状态 */ - private int status; + private RunStatusEnum status; /** * GSE TASK ID */ @@ -80,11 +94,28 @@ public class StepExecutionDetailDTO { /** * 步骤类型 */ - private Integer stepType; + private TaskStepTypeEnum stepType; + /** + * 步骤执行模式 + */ + private StepRunModeEnum runMode; + /** + * 步骤包含的滚动任务;如果非滚动步骤，那么该值为空 + */ + private List rollingTasks; - public void calculateTotalTime() { - if (this.endTime != null && this.startTime != null) { - this.totalTime = this.endTime - this.startTime; - } + public StepExecutionDetailDTO(StepInstanceBaseDTO stepInstance) { + this.stepInstance = stepInstance; + this.stepInstanceId = stepInstance.getId(); + this.executeCount = stepInstance.getExecuteCount(); + this.name = stepInstance.getName(); + this.setFinished(RunStatusEnum.isFinishedStatus(stepInstance.getStatus())); + this.status = stepInstance.getStatus(); + this.startTime = stepInstance.getStartTime(); + this.endTime = stepInstance.getEndTime(); + this.totalTime = TaskCostCalculator.calculate(stepInstance.getStartTime(), stepInstance.getEndTime(), + stepInstance.getTotalTime()); + this.lastStep = stepInstance.isLastStep(); + this.stepType = stepInstance.getStepType(); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepExecutionResultQuery.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepExecutionResultQuery.java index e43319facf..a75ae5b025 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepExecutionResultQuery.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepExecutionResultQuery.java @@ -25,7 +25,10 @@ package com.tencent.bk.job.execute.model; import com.tencent.bk.job.common.constant.Order; -import lombok.Data; +import lombok.Builder; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; import org.apache.commons.lang3.StringUtils; import org.jooq.generated.tables.GseTaskIpLog; @@ -33,7 +36,10 @@ import java.util.Map; import java.util.Set; -@Data +@Getter +@Setter +@ToString +@Builder public class StepExecutionResultQuery { public static final String ORDER_FIELD_TOTAL_TIME = "totalTime"; public static final String ORDER_FIELD_CLOUD_AREA_ID = "cloudAreaId"; @@ -55,6 +61,14 @@ public class StepExecutionResultQuery { * 执行次数 */ private Integer executeCount; + /** + * 滚动执行批次 + */ + private Integer batch; + /** + * 是否根据步骤实例的最新滚动批次过滤；如果为true，那么batch将使用滚动任务当前执行的批次 + */ + private Boolean filterByLatestBatch; /** * 执行日志关键词(脚本任务) */ @@ -66,7 +80,7 @@ public class StepExecutionResultQuery { /** * 执行结果分组 */ - private Integer resultType; + private Integer status; /** * 执行结果输出的自定义分组tag */ @@ -84,15 +98,23 @@ public class StepExecutionResultQuery { */ private Order order; - private Set matchIps; + private Set matchHostIds; public boolean hasIpCondition() { return StringUtils.isNotEmpty(logKeyword) || StringUtils.isNotEmpty(searchIp); } + public boolean hasBatchCondition() { + return batch != null && batch > 0; + } + public void transformOrderFieldToDbField() { if (orderField != null) { orderField = ORDER_FIELD_IN_DB.get(orderField); } } + + public boolean isFilterByLatestBatch() { + return this.filterByLatestBatch != null && this.filterByLatestBatch; + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceBaseDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceBaseDTO.java index 80d346cbe7..28397c7f30 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceBaseDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceBaseDTO.java @@ -48,7 +48,6 @@ public class StepInstanceBaseDTO { * 执行次数 */ protected int executeCount; - /** * 执行步骤id */ @@ -67,11 +66,15 @@ public class StepInstanceBaseDTO { */ protected String name; /** - * 步骤类型：1、执行脚本，2、传输文件，3、人工确认, 4、SQL执行 + * 步骤执行类型：1、执行脚本，2、传输文件，3、人工确认, 4、SQL执行 * * @see StepExecuteTypeEnum */ protected Integer executeType; + /** + * 步骤类型 + */ + protected TaskStepTypeEnum stepType; /** * 执行人 */ @@ -79,9 +82,8 @@ public class StepInstanceBaseDTO { /** * 执行状态 * - * @see RunStatusEnum */ - protected Integer status; + protected RunStatusEnum status; /** * 开始时间 */ @@ -91,29 +93,9 @@ public class StepInstanceBaseDTO { */ protected Long endTime; /** - * 总耗时，单位：毫秒秒 + * 总耗时，单位：毫秒 */ protected Long totalTime; - /** - * 总ip数量 - */ - protected int totalIPNum; - /** - * 没有agent - */ - protected int badIPNum; - /** - * 有agent - */ - protected int runIPNum; - /** - * 失败ip数量 - */ - protected int failIPNum; - /** - * 成功ip数量 - */ - protected int successIPNum; /** * 创建时间 */ @@ -126,10 +108,6 @@ public class StepInstanceBaseDTO { * 目标服务器 */ protected ServersDTO targetServers; - /** - * agent异常的服务器 - */ - protected String badIpList; /** * 目标服务器 */ @@ -146,28 +124,35 @@ public class StepInstanceBaseDTO { * 当前步骤在作业中的顺序 */ protected Integer stepOrder; + /** + * 滚动执行批次 + */ + protected int batch; + /** + * 滚动配置ID + */ + protected Long rollingConfigId; /** * 获取步骤类型 * - * @return + * @return 步骤类型 * @see TaskStepTypeEnum */ - public Integer getStepType() { - if (executeType != null) { - if (executeType.equals(StepExecuteTypeEnum.EXECUTE_SCRIPT.getValue()) - || executeType.equals(StepExecuteTypeEnum.EXECUTE_SQL.getValue())) { - return TaskStepTypeEnum.SCRIPT.getValue(); - } else if (executeType.equals(StepExecuteTypeEnum.SEND_FILE.getValue())) { - return TaskStepTypeEnum.FILE.getValue(); - } else if (executeType.equals(StepExecuteTypeEnum.MANUAL_CONFIRM.getValue())) { - return TaskStepTypeEnum.APPROVAL.getValue(); - } else { - return null; + public TaskStepTypeEnum getStepType() { + if (this.stepType == null) { + if (executeType != null) { + if (executeType.equals(StepExecuteTypeEnum.EXECUTE_SCRIPT.getValue()) + || executeType.equals(StepExecuteTypeEnum.EXECUTE_SQL.getValue())) { + this.stepType = TaskStepTypeEnum.SCRIPT; + } else if (executeType.equals(StepExecuteTypeEnum.SEND_FILE.getValue())) { + this.stepType = TaskStepTypeEnum.FILE; + } else if (executeType.equals(StepExecuteTypeEnum.MANUAL_CONFIRM.getValue())) { + this.stepType = TaskStepTypeEnum.APPROVAL; + } } - } else { - return null; } + return this.stepType; } /** @@ -177,15 +162,18 @@ public boolean isLastStep() { return this.stepNum.equals(1) || this.stepNum.equals(this.stepOrder); } - + /** + * 是否文件分发步骤 + */ public boolean isFileStep() { - return this.executeType != null && this.executeType.equals(StepExecuteTypeEnum.SEND_FILE.getValue()); + return getStepType() == TaskStepTypeEnum.FILE; } + /** + * 是否脚本执行步骤 + */ public boolean isScriptStep() { - return this.executeType != null - && (this.executeType.equals(StepExecuteTypeEnum.EXECUTE_SCRIPT.getValue()) - || this.executeType.equals(StepExecuteTypeEnum.EXECUTE_SQL.getValue())); + return getStepType() == TaskStepTypeEnum.SCRIPT; } public int getTargetServerTotalCount() { @@ -195,4 +183,18 @@ public int getTargetServerTotalCount() { return 0; } } + + /** + * 是否滚动步骤 + */ + public boolean isRollingStep() { + return this.rollingConfigId != null && this.rollingConfigId > 0; + } + + /** + * 是否滚动执行第一批次 + */ + public boolean isFirstRollingBatch() { + return this.batch == 1; + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceDTO.java index c92fec9bbb..ee04059ba5 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceDTO.java @@ -25,7 +25,7 @@ package com.tencent.bk.job.execute.model; import com.tencent.bk.job.common.constant.DuplicateHandlerEnum; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.util.function.LambdasUtil; import lombok.Getter; import lombok.Setter; @@ -35,7 +35,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.stream.Collectors; /** * 步骤实例 @@ -177,16 +176,12 @@ public StepInstanceDTO(StepInstanceBaseDTO stepInstanceBase) { this.startTime = stepInstanceBase.startTime; this.endTime = stepInstanceBase.endTime; this.totalTime = stepInstanceBase.totalTime; - this.totalIPNum = stepInstanceBase.totalIPNum; - this.badIPNum = stepInstanceBase.badIPNum; - this.runIPNum = stepInstanceBase.runIPNum; - this.failIPNum = stepInstanceBase.failIPNum; - this.successIPNum = stepInstanceBase.successIPNum; this.createTime = stepInstanceBase.createTime; this.ignoreError = stepInstanceBase.ignoreError; this.targetServers = stepInstanceBase.targetServers; - this.badIpList = stepInstanceBase.badIpList; this.ipList = stepInstanceBase.ipList; + this.rollingConfigId = stepInstanceBase.rollingConfigId; + this.batch = stepInstanceBase.getBatch(); } // -------------公共方法------------------// @@ -246,23 +241,42 @@ public void fillConfirmStepInfo(ConfirmStepInstanceDTO confirmStepInstance) { * * @return 不合法的主机 */ - public Set getInvalidIps() { - Set invalidIpSet = new HashSet<>(); + public Set getInvalidHosts() { + Set invalidHosts = new HashSet<>(); if (CollectionUtils.isNotEmpty(this.targetServers.getInvalidIpList())) { - invalidIpSet.addAll(this.targetServers.getInvalidIpList().stream().map(IpDTO::convertToStrIp) - .collect(Collectors.toSet())); + invalidHosts.addAll(this.targetServers.getInvalidIpList()); } if (isFileStep() && CollectionUtils.isNotEmpty(this.fileSourceList)) { this.fileSourceList.stream().filter(LambdasUtil.not(FileSourceDTO::isLocalUpload)) .forEach(fileSource -> { ServersDTO fileSourceServers = fileSource.getServers(); if (fileSourceServers != null && CollectionUtils.isNotEmpty(fileSourceServers.getInvalidIpList())) { - invalidIpSet.addAll(fileSourceServers.getInvalidIpList().stream().map(IpDTO::convertToStrIp) - .collect(Collectors.toSet())); + invalidHosts.addAll(fileSourceServers.getInvalidIpList()); } }); } - return invalidIpSet; + return invalidHosts; + } + + /** + * 判断步骤是否包含非法主机 + */ + public boolean hasInvalidHost() { + boolean hasInvalidHost = false; + if (CollectionUtils.isNotEmpty(this.targetServers.getInvalidIpList())) { + return true; + } + if (isFileStep() && CollectionUtils.isNotEmpty(this.fileSourceList)) { + hasInvalidHost = this.fileSourceList.stream() + .filter(LambdasUtil.not(FileSourceDTO::isLocalUpload)) + .anyMatch(fileSource -> { + ServersDTO fileSourceServers = fileSource.getServers(); + return fileSourceServers != null + && CollectionUtils.isNotEmpty(fileSourceServers.getInvalidIpList()); + }); + + } + return hasInvalidHost; } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceRollingTaskDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceRollingTaskDTO.java new file mode 100644 index 0000000000..92d3b1c7fe --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceRollingTaskDTO.java @@ -0,0 +1,71 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model; + +import com.tencent.bk.job.execute.common.constants.RunStatusEnum; +import lombok.Data; + +/** + * 步骤实例滚动任务 + */ +@Data +public class StepInstanceRollingTaskDTO { + /** + * id + */ + private Long id; + + /** + * 步骤实例ID + */ + private Long stepInstanceId; + + /** + * 步骤重试次数 + */ + private Integer executeCount; + + /** + * 滚动批次 + */ + private Integer batch; + /** + * 执行状态 + */ + protected RunStatusEnum status; + /** + * 开始时间 + */ + protected Long startTime; + /** + * 结束时间 + */ + protected Long endTime; + /** + * 总耗时，单位：毫秒秒 + */ + protected Long totalTime; +} + diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepRollingConfigDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepRollingConfigDTO.java new file mode 100644 index 0000000000..0080897aa8 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepRollingConfigDTO.java @@ -0,0 +1,66 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model; + +import com.tencent.bk.job.execute.model.esb.v3.EsbRollingConfigDTO; +import com.tencent.bk.job.execute.model.web.vo.RollingConfigVO; +import lombok.Data; +import org.apache.commons.lang3.StringUtils; + +/** + * 步骤滚动配置 + */ +@Data +public class StepRollingConfigDTO { + /** + * 滚动配置名称 + */ + private String name; + /** + * 滚动策略 + */ + private Integer mode; + /** + * 滚动表达式 + */ + private String expr; + + public static StepRollingConfigDTO fromRollingConfigVO(RollingConfigVO rollingConfigVO) { + StepRollingConfigDTO stepRollingConfigDTO = new StepRollingConfigDTO(); + stepRollingConfigDTO.setName(StringUtils.isBlank(rollingConfigVO.getName()) ? "default" : + rollingConfigVO.getName()); + stepRollingConfigDTO.setMode(rollingConfigVO.getMode()); + stepRollingConfigDTO.setExpr(rollingConfigVO.getExpr()); + return stepRollingConfigDTO; + } + + public static StepRollingConfigDTO fromEsbRollingConfig(EsbRollingConfigDTO rollingConfig) { + StepRollingConfigDTO stepRollingConfigDTO = new StepRollingConfigDTO(); + stepRollingConfigDTO.setName("default"); + stepRollingConfigDTO.setMode(rollingConfig.getMode()); + stepRollingConfigDTO.setExpr(rollingConfig.getExpression()); + return stepRollingConfigDTO; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskInstanceDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskInstanceDTO.java index 8e0c4f7683..a978690772 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskInstanceDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskInstanceDTO.java @@ -24,6 +24,7 @@ package com.tencent.bk.job.execute.model; +import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.common.constants.TaskStartupModeEnum; import com.tencent.bk.job.execute.common.constants.TaskTypeEnum; import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; @@ -84,14 +85,14 @@ public class TaskInstanceDTO { private Integer startupMode; /** - * 当前执行步骤id + * 当前执行的步骤实例ID */ - private long currentStepId; + private long currentStepInstanceId; /** * 状态 */ - private Integer status; + private RunStatusEnum status; /** * 开始时间 @@ -140,6 +141,13 @@ public class TaskInstanceDTO { */ private List variables; + /** + * 判断是否为执行方案实例 + */ + public boolean isPlanInstance() { + return this.taskId > 0; + } + } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskNotifyDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskNotifyDTO.java index bba2dc2820..a3a98c9fc3 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskNotifyDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskNotifyDTO.java @@ -29,6 +29,9 @@ import lombok.Getter; import lombok.Setter; +/** + * 作业执行结果消息通知 + */ @Getter @Setter @JsonInclude(JsonInclude.Include.NON_EMPTY) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/converter/TaskInstanceConverter.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/converter/TaskInstanceConverter.java index 2376f85035..b6af8c673c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/converter/TaskInstanceConverter.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/converter/TaskInstanceConverter.java @@ -28,7 +28,6 @@ import com.tencent.bk.job.common.model.dto.ResourceScope; import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.ApplicationContextRegister; -import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.common.constants.TaskStartupModeEnum; import com.tencent.bk.job.execute.common.constants.TaskTypeEnum; import com.tencent.bk.job.execute.model.TaskInstanceDTO; @@ -56,9 +55,9 @@ public static TaskInstanceVO convertToTaskInstanceVO(TaskInstanceDTO taskInstanc taskInstanceVO.setStartupMode(taskInstanceDTO.getStartupMode()); taskInstanceVO.setStartupModeDesc( i18nService.getI18n(TaskStartupModeEnum.getStartupMode(taskInstanceDTO.getStartupMode()).getI18nKey())); - taskInstanceVO.setStatus(taskInstanceDTO.getStatus()); + taskInstanceVO.setStatus(taskInstanceDTO.getStatus().getValue()); taskInstanceVO.setStatusDesc( - i18nService.getI18n(RunStatusEnum.valueOf(taskInstanceDTO.getStatus()).getI18nKey())); + i18nService.getI18n(taskInstanceDTO.getStatus().getI18nKey())); taskInstanceVO.setType(taskInstanceDTO.getType()); taskInstanceVO.setTypeDesc( i18nService.getI18n(Objects.requireNonNull(TaskTypeEnum.valueOf(taskInstanceDTO.getType())).getI18nKey())); @@ -85,9 +84,9 @@ public static ServiceTaskInstanceDTO convertToServiceTaskInstanceDTO(@NotNull Ta serviceTaskInstance.setType(taskInstanceDTO.getType()); serviceTaskInstance.setTypeDesc( i18nService.getI18n(Objects.requireNonNull(TaskTypeEnum.valueOf(taskInstanceDTO.getType())).getI18nKey())); - serviceTaskInstance.setStatus(taskInstanceDTO.getStatus()); + serviceTaskInstance.setStatus(taskInstanceDTO.getStatus().getValue()); serviceTaskInstance.setStatusDesc( - i18nService.getI18n(RunStatusEnum.valueOf(taskInstanceDTO.getStatus()).getI18nKey())); + i18nService.getI18n(taskInstanceDTO.getStatus().getI18nKey())); serviceTaskInstance.setDebugTask(taskInstanceDTO.isDebugTask()); serviceTaskInstance.setTaskId(taskInstanceDTO.getTaskId()); serviceTaskInstance.setTemplateId(taskInstanceDTO.getTaskTemplateId()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/db/RollingConfigDetailDO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/db/RollingConfigDetailDO.java new file mode 100644 index 0000000000..40d14b1d3a --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/db/RollingConfigDetailDO.java @@ -0,0 +1,102 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.db; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.annotation.PersistenceObject; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; +import java.util.Map; + +/** + * 滚动详细配置DO + */ +@Data +@JsonInclude(JsonInclude.Include.NON_EMPTY) +@NoArgsConstructor +@PersistenceObject +public class RollingConfigDetailDO { + /** + * 滚动配置名称 + */ + @JsonProperty("name") + private String name; + + /** + * 滚动区间包含的步骤实例ID列表 + */ + @JsonProperty("includeStepInstanceIdList") + private List includeStepInstanceIdList; + + /** + * 步骤滚动配置 + */ + @JsonProperty("stepRollingConfigs") + private Map stepRollingConfigs; + + /** + * 滚动策略 + * + * @see com.tencent.bk.job.common.constant.RollingModeEnum + */ + @JsonProperty("mode") + private Integer mode; + + /** + * 目标服务器滚动分批表达式 + */ + @JsonProperty("expr") + private String expr; + + /** + * 目标服务器滚动分批 + */ + @JsonProperty("hostsBatchList") + private List hostsBatchList; + + /** + * 滚动总批次 + */ + @JsonProperty("totalBatch") + private int totalBatch; + + /** + * 是否是滚动区间的第一个步骤 + */ + public boolean isFirstRollingStep(long stepInstanceId) { + return this.includeStepInstanceIdList.get(0).equals(stepInstanceId); + } + + /** + * 是否是滚动区间的最后一个步骤 + */ + public boolean isLastRollingStep(long stepInstanceId) { + return this.includeStepInstanceIdList.get(includeStepInstanceIdList.size() - 1).equals(stepInstanceId); + } + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/db/RollingHostsBatchDO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/db/RollingHostsBatchDO.java new file mode 100644 index 0000000000..5ce2f6f94c --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/db/RollingHostsBatchDO.java @@ -0,0 +1,56 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.db; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.common.annotation.PersistenceObject; +import com.tencent.bk.job.common.model.dto.HostDTO; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +/** + * 滚动执行-主机分批 DO + */ +@Data +@JsonInclude(JsonInclude.Include.NON_EMPTY) +@NoArgsConstructor +@PersistenceObject +public class RollingHostsBatchDO { + /** + * 滚动执行批次 + */ + private Integer batch; + /** + * 该批次的目标服务器 + */ + private List hosts; + + public RollingHostsBatchDO(Integer batch, List hosts) { + this.batch = batch; + this.hosts = hosts; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/db/StepRollingConfigDO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/db/StepRollingConfigDO.java new file mode 100644 index 0000000000..654ec001ca --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/db/StepRollingConfigDO.java @@ -0,0 +1,27 @@ +package com.tencent.bk.job.execute.model.db; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.annotation.PersistenceObject; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 步骤滚动配置 + */ +@Data +@JsonInclude(JsonInclude.Include.NON_EMPTY) +@NoArgsConstructor +@PersistenceObject +public class StepRollingConfigDO { + + /** + * 是否分批 + */ + @JsonProperty("batch") + private boolean batch; + + public StepRollingConfigDO(boolean batch) { + this.batch = batch; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/metrics/GseTasksCounter.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/metrics/GseTasksCounter.java index c47c4cc316..f8bd90bb9f 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/metrics/GseTasksCounter.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/metrics/GseTasksCounter.java @@ -24,7 +24,7 @@ package com.tencent.bk.job.execute.monitor.metrics; -import com.tencent.bk.job.execute.engine.GseTaskManager; +import com.tencent.bk.job.execute.engine.executor.GseTaskManager; import com.tencent.bk.job.execute.monitor.ExecuteMetricNames; import io.micrometer.core.instrument.FunctionCounter; import io.micrometer.core.instrument.MeterRegistry; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/AgentService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/AgentService.java index 453c19e1d8..57d19862a1 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/AgentService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/AgentService.java @@ -24,10 +24,11 @@ package com.tencent.bk.job.execute.service; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.model.ServersDTO; public interface AgentService { - String getLocalAgentBindIp(); + HostDTO getLocalAgentHost(); ServersDTO getLocalServersDTO(); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/AgentTaskService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/AgentTaskService.java new file mode 100644 index 0000000000..da6e6120ce --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/AgentTaskService.java @@ -0,0 +1,186 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.service; + +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.Order; +import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.model.AgentTaskDetailDTO; +import com.tencent.bk.job.execute.model.AgentTaskResultGroupBaseDTO; +import com.tencent.bk.job.execute.model.AgentTaskResultGroupDTO; +import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; + +import java.util.Collection; +import java.util.List; + +/** + * GSE Agent 任务 Service + */ +public interface AgentTaskService { + + /** + * 批量保存（insert/update) GSE Agent 任务 + * + * @param agentTasks GSE Agent 任务列表 + */ + void batchSaveAgentTasks(Collection agentTasks); + + /** + * 批量更新Agent任务 + * + * @param agentTasks Agent任务 + */ + void batchUpdateAgentTasks(Collection agentTasks); + + /** + * 获取执行成功的Agent任务数量 + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 步骤执行次数 + * @return 执行成功的Agent数量 + */ + int getSuccessAgentTaskCount(long stepInstanceId, int executeCount); + + + /** + * 根据GSE任务ID获取Agent任务 + * + * @param gseTaskId GSE任务ID + * @return Agent任务 + */ + List listAgentTasksByGseTaskId(Long gseTaskId); + + /** + * 获取Agent任务 + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @param batch 滚动执行批次；传入null或者0将忽略该参数 + * @return Agent任务 + */ + List listAgentTasks(Long stepInstanceId, + Integer executeCount, + Integer batch); + + /** + * 获取Agent任务详情并分组 + * + * @param stepInstance 步骤实例 + * @param executeCount 执行次数 + * @param batch 滚动执行批次；如果传入null或者0，将忽略该参数 + */ + List listAndGroupAgentTasks(StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch); + + /** + * 获取任务结果分组 + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @param batch 滚动执行批次；传入null或者0将忽略该参数 + * @return Agent任务 + */ + List listResultGroups(long stepInstanceId, + int executeCount, + Integer batch); + + /** + * 根据执行结果查询Agent任务详情(排序、限制返回数量) - 包含主机详情 + * + * @param stepInstance 步骤实例 + * @param executeCount 执行次数 + * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 + * @param status 任务状态 + * @param tag 用户自定义分组标签 + * @param limit 最大返回数量 + * @param orderField 排序字段 + * @param order 排序方式 + * @return Agent任务 + */ + List listAgentTaskDetailByResultGroup(StepInstanceBaseDTO stepInstance, + Integer executeCount, + Integer batch, + Integer status, + String tag, + Integer limit, + String orderField, + Order order); + + /** + * 根据结果分组获取Agent任务详情 - 包含主机详情 + * + * @param stepInstance 步骤实例 + * @param executeCount 执行次数 + * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 + * @param status 任务状态 + * @param tag 用户自定义分组标签 + * @return Agent任务 + */ + List listAgentTaskDetailByResultGroup(StepInstanceBaseDTO stepInstance, + Integer executeCount, + Integer batch, + Integer status, + String tag); + + /** + * 获取Agent任务详情 - 包含主机详情 + * + * @param stepInstance 步骤实例 + * @param executeCount 执行次数 + * @param batch 滚动执行批次；传入null或者0将忽略该参数 + * @return Agent任务 + */ + List listAgentTaskDetail(StepInstanceBaseDTO stepInstance, + Integer executeCount, + Integer batch); + + /** + * 获取Agent任务实际执行成功的executeCount值(重试场景,兼容历史数据) + * + * @param stepInstanceId 步骤实例ID + * @param cloudIp 云区域+ip + * @return Agent任务实际执行成功的executeCount值 + */ + @CompatibleImplementation(name = "rolling_execution", explain = "兼容历史数据", version = "3.7.x") + int getActualSuccessExecuteCount(long stepInstanceId, String cloudIp); + + /** + * 批量更新AgentTask的字段 + * + * @param stepInstanceId 条件 - 步骤实例ID + * @param executeCount 条件 - 重试次数 + * @param batch 条件 - 滚动执行批次；传入null将忽略该条件 + * @param actualExecuteCount 值 - Agent任务实际执行的步骤重试次数；如果传入null，则不更新 + * @param gseTaskId 值 - Agent任务对应的GSE_TASK_ID；如果传入null，则不更新 + */ + void updateAgentTaskFields(long stepInstanceId, + int executeCount, + Integer batch, + Integer actualExecuteCount, + Long gseTaskId); + + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/FileAgentTaskService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/FileAgentTaskService.java new file mode 100644 index 0000000000..5d2c7c6bd2 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/FileAgentTaskService.java @@ -0,0 +1,66 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.service; + +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.model.StepInstanceDTO; +import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; + +import java.util.List; + +/** + * GSE Agent 文件任务 Service + */ +public interface FileAgentTaskService extends AgentTaskService { + + /** + * 获取agent任务 + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @param batch 滚动执行批次；传入null或者0将忽略该参数 + * @param fileTaskMode 文件分发任务模式;传入null表示忽略该过滤条件 + * @return agent任务 + */ + List listAgentTasks(Long stepInstanceId, + Integer executeCount, + Integer batch, + FileTaskModeEnum fileTaskMode); + + /** + * @param stepInstance 步骤实例 + * @param executeCount 执行次数 + * @param batch 滚动执行批次；传入null或者0将忽略该参数 + * @param fileTaskMode 文件分发任务模式 + * @param host 主机 + * @return Agent任务 + */ + AgentTaskDTO getAgentTaskByHost(StepInstanceDTO stepInstance, + Integer executeCount, + Integer batch, + FileTaskModeEnum fileTaskMode, + HostDTO host); +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/GseTaskLogService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/GseTaskLogService.java deleted file mode 100644 index eacef25e55..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/GseTaskLogService.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.service; - -import com.tencent.bk.job.common.model.dto.IpDTO; -import com.tencent.bk.job.execute.engine.consts.IpStatus; -import com.tencent.bk.job.execute.model.AgentTaskResultGroupDTO; -import com.tencent.bk.job.execute.model.GseTaskIpLogDTO; -import com.tencent.bk.job.execute.model.GseTaskLogDTO; - -import java.util.Collection; -import java.util.List; - -public interface GseTaskLogService { - /** - * 获取 stepInstance 最后一次执行的日志 - * - * @param taskInstanceId - * @return - */ - GseTaskLogDTO getTaskLastRetryLog(long taskInstanceId); - - /** - * 保存 GseTaskLog - * - * @param gseTaskLog - */ - void saveGseTaskLog(GseTaskLogDTO gseTaskLog); - - GseTaskLogDTO getGseTaskLog(long stepInstanceId, int executeCount); - - void deleteGseTaskLog(long stepInstanceId, int executeCount); - - void clearAllIpLog(long stepInstanceId, int executeCount); - - void batchSaveIpLog(List ipLogList); - - void batchUpdateIpLog(long stepInstanceId, int executeCount, Collection cloudAreaIdAndIps, Long startTime - , Long endTime, IpStatus status); - - int getSuccessIpCount(long stepInstanceId, int executeCount); - - List getSuccessGseTaskIp(long stepInstanceId, int executeCount); - - /** - * 获取步骤执行结果分组信息-不包含ip详细信息 - * - * @param stepInstanceId 步骤实例ID - * @param executeCount 执行次数 - * @return - */ - List getIpLogStatInfo(long stepInstanceId, int executeCount); - - /** - * 获取步骤执行结果分组信息 - * - * @param stepInstanceId 步骤实例ID - * @param executeCount 执行次数 - * @return - */ - List getLogStatInfoWithIp(long stepInstanceId, int executeCount); - - List getIpLogByResultType(Long stepInstanceId, Integer executeCount, Integer resultType, - String tag); - - List getIpLogContentByResultType(Long stepInstanceId, Integer executeCount, Integer resultType, - String tag); - - /** - * 获取agent任务信息 - * - * @param stepInstanceId 步骤实例ID - * @param executeCount 执行次数 - * @param onlyTargetIp 是否仅返回目标服务器IP - * @return agent任务信息 - */ - List getIpLog(Long stepInstanceId, Integer executeCount, boolean onlyTargetIp); - - GseTaskIpLogDTO getIpLog(Long stepInstanceId, Integer executeCount, String cloudAreaIdAndIp); - - /** - * 获取文件任务源ip - * - * @param stepInstanceId 步骤实例ID - * @param executeCount 执行次数 - * @return 文件任务源ip - */ - List getTaskFileSourceIps(Long stepInstanceId, Integer executeCount); - - int getSuccessExecuteCount(long stepInstanceId, String cloudAreaAndIp); -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/GseTaskService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/GseTaskService.java new file mode 100644 index 0000000000..6e53081c98 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/GseTaskService.java @@ -0,0 +1,66 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.service; + +import com.tencent.bk.job.execute.model.GseTaskDTO; + +/** + * GSE 任务 Service + */ +public interface GseTaskService { + + /** + * 保存 GSE 任务 + * + * @param gseTask GSE 任务 + */ + Long saveGseTask(GseTaskDTO gseTask); + + /** + * 更新 GSE 任务 + * + * @param gseTask GSE 任务 + * @return 是否更新成功 + */ + boolean updateGseTask(GseTaskDTO gseTask); + + /** + * 获取 GSE 任务 + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 步骤执行次数 + * @param batch 滚动执行批次 + * @return GSE 任务 + */ + GseTaskDTO getGseTask(long stepInstanceId, int executeCount, Integer batch); + + /** + * 获取 GSE 任务 + * + * @param gseTaskId GSE任务ID + * @return GSE 任务 + */ + GseTaskDTO getGseTask(long gseTaskId); +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/HostService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/HostService.java index 4d06c8ecd0..4d0e01634f 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/HostService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/HostService.java @@ -25,8 +25,9 @@ package com.tencent.bk.job.execute.service; import com.tencent.bk.job.common.cc.model.CcInstanceDTO; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.manage.model.inner.ServiceHostDTO; +import com.tencent.bk.job.manage.model.inner.ServiceListAppHostResultDTO; import java.util.Collection; import java.util.List; @@ -43,7 +44,15 @@ public interface HostService { * @param hostIps 主机Ip列表 * @return 主机信息 */ - Map batchGetHosts(List hostIps); + Map batchGetHosts(List hostIps); + + /** + * 获取主机信息 + * + * @param host 主机 + * @return 主机信息 + */ + ServiceHostDTO getHost(HostDTO host); /** * 查询主机在白名单中允许的操作 @@ -52,26 +61,26 @@ public interface HostService { * @param host 主机 * @return 允许的操作 */ - List getHostAllowedAction(long appId, IpDTO host); + List getHostAllowedAction(long appId, HostDTO host); /** * 判断IP白名单是否配置该主机的对应操作 * - * @param appId 业务ID - * @param host 主机 - * @param action 操作 + * @param appId 业务ID + * @param cloudIp 主机IP + * @param action 操作 * @return 是否配置规则 */ - boolean isMatchWhiteIpRule(long appId, IpDTO host, String action); + boolean isMatchWhiteIpRule(long appId, String cloudIp, String action); /** - * 检查主机是否在业务下 + * 获取业务下的主机列表 * - * @param appId Job业务ID - * @param hostIps 主机列表 - * @return 非法的主机 + * @param appId Job业务ID + * @param hosts 主机列表 + * @return 主机信息 */ - List checkAppHosts(Long appId, Collection hostIps); + ServiceListAppHostResultDTO batchGetAppHosts(Long appId, Collection hosts); /** * 获取动态分组主机 @@ -80,7 +89,7 @@ public interface HostService { * @param groupId 动态分组ID * @return 主机列表 */ - List getIpByDynamicGroupId(long appId, String groupId); + List getIpByDynamicGroupId(long appId, String groupId); /** * 根据topo节点获取主机 @@ -89,7 +98,7 @@ public interface HostService { * @param ccInstances topo节点列表 * @return 主机列表 */ - List getIpByTopoNodes(long appId, List ccInstances); + List getIpByTopoNodes(long appId, List ccInstances); /** * 获取主机云区域名称 diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/LogExportService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/LogExportService.java index 5fb4081f17..e354805c34 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/LogExportService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/LogExportService.java @@ -27,12 +27,12 @@ import com.tencent.bk.job.execute.model.LogExportJobInfoDTO; /** - * @since 19/1/2021 12:00 + * 作业执行日志导出Service */ public interface LogExportService { - LogExportJobInfoDTO packageLogFile(String username, Long appId, Long stepInstanceId, String ip, int executeCount, - String logFileDir, String logFileName, Boolean repackage); + LogExportJobInfoDTO packageLogFile(String username, Long appId, Long stepInstanceId, Long hostId, String ip, + int executeCount, String logFileDir, String logFileName, Boolean repackage); - LogExportJobInfoDTO getExportInfo(Long appId, Long stepInstanceId, String ip); + LogExportJobInfoDTO getExportInfo(Long appId, Long stepInstanceId, Long hostId, String ip); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/LogService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/LogService.java index b036225931..dd103242a1 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/LogService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/LogService.java @@ -25,109 +25,55 @@ package com.tencent.bk.job.execute.service; import com.tencent.bk.job.common.exception.ServiceException; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.model.FileIpLogContent; -import com.tencent.bk.job.execute.model.ScriptIpLogContent; +import com.tencent.bk.job.execute.model.ScriptHostLogContent; import com.tencent.bk.job.logsvr.model.service.ServiceFileTaskLogDTO; -import com.tencent.bk.job.logsvr.model.service.ServiceIpLogDTO; -import com.tencent.bk.job.logsvr.model.service.ServiceIpLogsDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceHostLogDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceHostLogsDTO; import com.tencent.bk.job.logsvr.model.service.ServiceScriptLogDTO; import java.util.List; -import java.util.Map; /** * 日志服务 */ public interface LogService { - /** - * 写job系统日志(非用户脚本输出) - * - * @param jobCreateTime 任务创建时间 - * @param stepInstanceId 步骤实例ID - * @param executeCount 执行次数 - * @param cloudAreaIdAndIp 云区域ID:IP - * @param content 日志内容 - * @param offset 日志偏移 - 字节 - * @param logTimeInMillSeconds 日志时间 - * @throws ServiceException 写入失败，返回ServiceException - */ - void writeJobSystemScriptLog(long jobCreateTime, long stepInstanceId, int executeCount, - String cloudAreaIdAndIp, String content, int offset, - Long logTimeInMillSeconds) throws ServiceException; - - /** - * 批量写job系统日志(非用户脚本输出) - * - * @param jobCreateTime 任务创建时间 - * @param stepInstanceId 步骤实例ID - * @param executeCount 执行次数 - * @param ipsAndOffset 主机列表以及对应的日志偏移 - * @param content 日志内容 - * @param logTimeInMillSeconds 日志时间 - * @throws ServiceException 写入失败，返回ServiceException - */ - void batchWriteJobSystemScriptLog(long jobCreateTime, long stepInstanceId, int executeCount, - Map ipsAndOffset, String content, - Long logTimeInMillSeconds) throws ServiceException; - /** * 构造job系统日志 * - * @param cloudIp 云区域ID:IP + * @param host 主机 * @param content 日志原始内容 * @param offset 日志偏移 - 字节 * @param logTimeInMillSeconds 日志时间 * @return 系统日志 */ - ServiceScriptLogDTO buildSystemScriptLog(String cloudIp, String content, int offset, - Long logTimeInMillSeconds); + ServiceScriptLogDTO buildSystemScriptLog(HostDTO host, String content, int offset, Long logTimeInMillSeconds); /** * 写脚本执行日志 * - * @param jobCreateDate 任务创建时间 - * @param stepInstanceId 步骤实例ID - * @param executeCount 执行次数 - * @param scriptLog 脚本日志 - * @throws ServiceException 写入失败，返回ServiceException - */ - void writeScriptLog(String jobCreateDate, long stepInstanceId, int executeCount, - ServiceScriptLogDTO scriptLog) throws ServiceException; - - /** - * 写脚本执行日志 - * - * @param jobCreateDate 任务创建时间 + * @param jobCreateTime 任务创建时间 * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 + * @param batch 滚动执行批次;非滚动步骤传入null * @param scriptLogs 脚本日志 - * @throws ServiceException 写入失败，返回ServiceException */ - void batchWriteScriptLog(String jobCreateDate, long stepInstanceId, int executeCount, - List scriptLogs) throws ServiceException; + void batchWriteScriptLog(long jobCreateTime, long stepInstanceId, int executeCount, Integer batch, + List scriptLogs); - /** - * 删除步骤实例对应的日志 - * - * @param jobCreateDate 任务创建时间 - * @param stepInstanceId 步骤实例ID - * @param executeCount 执行次数 - * @throws ServiceException 删除失败，返回ServiceException - */ - void deleteStepLog(String jobCreateDate, long stepInstanceId, int executeCount) throws ServiceException; /** * 获取脚本执行日志 * * @param stepInstanceId 步骤实例 ID * @param executeCount 执行次数 - * @param ip 主机ip + * @param batch 滚动执行批次;非滚动步骤传入null + * @param host 主机 * @return 日志内容 - * @throws ServiceException 异常 */ - ScriptIpLogContent getScriptIpLogContent(long stepInstanceId, int executeCount, IpDTO ip) throws ServiceException; + ScriptHostLogContent getScriptHostLogContent(long stepInstanceId, int executeCount, Integer batch, HostDTO host); /** * 批量获取脚本执行日志 @@ -135,71 +81,75 @@ void batchWriteScriptLog(String jobCreateDate, long stepInstanceId, int executeC * @param jobCreateDateStr 作业创建时间 * @param stepInstanceId 步骤实例 ID * @param executeCount 执行次数 - * @param ips 主机列表,最大支持1000个 + * @param batch 滚动执行批次;非滚动步骤传入null + * @param hosts 主机列表,最大支持1000个 * @return 日志内容 - * @throws ServiceException 异常 */ - List batchGetScriptIpLogContent(String jobCreateDateStr, long stepInstanceId, int executeCount, - List ips) throws ServiceException; + List batchGetScriptHostLogContent(String jobCreateDateStr, long stepInstanceId, + int executeCount, Integer batch, List hosts); /** * 获取脚本执行日志 * * @param stepInstanceId 步骤实例 ID * @param executeCount 执行次数 - * @param ip 主机ip + * @param batch 滚动执行批次;非滚动步骤传入null + * @param host 主机 + * @param mode 文件传输模式 * @return 日志内容 * @throws ServiceException 异常 - * @parma mode 文件传输模式 */ - FileIpLogContent getFileIpLogContent(long stepInstanceId, int executeCount, IpDTO ip, - Integer mode) throws ServiceException; + FileIpLogContent getFileIpLogContent(long stepInstanceId, int executeCount, Integer batch, HostDTO host, + Integer mode); /** * 获取脚本执行日志 * * @param stepInstanceId 步骤实例 ID * @param executeCount 执行次数 + * @param batch 滚动执行批次;非滚动步骤传入null * @param taskIds 文件任务ID列表 * @return 日志内容 - * @throws ServiceException 异常 */ - List getFileLogContentByTaskIds(long stepInstanceId, int executeCount, - List taskIds) throws ServiceException; + List getFileLogContentByTaskIds(long stepInstanceId, int executeCount, Integer batch, + List taskIds); /** * 获取文件任务文件源日志 * * @param stepInstanceId 步骤实例 ID * @param executeCount 执行次数 + * @param batch 滚动执行批次;非滚动步骤传入null * @return 日志内容 - * @throws ServiceException */ List batchGetFileSourceIpLogContent(long stepInstanceId, - int executeCount) throws ServiceException; + int executeCount, + Integer batch); /** * 获取文件任务文件源日志 * * @param stepInstanceId 步骤实例 ID * @param executeCount 执行次数 - * @param ips 服务器列表 + * @param batch 滚动执行批次;非滚动步骤传入null + * @param hosts 主机列表 * @return 日志内容 - * @throws ServiceException */ - ServiceIpLogsDTO batchGetFileIpLogContent(long stepInstanceId, - int executeCount, List ips) throws ServiceException; + ServiceHostLogsDTO batchGetFileIpLogContent(long stepInstanceId, + int executeCount, + Integer batch, + List hosts); /** * 根据关键字获取对应的ip * * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 + * @param batch 滚动执行批次;非滚动步骤传入null * @param keyword 关键字 * @return ips - * @throws ServiceException 查询失败返回ServcieException */ - List getIpsByContentKeyword(long stepInstanceId, int executeCount, String keyword) throws ServiceException; + List getIpsByContentKeyword(long stepInstanceId, int executeCount, Integer batch, String keyword); /** * 写日志 @@ -207,14 +157,15 @@ ServiceIpLogsDTO batchGetFileIpLogContent(long stepInstanceId, * @param jobCreateTime 任务创建时间 * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 - * @param cloudAreaIdAndIp 云区域ID:IP + * @param batch 滚动执行批次;非滚动步骤传入null + * @param host 主机 * @param executionLog 文件任务执行日志 * @param logTimeInMillSeconds 日志时间 - * @throws ServiceException 写入失败，返回ServiceException */ - void writeFileLogWithTimestamp(long jobCreateTime, long stepInstanceId, int executeCount, - String cloudAreaIdAndIp, ServiceIpLogDTO executionLog, - Long logTimeInMillSeconds) throws ServiceException; + void writeFileLogWithTimestamp(long jobCreateTime, long stepInstanceId, int executeCount, Integer batch, + HostDTO host, ServiceHostLogDTO executionLog, + Long logTimeInMillSeconds); + /** * 写文件日志日志 @@ -222,6 +173,6 @@ void writeFileLogWithTimestamp(long jobCreateTime, long stepInstanceId, int exec * @param jobCreateTime 任务创建时间 * @param fileLogs 文件任务执行日志 */ - void writeFileLogs(long jobCreateTime, List fileLogs); + void writeFileLogs(long jobCreateTime, List fileLogs); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/NotifyService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/NotifyService.java index 57bd811f6b..d67d027091 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/NotifyService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/NotifyService.java @@ -24,6 +24,8 @@ package com.tencent.bk.job.execute.service; +import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; +import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.TaskNotifyDTO; /** @@ -50,4 +52,28 @@ public interface NotifyService { * @param taskNotifyDTO 通知 */ void notifyTaskConfirm(TaskNotifyDTO taskNotifyDTO); + + /** + * 发送作业执行失败通知给 MQ + * + * @param taskInstance 作业实例 + * @param stepInstance 步骤实例 + */ + void asyncSendMQFailTaskNotification(TaskInstanceDTO taskInstance, StepInstanceBaseDTO stepInstance); + + /** + * 发送作业执行成功通知给 MQ + * + * @param taskInstance 作业实例 + * @param stepInstance 步骤实例 + */ + void asyncSendMQSuccessTaskNotification(TaskInstanceDTO taskInstance, StepInstanceBaseDTO stepInstance); + + /** + * 发送人工确认通知给 MQ + * + * @param taskInstance 作业实例 + * @param stepInstance 步骤实例 + */ + void asyncSendMQConfirmNotification(TaskInstanceDTO taskInstance, StepInstanceBaseDTO stepInstance); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/RollingConfigService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/RollingConfigService.java new file mode 100644 index 0000000000..c9d9c39a48 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/RollingConfigService.java @@ -0,0 +1,56 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.service; + +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.model.FastTaskDTO; +import com.tencent.bk.job.execute.model.RollingConfigDTO; +import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; + +import java.util.List; + +/** + * 滚动配置服务 + */ +public interface RollingConfigService { + /** + * 根据滚动批次获取主机 + * + * @param stepInstance 步骤实例 + * @param batch 滚动执行批次 + * @return 主机列表 + */ + List getRollingServers(StepInstanceBaseDTO stepInstance, Integer batch); + + /** + * 保存快速执行作业滚动配置 + * + * @param fastTask 快速执行作业 + * @return 保存之后的滚动配置 + */ + RollingConfigDTO saveRollingConfigForFastJob(FastTaskDTO fastTask); + + RollingConfigDTO getRollingConfig(long rollingConfigId); +} diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/CreateBizSetReq.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ScriptAgentTaskService.java similarity index 64% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/CreateBizSetReq.java rename to src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ScriptAgentTaskService.java index 830d3030ca..67cb8cfa34 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/CreateBizSetReq.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ScriptAgentTaskService.java @@ -22,32 +22,27 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.common.cc.model.bizset; +package com.tencent.bk.job.execute.service; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.tencent.bk.job.common.esb.model.EsbReq; -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; /** - * CMDB接口请求实体类，用于创建业务集 + * GSE Agent 脚本任务 Service */ -@Setter -@Getter -@ToString -public class CreateBizSetReq extends EsbReq { - +public interface ScriptAgentTaskService extends AgentTaskService { /** - * 业务集字段集合 + * 根据主机ID获取Agent任务 + * + * @param stepInstance 步骤实例 + * @param executeCount 执行次数 + * @param batch 滚动执行批次；传入null或者0将忽略该参数 + * @param host 主机 + * @return Agent任务 */ - @JsonProperty("bk_biz_set_attr") - private BizSetAttr attr; - - /** - * 业务集的业务选择范围 - */ - @JsonProperty("bk_scope") - private BizSetScope scope; - + AgentTaskDTO getAgentTaskByHost(StepInstanceBaseDTO stepInstance, + Integer executeCount, + Integer batch, + HostDTO host); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceRollingTaskService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceRollingTaskService.java new file mode 100644 index 0000000000..3bdce59ac7 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceRollingTaskService.java @@ -0,0 +1,95 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.service; + +import com.tencent.bk.job.execute.common.constants.RunStatusEnum; +import com.tencent.bk.job.execute.model.StepInstanceRollingTaskDTO; + +import java.util.List; + +/** + * 步骤滚动任务 Service + */ +public interface StepInstanceRollingTaskService { + + /** + * 查询步骤滚动任务 + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 步骤执行次数 + * @param batch 滚动批次 + * @return 步骤滚动任务 + */ + StepInstanceRollingTaskDTO queryRollingTask(long stepInstanceId, + int executeCount, + int batch); + + /** + * 查询最新的步骤滚动任务 + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 步骤执行次数 + * @return 步骤滚动任务 + */ + List listLatestRollingTasks(long stepInstanceId, int executeCount); + + /** + * 根据批次查询滚动任务 + * + * @param stepInstanceId 步骤实例ID + * @param batch 滚动批次 + * @return 步骤滚动任务 + */ + List listRollingTasksByBatch(long stepInstanceId, Integer batch); + + /** + * 保存步骤滚动任务 + * + * @param rollingTask 滚动任务 + * @return 步骤滚动任务ID + */ + long saveRollingTask(StepInstanceRollingTaskDTO rollingTask); + + /** + * 更新滚动任务 + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 步骤执行次数 + * @param batch 滚动执行批次 + * @param status 任务状态；如果不更新传入null + * @param startTime 任务开始时间；如果不更新传入null + * @param endTime 任务结束时间；如果不更新传入null + * @param totalTime 任务执行总时间；如果不更新传入null + */ + void updateRollingTask(long stepInstanceId, + int executeCount, + int batch, + RunStatusEnum status, + Long startTime, + Long endTime, + Long totalTime); + + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceService.java new file mode 100644 index 0000000000..2050800eab --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceService.java @@ -0,0 +1,89 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.service; + +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; + +import java.util.Map; +import java.util.function.Function; + +/** + * 作业步骤执行实例 Service + */ +public interface StepInstanceService { + /** + * 更新步骤实例的当前滚动执行批次 + * + * @param stepInstanceId 步骤实例ID + * @param batch 滚动执行批次 + */ + void updateStepCurrentBatch(long stepInstanceId, int batch); + + /** + * 更新步骤实例的当前滚动执行批次 + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + */ + void updateStepCurrentExecuteCount(long stepInstanceId, int executeCount); + + /** + * 更新步骤实例的滚动配置ID + * + * @param stepInstanceId 步骤实例ID + * @param rollingConfigId 滚动配置ID + */ + void updateStepRollingConfigId(long stepInstanceId, long rollingConfigId); + + /** + * 获取下一个步骤实例 + * + * @param taskInstanceId 作业实例ID + * @param currentStepOrder 当前步骤的顺序 + * @return 步骤实例；如果当前为最后一个步骤实例，那么返回null + */ + StepInstanceBaseDTO getNextStepInstance(long taskInstanceId, int currentStepOrder); + + /** + * 获取步骤包含的主机(源+目标) + * + * @param stepInstance 步骤实例 + * @return 步骤实例包含的主机列表 + */ + Map computeStepHosts(StepInstanceBaseDTO stepInstance, + Function keyMapper); + + /** + * 获取步骤包含的主机(源+目标) + * + * @param stepInstanceId 步骤实例ID + * @return 步骤实例包含的主机列表 + */ + Map computeStepHosts(long stepInstanceId, + Function keyMapper); + + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskExecuteService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskExecuteService.java index 23d1a2cc0d..d7ae6e40a2 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskExecuteService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskExecuteService.java @@ -27,7 +27,7 @@ import com.tencent.bk.job.common.exception.ServiceException; import com.tencent.bk.job.execute.constants.TaskOperationEnum; import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; -import com.tencent.bk.job.execute.model.StepInstanceDTO; +import com.tencent.bk.job.execute.model.FastTaskDTO; import com.tencent.bk.job.execute.model.StepOperationDTO; import com.tencent.bk.job.execute.model.TaskExecuteParam; import com.tencent.bk.job.execute.model.TaskInstanceDTO; @@ -39,43 +39,35 @@ */ public interface TaskExecuteService { /** - * 创建快速作业实例 + * 快速执行作业 * - * @param taskInstance 作业实例 - * @param stepInstance 步骤实例 + * @param fastTask 快速执行作业 * @return 作业实例 ID - * @throws ServiceException */ - Long createTaskInstanceFast(TaskInstanceDTO taskInstance, StepInstanceDTO stepInstance) - throws ServiceException; + Long executeFastTask(FastTaskDTO fastTask); /** - * 创建重做的快速作业实例 + * 重做快速作业实例 * - * @param taskInstance 作业实例 - * @param stepInstance 步骤实例 + * @param fastTask 快速作业 * @return 作业实例 ID - * @throws ServiceException */ - Long createTaskInstanceForFastTaskRedo(TaskInstanceDTO taskInstance, StepInstanceDTO stepInstance) - throws ServiceException; + Long redoFastTask(FastTaskDTO fastTask); /** * 启动作业 * * @param taskInstanceId 作业实例 ID - * @throws ServiceException */ - void startTask(long taskInstanceId) throws ServiceException; + void startTask(long taskInstanceId); /** * 创建作业实例 * * @param executeParam 作业执行参数 * @return 创建的作业实例 - * @throws ServiceException */ - TaskInstanceDTO createTaskInstanceForTask(TaskExecuteParam executeParam) throws ServiceException; + TaskInstanceDTO executeJobPlan(TaskExecuteParam executeParam); /** * 创建重做作业实例 diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskInstanceService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskInstanceService.java index fd9347805e..52c03aaee9 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskInstanceService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskInstanceService.java @@ -79,6 +79,14 @@ public interface TaskInstanceService { */ StepInstanceDTO getStepInstanceDetail(long stepInstanceId); + /** + * 获取作业的第一个步骤实例 + * + * @param taskInstanceId 作业实例ID + * @return 作业第一个步骤实例 + */ + StepInstanceBaseDTO getFirstStepInstance(long taskInstanceId); + void updateTaskStatus(long taskInstanceId, int status); List getTaskStepIdList(long taskInstanceId); @@ -101,7 +109,7 @@ public interface TaskInstanceService { * * @param taskInstanceId 作业实例ID */ - void resetTaskExecuteInfoForResume(long taskInstanceId); + void resetTaskExecuteInfoForRetry(long taskInstanceId); void resetStepStatus(long stepInstanceId); @@ -117,12 +125,15 @@ public interface TaskInstanceService { void updateStepEndTime(long stepInstanceId, Long endTime); - void addTaskExecuteCount(long taskInstanceId); + /** + * 步骤重试次数+1 + * + * @param stepInstanceId 步骤实例ID + */ + void addStepInstanceExecuteCount(long stepInstanceId); void updateStepTotalTime(long stepInstanceId, long totalTime); - void updateStepStatInfo(long stepInstanceId, int runIPNum, int successIPNum, int failIPNum); - /** * 更新作业的执行信息 * @@ -148,20 +159,6 @@ void updateTaskExecutionInfo(long taskInstanceId, RunStatusEnum status, Long cur void updateStepExecutionInfo(long stepInstanceId, RunStatusEnum status, Long startTime, Long endTime, Long totalTime); - /** - * 更新步骤的执行信息 - * - * @param stepInstanceId 步骤实例ID - * @param status 步骤执行状态 - * @param startTime 开始时间 - * @param endTime 结束时间 - * @param totalTime 总耗时 - * @param runIPNum 运行中的ip - * @param successIPNum 执行成功的ip - * @param failIPNum 失败的ip - */ - void updateStepExecutionInfo(long stepInstanceId, RunStatusEnum status, Long startTime, Long endTime, - Long totalTime, Integer runIPNum, Integer successIPNum, Integer failIPNum); /** * 更新解析之后的脚本参数 diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskOperationLogService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskOperationLogService.java index 7e7713db1b..fae4360820 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskOperationLogService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskOperationLogService.java @@ -35,14 +35,14 @@ public interface TaskOperationLogService { /** * 保存用户操作日志 * - * @param operationLog + * @param operationLog 操作日志 */ long saveOperationLog(OperationLogDTO operationLog); /** * 获取用户操作日志 * - * @param taskInstanceId + * @param taskInstanceId 作业实例ID */ List listOperationLog(long taskInstanceId); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskResultService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskResultService.java index d8809f2d00..92ba34f03b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskResultService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskResultService.java @@ -27,8 +27,13 @@ import com.tencent.bk.job.common.exception.ServiceException; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; -import com.tencent.bk.job.common.model.dto.IpDTO; -import com.tencent.bk.job.execute.model.*; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.model.StepExecutionDetailDTO; +import com.tencent.bk.job.execute.model.StepExecutionRecordDTO; +import com.tencent.bk.job.execute.model.StepExecutionResultQuery; +import com.tencent.bk.job.execute.model.TaskExecuteResultDTO; +import com.tencent.bk.job.execute.model.TaskInstanceDTO; +import com.tencent.bk.job.execute.model.TaskInstanceQuery; import com.tencent.bk.job.execute.model.inner.ServiceCronTaskExecuteResultStatistics; import java.util.List; @@ -99,14 +104,20 @@ Map getCronTaskExecuteResultStatis * @param appId 业务ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 + * @param batch 滚动执行批次;如果传入null或0，忽略该参数 * @param resultType 结果 * @param tag 结果输出分类标签 * @param keyword 脚本日志关键字 - * @return + * @return 主机列表 */ - List getHostsByResultType(String username, Long appId, Long stepInstanceId, - Integer executeCount, Integer resultType, - String tag, String keyword); + List getHostsByResultType(String username, + Long appId, + Long stepInstanceId, + Integer batch, + Integer executeCount, + Integer resultType, + String tag, + String keyword); /** * 获取步骤执行历史 @@ -114,8 +125,12 @@ List getHostsByResultType(String username, Long appId, Long stepInstanceI * @param username 用户名 * @param appId 业务ID * @param stepInstanceId 步骤实例ID + * @param batch 滚动执行批次;如果传入，则会按照batch过滤步骤执行历史 * @return 执行历史 */ - List listStepExecutionHistory(String username, Long appId, Long stepInstanceId); + List listStepExecutionHistory(String username, + Long appId, + Long stepInstanceId, + Integer batch); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/AbstractAgentTaskServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/AbstractAgentTaskServiceImpl.java new file mode 100644 index 0000000000..f94265ce6a --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/AbstractAgentTaskServiceImpl.java @@ -0,0 +1,80 @@ +package com.tencent.bk.job.execute.service.impl; + +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.model.AgentTaskDetailDTO; +import com.tencent.bk.job.execute.model.AgentTaskResultGroupDTO; +import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; +import com.tencent.bk.job.execute.service.AgentTaskService; +import com.tencent.bk.job.execute.service.HostService; +import com.tencent.bk.job.execute.service.StepInstanceService; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Agent 任务Service的公共实现 + */ +public abstract class AbstractAgentTaskServiceImpl implements AgentTaskService { + private final StepInstanceService stepInstanceService; + + private final HostService hostService; + + public AbstractAgentTaskServiceImpl(StepInstanceService stepInstanceService, + HostService hostService) { + this.stepInstanceService = stepInstanceService; + this.hostService = hostService; + } + + protected final List fillHostDetail(StepInstanceBaseDTO stepInstance, + List agentTasks) { + if (CollectionUtils.isEmpty(agentTasks)) { + return Collections.emptyList(); + } + + List agentTaskDetailList; + // 历史版本AgentTask会包含ip信息,从当前版本开始AgentTask不会包含ip信息，需要从StepInstance反查 + boolean hasIpInfo = StringUtils.isNotEmpty(agentTasks.get(0).getCloudIp()); + if (!hasIpInfo) { + Map hosts = stepInstanceService.computeStepHosts(stepInstance, HostDTO::getHostId); + agentTaskDetailList = agentTasks.stream() + .map(agentTask -> { + AgentTaskDetailDTO agentTaskDetail = new AgentTaskDetailDTO(agentTask); + HostDTO host = hosts.get(agentTask.getHostId()); + agentTaskDetail.setCloudIp(host.toCloudIp()); + agentTaskDetail.setBkCloudId(host.getBkCloudId()); + agentTaskDetail.setIp(host.getIp()); + agentTaskDetail.setBkCloudName(hostService.getCloudAreaName(host.getBkCloudId())); + return agentTaskDetail; + }).collect(Collectors.toList()); + } else { + agentTaskDetailList = agentTasks.stream() + .map(agentTask -> { + AgentTaskDetailDTO agentTaskDetail = new AgentTaskDetailDTO(agentTask); + agentTaskDetail.setBkCloudName(hostService.getCloudAreaName(agentTaskDetail.getBkCloudId())); + return agentTaskDetail; + }).collect(Collectors.toList()); + } + return agentTaskDetailList; + } + + protected final List groupAgentTasks(List agentTasks) { + List resultGroups = new ArrayList<>(); + agentTasks.stream() + .collect(Collectors.groupingBy( + agentTask -> new AgentTaskResultGroupDTO(agentTask.getStatus().getValue(), agentTask.getTag()))) + .forEach((resultGroup, groupedAgentTasks) -> { + resultGroup.setTotalAgentTasks(groupedAgentTasks.size()); + resultGroup.setAgentTasks(groupedAgentTasks); + resultGroups.add(resultGroup); + }); + return resultGroups; + } + + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/AgentServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/AgentServiceImpl.java index 5f7cb97a32..dc25a414aa 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/AgentServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/AgentServiceImpl.java @@ -25,10 +25,12 @@ package com.tencent.bk.job.execute.service.impl; import com.tencent.bk.job.common.gse.service.QueryAgentStatusClient; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.engine.consts.Consts; import com.tencent.bk.job.execute.model.ServersDTO; import com.tencent.bk.job.execute.service.AgentService; +import com.tencent.bk.job.execute.service.HostService; +import com.tencent.bk.job.manage.model.inner.ServiceHostDTO; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -48,36 +50,39 @@ @Slf4j public class AgentServiceImpl implements AgentService { private final QueryAgentStatusClient queryAgentStatusClient; - private volatile String agentBindIp; + private final HostService hostService; + private volatile HostDTO agentHost; @Autowired - public AgentServiceImpl(QueryAgentStatusClient queryAgentStatusClient) { + public AgentServiceImpl(QueryAgentStatusClient queryAgentStatusClient, + HostService hostService) { this.queryAgentStatusClient = queryAgentStatusClient; + this.hostService = hostService; } @Override - public String getLocalAgentBindIp() { - if (StringUtils.isNotBlank(agentBindIp)) { - return agentBindIp; + public HostDTO getLocalAgentHost() { + if (agentHost != null) { + return agentHost; } - return getAgentBindIp(); + return getAgentBindHost(); } @Override public ServersDTO getLocalServersDTO() { - List ipDTOList = new ArrayList<>(); - ipDTOList.add(new IpDTO((long) Consts.DEFAULT_CLOUD_ID, getLocalAgentBindIp())); + List hostDTOList = new ArrayList<>(); + hostDTOList.add(getLocalAgentHost()); ServersDTO servers = new ServersDTO(); - servers.setStaticIpList(ipDTOList); - servers.setIpList(ipDTOList); + servers.setStaticIpList(hostDTOList); + servers.setIpList(hostDTOList); return servers; } - private String getAgentBindIp() { - log.info("Get local agent bind ip!"); + private HostDTO getAgentBindHost() { + log.info("Get local agent bind host!"); synchronized (this) { - if (StringUtils.isNotBlank(agentBindIp)) { - return agentBindIp; + if (agentHost != null) { + return agentHost; } String physicalMachineMultiIp; String nodeIP = System.getenv("BK_JOB_NODE_IP"); @@ -92,9 +97,14 @@ private String getAgentBindIp() { } physicalMachineMultiIp = sj.toString(); } - agentBindIp = queryAgentStatusClient.getHostIpByAgentStatus(physicalMachineMultiIp, Consts.DEFAULT_CLOUD_ID); + String agentBindIp = queryAgentStatusClient.getHostIpByAgentStatus(physicalMachineMultiIp, + Consts.DEFAULT_CLOUD_ID); log.info("Local agent bind ip is {}", agentBindIp); - return agentBindIp; + String agentBindCloudIp = Consts.DEFAULT_CLOUD_ID + ":" + agentBindIp; + ServiceHostDTO host = hostService.getHost(HostDTO.fromCloudIp(agentBindCloudIp)); + agentHost = HostDTO.fromHostIdAndCloudIp(host.getHostId(), agentBindCloudIp); + agentHost.setAgentId(agentHost.getFinalAgentId()); + return agentHost; } } @@ -108,8 +118,8 @@ private Map getMachineIP() { while (allNetInterfaces.hasMoreElements()) {// 循环网卡获取网卡的IP地址 NetworkInterface netInterface = allNetInterfaces.nextElement(); String netInterfaceName = netInterface.getName(); - if (StringUtils.isBlank(netInterfaceName) || "lo".equalsIgnoreCase(netInterfaceName)) {// 过滤掉127 - // .0.0.1的IP + if (StringUtils.isBlank(netInterfaceName) || "lo".equalsIgnoreCase(netInterfaceName)) { + // 过滤掉127.0.0.1的IP continue; } Enumeration addresses = netInterface.getInetAddresses(); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileAgentTaskServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileAgentTaskServiceImpl.java new file mode 100644 index 0000000000..a199304d75 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileAgentTaskServiceImpl.java @@ -0,0 +1,245 @@ +package com.tencent.bk.job.execute.service.impl; + +import com.tencent.bk.job.common.constant.Order; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.dao.FileAgentTaskDAO; +import com.tencent.bk.job.execute.dao.GseTaskIpLogDAO; +import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.model.AgentTaskDetailDTO; +import com.tencent.bk.job.execute.model.AgentTaskResultGroupBaseDTO; +import com.tencent.bk.job.execute.model.AgentTaskResultGroupDTO; +import com.tencent.bk.job.execute.model.FileSourceDTO; +import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; +import com.tencent.bk.job.execute.model.StepInstanceDTO; +import com.tencent.bk.job.execute.service.FileAgentTaskService; +import com.tencent.bk.job.execute.service.HostService; +import com.tencent.bk.job.execute.service.StepInstanceService; +import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +@Service +@Slf4j +public class FileAgentTaskServiceImpl + extends AbstractAgentTaskServiceImpl + implements FileAgentTaskService { + + private final FileAgentTaskDAO fileAgentTaskDAO; + private final GseTaskIpLogDAO gseTaskIpLogDAO; + + @Autowired + public FileAgentTaskServiceImpl(FileAgentTaskDAO fileAgentTaskDAO, + StepInstanceService stepInstanceService, + HostService hostService, + GseTaskIpLogDAO gseTaskIpLogDAO) { + super(stepInstanceService, hostService); + this.fileAgentTaskDAO = fileAgentTaskDAO; + this.gseTaskIpLogDAO = gseTaskIpLogDAO; + } + + @Override + public void batchSaveAgentTasks(Collection agentTasks) { + if (CollectionUtils.isEmpty(agentTasks)) { + return; + } + fileAgentTaskDAO.batchSaveAgentTasks(agentTasks); + } + + @Override + public void batchUpdateAgentTasks(Collection agentTasks) { + if (CollectionUtils.isEmpty(agentTasks)) { + return; + } + fileAgentTaskDAO.batchUpdateAgentTasks(agentTasks); + } + + @Override + public int getSuccessAgentTaskCount(long stepInstanceId, int executeCount) { + if (isStepInstanceRecordExist(stepInstanceId)) { + return fileAgentTaskDAO.getSuccessAgentTaskCount(stepInstanceId, executeCount); + } else { + return gseTaskIpLogDAO.getSuccessAgentTaskCount(stepInstanceId, executeCount); + } + } + + @Override + public List listAndGroupAgentTasks(StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch) { + List resultGroups = new ArrayList<>(); + + List agentTasks = listAgentTasks(stepInstance.getId(), executeCount, batch, + FileTaskModeEnum.DOWNLOAD); + if (CollectionUtils.isEmpty(agentTasks)) { + return resultGroups; + } + + List agentTaskDetailList = fillHostDetail(stepInstance, agentTasks); + + resultGroups = groupAgentTasks(agentTaskDetailList); + + return resultGroups.stream().sorted().collect(Collectors.toList()); + } + + @Override + public List listResultGroups(long stepInstanceId, + int executeCount, + Integer batch) { + List resultGroups; + resultGroups = fileAgentTaskDAO.listResultGroups(stepInstanceId, executeCount, batch); + if (CollectionUtils.isEmpty(resultGroups)) { + // 兼容历史数据 + resultGroups = gseTaskIpLogDAO.listResultGroups(stepInstanceId, executeCount); + } + return resultGroups; + } + + @Override + public List listAgentTaskDetailByResultGroup(StepInstanceBaseDTO stepInstance, + Integer executeCount, + Integer batch, + Integer status, + String tag) { + List agentTasks = fileAgentTaskDAO.listAgentTaskByResultGroup(stepInstance.getId(), + executeCount, batch, status); + if (CollectionUtils.isEmpty(agentTasks)) { + // 兼容历史数据 + agentTasks = gseTaskIpLogDAO.listAgentTaskByResultGroup(stepInstance.getId(), executeCount, status, tag); + } + return fillHostDetail(stepInstance, agentTasks); + } + + @Override + public List listAgentTaskDetailByResultGroup(StepInstanceBaseDTO stepInstance, + Integer executeCount, + Integer batch, + Integer status, + String tag, + Integer limit, + String orderField, + Order order) { + List agentTasks = fileAgentTaskDAO.listAgentTaskByResultGroup(stepInstance.getId(), + executeCount, batch, status, limit, orderField, order); + if (CollectionUtils.isEmpty(agentTasks)) { + // 兼容历史数据 + agentTasks = gseTaskIpLogDAO.listAgentTaskByResultGroup(stepInstance.getId(), executeCount, status, tag, + limit, orderField, order); + } + return fillHostDetail(stepInstance, agentTasks); + } + + @Override + public List listAgentTasks(Long stepInstanceId, Integer executeCount, Integer batch, + FileTaskModeEnum fileTaskMode) { + List agentTasks = fileAgentTaskDAO.listAgentTasks(stepInstanceId, executeCount, batch, + fileTaskMode); + // 兼容历史数据 + if (CollectionUtils.isEmpty(agentTasks)) { + agentTasks = gseTaskIpLogDAO.listAgentTasks(stepInstanceId, executeCount); + if (CollectionUtils.isNotEmpty(agentTasks) && fileTaskMode != null) { + agentTasks = agentTasks.stream().filter(agentTask -> agentTask.getFileTaskMode() == fileTaskMode) + .collect(Collectors.toList()); + } + } + + return agentTasks; + } + + @Override + public List listAgentTasksByGseTaskId(Long gseTaskId) { + return fileAgentTaskDAO.listAgentTasksByGseTaskId(gseTaskId); + } + + @Override + public AgentTaskDTO getAgentTaskByHost(StepInstanceDTO stepInstance, Integer executeCount, Integer batch, + FileTaskModeEnum fileTaskMode, HostDTO host) { + AgentTaskDTO agentTask = null; + Long hostId = host.getHostId(); + if (hostId != null) { + // 根据hostId查询 + agentTask = fileAgentTaskDAO.getAgentTaskByHostId(stepInstance.getId(), executeCount, batch, + fileTaskMode, hostId); + } else if (StringUtils.isNotEmpty(host.toCloudIp())) { + // 根据ip查询的模式，有两种情况，数据可能在gse_file_agent_task/gse_task_ip_log表中，优先查询gse_file_agent_task + HostDTO queryHost = getStepHostByIp(stepInstance, host.toCloudIp()); + if (queryHost != null) { + agentTask = fileAgentTaskDAO.getAgentTaskByHostId(stepInstance.getId(), executeCount, batch, + fileTaskMode, queryHost.getHostId()); + } else { + // 根据ip查询gse_task_ip_log表中的数据 + agentTask = gseTaskIpLogDAO.getAgentTaskByIp(stepInstance.getId(), executeCount, host.toCloudIp()); + } + } + return agentTask; + } + + private HostDTO getStepHostByIp(StepInstanceDTO stepInstance, String cloudIp) { + HostDTO queryHost = stepInstance.getTargetServers().getIpList().stream() + .filter(targetHost -> cloudIp.equals(targetHost.toCloudIp())) + .findFirst() + .orElse(null); + if (queryHost == null) { + if (CollectionUtils.isNotEmpty(stepInstance.getResolvedFileSourceList())) { + for (FileSourceDTO fileSource : stepInstance.getResolvedFileSourceList()) { + if (fileSource.getServers() != null + && CollectionUtils.isNotEmpty(fileSource.getServers().getIpList())) { + queryHost = fileSource.getServers().getIpList().stream() + .filter(sourceHost -> cloudIp.equals(sourceHost.toCloudIp())) + .findFirst() + .orElse(null); + if (queryHost != null) { + break; + } + } + } + } + } + + return queryHost; + } + + @Override + public List listAgentTasks(Long stepInstanceId, Integer executeCount, Integer batch) { + List agentTasks = fileAgentTaskDAO.listAgentTasks(stepInstanceId, executeCount, batch, null); + if (CollectionUtils.isEmpty(agentTasks)) { + // 兼容历史数据 + agentTasks = gseTaskIpLogDAO.listAgentTasks(stepInstanceId, executeCount); + } + return agentTasks; + } + + @Override + public int getActualSuccessExecuteCount(long stepInstanceId, String cloudIp) { + // 兼容历史数据 + return gseTaskIpLogDAO.getActualSuccessExecuteCount(stepInstanceId, cloudIp); + } + + @Override + public List listAgentTaskDetail(StepInstanceBaseDTO stepInstance, + Integer executeCount, + Integer batch) { + List agentTasks = listAgentTasks(stepInstance.getId(), executeCount, batch); + return fillHostDetail(stepInstance, agentTasks); + } + + private boolean isStepInstanceRecordExist(long stepInstanceId) { + return fileAgentTaskDAO.isStepInstanceRecordExist(stepInstanceId); + } + + @Override + public void updateAgentTaskFields(long stepInstanceId, + int executeCount, + Integer batch, + Integer actualExecuteCount, + Long gseTaskId) { + fileAgentTaskDAO.updateAgentTaskFields(stepInstanceId, executeCount, batch, actualExecuteCount, gseTaskId); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/GseTaskLogServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/GseTaskLogServiceImpl.java deleted file mode 100644 index 5c7a862e60..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/GseTaskLogServiceImpl.java +++ /dev/null @@ -1,234 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.service.impl; - -import com.tencent.bk.job.common.model.dto.IpDTO; -import com.tencent.bk.job.common.util.BatchUtil; -import com.tencent.bk.job.common.util.date.DateUtils; -import com.tencent.bk.job.execute.common.constants.FileDistModeEnum; -import com.tencent.bk.job.execute.dao.GseTaskIpLogDAO; -import com.tencent.bk.job.execute.dao.GseTaskLogDAO; -import com.tencent.bk.job.execute.dao.StepInstanceDAO; -import com.tencent.bk.job.execute.dao.TaskInstanceDAO; -import com.tencent.bk.job.execute.engine.consts.IpStatus; -import com.tencent.bk.job.execute.model.*; -import com.tencent.bk.job.execute.service.GseTaskLogService; -import com.tencent.bk.job.execute.service.LogService; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import java.time.ZoneId; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; - -@Service -@Slf4j -public class GseTaskLogServiceImpl implements GseTaskLogService { - private final TaskInstanceDAO taskInstanceDao; - private final StepInstanceDAO stepInstanceDao; - private final GseTaskLogDAO gseTaskLogDao; - private final GseTaskIpLogDAO gseTaskIpLogDao; - - private final LogService logService; - - @Autowired - public GseTaskLogServiceImpl(TaskInstanceDAO taskInstanceDao, StepInstanceDAO stepInstanceDao, - GseTaskLogDAO gseTaskLogDao, GseTaskIpLogDAO gseTaskIpLogDao, LogService logService) { - this.taskInstanceDao = taskInstanceDao; - this.stepInstanceDao = stepInstanceDao; - this.gseTaskLogDao = gseTaskLogDao; - this.gseTaskIpLogDao = gseTaskIpLogDao; - this.logService = logService; - } - - @Override - public GseTaskLogDTO getTaskLastRetryLog(long taskInstanceId) { - long currentStepId = taskInstanceDao.getTaskInstance(taskInstanceId).getCurrentStepId(); - return gseTaskLogDao.getStepLastExecuteLog(currentStepId); - } - - @Override - public void saveGseTaskLog(GseTaskLogDTO gseTaskLog) { - gseTaskLogDao.saveGseTaskLog(gseTaskLog); - } - - @Override - public GseTaskLogDTO getGseTaskLog(long stepInstanceId, int executeCount) { - return gseTaskLogDao.getGseTaskLog(stepInstanceId, executeCount); - } - - @Override - public void deleteGseTaskLog(long stepInstanceId, int executeCount) { - gseTaskLogDao.deleteGseTaskLog(stepInstanceId, executeCount); - } - - @Override - public void clearAllIpLog(long stepInstanceId, int executeCount) { - gseTaskIpLogDao.deleteAllIpLog(stepInstanceId, executeCount); - - // 删除日志文件 - StepInstanceBaseDTO stepInstance = stepInstanceDao.getStepInstanceBase(stepInstanceId); - if (null == stepInstance) { - return; - } - - TaskInstanceDTO taskInstance = taskInstanceDao.getTaskInstance(stepInstance.getTaskInstanceId()); - if (null == taskInstance) { - return; - } - logService.deleteStepLog(DateUtils.formatUnixTimestamp(taskInstance.getCreateTime(), ChronoUnit.MILLIS, - "yyyy_MM_dd", ZoneId.of("UTC")), stepInstanceId, executeCount); - } - - @Override - public void batchSaveIpLog(List ipLogList) { - if (ipLogList.size() == 0) { - return; - } - if (ipLogList.size() <= 1000) { - gseTaskIpLogDao.batchSaveIpLog(ipLogList); - } else { - List> batches = BatchUtil.buildBatchList(ipLogList, 1000); - batches.parallelStream().forEach(gseTaskIpLogDao::batchSaveIpLog); - } - } - - @Override - public void batchUpdateIpLog(long stepInstanceId, int executeCount, Collection cloudAreaIdAndIps, - Long startTime, Long endTime, IpStatus status) { - if (cloudAreaIdAndIps == null || cloudAreaIdAndIps.size() == 0) { - return; - } - gseTaskIpLogDao.batchUpdateIpLog(stepInstanceId, executeCount, cloudAreaIdAndIps, startTime, endTime, status); - } - - @Override - public int getSuccessIpCount(long stepInstanceId, int executeCount) { - return gseTaskIpLogDao.getSuccessIpCount(stepInstanceId, executeCount); - } - - @Override - public List getSuccessGseTaskIp(long stepInstanceId, int executeCount) { - return gseTaskIpLogDao.getSuccessGseTaskIp(stepInstanceId, executeCount); - } - - @Override - public List getIpLogStatInfo(long stepInstanceId, int executeCount) { - List resultGroups = new ArrayList<>(); - - List baseResultGroups = gseTaskIpLogDao.getResultGroups(stepInstanceId, executeCount); - for (ResultGroupBaseDTO baseResultGroup : baseResultGroups) { - AgentTaskResultGroupDTO resultGroup = new AgentTaskResultGroupDTO(); - resultGroup.setResultType(IpStatus.valueOf(baseResultGroup.getResultType())); - resultGroup.setTag(baseResultGroup.getTag()); - resultGroup.setCount(baseResultGroup.getAgentTaskCount()); - resultGroups.add(resultGroup); - } - return resultGroups; - } - - @Override - public List getLogStatInfoWithIp(long stepInstanceId, int executeCount) { - List resultGroups = new ArrayList<>(); - - List baseResultGroups = gseTaskIpLogDao.getResultGroups(stepInstanceId, executeCount); - for (ResultGroupBaseDTO baseResultGroup : baseResultGroups) { - AgentTaskResultGroupDTO resultGroup = new AgentTaskResultGroupDTO(); - resultGroup.setResultType(IpStatus.valueOf(baseResultGroup.getResultType())); - resultGroup.setTag(baseResultGroup.getTag()); - resultGroup.setCount(baseResultGroup.getAgentTaskCount()); - List gseTaskIpLogList = getIpLogByResultType(stepInstanceId, executeCount, - baseResultGroup.getResultType(), baseResultGroup.getTag()); - List ipList = new ArrayList<>(); - for (GseTaskIpLogDTO gseTaskIpLog : gseTaskIpLogList) { - ipList.add(IpDTO.fromCloudAreaIdAndIpStr(gseTaskIpLog.getCloudAreaAndIp())); - } - resultGroup.setIpList(ipList); - resultGroups.add(resultGroup); - } - return resultGroups; - } - - - @Override - public List getIpLogByResultType(Long stepInstanceId, Integer executeCount, Integer resultType, - String tag) { - return gseTaskIpLogDao.getIpLogByResultType(stepInstanceId, executeCount, resultType, tag); - } - - @Override - public List getIpLogContentByResultType(Long stepInstanceId, Integer executeCount, - Integer resultType, String tag) { - List ipLogByResultType = getIpLogByResultType(stepInstanceId, executeCount, resultType, tag); - StepInstanceBaseDTO stepInstance = stepInstanceDao.getStepInstanceBase(stepInstanceId); - if (stepInstance == null) { - return Collections.emptyList(); - } - for (GseTaskIpLogDTO gseTaskIpLog : ipLogByResultType) { - long startTime = System.currentTimeMillis(); - if (stepInstance.isScriptStep()) { - ScriptIpLogContent scriptIpLogContent = logService.getScriptIpLogContent(stepInstanceId, executeCount - , IpDTO.fromCloudAreaIdAndIpStr(gseTaskIpLog.getCloudAreaAndIp())); - gseTaskIpLog.setLogContent(scriptIpLogContent == null ? "" : scriptIpLogContent.getContent()); - } else if (stepInstance.isFileStep()) { - FileIpLogContent fileIpLogContent = logService.getFileIpLogContent(stepInstanceId, executeCount, - IpDTO.fromCloudAreaIdAndIpStr(gseTaskIpLog.getCloudAreaAndIp()), - FileDistModeEnum.DOWNLOAD.getValue()); - gseTaskIpLog.setLogContent(fileIpLogContent == null ? "" : fileIpLogContent.getContent()); - } - log.debug("stepInstanceId={}|ip={}|time={} ms", stepInstanceId, gseTaskIpLog.getCloudAreaAndIp(), - (System.currentTimeMillis() - startTime)); - - } - - return ipLogByResultType; - } - - @Override - public List getIpLog(Long stepInstanceId, Integer executeCount, boolean onlyTargetIp) { - return gseTaskIpLogDao.getIpLog(stepInstanceId, executeCount, onlyTargetIp); - } - - @Override - public GseTaskIpLogDTO getIpLog(Long stepInstanceId, Integer executeCount, String cloudAreaIdAndIp) { - return gseTaskIpLogDao.getIpLogByIp(stepInstanceId, executeCount, cloudAreaIdAndIp); - } - - @Override - public List getTaskFileSourceIps(Long stepInstanceId, Integer executeCount) { - return gseTaskIpLogDao.getTaskFileSourceIps(stepInstanceId, executeCount) - .stream().map(IpDTO::fromCloudAreaIdAndIpStr).collect(Collectors.toList()); - } - - @Override - public int getSuccessExecuteCount(long stepInstanceId, String cloudAreaAndIp) { - return gseTaskIpLogDao.getSuccessRetryCount(stepInstanceId, cloudAreaAndIp); - } -} diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/util/VersionCompatUtil.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/GseTaskServiceImpl.java similarity index 58% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/util/VersionCompatUtil.java rename to src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/GseTaskServiceImpl.java index 7d34ded66b..f9814234c4 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/util/VersionCompatUtil.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/GseTaskServiceImpl.java @@ -22,32 +22,43 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.common.cc.util; +package com.tencent.bk.job.execute.service.impl; +import com.tencent.bk.job.execute.dao.GseTaskDAO; +import com.tencent.bk.job.execute.model.GseTaskDTO; +import com.tencent.bk.job.execute.service.GseTaskService; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +@Service @Slf4j -public class VersionCompatUtil { - - /* - * 转换maintainers，对接CC3.0之后maintainer用","分隔，CC2.0使用";"分隔；需要兼容 - */ - public static String convertMaintainers(String maintainers) { - if (StringUtils.isBlank(maintainers)) { - return ""; - } - String[] maintainerArray = maintainers.split("[,;]"); - StringBuilder sb = new StringBuilder(); - for (String maintainer : maintainerArray) { - if (StringUtils.isNotBlank(maintainer)) { - sb.append(maintainer).append(";"); - } else { - log.info("ignore an empty maintainer:{}", maintainer); - } - } - String newMaintainers = sb.toString(); - newMaintainers = newMaintainers.substring(0, newMaintainers.length() - 1); - return newMaintainers; +public class GseTaskServiceImpl implements GseTaskService { + + private final GseTaskDAO gseTaskDAO; + + @Autowired + public GseTaskServiceImpl(GseTaskDAO gseTaskDAO) { + this.gseTaskDAO = gseTaskDAO; + } + + @Override + public Long saveGseTask(GseTaskDTO gseTask) { + return gseTaskDAO.saveGseTask(gseTask); + } + + @Override + public boolean updateGseTask(GseTaskDTO gseTask) { + return gseTaskDAO.updateGseTask(gseTask); + } + + @Override + public GseTaskDTO getGseTask(long stepInstanceId, int executeCount, Integer batch) { + return gseTaskDAO.getGseTask(stepInstanceId, executeCount, batch); + } + + @Override + public GseTaskDTO getGseTask(long gseTaskId) { + return gseTaskDAO.getGseTask(gseTaskId); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/HostServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/HostServiceImpl.java index 7bafab54ad..ce1c6fd5ca 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/HostServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/HostServiceImpl.java @@ -35,7 +35,7 @@ import com.tencent.bk.job.common.cc.sdk.IBizCmdbClient; import com.tencent.bk.job.common.model.InternalResponse; import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.model.dto.ResourceScope; import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.json.JsonUtils; @@ -43,6 +43,7 @@ import com.tencent.bk.job.execute.client.WhiteIpResourceClient; import com.tencent.bk.job.execute.service.HostService; import com.tencent.bk.job.manage.model.inner.ServiceHostDTO; +import com.tencent.bk.job.manage.model.inner.ServiceListAppHostResultDTO; import com.tencent.bk.job.manage.model.inner.ServiceWhiteIPInfo; import com.tencent.bk.job.manage.model.inner.request.ServiceBatchGetHostsReq; import com.tencent.bk.job.manage.model.inner.request.ServiceCheckAppHostsReq; @@ -71,7 +72,7 @@ public class HostServiceImpl implements HostService { private final AppScopeMappingService appScopeMappingService; private volatile boolean isWhiteIpConfigLoaded = false; - private final Map whiteIpConfig = new ConcurrentHashMap<>(); + private final Map whiteIpConfig = new ConcurrentHashMap<>(); private final LoadingCache cloudAreaNameCache = CacheBuilder.newBuilder() .maximumSize(10000).expireAfterWrite(1, TimeUnit.HOURS). @@ -106,25 +107,35 @@ public HostServiceImpl(WhiteIpResourceClient whiteIpResourceClient, } @Override - public Map batchGetHosts(List hostIps) { + public Map batchGetHosts(List hostIps) { List hosts = hostResourceClient.batchGetHosts( new ServiceBatchGetHostsReq(hostIps)).getData(); - Map hostMap = new HashMap<>(); - hosts.forEach(host -> hostMap.put(new IpDTO(host.getCloudAreaId(), host.getIp()), host)); + Map hostMap = new HashMap<>(); + hosts.forEach(host -> hostMap.put(new HostDTO(host.getCloudAreaId(), host.getIp()), host)); return hostMap; } @Override - public List getHostAllowedAction(long appId, IpDTO host) { + public ServiceHostDTO getHost(HostDTO host) { + List hosts = hostResourceClient.batchGetHosts( + new ServiceBatchGetHostsReq(Collections.singletonList(host))).getData(); + if (CollectionUtils.isEmpty(hosts)) { + return null; + } + return hosts.get(0); + } + + @Override + public List getHostAllowedAction(long appId, HostDTO host) { try { InternalResponse> resp = whiteIpResourceClient.getWhiteIPActionScopes(appId, host.getIp(), - host.getCloudAreaId()); + host.getBkCloudId()); if (!resp.isSuccess()) { log.warn("Get white ip action scopes return fail resp, appId:{}, host:{}", appId, - host.convertToStrIp()); + host.toCloudIp()); return Collections.emptyList(); } - log.info("Get white ip action scopes, appId:{}, host:{}, resp:{}", appId, host.convertToStrIp(), resp); + log.info("Get white ip action scopes, appId:{}, host:{}, resp:{}", appId, host.toCloudIp(), resp); return resp.getData(); } catch (Exception e) { log.warn("GetHostAllowedAction fail!", e); @@ -147,7 +158,7 @@ public void syncWhiteIpConfig() { List whiteIpInfos = resp.getData(); whiteIpConfig.clear(); whiteIpInfos.forEach(whiteIpInfo -> - whiteIpConfig.put(new IpDTO(whiteIpInfo.getCloudId(), whiteIpInfo.getIp()), whiteIpInfo)); + whiteIpConfig.put(whiteIpInfo.getCloudId() + ":" + whiteIpInfo.getIp(), whiteIpInfo)); long cost = System.currentTimeMillis() - start; if (cost > 1000L) { @@ -157,12 +168,12 @@ public void syncWhiteIpConfig() { } @Override - public boolean isMatchWhiteIpRule(long appId, IpDTO host, String action) { + public boolean isMatchWhiteIpRule(long appId, String cloudIp, String action) { try { if (!isWhiteIpConfigLoaded) { syncWhiteIpConfig(); } - ServiceWhiteIPInfo whiteIpInfo = whiteIpConfig.get(host); + ServiceWhiteIPInfo whiteIpInfo = whiteIpConfig.get(cloudIp); if (whiteIpInfo == null) { return false; } @@ -181,19 +192,20 @@ public boolean isMatchWhiteIpRule(long appId, IpDTO host, String action) { } @Override - public List checkAppHosts(Long appId, Collection hostIps) { - InternalResponse> response = - hostResourceClient.checkAppHosts(appId, new ServiceCheckAppHostsReq(new ArrayList<>(hostIps))); + public ServiceListAppHostResultDTO batchGetAppHosts(Long appId, + Collection hosts) { + InternalResponse response = + hostResourceClient.batchGetAppHosts(appId, new ServiceCheckAppHostsReq(new ArrayList<>(hosts))); return response.getData(); } @Override - public List getIpByDynamicGroupId(long appId, String groupId) { + public List getIpByDynamicGroupId(long appId, String groupId) { IBizCmdbClient bizCmdbClient = CmdbClientFactory.getCmdbClient(); ResourceScope resourceScope = appScopeMappingService.getScopeByAppId(appId); List cmdbGroupHostList = bizCmdbClient.getDynamicGroupIp(Long.parseLong(resourceScope.getId()), groupId); - List ips = new ArrayList<>(); + List ips = new ArrayList<>(); if (cmdbGroupHostList == null || cmdbGroupHostList.isEmpty()) { return ips; } @@ -210,7 +222,7 @@ public List getIpByDynamicGroupId(long appId, String groupId) { "hostIp={}", appId, groupId, hostProp.getIp()); continue; } - IpDTO ip = new IpDTO(hostCloudId.getInstanceId(), hostProp.getIp()); + HostDTO ip = new HostDTO(hostCloudId.getInstanceId(), hostProp.getIp()); ips.add(ip); } log.info("Get hosts by groupId, appId={}, groupId={}, hosts={}", appId, groupId, ips); @@ -218,17 +230,17 @@ public List getIpByDynamicGroupId(long appId, String groupId) { } @Override - public List getIpByTopoNodes(long appId, List ccInstances) { + public List getIpByTopoNodes(long appId, List ccInstances) { IBizCmdbClient bizCmdbClient = CmdbClientFactory.getCmdbClient(); ResourceScope resourceScope = appScopeMappingService.getScopeByAppId(appId); long bizId = Long.parseLong(resourceScope.getId()); List appHosts = bizCmdbClient.getHosts(bizId, ccInstances); - List ips = new ArrayList<>(); + List ips = new ArrayList<>(); if (appHosts == null || appHosts.isEmpty()) { return ips; } for (ApplicationHostDTO hostProp : appHosts) { - IpDTO ip = new IpDTO(hostProp.getCloudAreaId(), hostProp.getIp()); + HostDTO ip = new HostDTO(hostProp.getCloudAreaId(), hostProp.getIp()); ips.add(ip); } log.info("Get hosts by cc topo nodes, appId={}, nodes={}, hosts={}", appId, ccInstances, ips); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogExportServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogExportServiceImpl.java index 37baac3560..b13fc61fb2 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogExportServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogExportServiceImpl.java @@ -24,27 +24,26 @@ package com.tencent.bk.job.execute.service.impl; -import brave.Tracing; import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.tencent.bk.job.common.artifactory.sdk.ArtifactoryClient; import com.tencent.bk.job.common.constant.JobConstants; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.redis.util.LockUtils; +import com.tencent.bk.job.common.trace.executors.TraceableExecutorService; import com.tencent.bk.job.common.util.BatchUtil; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.common.util.file.ZipUtil; import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.common.trace.executors.TraceableExecutorService; import com.tencent.bk.job.execute.config.ArtifactoryConfig; import com.tencent.bk.job.execute.config.LogExportConfig; import com.tencent.bk.job.execute.constants.LogExportStatusEnum; -import com.tencent.bk.job.execute.model.GseTaskIpLogDTO; +import com.tencent.bk.job.execute.model.AgentTaskDTO; import com.tencent.bk.job.execute.model.LogExportJobInfoDTO; -import com.tencent.bk.job.execute.model.ScriptIpLogContent; +import com.tencent.bk.job.execute.model.ScriptHostLogContent; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; -import com.tencent.bk.job.execute.service.GseTaskLogService; import com.tencent.bk.job.execute.service.LogExportService; import com.tencent.bk.job.execute.service.LogService; +import com.tencent.bk.job.execute.service.ScriptAgentTaskService; import com.tencent.bk.job.execute.service.TaskInstanceService; import lombok.Data; import lombok.extern.slf4j.Slf4j; @@ -52,6 +51,7 @@ import org.apache.commons.lang3.StringUtils; import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cloud.sleuth.Tracer; import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.stereotype.Service; import org.springframework.util.StopWatch; @@ -78,7 +78,6 @@ @Service public class LogExportServiceImpl implements LogExportService { private static final String EXPORT_KEY_PREFIX = "execute:log:export:"; - private final GseTaskLogService gseTaskLogService; private final LogService logService; private final TraceableExecutorService logExportExecutor; private final StringRedisTemplate redisTemplate; @@ -86,34 +85,37 @@ public class LogExportServiceImpl implements LogExportService { private final ArtifactoryClient artifactoryClient; private final ArtifactoryConfig artifactoryConfig; private final LogExportConfig logExportConfig; + private final ScriptAgentTaskService scriptAgentTaskService; @Autowired - public LogExportServiceImpl(GseTaskLogService gseTaskLogService, LogService logService, Tracing tracing, + public LogExportServiceImpl(LogService logService, + Tracer tracer, StringRedisTemplate redisTemplate, TaskInstanceService taskInstanceService, ArtifactoryClient artifactoryClient, ArtifactoryConfig artifactoryConfig, - LogExportConfig logExportConfig) { - this.gseTaskLogService = gseTaskLogService; + LogExportConfig logExportConfig, + ScriptAgentTaskService scriptAgentTaskService) { this.logService = logService; this.redisTemplate = redisTemplate; this.taskInstanceService = taskInstanceService; this.artifactoryClient = artifactoryClient; this.artifactoryConfig = artifactoryConfig; this.logExportConfig = logExportConfig; + this.scriptAgentTaskService = scriptAgentTaskService; ThreadFactory threadFactory = new ThreadFactoryBuilder().setNameFormat("log-export-thread-%d").build(); this.logExportExecutor = new TraceableExecutorService(new ThreadPoolExecutor(10, - 100, 60, TimeUnit.SECONDS, new LinkedBlockingQueue<>(), threadFactory), tracing); + 100, 60, TimeUnit.SECONDS, new LinkedBlockingQueue<>(), threadFactory), tracer); } @Override - public LogExportJobInfoDTO packageLogFile(String username, Long appId, Long stepInstanceId, String ip, - int executeCount, + public LogExportJobInfoDTO packageLogFile(String username, Long appId, Long stepInstanceId, Long hostId, + String cloudIp, int executeCount, String logFileDir, String logFileName, Boolean repackage) { - log.debug("Package log file for {}|{}|{}|{}|{}|{}|{}|{}", username, appId, stepInstanceId, ip, executeCount, + log.debug("Package log file for {}|{}|{}|{}|{}|{}|{}|{}", username, appId, stepInstanceId, hostId, executeCount, logFileDir, logFileName, repackage); LogExportJobInfoDTO exportJobInfo = new LogExportJobInfoDTO(); - exportJobInfo.setJobKey(getExportJobKey(appId, stepInstanceId, ip)); + exportJobInfo.setJobKey(getExportJobKey(appId, stepInstanceId, hostId, cloudIp)); exportJobInfo.setStatus(LogExportStatusEnum.INIT); if (repackage) { @@ -123,10 +125,10 @@ public LogExportJobInfoDTO packageLogFile(String username, Long appId, Long step } saveExportInfo(exportJobInfo); - boolean isGetByIp = StringUtils.isNotBlank(ip); + boolean isGetByHost = hostId != null || StringUtils.isNotBlank(cloudIp); - if (isGetByIp) { - doPackage(exportJobInfo, stepInstanceId, ip, executeCount, logFileDir, logFileName); + if (isGetByHost) { + doPackage(exportJobInfo, stepInstanceId, hostId, cloudIp, executeCount, logFileDir, logFileName); } else { logExportExecutor.execute(() -> { String requestId = UUID.randomUUID().toString(); @@ -139,12 +141,13 @@ public LogExportJobInfoDTO packageLogFile(String username, Long appId, Long step exportJobInfo.setStatus(LogExportStatusEnum.PROCESSING); saveExportInfo(exportJobInfo); - doPackage(exportJobInfo, stepInstanceId, ip, executeCount, logFileDir, logFileName); + doPackage(exportJobInfo, stepInstanceId, hostId, cloudIp, executeCount, logFileDir, + logFileName); } else { - log.error("Job already running!|{}|{}|{}|{}", requestId, appId, stepInstanceId, ip); + log.error("Job already running!|{}|{}|{}|{}", requestId, appId, stepInstanceId, hostId); } } catch (Exception e) { - log.error("Error while package log file!|{}|{}|{}|{}", requestId, stepInstanceId, ip, + log.error("Error while package log file!|{}|{}|{}|{}", requestId, stepInstanceId, hostId, executeCount, e); markJobFailed(exportJobInfo); } finally { @@ -162,8 +165,8 @@ private void markJobFailed(LogExportJobInfoDTO exportJobInfo) { } @Override - public LogExportJobInfoDTO getExportInfo(Long appId, Long stepInstanceId, String ip) { - return JsonUtils.fromJson(redisTemplate.opsForValue().get(getExportJobKey(appId, stepInstanceId, ip)), + public LogExportJobInfoDTO getExportInfo(Long appId, Long stepInstanceId, Long hostId, String ip) { + return JsonUtils.fromJson(redisTemplate.opsForValue().get(getExportJobKey(appId, stepInstanceId, hostId, ip)), LogExportJobInfoDTO.class); } @@ -175,54 +178,62 @@ private void deleteExportInfo(String jobKey) { redisTemplate.delete(jobKey); } - private String getExportJobKey(Long appId, Long stepInstanceId, String ip) { - return EXPORT_KEY_PREFIX + appId + ":" + stepInstanceId + ":" + ip; + private String getExportJobKey(Long appId, Long stepInstanceId, Long hostId, String cloudIp) { + String key = EXPORT_KEY_PREFIX + appId + ":" + stepInstanceId + ":"; + if (hostId != null) { + key = key + hostId; + } else { + key = key + cloudIp; + } + return key; } - private void doPackage(LogExportJobInfoDTO exportJobInfo, Long stepInstanceId, String ip, int executeCount, - String logFileDir, String logFileName) { - boolean isGetByIp = StringUtils.isNotBlank(ip); + private void doPackage(LogExportJobInfoDTO exportJobInfo, long stepInstanceId, Long hostId, + String ip, int executeCount, String logFileDir, String logFileName) { + StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(stepInstanceId); + boolean isGetByHost = hostId != null || StringUtils.isNotBlank(ip); File logFile = new File(logFileDir + logFileName); StopWatch watch = new StopWatch("exportJobLog"); watch.start("listJobIps"); - List gseTaskIpLogs = new ArrayList<>(); - if (isGetByIp) { - GseTaskIpLogDTO gseTaskIpLog = gseTaskLogService.getIpLog(stepInstanceId, executeCount, ip); - if (gseTaskIpLog != null) { - gseTaskIpLogs.add(gseTaskIpLog); + List gseAgentTasks = new ArrayList<>(); + if (isGetByHost) { + HostDTO host = HostDTO.fromHostIdAndCloudIp(hostId, ip); + AgentTaskDTO agentTask = scriptAgentTaskService.getAgentTaskByHost(stepInstance, executeCount, null, + host); + if (agentTask != null) { + gseAgentTasks.add(agentTask); } } else { - gseTaskIpLogs = gseTaskLogService.getIpLog(stepInstanceId, executeCount, true); + gseAgentTasks = scriptAgentTaskService.listAgentTasks(stepInstanceId, executeCount, null); } watch.stop(); - if (gseTaskIpLogs == null || gseTaskIpLogs.isEmpty()) { + if (gseAgentTasks == null || gseAgentTasks.isEmpty()) { log.warn("Gse task ips are empty! stepInstanceId={}", stepInstanceId); markJobFailed(exportJobInfo); return; } - Collection querys = buildLogBatchQuery(stepInstanceId, gseTaskIpLogs); + Collection querys = buildLogBatchQuery(stepInstanceId, gseAgentTasks); watch.start("getLogContent"); - StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(stepInstanceId); String jobCreateDate = DateUtils.formatUnixTimestamp(stepInstance.getCreateTime(), ChronoUnit.MILLIS, "yyyy_MM_dd", ZoneId.of("UTC")); try (PrintWriter out = new PrintWriter(logFile, "UTF-8")) { for (LogBatchQuery query : querys) { - for (List ips : query.getIpBatches()) { - List scriptIpLogContentList = - logService.batchGetScriptIpLogContent(jobCreateDate, stepInstanceId, query.getExecuteCount(), - ips); - for (ScriptIpLogContent scriptIpLogContent : scriptIpLogContentList) { - if (scriptIpLogContent != null && StringUtils.isNotEmpty(scriptIpLogContent.getContent())) { - String[] logList = scriptIpLogContent.getContent().split("\n"); + for (List hosts : query.getHostBatches()) { + List scriptHostLogContentList = + logService.batchGetScriptHostLogContent(jobCreateDate, stepInstanceId, query.getExecuteCount(), + null, hosts); + for (ScriptHostLogContent scriptHostLogContent : scriptHostLogContentList) { + if (scriptHostLogContent != null && StringUtils.isNotEmpty(scriptHostLogContent.getContent())) { + String[] logList = scriptHostLogContent.getContent().split("\n"); for (String log : logList) { - if (isGetByIp) { + if (isGetByHost) { out.println(log); } else { - out.println(scriptIpLogContent.getIp() + " | " + log); + out.println(scriptHostLogContent.getIp() + " | " + log); } } } @@ -285,14 +296,20 @@ private void doPackage(LogExportJobInfoDTO exportJobInfo, Long stepInstanceId, S } } - private Collection buildLogBatchQuery(long stepInstanceId, List gseTaskIpLogs) { + private Collection buildLogBatchQuery(long stepInstanceId, List agentTasks) { Map batchQueryGroups = new HashMap<>(); - gseTaskIpLogs.forEach(gseTaskIpLog -> { - LogBatchQuery query = batchQueryGroups.computeIfAbsent(gseTaskIpLog.getExecuteCount(), + agentTasks.forEach(agentTask -> { + LogBatchQuery query = batchQueryGroups.computeIfAbsent(agentTask.getExecuteCount(), (executeCount) -> new LogBatchQuery(stepInstanceId, executeCount)); - query.addIp(new IpDTO(gseTaskIpLog.getCloudAreaId(), gseTaskIpLog.getIp())); + HostDTO queryHost = null; + if (agentTask.getHostId() != null) { + queryHost = HostDTO.fromHostId(agentTask.getHostId()); + } else if (StringUtils.isNotEmpty(agentTask.getCloudIp())) { + queryHost = HostDTO.fromCloudIp(agentTask.getCloudIp()); + } + query.addHost(queryHost); }); - batchQueryGroups.values().forEach(LogBatchQuery::batchIps); + batchQueryGroups.values().forEach(LogBatchQuery::batchHosts); return batchQueryGroups.values(); } @@ -301,23 +318,23 @@ private static class LogBatchQuery { private static final int MAX_BATCH_IPS = 1000; private long stepInstanceId; private int executeCount; - private List ips = new ArrayList<>(); - private List> ipBatches; + private List hosts = new ArrayList<>(); + private List> hostBatches; LogBatchQuery(long stepInstanceId, int executeCount) { this.stepInstanceId = stepInstanceId; this.executeCount = executeCount; } - void addIp(IpDTO ip) { - if (ips == null) { - ips = new ArrayList<>(); + void addHost(HostDTO host) { + if (hosts == null) { + hosts = new ArrayList<>(); } - ips.add(ip); + hosts.add(host); } - void batchIps() { - ipBatches = BatchUtil.buildBatchList(ips, MAX_BATCH_IPS); + void batchHosts() { + hostBatches = BatchUtil.buildBatchList(hosts, MAX_BATCH_IPS); } } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogServiceImpl.java index 39fcda48f6..7647e6c621 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogServiceImpl.java @@ -25,40 +25,42 @@ package com.tencent.bk.job.execute.service.impl; import com.tencent.bk.job.common.exception.InternalException; -import com.tencent.bk.job.common.exception.ServiceException; import com.tencent.bk.job.common.model.InternalResponse; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.execute.client.LogServiceResourceClient; import com.tencent.bk.job.execute.common.constants.FileDistModeEnum; import com.tencent.bk.job.execute.common.constants.FileDistStatusEnum; -import com.tencent.bk.job.execute.dao.GseTaskIpLogDAO; -import com.tencent.bk.job.execute.dao.StepInstanceDAO; -import com.tencent.bk.job.execute.engine.consts.IpStatus; +import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; +import com.tencent.bk.job.execute.model.AgentTaskDTO; import com.tencent.bk.job.execute.model.FileIpLogContent; -import com.tencent.bk.job.execute.model.GseTaskIpLogDTO; -import com.tencent.bk.job.execute.model.ScriptIpLogContent; +import com.tencent.bk.job.execute.model.ScriptHostLogContent; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; +import com.tencent.bk.job.execute.model.StepInstanceDTO; +import com.tencent.bk.job.execute.service.FileAgentTaskService; import com.tencent.bk.job.execute.service.LogService; +import com.tencent.bk.job.execute.service.ScriptAgentTaskService; +import com.tencent.bk.job.execute.service.StepInstanceService; +import com.tencent.bk.job.execute.service.TaskInstanceService; +import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; import com.tencent.bk.job.logsvr.consts.LogTypeEnum; -import com.tencent.bk.job.logsvr.model.service.BatchSaveLogRequest; -import com.tencent.bk.job.logsvr.model.service.FileLogQueryRequest; -import com.tencent.bk.job.logsvr.model.service.SaveLogRequest; -import com.tencent.bk.job.logsvr.model.service.ScriptLogQueryRequest; +import com.tencent.bk.job.logsvr.model.service.ServiceBatchSaveLogRequest; +import com.tencent.bk.job.logsvr.model.service.ServiceFileLogQueryRequest; import com.tencent.bk.job.logsvr.model.service.ServiceFileTaskLogDTO; -import com.tencent.bk.job.logsvr.model.service.ServiceIpLogDTO; -import com.tencent.bk.job.logsvr.model.service.ServiceIpLogsDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceHostLogDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceHostLogsDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceSaveLogRequest; import com.tencent.bk.job.logsvr.model.service.ServiceScriptLogDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceScriptLogQueryRequest; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import java.nio.charset.StandardCharsets; import java.time.ZoneId; import java.time.temporal.ChronoUnit; -import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; @@ -68,56 +70,26 @@ @Slf4j public class LogServiceImpl implements LogService { private final LogServiceResourceClient logServiceResourceClient; - private final StepInstanceDAO stepInstanceDAO; - private final GseTaskIpLogDAO gseTaskIpLogDAO; + private final TaskInstanceService taskInstanceService; + private final ScriptAgentTaskService scriptAgentTaskService; + private final FileAgentTaskService fileAgentTaskService; + private final StepInstanceService stepInstanceService; @Autowired - public LogServiceImpl(LogServiceResourceClient logServiceResourceClient, StepInstanceDAO stepInstanceDAO, - GseTaskIpLogDAO gseTaskIpLogDAO) { + public LogServiceImpl(LogServiceResourceClient logServiceResourceClient, + TaskInstanceService taskInstanceService, + ScriptAgentTaskService scriptAgentTaskService, + FileAgentTaskService fileAgentTaskService, + StepInstanceService stepInstanceService) { this.logServiceResourceClient = logServiceResourceClient; - this.stepInstanceDAO = stepInstanceDAO; - this.gseTaskIpLogDAO = gseTaskIpLogDAO; + this.taskInstanceService = taskInstanceService; + this.scriptAgentTaskService = scriptAgentTaskService; + this.fileAgentTaskService = fileAgentTaskService; + this.stepInstanceService = stepInstanceService; } @Override - public void deleteStepLog(String jobCreateDate, long stepInstanceId, int executeCount) throws ServiceException { - logServiceResourceClient.deleteStepContent(stepInstanceId, executeCount, jobCreateDate); - } - - @Override - public void writeJobSystemScriptLog(long jobCreateTime, long stepInstanceId, int executeCount, - String cloudAreaAndIp, - String content, int offset, Long logTimeInMillSeconds) { - if (StringUtils.isEmpty(content)) { - return; - } - writeScriptLog(DateUtils.formatUnixTimestamp(jobCreateTime, ChronoUnit.MILLIS, "yyyy_MM_dd", ZoneId.of("UTC")), - stepInstanceId, executeCount, buildSystemScriptLog(cloudAreaAndIp, content, offset, logTimeInMillSeconds)); - } - - @Override - public void batchWriteJobSystemScriptLog(long jobCreateTime, long stepInstanceId, int executeCount, - Map ipsAndOffset, String content, - Long logTimeInMillSeconds) throws ServiceException { - BatchSaveLogRequest request = new BatchSaveLogRequest(); - request.setJobCreateDate(DateUtils.formatUnixTimestamp(jobCreateTime, ChronoUnit.MILLIS, - "yyyy_MM_dd", ZoneId.of("UTC"))); - request.setLogType(LogTypeEnum.SCRIPT.getValue()); - List logs = new ArrayList<>(ipsAndOffset.size()); - ipsAndOffset.forEach((cloudIp, offset) -> logs.add(buildServiceLogDTO(stepInstanceId, - executeCount, new ServiceScriptLogDTO(cloudIp, offset, content)))); - request.setLogs(logs); - - InternalResponse resp = logServiceResourceClient.saveLogs(request); - if (!resp.isSuccess()) { - log.error("Batch write system script log content fail, stepInstanceId:{}, executeCount:{}", stepInstanceId - , executeCount); - throw new InternalException(resp.getCode()); - } - } - - @Override - public ServiceScriptLogDTO buildSystemScriptLog(String cloudIp, String content, int offset, + public ServiceScriptLogDTO buildSystemScriptLog(HostDTO host, String content, int offset, Long logTimeInMillSeconds) { String logDateTime; if (logTimeInMillSeconds != null) { @@ -127,112 +99,117 @@ public ServiceScriptLogDTO buildSystemScriptLog(String cloudIp, String content, logDateTime = DateUtils.formatUnixTimestamp(System.currentTimeMillis(), ChronoUnit.MILLIS, "yyyy-MM-dd HH:mm:ss", ZoneId.systemDefault()); } - String logContentWithDateTime = "[" + - logDateTime + - "] " + content + "\n"; + String logContentWithDateTime = "[" + logDateTime + "] " + content + "\n"; int length = logContentWithDateTime.getBytes(StandardCharsets.UTF_8).length; - return new ServiceScriptLogDTO(cloudIp, offset + length, logContentWithDateTime); + return new ServiceScriptLogDTO(host, offset + length, logContentWithDateTime); } @Override - public void writeScriptLog(String jobCreateDate, long stepInstanceId, int executeCount, - ServiceScriptLogDTO scriptLog) throws ServiceException { - if (scriptLog == null || StringUtils.isEmpty(scriptLog.getContent())) { - return; - } - SaveLogRequest request = new SaveLogRequest(); - request.setStepInstanceId(stepInstanceId); - request.setExecuteCount(executeCount); - request.setIp(scriptLog.getCloudIp()); - request.setJobCreateDate(jobCreateDate); - request.setScriptLog(scriptLog); - request.setLogType(LogTypeEnum.SCRIPT.getValue()); - InternalResponse resp = logServiceResourceClient.saveLog(request); - if (!resp.isSuccess()) { - log.error("Write log content fail, stepInstanceId:{}, executeCount:{}, ip:{}", stepInstanceId, executeCount, - scriptLog.getCloudIp()); - throw new InternalException(resp.getCode()); - } - } - - @Override - public void batchWriteScriptLog(String jobCreateDate, long stepInstanceId, int executeCount, - List scriptLogs) throws ServiceException { + public void batchWriteScriptLog(long jobCreateTime, long stepInstanceId, int executeCount, Integer batch, + List scriptLogs) { if (CollectionUtils.isEmpty(scriptLogs)) { return; } - BatchSaveLogRequest request = new BatchSaveLogRequest(); + String jobCreateDate = DateUtils.formatUnixTimestamp(jobCreateTime, ChronoUnit.MILLIS, + "yyyy_MM_dd", ZoneId.of("UTC")); + ServiceBatchSaveLogRequest request = new ServiceBatchSaveLogRequest(); request.setJobCreateDate(jobCreateDate); request.setLogType(LogTypeEnum.SCRIPT.getValue()); - List logs = scriptLogs.stream().map(scriptLog -> buildServiceLogDTO(stepInstanceId, - executeCount, scriptLog)).collect(Collectors.toList()); + List logs = scriptLogs.stream() + .map(scriptLog -> buildServiceLogDTO(stepInstanceId, executeCount, batch, scriptLog)) + .collect(Collectors.toList()); request.setLogs(logs); InternalResponse resp = logServiceResourceClient.saveLogs(request); if (!resp.isSuccess()) { - log.error("Batch write log content fail, stepInstanceId:{}, executeCount:{}", stepInstanceId, executeCount); + log.error("Batch write log content fail, stepInstanceId:{}, executeCount:{}, batch: {}", + stepInstanceId, executeCount, batch); throw new InternalException(resp.getCode()); } } - private ServiceIpLogDTO buildServiceLogDTO(long stepInstanceId, int executeCount, ServiceScriptLogDTO scriptLog) { - ServiceIpLogDTO logDTO = new ServiceIpLogDTO(); + private ServiceHostLogDTO buildServiceLogDTO(long stepInstanceId, int executeCount, Integer batch, + ServiceScriptLogDTO scriptLog) { + ServiceHostLogDTO logDTO = new ServiceHostLogDTO(); logDTO.setStepInstanceId(stepInstanceId); logDTO.setExecuteCount(executeCount); + logDTO.setBatch(batch); + logDTO.setHostId(scriptLog.getHostId()); logDTO.setIp(scriptLog.getCloudIp()); logDTO.setScriptLog(scriptLog); return logDTO; } @Override - public ScriptIpLogContent getScriptIpLogContent(long stepInstanceId, int executeCount, - IpDTO ip) throws ServiceException { - StepInstanceBaseDTO stepInstance = stepInstanceDAO.getStepInstanceBase(stepInstanceId); + public ScriptHostLogContent getScriptHostLogContent(long stepInstanceId, int executeCount, Integer batch, + HostDTO host) { + StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(stepInstanceId); // 如果存在重试，那么该ip可能是之前已经执行过的，查询日志的时候需要获取到对应的executeCount int actualExecuteCount = executeCount; - GseTaskIpLogDTO gseTaskIpLog = gseTaskIpLogDAO.getIpLogByIp(stepInstanceId, executeCount, ip.convertToStrIp()); - if (gseTaskIpLog == null) { + AgentTaskDTO agentTask = scriptAgentTaskService.getAgentTaskByHost(stepInstance, executeCount, batch, host); + if (agentTask == null) { return null; } - if (gseTaskIpLog.getStatus() == IpStatus.LAST_SUCCESS.getValue()) { - actualExecuteCount = gseTaskIpLogDAO.getSuccessRetryCount(stepInstanceId, ip.convertToStrIp()); + + if (executeCount > 0 && agentTask.getActualExecuteCount() != null) { + actualExecuteCount = agentTask.getActualExecuteCount(); + } else if (agentTask.getStatus() == AgentTaskStatusEnum.LAST_SUCCESS) { + // 兼容历史数据 + actualExecuteCount = scriptAgentTaskService.getActualSuccessExecuteCount(stepInstanceId, host.toCloudIp()); } + String taskCreateDateStr = DateUtils.formatUnixTimestamp(stepInstance.getCreateTime(), ChronoUnit.MILLIS, "yyyy_MM_dd", ZoneId.of("UTC")); - InternalResponse resp = logServiceResourceClient.getScriptIpLogContent(stepInstanceId, - actualExecuteCount, - ip.convertToStrIp(), taskCreateDateStr); + InternalResponse resp; + if (agentTask.getHostId() != null) { + resp = logServiceResourceClient.getScriptHostLogByHostId(taskCreateDateStr, + stepInstanceId, actualExecuteCount, agentTask.getHostId(), batch); + } else { + // 兼容ip查询 + resp = logServiceResourceClient.getScriptHostLogByIp(taskCreateDateStr, + stepInstanceId, actualExecuteCount, agentTask.getCloudIp(), batch); + } if (!resp.isSuccess()) { - log.error("Get script log content by ip error, stepInstanceId={}, executeCount={}, ip={}", stepInstanceId, - actualExecuteCount, ip); + log.error("Get script log content by host error, stepInstanceId={}, executeCount={}, batch={}, host={}", + stepInstanceId, actualExecuteCount, batch, host); throw new InternalException(resp.getCode()); } - return convertToScriptIpLogContent(resp.getData(), gseTaskIpLog); + return convertToScriptIpLogContent(stepInstanceId, executeCount, resp.getData(), agentTask); } - private ScriptIpLogContent convertToScriptIpLogContent(ServiceIpLogDTO logDTO, GseTaskIpLogDTO gseTaskIpLog) { - if (logDTO == null) { - return null; - } - int ipStatus = gseTaskIpLog.getStatus(); - boolean isFinished = ipStatus != IpStatus.RUNNING.getValue() && ipStatus != IpStatus.WAITING.getValue(); - String scriptContent = logDTO.getScriptLog() != null ? + private ScriptHostLogContent convertToScriptIpLogContent(long stepInstanceId, int executeCount, + ServiceHostLogDTO logDTO, AgentTaskDTO agentTask) { + // 日志是否拉取完成 + boolean isFinished = agentTask.getStatus().isFinished(); + String scriptContent = logDTO != null && logDTO.getScriptLog() != null ? logDTO.getScriptLog().getContent() : ""; - return new ScriptIpLogContent(logDTO.getStepInstanceId(), logDTO.getExecuteCount(), - logDTO.getIp(), scriptContent, isFinished); + Long hostId = logDTO != null ? logDTO.getHostId() : null; + String ip = logDTO != null ? logDTO.getIp() : null; + return new ScriptHostLogContent(stepInstanceId, executeCount, hostId, ip, scriptContent, isFinished); } @Override - public List batchGetScriptIpLogContent(String jobCreateDateStr, long stepInstanceId, - int executeCount, - List ips) throws ServiceException { - - ScriptLogQueryRequest query = new ScriptLogQueryRequest(); - query.setIps(ips.stream().map(IpDTO::convertToStrIp).collect(Collectors.toList())); - InternalResponse> resp = - logServiceResourceClient.batchGetScriptLogContent(stepInstanceId, executeCount, jobCreateDateStr, query); + public List batchGetScriptHostLogContent(String jobCreateDateStr, + long stepInstanceId, + int executeCount, + Integer batch, + List hosts) { + + ServiceScriptLogQueryRequest query = new ServiceScriptLogQueryRequest(); + query.setBatch(batch); + + StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(stepInstanceId); + if (isQueryByHostIdCondition(stepInstance)) { + query.setHostIds(buildHostIdQueryCondition(stepInstance, hosts)); + } else { + // 兼容历史数据的查询（使用ip，没有hostId) + query.setIps(hosts.stream().map(HostDTO::toCloudIp).distinct().collect(Collectors.toList())); + } + + InternalResponse> resp = + logServiceResourceClient.listScriptLogs(jobCreateDateStr, stepInstanceId, executeCount, query); if (!resp.isSuccess()) { - log.error("Get script log content by ips error, stepInstanceId={}, executeCount={}, ips={}", stepInstanceId, - executeCount, ips); + log.error("Get script log content by ips error, stepInstanceId={}, executeCount={}, batch={}, ips={}", + stepInstanceId, executeCount, batch, hosts); throw new InternalException(resp.getCode()); } if (CollectionUtils.isEmpty(resp.getData())) { @@ -241,40 +218,75 @@ public List batchGetScriptIpLogContent(String jobCreateDateS return resp.getData().stream().map(logDTO -> { String scriptContent = logDTO.getScriptLog() != null ? logDTO.getScriptLog().getContent() : ""; - return new ScriptIpLogContent(logDTO.getStepInstanceId(), logDTO.getExecuteCount(), + return new ScriptHostLogContent(logDTO.getStepInstanceId(), logDTO.getExecuteCount(), logDTO.getHostId(), logDTO.getIp(), scriptContent, true); }).collect(Collectors.toList()); } + private boolean isQueryByHostIdCondition(StepInstanceBaseDTO stepInstance) { + return stepInstance.getTargetServers().getIpList().stream() + .anyMatch(host -> host.getHostId() != null); + } + + private List buildHostIdQueryCondition(StepInstanceBaseDTO stepInstance, List hosts) { + boolean hostIdParamNull = hosts.get(0).getHostId() == null; + List hostIds; + if (hostIdParamNull) { + // 需要把ip查询参数转换为基于hostId的查询参数 + Map ip2Hosts = stepInstanceService.computeStepHosts(stepInstance, HostDTO::toCloudIp); + hostIds = hosts.stream() + .map(host -> ip2Hosts.get(host.toCloudIp()).getHostId()) + .distinct() + .collect(Collectors.toList()); + } else { + hostIds = hosts.stream() + .map(HostDTO::getHostId) + .distinct() + .collect(Collectors.toList()); + } + + return hostIds; + } + @Override - public FileIpLogContent getFileIpLogContent(long stepInstanceId, int executeCount, IpDTO ip, - Integer mode) throws ServiceException { - StepInstanceBaseDTO stepInstance = stepInstanceDAO.getStepInstanceBase(stepInstanceId); + public FileIpLogContent getFileIpLogContent(long stepInstanceId, int executeCount, Integer batch, HostDTO host, + Integer mode) { + StepInstanceDTO stepInstance = taskInstanceService.getStepInstanceDetail(stepInstanceId); // 如果存在重试，那么该ip可能是之前已经执行过的，查询日志的时候需要获取到对应的executeCount int actualExecuteCount = executeCount; - GseTaskIpLogDTO gseTaskIpLog = gseTaskIpLogDAO.getIpLogByIp(stepInstanceId, executeCount, ip.convertToStrIp()); - if (gseTaskIpLog == null) { + AgentTaskDTO agentTask = fileAgentTaskService.getAgentTaskByHost(stepInstance, executeCount, batch, + FileTaskModeEnum.getFileTaskMode(mode), host); + if (agentTask == null) { return null; } - if (gseTaskIpLog.getStatus() == IpStatus.LAST_SUCCESS.getValue()) { - actualExecuteCount = gseTaskIpLogDAO.getSuccessRetryCount(stepInstanceId, ip.convertToStrIp()); + if (executeCount > 0 && agentTask.getActualExecuteCount() != null) { + actualExecuteCount = agentTask.getActualExecuteCount(); + } else if (agentTask.getStatus() == AgentTaskStatusEnum.LAST_SUCCESS) { + // 兼容历史数据 + actualExecuteCount = scriptAgentTaskService.getActualSuccessExecuteCount(stepInstanceId, host.toCloudIp()); } String taskCreateDateStr = DateUtils.formatUnixTimestamp(stepInstance.getCreateTime(), ChronoUnit.MILLIS, "yyyy_MM_dd", ZoneId.of("UTC")); - InternalResponse resp = logServiceResourceClient.getFileIpLogContent(stepInstanceId, - actualExecuteCount, - ip.convertToStrIp(), taskCreateDateStr, mode); + InternalResponse resp; + if (agentTask.getHostId() != null) { + resp = logServiceResourceClient.getFileHostLogByHostId(taskCreateDateStr, + stepInstanceId, actualExecuteCount, agentTask.getHostId(), mode, batch); + } else { + // 兼容ip查询 + resp = logServiceResourceClient.getFileHostLogByIp(taskCreateDateStr, + stepInstanceId, actualExecuteCount, agentTask.getCloudIp(), mode, batch); + } + if (!resp.isSuccess()) { - log.error("Get file log content by ip error, stepInstanceId={}, executeCount={}, ip={}", stepInstanceId, - actualExecuteCount, ip); + log.error("Get file log content by ip error, stepInstanceId={}, executeCount={}, batch={}, host={}", + stepInstanceId, actualExecuteCount, batch, host); throw new InternalException(resp.getCode()); } List fileTaskLogs = (resp.getData() == null) ? null : resp.getData().getFileTaskLogs(); - int ipStatus = gseTaskIpLog.getStatus(); - boolean isFinished = (ipStatus != IpStatus.RUNNING.getValue() && ipStatus != IpStatus.WAITING.getValue()) || - isAllFileTasksFinished(fileTaskLogs); + AgentTaskStatusEnum agentTaskStatus = agentTask.getStatus(); + boolean isFinished = agentTaskStatus.isFinished() || isAllFileTasksFinished(fileTaskLogs); return new FileIpLogContent(stepInstanceId, executeCount, null, fileTaskLogs, isFinished); } @@ -295,17 +307,16 @@ private boolean isFileTaskNotFinished(ServiceFileTaskLogDTO fileTaskLog) { } @Override - public List getFileLogContentByTaskIds(long stepInstanceId, int executeCount, - List taskIds) throws ServiceException { - StepInstanceBaseDTO stepInstance = stepInstanceDAO.getStepInstanceBase(stepInstanceId); + public List getFileLogContentByTaskIds(long stepInstanceId, int executeCount, Integer batch, + List taskIds) { + StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(stepInstanceId); String taskCreateDateStr = DateUtils.formatUnixTimestamp(stepInstance.getCreateTime(), ChronoUnit.MILLIS, "yyyy_MM_dd", ZoneId.of("UTC")); - InternalResponse resp = logServiceResourceClient.getFileLogContentListByTaskIds(stepInstanceId, - executeCount, - taskCreateDateStr, taskIds); + InternalResponse resp = logServiceResourceClient.listFileHostLogsByTaskIds( + taskCreateDateStr, stepInstanceId, executeCount, batch, taskIds); if (!resp.isSuccess()) { - log.error("Get file log content by ids error, stepInstanceId={}, executeCount={}, taskIds={}", - stepInstanceId, executeCount, taskIds); + log.error("Get file log content by ids error, stepInstanceId={}, executeCount={}, batch={}, taskIds={}", + stepInstanceId, executeCount, batch, taskIds); throw new InternalException(resp.getCode()); } if (resp.getData() == null) { @@ -316,33 +327,40 @@ public List getFileLogContentByTaskIds(long stepInstanceI @Override public List batchGetFileSourceIpLogContent(long stepInstanceId, - int executeCount) throws ServiceException { - StepInstanceBaseDTO stepInstance = stepInstanceDAO.getStepInstanceBase(stepInstanceId); + int executeCount, + Integer batch) { + StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(stepInstanceId); String taskCreateDateStr = DateUtils.formatUnixTimestamp(stepInstance.getCreateTime(), ChronoUnit.MILLIS, "yyyy_MM_dd", ZoneId.of("UTC")); - InternalResponse> resp = logServiceResourceClient.getFileLogContent(stepInstanceId, - executeCount, taskCreateDateStr, FileDistModeEnum.UPLOAD.getValue(), null); + InternalResponse> resp = logServiceResourceClient.listFileHostLogs( + taskCreateDateStr, stepInstanceId, executeCount, batch, FileDistModeEnum.UPLOAD.getValue(), null, null); if (!resp.isSuccess()) { - log.error("Get file source log content error, stepInstanceId={}, executeCount={}", stepInstanceId, - executeCount); + log.error("Get file source log content error, stepInstanceId={}, executeCount={}, batch={}", + stepInstanceId, executeCount, batch); return Collections.emptyList(); } return resp.getData(); } @Override - public ServiceIpLogsDTO batchGetFileIpLogContent(long stepInstanceId, int executeCount, - List ips) throws ServiceException { - StepInstanceBaseDTO stepInstance = stepInstanceDAO.getStepInstanceBase(stepInstanceId); + public ServiceHostLogsDTO batchGetFileIpLogContent(long stepInstanceId, int executeCount, Integer batch, + List hosts) { + StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(stepInstanceId); String taskCreateDateStr = DateUtils.formatUnixTimestamp(stepInstance.getCreateTime(), ChronoUnit.MILLIS, "yyyy_MM_dd", ZoneId.of("UTC")); - FileLogQueryRequest request = new FileLogQueryRequest(); + ServiceFileLogQueryRequest request = new ServiceFileLogQueryRequest(); request.setStepInstanceId(stepInstanceId); request.setExecuteCount(executeCount); + request.setBatch(batch); request.setJobCreateDate(taskCreateDateStr); - request.setIps(ips.stream().map(IpDTO::convertToStrIp).collect(Collectors.toList())); - InternalResponse resp = logServiceResourceClient.getFileLogContent(request); + if (isQueryByHostIdCondition(stepInstance)) { + request.setHostIds(buildHostIdQueryCondition(stepInstance, hosts)); + } else { + request.setIps(hosts.stream().map(HostDTO::toCloudIp).distinct().collect(Collectors.toList())); + } + + InternalResponse resp = logServiceResourceClient.listFileHostLogs(request); if (!resp.isSuccess()) { log.error("Get file log content error, request={}", request); return null; @@ -351,13 +369,13 @@ public ServiceIpLogsDTO batchGetFileIpLogContent(long stepInstanceId, int execut } @Override - public List getIpsByContentKeyword(long stepInstanceId, int executeCount, - String keyword) throws ServiceException { - StepInstanceBaseDTO stepInstance = stepInstanceDAO.getStepInstanceBase(stepInstanceId); + public List getIpsByContentKeyword(long stepInstanceId, int executeCount, Integer batch, + String keyword) { + StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(stepInstanceId); String taskCreateDateStr = DateUtils.formatUnixTimestamp(stepInstance.getCreateTime(), ChronoUnit.MILLIS, "yyyy_MM_dd", ZoneId.of("UTC")); - InternalResponse> resp = logServiceResourceClient.getIpsByKeyword(stepInstanceId, executeCount, - taskCreateDateStr, keyword); + InternalResponse> resp = logServiceResourceClient.getIpsByKeyword(taskCreateDateStr, + stepInstanceId, executeCount, batch, keyword); if (!resp.isSuccess()) { log.error("Search ips by keyword error, stepInstanceId={}, executeCount={}, keyword={}", stepInstanceId, @@ -368,10 +386,17 @@ public List getIpsByContentKeyword(long stepInstanceId, int executeCount, } @Override - public void writeFileLogWithTimestamp(long jobCreateTime, long stepInstanceId, int executeCount, - String cloudAreaIdAndIp, - ServiceIpLogDTO executionLog, - Long logTimeInMillSeconds) throws ServiceException { + public void writeFileLogWithTimestamp(long jobCreateTime, + long stepInstanceId, + int executeCount, + Integer batch, + HostDTO host, + ServiceHostLogDTO executionLog, + Long logTimeInMillSeconds) { + + if (executionLog == null || CollectionUtils.isEmpty(executionLog.getFileTaskLogs())) { + return; + } String logDateTime = "["; if (logTimeInMillSeconds != null) { @@ -385,11 +410,13 @@ public void writeFileLogWithTimestamp(long jobCreateTime, long stepInstanceId, i for (ServiceFileTaskLogDTO fileTaskLog : executionLog.getFileTaskLogs()) { fileTaskLog.setContent(logDateTime + fileTaskLog.getContent() + "\n"); } - SaveLogRequest request = new SaveLogRequest(); + ServiceSaveLogRequest request = new ServiceSaveLogRequest(); request.setLogType(LogTypeEnum.FILE.getValue()); request.setStepInstanceId(stepInstanceId); request.setExecuteCount(executeCount); - request.setIp(cloudAreaIdAndIp); + request.setBatch(batch); + request.setIp(host.getIp()); + request.setHostId(host.getHostId()); request.setJobCreateDate(DateUtils.formatUnixTimestamp(jobCreateTime, ChronoUnit.MILLIS, "yyyy_MM_dd", ZoneId.of("UTC"))); request.setFileTaskLogs(executionLog.getFileTaskLogs()); @@ -397,8 +424,22 @@ public void writeFileLogWithTimestamp(long jobCreateTime, long stepInstanceId, i } @Override - public void writeFileLogs(long jobCreateTime, List fileLogs) throws ServiceException { - BatchSaveLogRequest request = new BatchSaveLogRequest(); + public void writeFileLogs(long jobCreateTime, List fileLogs) { + if (CollectionUtils.isEmpty(fileLogs)) { + return; + } + + String logDateTime = "[" + DateUtils.formatUnixTimestamp(System.currentTimeMillis(), ChronoUnit.MILLIS, + "yyyy-MM-dd HH:mm:ss", ZoneId.systemDefault()) + "]"; + fileLogs.forEach(fileLog -> { + if (CollectionUtils.isNotEmpty(fileLog.getFileTaskLogs())) { + fileLog.getFileTaskLogs().stream() + .filter(fileTaskLog -> StringUtils.isNotEmpty(fileTaskLog.getContent())) + .forEach(fileTaskLog -> fileTaskLog.setContent(logDateTime + fileTaskLog.getContent() + "\n")); + } + }); + + ServiceBatchSaveLogRequest request = new ServiceBatchSaveLogRequest(); request.setJobCreateDate(DateUtils.formatUnixTimestamp(jobCreateTime, ChronoUnit.MILLIS, "yyyy_MM_dd", ZoneId.of("UTC"))); request.setLogs(fileLogs); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/NotifyServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/NotifyServiceImpl.java index beb6c23223..4ad1c9a8e4 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/NotifyServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/NotifyServiceImpl.java @@ -31,21 +31,25 @@ import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.execute.client.ServiceNotificationResourceClient; import com.tencent.bk.job.execute.client.ServiceUserResourceClient; +import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; import com.tencent.bk.job.execute.common.constants.TaskStartupModeEnum; import com.tencent.bk.job.execute.config.JobExecuteConfig; +import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; import com.tencent.bk.job.execute.model.NotifyDTO; +import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.TaskNotifyDTO; import com.tencent.bk.job.execute.service.ApplicationService; +import com.tencent.bk.job.execute.service.FileAgentTaskService; import com.tencent.bk.job.execute.service.NotifyService; +import com.tencent.bk.job.execute.service.ScriptAgentTaskService; import com.tencent.bk.job.execute.service.TaskInstanceService; import com.tencent.bk.job.manage.common.consts.notify.ExecuteStatusEnum; import com.tencent.bk.job.manage.common.consts.notify.NotifyConsts; import com.tencent.bk.job.manage.common.consts.notify.ResourceTypeEnum; import com.tencent.bk.job.manage.common.consts.notify.TriggerTypeEnum; import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; -import com.tencent.bk.job.manage.model.inner.ServiceNotificationMessage; import com.tencent.bk.job.manage.model.inner.ServiceNotificationTriggerDTO; import com.tencent.bk.job.manage.model.inner.ServiceTemplateNotificationDTO; import com.tencent.bk.job.manage.model.inner.ServiceTriggerTemplateNotificationDTO; @@ -63,7 +67,6 @@ import java.text.DecimalFormat; import java.time.LocalDateTime; import java.time.temporal.ChronoUnit; -import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -88,18 +91,29 @@ public class NotifyServiceImpl implements NotifyService { private final ApplicationService applicationService; private final TaskInstanceService taskInstanceService; private final MessageI18nService i18nService; + private final ScriptAgentTaskService scriptAgentTaskService; + private final FileAgentTaskService fileAgentTaskService; + private final TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher; @Autowired public NotifyServiceImpl(JobExecuteConfig jobExecuteConfig, ServiceNotificationResourceClient notificationResourceClient, - ServiceUserResourceClient userResourceClient, ApplicationService applicationService, - TaskInstanceService taskInstanceService, MessageI18nService i18nService) { + ServiceUserResourceClient userResourceClient, + ApplicationService applicationService, + TaskInstanceService taskInstanceService, + MessageI18nService i18nService, + ScriptAgentTaskService scriptAgentTaskService, + FileAgentTaskService fileAgentTaskService, + TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher) { this.jobExecuteConfig = jobExecuteConfig; this.notificationResourceClient = notificationResourceClient; this.userResourceClient = userResourceClient; this.applicationService = applicationService; this.taskInstanceService = taskInstanceService; this.i18nService = i18nService; + this.scriptAgentTaskService = scriptAgentTaskService; + this.fileAgentTaskService = fileAgentTaskService; + this.taskExecuteMQEventDispatcher = taskExecuteMQEventDispatcher; } private static void loadNotifyTemplate() { @@ -126,14 +140,14 @@ private static String formatCostTime(Long costInMills) { return ""; } if (costInMills < 60_000) { - String cost = ""; + String cost; + DecimalFormat df; if (costInMills < 1_000) { - DecimalFormat df = new DecimalFormat("0.000"); - cost = df.format(costInMills / 1000.0d); + df = new DecimalFormat("0.000"); } else { - DecimalFormat df = new DecimalFormat("#.000"); - cost = df.format(costInMills / 1000.0d); + df = new DecimalFormat("#.000"); } + cost = df.format(costInMills / 1000.0d); if (isChineseLocale()) { return cost + " 秒"; } else { @@ -163,9 +177,6 @@ private static String formatCostTime(Long costInMills) { } private static boolean isChineseLocale() { -// Locale locale = LocaleContextHolder.getLocale(); -// log.info("locale={}", locale); -// return (locale == null || locale.equals(Locale.SIMPLIFIED_CHINESE) || locale.equals(Locale.CHINA)); return true; } @@ -194,61 +205,6 @@ private ServiceNotificationTriggerDTO buildQueryTrigger(TaskNotifyDTO taskNotify return trigger; } - private String buildContentTemplateKey(String executeStatusKey, String channelKey) { - return "task." + executeStatusKey + ".msg.content.template." + channelKey; - } - - private String buildTitleTemplateKey(String executeStatusKey, String channelKey) { - return "task." + executeStatusKey + ".msg.title.template." + channelKey; - } - - private String getExecuteStatusKey(ExecuteStatusEnum executeStatus) { - String executeStatusKey = ""; - if (executeStatus == ExecuteStatusEnum.FAIL) { - executeStatusKey = "fail"; - } else if (executeStatus == ExecuteStatusEnum.SUCCESS) { - executeStatusKey = "success"; - } else if (executeStatus == ExecuteStatusEnum.READY) { - executeStatusKey = "waiting"; - } - return executeStatusKey; - } - - private String getChannelKey(String channel) { - String channelKey = ""; - if (StringUtils.isEmpty(channel)) { - return "common"; - } - if (channel.equalsIgnoreCase("weixin")) { - channelKey = "weixin"; - } else if (channel.equalsIgnoreCase("work-weixin")) { - channelKey = "weixin"; - } else if (channel.equalsIgnoreCase("sms")) { - channelKey = "sms"; - } else if (channel.equalsIgnoreCase("mail")) { - channelKey = "mail"; - } else { - channelKey = "common"; - } - return channelKey; - } - - private String replaceTemplatePlaceHolder(String template, Map replacements) { - String result = template; - for (Map.Entry replacement : replacements.entrySet()) { - result = result.replaceAll("\\{" + replacement.getKey() + "}", replacement.getValue()); - } - return result; - } - - private String getTemplate(String key) { - if (isChineseLocale()) { - return templatePropsZH.getProperty(key); - } else { - return templatePropsEN.getProperty(key); - } - } - private String buildJobExecuteDetailUrl(Long taskInstanceId) { return jobExecuteConfig.getJobWebUrl() + "/api_execute/" + taskInstanceId; } @@ -295,7 +251,7 @@ private Map getTemplateVariablesMap(TaskNotifyDTO taskNotifyDTO, ChronoUnit.MILLIS)); List stepIdList = taskInstanceService.getTaskStepIdList(taskInstanceDTO.getId()); variablesMap.put("task.step.total_seq_cnt", "" + stepIdList.size()); - long currentStepId = taskInstanceDTO.getCurrentStepId(); + long currentStepId = taskInstanceDTO.getCurrentStepInstanceId(); variablesMap.put("task.step.current_seq_id", "" + (stepIdList.indexOf(currentStepId) + 1)); StepInstanceDTO stepInstanceDTO = taskInstanceService.getStepInstanceDetail(currentStepId); if (executeStatus == ExecuteStatusEnum.FAIL || executeStatus == ExecuteStatusEnum.SUCCESS) { @@ -305,8 +261,22 @@ private Map getTemplateVariablesMap(TaskNotifyDTO taskNotifyDTO, if (taskInstanceDTO.getTotalTime() != null) { variablesMap.put("task.total_duration", "" + taskInstanceDTO.getTotalTime() / 1000.0); } - variablesMap.put("task.step.failed_cnt", "" + stepInstanceDTO.getFailIPNum()); - variablesMap.put("task.step.success_cnt", "" + stepInstanceDTO.getSuccessIPNum()); + int totalTargetIpCount = stepInstanceDTO.getTargetServers().getIpList().size(); + if (executeStatus == ExecuteStatusEnum.SUCCESS) { + variablesMap.put("task.step.success_cnt", "" + totalTargetIpCount); + variablesMap.put("task.step.failed_cnt", "" + 0); + } else { + int successIpCount = 0; + if (stepInstanceDTO.isScriptStep()) { + successIpCount = scriptAgentTaskService.getSuccessAgentTaskCount(stepInstanceDTO.getId(), + stepInstanceDTO.getExecuteCount()); + } else if (stepInstanceDTO.isFileStep()) { + successIpCount = fileAgentTaskService.getSuccessAgentTaskCount(stepInstanceDTO.getId(), + stepInstanceDTO.getExecuteCount()); + } + variablesMap.put("task.step.failed_cnt", String.valueOf(totalTargetIpCount - successIpCount)); + variablesMap.put("task.step.success_cnt", String.valueOf(successIpCount)); + } } // 国际化处理 Long appId = taskNotifyDTO.getAppId(); @@ -341,11 +311,11 @@ private Map getTemplateVariablesMap(TaskNotifyDTO taskNotifyDTO, variablesMap.put("task.operator", taskNotifyDTO.getOperator()); variablesMap.put("current.date", DateUtils.formatLocalDateTime(LocalDateTime.now(), "yyyy-MM-dd")); variablesMap.put("task.step.name", stepInstanceDTO.getName()); - if (stepInstanceDTO.getStepType().equals(TaskStepTypeEnum.SCRIPT.getType())) { + if (stepInstanceDTO.getStepType() == TaskStepTypeEnum.SCRIPT) { variablesMap.put("task.step.type", i18nService.getI18n("task.step.type.name.script")); - } else if (stepInstanceDTO.getStepType().equals(TaskStepTypeEnum.FILE.getType())) { + } else if (stepInstanceDTO.getStepType() == TaskStepTypeEnum.FILE) { variablesMap.put("task.step.type", i18nService.getI18n("task.step.type.name.file")); - } else if (stepInstanceDTO.getStepType().equals(TaskStepTypeEnum.APPROVAL.getType())) { + } else if (stepInstanceDTO.getStepType() == TaskStepTypeEnum.APPROVAL) { variablesMap.put("task.step.type", i18nService.getI18n("task.step.type.name.manual_confirm")); } if (executeStatus == ExecuteStatusEnum.SUCCESS) { @@ -366,32 +336,6 @@ private Map getTemplateVariablesMap(TaskNotifyDTO taskNotifyDTO, return variablesMap; } - private Map buildNotificationMessage(ExecuteStatusEnum executeStatus, - Collection notifyChannels, - Map placeholders) { - Map messageMap = new HashMap<>(); - for (String notifyChannel : notifyChannels) { - String executeStatusKey = getExecuteStatusKey(executeStatus); - String channelKey = getChannelKey(notifyChannel); - String contentTemplateKey = buildContentTemplateKey(executeStatusKey, channelKey); - String titleTemplateKey = buildTitleTemplateKey(executeStatusKey, channelKey); - String title = getTemplate(titleTemplateKey); - String contentTemplate = getTemplate(contentTemplateKey); - - if (StringUtils.isBlank(contentTemplate)) { - title = getTemplate(buildTitleTemplateKey(executeStatusKey, "common")); - contentTemplate = getTemplate(buildContentTemplateKey(executeStatusKey, "common")); - } - - String content = replaceTemplatePlaceHolder(contentTemplate, placeholders); - - ServiceNotificationMessage notificationMessage = new ServiceNotificationMessage(title, content); - messageMap.put(notifyChannel, notificationMessage); - } - - return messageMap; - } - @Override public void notifyTaskConfirm(TaskNotifyDTO taskNotifyDTO) { if (taskNotifyDTO.getNotifyDTO() != null) { @@ -412,4 +356,83 @@ public void notifyTaskConfirm(TaskNotifyDTO taskNotifyDTO) { notificationResourceClient.sendTemplateNotification(serviceTemplateNotificationDTO); } } + + @Override + public void asyncSendMQFailTaskNotification(TaskInstanceDTO taskInstance, + StepInstanceBaseDTO stepInstance) { + TaskNotifyDTO taskNotifyDTO = buildCommonTaskNotification(taskInstance, stepInstance); + taskNotifyDTO.setResourceExecuteStatus(ExecuteStatusEnum.FAIL.getStatus()); + taskNotifyDTO.setStepName(stepInstance.getName()); + setResourceInfo(taskInstance, stepInstance, taskNotifyDTO); + taskExecuteMQEventDispatcher.asyncSendNotifyMsg(taskNotifyDTO); + } + + @Override + public void asyncSendMQSuccessTaskNotification(TaskInstanceDTO taskInstance, + StepInstanceBaseDTO stepInstance) { + TaskNotifyDTO taskNotifyDTO = buildCommonTaskNotification(taskInstance, stepInstance); + taskNotifyDTO.setResourceExecuteStatus(ExecuteStatusEnum.SUCCESS.getStatus()); + taskNotifyDTO.setCost(taskInstance.getTotalTime()); + setResourceInfo(taskInstance, stepInstance, taskNotifyDTO); + taskExecuteMQEventDispatcher.asyncSendNotifyMsg(taskNotifyDTO); + } + + @Override + public void asyncSendMQConfirmNotification(TaskInstanceDTO taskInstance, + StepInstanceBaseDTO stepInstance) { + StepInstanceDTO stepInstanceDetail = taskInstanceService.getStepInstanceDetail(stepInstance.getId()); + if (stepInstanceDetail == null) { + log.warn("StepInstance is not exist, stepInstanceId: {}", stepInstance.getId()); + return; + } + TaskNotifyDTO taskNotifyDTO = buildCommonTaskNotification(taskInstance, stepInstance); + taskNotifyDTO.setResourceExecuteStatus(ExecuteStatusEnum.READY.getStatus()); + taskNotifyDTO.setStepName(stepInstance.getName()); + taskNotifyDTO.setConfirmMessage(stepInstanceDetail.getConfirmMessage()); + NotifyDTO notifyDTO = new NotifyDTO(); + notifyDTO.setReceiverUsers(stepInstanceDetail.getConfirmUsers()); + notifyDTO.setReceiverRoles(stepInstanceDetail.getConfirmRoles()); + notifyDTO.setChannels(stepInstanceDetail.getNotifyChannels()); + notifyDTO.setTriggerUser(stepInstance.getOperator()); + taskNotifyDTO.setNotifyDTO(notifyDTO); + taskNotifyDTO.setResourceId(String.valueOf(taskInstance.getTaskId())); + taskNotifyDTO.setResourceType(ResourceTypeEnum.JOB.getType()); + taskNotifyDTO.setOperator(stepInstance.getOperator()); + taskExecuteMQEventDispatcher.asyncSendNotifyMsg(taskNotifyDTO); + } + + + private void setResourceInfo(TaskInstanceDTO taskInstance, StepInstanceBaseDTO stepInstance, + TaskNotifyDTO taskNotifyDTO) { + Long taskPlanId = taskInstance.getTaskId(); + taskNotifyDTO.setResourceId(String.valueOf(taskPlanId)); + if (taskPlanId == -1L) { + if (stepInstance.getExecuteType().equals(StepExecuteTypeEnum.EXECUTE_SCRIPT.getValue())) { + taskNotifyDTO.setResourceType(ResourceTypeEnum.SCRIPT.getType()); + } else if (stepInstance.getExecuteType().equals(StepExecuteTypeEnum.SEND_FILE.getValue())) { + taskNotifyDTO.setResourceType(ResourceTypeEnum.FILE.getType()); + } else { + log.warn("notify resourceType not supported yet:{}, use Job", stepInstance.getExecuteType()); + taskNotifyDTO.setResourceType(ResourceTypeEnum.JOB.getType()); + } + } else { + taskNotifyDTO.setResourceType(ResourceTypeEnum.JOB.getType()); + } + } + + private TaskNotifyDTO buildCommonTaskNotification(TaskInstanceDTO taskInstance, StepInstanceBaseDTO stepInstance) { + TaskNotifyDTO taskNotifyDTO = new TaskNotifyDTO(); + taskNotifyDTO.setAppId(taskInstance.getAppId()); + String operator = getOperator(taskInstance.getOperator(), stepInstance.getOperator()); + taskNotifyDTO.setOperator(operator); + taskNotifyDTO.setTaskInstanceId(taskInstance.getId()); + taskNotifyDTO.setStartupMode(taskInstance.getStartupMode()); + taskNotifyDTO.setTaskId(taskInstance.getTaskId()); + taskNotifyDTO.setTaskInstanceName(taskInstance.getName()); + return taskNotifyDTO; + } + + private String getOperator(String taskOperator, String stepOperator) { + return StringUtils.isNotEmpty(stepOperator) ? stepOperator : taskOperator; + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/RollingConfigServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/RollingConfigServiceImpl.java new file mode 100644 index 0000000000..690bbddcef --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/RollingConfigServiceImpl.java @@ -0,0 +1,128 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.service.impl; + +import com.google.common.collect.Lists; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.dao.RollingConfigDAO; +import com.tencent.bk.job.execute.engine.rolling.RollingBatchServersResolver; +import com.tencent.bk.job.execute.engine.rolling.RollingServerBatch; +import com.tencent.bk.job.execute.model.FastTaskDTO; +import com.tencent.bk.job.execute.model.RollingConfigDTO; +import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; +import com.tencent.bk.job.execute.model.StepInstanceDTO; +import com.tencent.bk.job.execute.model.StepRollingConfigDTO; +import com.tencent.bk.job.execute.model.db.RollingConfigDetailDO; +import com.tencent.bk.job.execute.model.db.RollingHostsBatchDO; +import com.tencent.bk.job.execute.model.db.StepRollingConfigDO; +import com.tencent.bk.job.execute.service.RollingConfigService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Service +@Slf4j +public class RollingConfigServiceImpl implements RollingConfigService { + + private final RollingConfigDAO rollingConfigDAO; + + @Autowired + public RollingConfigServiceImpl(RollingConfigDAO rollingConfigDAO) { + this.rollingConfigDAO = rollingConfigDAO; + } + + @Override + public List getRollingServers(StepInstanceBaseDTO stepInstance, Integer batch) { + long rollingConfigId = stepInstance.getRollingConfigId(); + long stepInstanceId = stepInstance.getId(); + + RollingConfigDTO rollingConfig = + rollingConfigDAO.queryRollingConfigById(rollingConfigId); + if (rollingConfig.isBatchRollingStep(stepInstanceId)) { + if (batch == null || batch == 0) { + // 忽略滚动批次，返回当前步骤的所有目标服务器 + return stepInstance.getTargetServers().getIpList(); + } else { + return rollingConfig.getConfigDetail().getHostsBatchList() + .stream().filter(serverBatch -> serverBatch.getBatch().equals(batch)) + .findFirst().orElseThrow(() -> new InternalException(ErrorCode.INTERNAL_ERROR)).getHosts(); + } + } else { + return stepInstance.getTargetServers().getIpList(); + } + } + + @Override + public RollingConfigDTO saveRollingConfigForFastJob(FastTaskDTO fastTask) { + StepInstanceDTO stepInstance = fastTask.getStepInstance(); + + RollingConfigDTO taskInstanceRollingConfig = new RollingConfigDTO(); + taskInstanceRollingConfig.setTaskInstanceId(fastTask.getTaskInstance().getId()); + + StepRollingConfigDTO rollingConfig = fastTask.getRollingConfig(); + + String rollingConfigName = StringUtils.isBlank(rollingConfig.getName()) ? "default" : rollingConfig.getName(); + taskInstanceRollingConfig.setConfigName(rollingConfigName); + + RollingConfigDetailDO rollingConfigDetailDO = new RollingConfigDetailDO(); + rollingConfigDetailDO.setName(rollingConfigName); + rollingConfigDetailDO.setMode(rollingConfig.getMode()); + rollingConfigDetailDO.setExpr(rollingConfig.getExpr()); + + RollingBatchServersResolver resolver = + new RollingBatchServersResolver(fastTask.getStepInstance().getTargetServers().getIpList(), + rollingConfig.getExpr()); + List serversBatchList = resolver.resolve(); + rollingConfigDetailDO.setHostsBatchList( + serversBatchList.stream() + .map(rollingServerBatch -> + new RollingHostsBatchDO(rollingServerBatch.getBatch(), rollingServerBatch.getServers())) + .collect(Collectors.toList())); + rollingConfigDetailDO.setTotalBatch(rollingConfigDetailDO.getHostsBatchList().size()); + taskInstanceRollingConfig.setConfigDetail(rollingConfigDetailDO); + + rollingConfigDetailDO.setIncludeStepInstanceIdList(Lists.newArrayList(stepInstance.getId())); + Map stepRollingConfigs = new HashMap<>(); + stepRollingConfigs.put(stepInstance.getId(), new StepRollingConfigDO(true)); + rollingConfigDetailDO.setStepRollingConfigs(stepRollingConfigs); + + Long rollingConfigId= rollingConfigDAO.saveRollingConfig(taskInstanceRollingConfig); + taskInstanceRollingConfig.setId(rollingConfigId); + return taskInstanceRollingConfig; + } + + @Override + public RollingConfigDTO getRollingConfig(long rollingConfigId) { + return rollingConfigDAO.queryRollingConfigById(rollingConfigId); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptAgentTaskServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptAgentTaskServiceImpl.java new file mode 100644 index 0000000000..0a9656d90b --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptAgentTaskServiceImpl.java @@ -0,0 +1,206 @@ +package com.tencent.bk.job.execute.service.impl; + +import com.tencent.bk.job.common.constant.Order; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.dao.GseTaskIpLogDAO; +import com.tencent.bk.job.execute.dao.ScriptAgentTaskDAO; +import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.model.AgentTaskDetailDTO; +import com.tencent.bk.job.execute.model.AgentTaskResultGroupBaseDTO; +import com.tencent.bk.job.execute.model.AgentTaskResultGroupDTO; +import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; +import com.tencent.bk.job.execute.service.HostService; +import com.tencent.bk.job.execute.service.ScriptAgentTaskService; +import com.tencent.bk.job.execute.service.StepInstanceService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +@Service +@Slf4j +public class ScriptAgentTaskServiceImpl + extends AbstractAgentTaskServiceImpl + implements ScriptAgentTaskService { + + private final ScriptAgentTaskDAO scriptAgentTaskDAO; + private final GseTaskIpLogDAO gseTaskIpLogDAO; + + @Autowired + public ScriptAgentTaskServiceImpl(ScriptAgentTaskDAO scriptAgentTaskDAO, + StepInstanceService stepInstanceService, + HostService hostService, + GseTaskIpLogDAO gseTaskIpLogDAO) { + super(stepInstanceService, hostService); + this.scriptAgentTaskDAO = scriptAgentTaskDAO; + this.gseTaskIpLogDAO = gseTaskIpLogDAO; + } + + @Override + public void batchSaveAgentTasks(Collection agentTasks) { + if (CollectionUtils.isEmpty(agentTasks)) { + return; + } + scriptAgentTaskDAO.batchSaveAgentTasks(agentTasks); + } + + @Override + public void batchUpdateAgentTasks(Collection agentTasks) { + if (CollectionUtils.isEmpty(agentTasks)) { + return; + } + scriptAgentTaskDAO.batchUpdateAgentTasks(agentTasks); + } + + @Override + public int getSuccessAgentTaskCount(long stepInstanceId, int executeCount) { + if (isStepInstanceRecordExist(stepInstanceId)) { + return scriptAgentTaskDAO.getSuccessAgentTaskCount(stepInstanceId, executeCount); + } else { + return gseTaskIpLogDAO.getSuccessAgentTaskCount(stepInstanceId, executeCount); + } + } + + @Override + public List listAgentTasks(Long stepInstanceId, Integer executeCount, Integer batch) { + List agentTasks = scriptAgentTaskDAO.listAgentTasks(stepInstanceId, executeCount, batch); + if (CollectionUtils.isEmpty(agentTasks)) { + // 兼容历史数据 + agentTasks = gseTaskIpLogDAO.listAgentTasks(stepInstanceId, executeCount); + } + return agentTasks; + } + + @Override + public List listAgentTasksByGseTaskId(Long gseTaskId) { + return scriptAgentTaskDAO.listAgentTasksByGseTaskId(gseTaskId); + } + + @Override + public AgentTaskDTO getAgentTaskByHost(StepInstanceBaseDTO stepInstance, + Integer executeCount, + Integer batch, + HostDTO host) { + AgentTaskDTO agentTask = null; + Long hostId = host.getHostId(); + if (hostId != null) { + // 根据hostId查询 + agentTask = scriptAgentTaskDAO.getAgentTaskByHostId(stepInstance.getId(), executeCount, batch, hostId); + } else if (StringUtils.isNotEmpty(host.toCloudIp())) { + // 根据ip查询的模式，有两种情况，数据可能在gse_script_agent_task/gse_task_ip_log表中，优先查询gse_script_agent_task + HostDTO queryHost = getStepHostByIp(stepInstance, host.toCloudIp()); + if (queryHost != null && queryHost.getHostId() != null) { + agentTask = scriptAgentTaskDAO.getAgentTaskByHostId(stepInstance.getId(), executeCount, batch, + queryHost.getHostId()); + } else { + // 根据ip查询gse_task_ip_log表中的数据 + agentTask = gseTaskIpLogDAO.getAgentTaskByIp(stepInstance.getId(), executeCount, host.toCloudIp()); + } + } + return agentTask; + } + + private HostDTO getStepHostByIp(StepInstanceBaseDTO stepInstance, String cloudIp) { + return stepInstance.getTargetServers().getIpList().stream() + .filter(targetHost -> cloudIp.equals(targetHost.toCloudIp())) + .findFirst() + .orElse(null); + } + + @Override + public int getActualSuccessExecuteCount(long stepInstanceId, String cloudIp) { + // 兼容历史数据 + return gseTaskIpLogDAO.getActualSuccessExecuteCount(stepInstanceId, cloudIp); + } + + @Override + public List listAndGroupAgentTasks(StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch) { + List resultGroups = new ArrayList<>(); + List agentTasks = listAgentTasks(stepInstance.getId(), executeCount, batch); + if (CollectionUtils.isEmpty(agentTasks)) { + return resultGroups; + } + + List agentTaskDetailList = fillHostDetail(stepInstance, agentTasks); + resultGroups = groupAgentTasks(agentTaskDetailList); + + return resultGroups.stream().sorted().collect(Collectors.toList()); + } + + @Override + public List listResultGroups(long stepInstanceId, + int executeCount, + Integer batch) { + List resultGroups; + resultGroups = scriptAgentTaskDAO.listResultGroups(stepInstanceId, executeCount, batch); + if (CollectionUtils.isEmpty(resultGroups)) { + // 兼容历史数据 + resultGroups = gseTaskIpLogDAO.listResultGroups(stepInstanceId, executeCount); + } + return resultGroups; + } + + @Override + public List listAgentTaskDetailByResultGroup(StepInstanceBaseDTO stepInstance, + Integer executeCount, + Integer batch, + Integer status, + String tag) { + List agentTasks = scriptAgentTaskDAO.listAgentTaskByResultGroup(stepInstance.getId(), + executeCount, batch, status, tag); + if (CollectionUtils.isEmpty(agentTasks)) { + // 兼容历史数据 + agentTasks = gseTaskIpLogDAO.listAgentTaskByResultGroup(stepInstance.getId(), executeCount, status, tag); + } + return fillHostDetail(stepInstance, agentTasks); + } + + + @Override + public List listAgentTaskDetailByResultGroup(StepInstanceBaseDTO stepInstance, + Integer executeCount, + Integer batch, + Integer status, + String tag, + Integer limit, + String orderField, + Order order) { + List agentTasks = scriptAgentTaskDAO.listAgentTaskByResultGroup(stepInstance.getId(), + executeCount, batch, status, tag, limit, orderField, order); + if (CollectionUtils.isEmpty(agentTasks)) { + // 兼容历史数据 + agentTasks = gseTaskIpLogDAO.listAgentTaskByResultGroup(stepInstance.getId(), executeCount, status, tag, + limit, orderField, order); + } + return fillHostDetail(stepInstance, agentTasks); + } + + @Override + public List listAgentTaskDetail(StepInstanceBaseDTO stepInstance, + Integer executeCount, + Integer batch) { + List agentTasks = listAgentTasks(stepInstance.getId(), executeCount, batch); + return fillHostDetail(stepInstance, agentTasks); + } + + private boolean isStepInstanceRecordExist(long stepInstanceId) { + return scriptAgentTaskDAO.isStepInstanceRecordExist(stepInstanceId); + } + + @Override + public void updateAgentTaskFields(long stepInstanceId, + int executeCount, + Integer batch, + Integer actualExecuteCount, + Long gseTaskId) { + scriptAgentTaskDAO.updateAgentTaskFields(stepInstanceId, executeCount, batch, actualExecuteCount, gseTaskId); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceRollingTaskServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceRollingTaskServiceImpl.java new file mode 100644 index 0000000000..d2cd2c3e56 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceRollingTaskServiceImpl.java @@ -0,0 +1,116 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.service.impl; + +import com.tencent.bk.job.execute.common.constants.RunStatusEnum; +import com.tencent.bk.job.execute.dao.StepInstanceRollingTaskDAO; +import com.tencent.bk.job.execute.model.StepInstanceRollingTaskDTO; +import com.tencent.bk.job.execute.service.StepInstanceRollingTaskService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Service +@Slf4j +public class StepInstanceRollingTaskServiceImpl implements StepInstanceRollingTaskService { + + private final StepInstanceRollingTaskDAO stepInstanceRollingTaskDAO; + + @Autowired + public StepInstanceRollingTaskServiceImpl( + StepInstanceRollingTaskDAO stepInstanceRollingTaskDAO) { + this.stepInstanceRollingTaskDAO = stepInstanceRollingTaskDAO; + } + + @Override + public StepInstanceRollingTaskDTO queryRollingTask(long stepInstanceId, + int executeCount, int batch) { + return stepInstanceRollingTaskDAO.queryRollingTask(stepInstanceId, executeCount, batch); + } + + @Override + public List listLatestRollingTasks(long stepInstanceId, int executeCount) { + List stepInstanceRollingTasks = + stepInstanceRollingTaskDAO.listRollingTasks(stepInstanceId, null, null); + if (CollectionUtils.isEmpty(stepInstanceRollingTasks)) { + return stepInstanceRollingTasks; + } + if (executeCount == 0) { + return stepInstanceRollingTasks.stream() + .filter(rollingTask -> rollingTask.getExecuteCount() == 0) + .collect(Collectors.toList()); + } + + Map latestRollingTasks = new HashMap<>(); + for (StepInstanceRollingTaskDTO stepInstanceRollingTask : stepInstanceRollingTasks) { + if (stepInstanceRollingTask.getExecuteCount() > executeCount) { + continue; + } + String key = stepInstanceRollingTask.getStepInstanceId() + ":" + stepInstanceRollingTask.getBatch(); + StepInstanceRollingTaskDTO existStepInstanceRollingTask = latestRollingTasks.get(key); + if (existStepInstanceRollingTask == null) { + latestRollingTasks.put(key, stepInstanceRollingTask); + } else { + if (stepInstanceRollingTask.getExecuteCount() > existStepInstanceRollingTask.getExecuteCount()) { + // 覆盖旧的滚动任务 + latestRollingTasks.put(key, stepInstanceRollingTask); + } + } + } + return latestRollingTasks.values().stream() + .sorted(Comparator.comparing(StepInstanceRollingTaskDTO::getBatch)) + .collect(Collectors.toList()); + } + + @Override + public List listRollingTasksByBatch(long stepInstanceId, Integer batch) { + return stepInstanceRollingTaskDAO.listRollingTasks(stepInstanceId, null, batch); + } + + @Override + public long saveRollingTask(StepInstanceRollingTaskDTO rollingTask) { + return stepInstanceRollingTaskDAO.saveRollingTask(rollingTask); + } + + @Override + public void updateRollingTask(long stepInstanceId, + int executeCount, + int batch, + RunStatusEnum status, + Long startTime, + Long endTime, + Long totalTime) { + stepInstanceRollingTaskDAO.updateRollingTask(stepInstanceId, executeCount, batch, status, startTime, + endTime, totalTime); + } + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceServiceImpl.java new file mode 100644 index 0000000000..6a5dd2b078 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceServiceImpl.java @@ -0,0 +1,111 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.service.impl; + +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.dao.StepInstanceDAO; +import com.tencent.bk.job.execute.model.FileSourceDTO; +import com.tencent.bk.job.execute.model.FileStepInstanceDTO; +import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; +import com.tencent.bk.job.execute.model.StepInstanceDTO; +import com.tencent.bk.job.execute.service.StepInstanceService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Slf4j +@Service +public class StepInstanceServiceImpl implements StepInstanceService { + + private final StepInstanceDAO stepInstanceDAO; + + @Autowired + public StepInstanceServiceImpl(StepInstanceDAO stepInstanceDAO) { + this.stepInstanceDAO = stepInstanceDAO; + } + + @Override + public void updateStepCurrentBatch(long stepInstanceId, int batch) { + stepInstanceDAO.updateStepCurrentBatch(stepInstanceId, batch); + } + + @Override + public void updateStepCurrentExecuteCount(long stepInstanceId, int executeCount) { + stepInstanceDAO.updateStepCurrentExecuteCount(stepInstanceId, executeCount); + } + + @Override + public void updateStepRollingConfigId(long stepInstanceId, long rollingConfigId) { + stepInstanceDAO.updateStepRollingConfigId(stepInstanceId, rollingConfigId); + } + + @Override + public StepInstanceBaseDTO getNextStepInstance(long taskInstanceId, + int currentStepOrder) { + return stepInstanceDAO.getNextStepInstance(taskInstanceId, currentStepOrder); + } + + @Override + public Map computeStepHosts(StepInstanceBaseDTO stepInstance, + Function keyMapper) { + Map hosts = + stepInstance.getTargetServers().getIpList().stream() + .collect(Collectors.toMap(keyMapper, host -> host)); + if (stepInstance.isFileStep()) { + List fileSourceList; + if (stepInstance instanceof StepInstanceDTO) { + fileSourceList = ((StepInstanceDTO) stepInstance).getFileSourceList(); + } else { + FileStepInstanceDTO fileStepInstance = stepInstanceDAO.getFileStepInstance(stepInstance.getId()); + fileSourceList = fileStepInstance.getFileSourceList(); + } + + if (CollectionUtils.isNotEmpty(fileSourceList)) { + fileSourceList.forEach( + fileSource -> { + if (fileSource.getServers() != null + && CollectionUtils.isNotEmpty(fileSource.getServers().getIpList())) { + fileSource.getServers().getIpList().forEach(host -> hosts.put(keyMapper.apply(host), host)); + } + }); + } + } + + return hosts; + } + + @Override + public Map computeStepHosts(long stepInstanceId, + Function keyMapper) { + StepInstanceBaseDTO stepInstance = stepInstanceDAO.getStepInstanceBase(stepInstanceId); + return computeStepHosts(stepInstance, keyMapper); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java index 2e65cafd1c..f19d228280 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java @@ -24,8 +24,6 @@ package com.tencent.bk.job.execute.service.impl; -import brave.Tracing; -import com.google.common.collect.Lists; import com.tencent.bk.job.common.cc.model.CcInstanceDTO; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.TaskVariableTypeEnum; @@ -41,7 +39,8 @@ import com.tencent.bk.job.common.iam.model.AuthResult; import com.tencent.bk.job.common.model.InternalResponse; import com.tencent.bk.job.common.model.dto.AppResourceScope; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.common.trace.executors.TraceableExecutorService; import com.tencent.bk.job.common.util.ArrayUtil; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.common.util.json.JsonUtils; @@ -51,23 +50,27 @@ import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; import com.tencent.bk.job.execute.common.constants.TaskStartupModeEnum; import com.tencent.bk.job.execute.common.constants.TaskTypeEnum; -import com.tencent.bk.job.common.trace.executors.TraceableExecutorService; import com.tencent.bk.job.execute.config.JobExecuteConfig; import com.tencent.bk.job.execute.constants.ScriptSourceEnum; import com.tencent.bk.job.execute.constants.StepOperationEnum; import com.tencent.bk.job.execute.constants.TaskOperationEnum; import com.tencent.bk.job.execute.constants.UserOperationEnum; -import com.tencent.bk.job.execute.engine.TaskExecuteControlMsgSender; import com.tencent.bk.job.execute.engine.evict.TaskEvictPolicyExecutor; +import com.tencent.bk.job.execute.engine.listener.event.JobEvent; +import com.tencent.bk.job.execute.engine.listener.event.StepEvent; +import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; import com.tencent.bk.job.execute.engine.util.TimeoutUtils; import com.tencent.bk.job.execute.model.AccountDTO; import com.tencent.bk.job.execute.model.DynamicServerGroupDTO; import com.tencent.bk.job.execute.model.DynamicServerTopoNodeDTO; +import com.tencent.bk.job.execute.model.FastTaskDTO; import com.tencent.bk.job.execute.model.FileDetailDTO; import com.tencent.bk.job.execute.model.FileSourceDTO; import com.tencent.bk.job.execute.model.OperationLogDTO; +import com.tencent.bk.job.execute.model.RollingConfigDTO; import com.tencent.bk.job.execute.model.ServersDTO; +import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.StepOperationDTO; import com.tencent.bk.job.execute.model.TaskExecuteParam; @@ -76,7 +79,9 @@ import com.tencent.bk.job.execute.service.AccountService; import com.tencent.bk.job.execute.service.DangerousScriptCheckService; import com.tencent.bk.job.execute.service.HostService; +import com.tencent.bk.job.execute.service.RollingConfigService; import com.tencent.bk.job.execute.service.ScriptService; +import com.tencent.bk.job.execute.service.StepInstanceService; import com.tencent.bk.job.execute.service.TaskExecuteService; import com.tencent.bk.job.execute.service.TaskInstanceService; import com.tencent.bk.job.execute.service.TaskInstanceVariableService; @@ -93,6 +98,7 @@ import com.tencent.bk.job.manage.common.consts.whiteip.ActionScopeEnum; import com.tencent.bk.job.manage.model.inner.ServiceAccountDTO; import com.tencent.bk.job.manage.model.inner.ServiceHostInfoDTO; +import com.tencent.bk.job.manage.model.inner.ServiceListAppHostResultDTO; import com.tencent.bk.job.manage.model.inner.ServiceScriptCheckResultItemDTO; import com.tencent.bk.job.manage.model.inner.ServiceScriptDTO; import com.tencent.bk.job.manage.model.inner.ServiceTaskApprovalStepDTO; @@ -112,6 +118,7 @@ import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cloud.sleuth.Tracer; import org.springframework.stereotype.Service; import org.springframework.util.StopWatch; @@ -144,17 +151,19 @@ public class TaskExecuteServiceImpl implements TaskExecuteService { private final AccountService accountService; private final ScriptService scriptService; - private final TaskExecuteControlMsgSender controlMsgSender; + private final TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher; private final TaskPlanService taskPlanService; private final TaskInstanceVariableService taskInstanceVariableService; private final QueryAgentStatusClient queryAgentStatusClient; private final TaskOperationLogService taskOperationLogService; private final TaskInstanceService taskInstanceService; + private final StepInstanceService stepInstanceService; private final HostService hostService; private final ServiceUserResourceClient userResource; private final ExecuteAuthService executeAuthService; private final ExecutorService GET_HOSTS_BY_TOPO_EXECUTOR; private final DangerousScriptCheckService dangerousScriptCheckService; + private final RollingConfigService rollingConfigService; private final JobExecuteConfig jobExecuteConfig; private final TaskEvictPolicyExecutor taskEvictPolicyExecutor; @@ -163,56 +172,48 @@ public class TaskExecuteServiceImpl implements TaskExecuteService { @Autowired public TaskExecuteServiceImpl(AccountService accountService, TaskInstanceService taskInstanceService, - TaskExecuteControlMsgSender controlMsgSender, + TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, TaskPlanService taskPlanService, TaskInstanceVariableService taskInstanceVariableService, QueryAgentStatusClient queryAgentStatusClient, TaskOperationLogService taskOperationLogService, ScriptService scriptService, + StepInstanceService stepInstanceService, HostService hostService, ServiceUserResourceClient userResource, ExecuteAuthService executeAuthService, - Tracing tracing, + Tracer tracer, DangerousScriptCheckService dangerousScriptCheckService, JobExecuteConfig jobExecuteConfig, - TaskEvictPolicyExecutor taskEvictPolicyExecutor) { + TaskEvictPolicyExecutor taskEvictPolicyExecutor, + RollingConfigService rollingConfigService) { this.accountService = accountService; this.taskInstanceService = taskInstanceService; - this.controlMsgSender = controlMsgSender; + this.taskExecuteMQEventDispatcher = taskExecuteMQEventDispatcher; this.taskPlanService = taskPlanService; this.taskInstanceVariableService = taskInstanceVariableService; this.queryAgentStatusClient = queryAgentStatusClient; this.taskOperationLogService = taskOperationLogService; this.scriptService = scriptService; + this.stepInstanceService = stepInstanceService; this.hostService = hostService; this.userResource = userResource; this.executeAuthService = executeAuthService; this.GET_HOSTS_BY_TOPO_EXECUTOR = new TraceableExecutorService(new ThreadPoolExecutor(50, - 100, 60, TimeUnit.SECONDS, new LinkedBlockingQueue()), tracing); + 100, 60, TimeUnit.SECONDS, new LinkedBlockingQueue()), tracer); this.dangerousScriptCheckService = dangerousScriptCheckService; + this.rollingConfigService = rollingConfigService; this.jobExecuteConfig = jobExecuteConfig; this.taskEvictPolicyExecutor = taskEvictPolicyExecutor; } - private static List getHostsContainsNotAllowedAction(Map> hostBindActions, - Map> hostAllowedActions) { - List invalidHosts = new ArrayList<>(); - for (Map.Entry> binding : hostBindActions.entrySet()) { - IpDTO host = binding.getKey(); - if (!hostAllowedActions.containsKey(host) - || !hostAllowedActions.get(host).containsAll(binding.getValue())) { - invalidHosts.add(host); - } - } - return invalidHosts; - } - @Override - public Long createTaskInstanceFast(TaskInstanceDTO taskInstance, - StepInstanceDTO stepInstance) throws ServiceException { - log.info("Begin to create task instance and step instance for fast-execution-task, taskInstance: {}, " + - "stepInstance: {}", taskInstance, stepInstance); - StopWatch watch = new StopWatch("createTaskInstanceFast"); + public Long executeFastTask(FastTaskDTO fastTask) { + log.info("Begin to execute fast task: {}", fastTask); + TaskInstanceDTO taskInstance = fastTask.getTaskInstance(); + StepInstanceDTO stepInstance = fastTask.getStepInstance(); + + StopWatch watch = new StopWatch("executeFastTask"); // 检查任务是否应当被驱逐 checkTaskEvict(taskInstance); standardizeStepDynamicGroupId(Collections.singletonList(stepInstance)); @@ -222,17 +223,20 @@ public Long createTaskInstanceFast(TaskInstanceDTO taskInstance, watch.start("checkAndSetScriptInfoIfScriptTask"); checkAndSetScriptInfoForFast(taskInstance, stepInstance); watch.stop(); + // 设置账号信息 watch.start("setAccountInfo"); checkAndSetAccountInfo(stepInstance, taskInstance.getAppId()); watch.stop(); - // 获取ip列表 + + // 获取主机列表 watch.start("setServerInfoFastJob"); setServerInfoFastJob(stepInstance); watch.stop(); - //检查ip + + //检查主机 watch.start("checkHosts"); - checkHosts(stepInstance, shouldIgnoreInvalidHost(taskInstance)); + checkAndSetHosts(Collections.singletonList(stepInstance)); watch.stop(); // 检查步骤约束 @@ -240,14 +244,15 @@ public Long createTaskInstanceFast(TaskInstanceDTO taskInstance, checkStepInstanceConstraint(taskInstance, Collections.singletonList(stepInstance)); watch.stop(); + // 鉴权 watch.start("authFastExecute"); authFastExecute(taskInstance, stepInstance); watch.stop(); + // 保存作业、步骤实例 watch.start("saveInstance"); Long taskInstanceId = taskInstanceService.addTaskInstance(taskInstance); taskInstance.setId(taskInstanceId); - stepInstance.setTaskInstanceId(taskInstanceId); stepInstance.setStepNum(1); stepInstance.setStepOrder(1); @@ -255,10 +260,26 @@ public Long createTaskInstanceFast(TaskInstanceDTO taskInstance, stepInstance.setId(stepInstanceId); watch.stop(); + // 保存滚动配置 + if (fastTask.isRollingEnabled()) { + watch.start("saveRollingConfig"); + RollingConfigDTO rollingConfig = rollingConfigService.saveRollingConfigForFastJob(fastTask); + long rollingConfigId = rollingConfig.getId(); + stepInstanceService.updateStepRollingConfigId(stepInstanceId, rollingConfigId); + watch.stop(); + } + + // 记录操作日志 watch.start("saveOperationLog"); taskOperationLogService.saveOperationLog(buildTaskOperationLog(taskInstance, taskInstance.getOperator(), UserOperationEnum.START)); watch.stop(); + + // 启动作业 + watch.start("startJob"); + startTask(taskInstanceId); + watch.stop(); + return taskInstanceId; } finally { if (watch.isRunning()) { @@ -439,7 +460,7 @@ private void checkScriptStatusExecutable(ServiceScriptDTO script) { log.warn("Script status is {}, should not execute! ScriptId: {}, scriptVersionId={}", scriptStatus, script.getId(), script.getScriptVersionId()); throw new FailedPreconditionException(ErrorCode.SCRIPT_NOT_EXECUTABLE_STATUS, - new String[] { + new String[]{ "{" + scriptStatus.getStatusI18nKey() + "}" }); } @@ -539,9 +560,9 @@ private void filterServerDoNotRequireAuth(long appId, ServersDTO servers, Action if (CollectionUtils.isNotEmpty(servers.getStaticIpList())) { servers.setStaticIpList(servers.getStaticIpList().stream() .filter(host -> { - boolean isWhiteIp = hostService.isMatchWhiteIpRule(appId, host, action.name()); + boolean isWhiteIp = hostService.isMatchWhiteIpRule(appId, host.toCloudIp(), action.name()); if (isWhiteIp) { - log.info("Host: {} is white ip, skip auth!", host.convertToStrIp()); + log.info("Host: {} is white ip, skip auth!", host.toCloudIp()); } return !isWhiteIp; }) @@ -549,7 +570,7 @@ private void filterServerDoNotRequireAuth(long appId, ServersDTO servers, Action boolean isValidIp = servers.getInvalidIpList() == null || !servers.getInvalidIpList().contains(host); if (!isValidIp) { - log.info("Host: {} is invalid ip, skip auth!", host.convertToStrIp()); + log.info("Host: {} is invalid ip, skip auth!", host.toCloudIp()); } return isValidIp; }) @@ -624,60 +645,126 @@ private void setServerInfoFastJob(StepInstanceDTO stepInstance) { } } - private String convertToIpListStr(Collection ips) { + private String convertToIpListStr(Collection ips) { return StringUtils.join(ips.stream().map(ipDTO -> - ipDTO.getCloudAreaId() + ":" + ipDTO.getIp()).collect(Collectors.toList()), ","); + ipDTO.getBkCloudId() + ":" + ipDTO.getIp()).collect(Collectors.toList()), ","); } /** - * 检查主机的合法性 + * 设置主机信息并检查主机的合法性并设置主机信息 * - * @param stepInstanceList 步骤列表 - * @param ignoreInvalidHost 是否忽略不合法主机 + * @param stepInstanceList 步骤列表 * @throws ServiceException 如果包含不合法的主机，抛出异常 */ - private void checkHosts(List stepInstanceList, boolean ignoreInvalidHost) - throws ServiceException { - long appId = stepInstanceList.get(0).getAppId(); + private void checkAndSetHosts(List stepInstanceList) throws ServiceException { + // 检查步骤引用的主机不为空 + stepInstanceList.forEach(this::checkStepInstanceHostNonEmpty); - Set checkHosts = new HashSet<>(); + long appId = stepInstanceList.get(0).getAppId(); + Set checkHosts = new HashSet<>(); addNeedCheckHosts(stepInstanceList, checkHosts); if (checkHosts.isEmpty()) { return; } - // 检查是否在当前业务下 - Collection unavailableHosts = checkHostsNotInApp(appId, checkHosts); - if (unavailableHosts.isEmpty()) { - return; + // 检查主机是否存在 + ServiceListAppHostResultDTO hosts = hostService.batchGetAppHosts(appId, checkHosts); + if (CollectionUtils.isNotEmpty(hosts.getNotExistHosts())) { + throwHostInvalidException(hosts.getNotExistHosts(), appId); } + // 设置主机信息 + fillStepHostDetail(stepInstanceList, hosts); + // 检查是否在白名单配置 - List invalidHosts = checkHostsNotAllowedInWhiteIpConfig(appId, stepInstanceList, unavailableHosts); - if (!invalidHosts.isEmpty()) { - log.warn("Contains invalid host, invalidHost: {}", JsonUtils.toJson(invalidHosts)); - // 如果不允许忽略非法主机，或者全部主机都非法，那么直接拒绝 - if (!ignoreInvalidHost || (invalidHosts.size() == checkHosts.size())) { + if (CollectionUtils.isNotEmpty(hosts.getNotInAppHosts())) { + List invalidHosts = checkHostsNotAllowedInWhiteIpConfig(appId, stepInstanceList, + hosts.getNotInAppHosts()); + if (!invalidHosts.isEmpty()) { + log.warn("Contains invalid host, invalidHost: {}", JsonUtils.toJson(invalidHosts)); throwHostInvalidException(invalidHosts, appId); } - // 包含非法IP，需要继续走完流程，但是不下发任务 - setInvalidHostsForStepInstance(stepInstanceList, invalidHosts); } } - private void addNeedCheckHosts(List stepInstanceList, Set checkHosts) { + private void checkStepInstanceHostNonEmpty(StepInstanceDTO stepInstance) { + if (!isStepContainsHostProps(stepInstance)) { + return; + } + ServersDTO targetServers = stepInstance.getTargetServers(); + if (targetServers == null || CollectionUtils.isEmpty(targetServers.getIpList())) { + log.warn("Empty target server, stepInstanceId: {}", stepInstance.getId()); + throw new FailedPreconditionException(ErrorCode.SERVER_EMPTY); + } + if (stepInstance.isFileStep()) { + List fileSourceList = stepInstance.getFileSourceList(); + for (FileSourceDTO fileSource : fileSourceList) { + // 远程文件分发需要判断文件源主机是否为空 + if (TaskFileTypeEnum.SERVER.getType() == fileSource.getFileType()) { + ServersDTO servers = fileSource.getServers(); + if (servers != null && CollectionUtils.isEmpty(servers.getIpList())) { + log.warn("Empty file source server, stepInstanceId: {}", stepInstance.getId()); + throw new FailedPreconditionException(ErrorCode.SERVER_EMPTY); + } + } + } + } + } + + private boolean isStepContainsHostProps(StepInstanceBaseDTO stepInstance) { + // 判断步骤是否包含主机信息 + return !stepInstance.getExecuteType().equals(MANUAL_CONFIRM.getValue()); + } + + private void fillStepHostDetail(List stepInstanceList, ServiceListAppHostResultDTO hosts) { + Map hostMap = new HashMap<>(); + if (CollectionUtils.isNotEmpty(hosts.getValidHosts())) { + hosts.getValidHosts().forEach(host -> { + hostMap.put("hostId:" + host.getHostId(), host); + hostMap.put("hostIp:" + host.toCloudIp(), host); + }); + } + if (CollectionUtils.isNotEmpty(hosts.getNotInAppHosts())) { + hosts.getNotInAppHosts().forEach(host -> { + hostMap.put("hostId:" + host.getHostId(), host); + hostMap.put("hostIp:" + host.toCloudIp(), host); + }); + } + for (StepInstanceDTO stepInstance : stepInstanceList) { - if (stepInstance.getExecuteType().equals(MANUAL_CONFIRM.getValue())) { + if (!isStepContainsHostProps(stepInstance)) { continue; } - checkHosts.addAll(stepInstance.getTargetServers().getIpList()); - if (stepInstance.getExecuteType().equals(SEND_FILE.getValue())) { - List fileSourceList = stepInstance.getFileSourceList(); - if (fileSourceList != null) { - for (FileSourceDTO fileSource : fileSourceList) { - ServersDTO servers = fileSource.getServers(); - if (servers != null && servers.getIpList() != null) { - checkHosts.addAll(servers.getIpList()); + // 目标主机设置主机详情 + fillTargetHostDetail(stepInstance, hostMap); + // 文件源设置主机详情 + fillFileSourceHostDetail(stepInstance, hostMap); + } + } + + private void fillTargetHostDetail(StepInstanceDTO stepInstance, Map hostMap) { + if (CollectionUtils.isNotEmpty(stepInstance.getTargetServers().getStaticIpList())) { + stepInstance.getTargetServers().getStaticIpList() + .forEach(host -> fillHostDetail(host, hostMap)); + } + if (CollectionUtils.isNotEmpty(stepInstance.getTargetServers().getIpList())) { + stepInstance.getTargetServers().getIpList() + .forEach(host -> fillHostDetail(host, hostMap)); + } + } + + private void fillFileSourceHostDetail(StepInstanceDTO stepInstance, Map hostMap) { + if (stepInstance.getExecuteType().equals(SEND_FILE.getValue())) { + List fileSourceList = stepInstance.getFileSourceList(); + if (fileSourceList != null) { + for (FileSourceDTO fileSource : fileSourceList) { + ServersDTO servers = fileSource.getServers(); + if (servers != null) { + if (CollectionUtils.isNotEmpty(servers.getStaticIpList())) { + servers.getStaticIpList().forEach(host -> fillHostDetail(host, hostMap)); + } + if (CollectionUtils.isNotEmpty(servers.getIpList())) { + servers.getIpList().forEach(host -> fillHostDetail(host, hostMap)); } } } @@ -685,33 +772,45 @@ private void addNeedCheckHosts(List stepInstanceList, Set stepInstanceList, List invalidHosts) { - stepInstanceList.forEach(stepInstance -> { - if (stepInstance.getExecuteType().equals(MANUAL_CONFIRM.getValue())) { - return; + private void fillHostDetail(HostDTO host, Map hostMap) { + if (host.getHostId() != null) { + HostDTO hostDetail = hostMap.get("hostId:" + host.getHostId()); + host.setBkCloudId(hostDetail.getBkCloudId()); + host.setIp(hostDetail.getIp()); + // 兼容没有agent_id的主机，按照与GSE的约定，按照{云区域ID:ip}的方式构造agent_id + host.setAgentId(StringUtils.isEmpty(hostDetail.getAgentId()) ? host.toCloudIp() : hostDetail.getAgentId()); + } else { + HostDTO hostDetail = hostMap.get("hostIp:" + host.toCloudIp()); + // 兼容没有agent_id的主机，按照与GSE的约定，按照{云区域ID:ip}的方式构造agent_id + host.setAgentId(StringUtils.isEmpty(hostDetail.getAgentId()) ? host.toCloudIp() : hostDetail.getAgentId()); + host.setHostId(hostDetail.getHostId()); + } + } + + private void addNeedCheckHosts(List stepInstanceList, Set checkHosts) { + for (StepInstanceDTO stepInstance : stepInstanceList) { + if (!isStepContainsHostProps(stepInstance)) { + continue; } + checkHosts.addAll(stepInstance.getTargetServers().getIpList()); if (stepInstance.getExecuteType().equals(SEND_FILE.getValue())) { List fileSourceList = stepInstance.getFileSourceList(); if (fileSourceList != null) { for (FileSourceDTO fileSource : fileSourceList) { ServersDTO servers = fileSource.getServers(); if (servers != null && servers.getIpList() != null) { - servers.setInvalidIpList(servers.getIpList().stream() - .filter(invalidHosts::contains).collect(Collectors.toList())); + checkHosts.addAll(servers.getIpList()); } } } } - ServersDTO targetServers = stepInstance.getTargetServers(); - targetServers.setInvalidIpList(targetServers.getIpList().stream() - .filter(invalidHosts::contains).collect(Collectors.toList())); - }); + } } - private List checkHostsNotAllowedInWhiteIpConfig(long appId, List stepInstanceList, - Collection unavailableHosts) { - Map> hostAllowActionsMap = new HashMap<>(); - for (IpDTO host : unavailableHosts) { + private List checkHostsNotAllowedInWhiteIpConfig(long appId, List stepInstanceList, + Collection unavailableHosts) { + Map> hostAllowActionsMap = new HashMap<>(); + for (HostDTO host : unavailableHosts) { List allowActions = hostService.getHostAllowedAction(appId, host); if (allowActions != null && !allowActions.isEmpty()) { hostAllowActionsMap.put(host, allowActions); @@ -724,16 +823,29 @@ private List checkHostsNotAllowedInWhiteIpConfig(long appId, List> hostBindActionsMap = getHostBindActions(stepInstanceList, unavailableHosts); + Map> hostBindActionsMap = getHostBindActions(stepInstanceList, unavailableHosts); log.debug("Host bind actions:{}", hostBindActionsMap); return getHostsContainsNotAllowedAction(hostBindActionsMap, hostAllowActionsMap); } - private Map> getHostBindActions(List stepInstanceList, - Collection unavailableHosts) { - Map> hostBindActionsMap = new HashMap<>(); - for (IpDTO host : unavailableHosts) { + private static List getHostsContainsNotAllowedAction(Map> hostBindActions, + Map> hostAllowedActions) { + List invalidHosts = new ArrayList<>(); + for (Map.Entry> binding : hostBindActions.entrySet()) { + HostDTO host = binding.getKey(); + if (!hostAllowedActions.containsKey(host) + || !hostAllowedActions.get(host).containsAll(binding.getValue())) { + invalidHosts.add(host); + } + } + return invalidHosts; + } + + private Map> getHostBindActions(List stepInstanceList, + Collection unavailableHosts) { + Map> hostBindActionsMap = new HashMap<>(); + for (HostDTO host : unavailableHosts) { for (StepInstanceDTO stepInstance : stepInstanceList) { if (stepInstance.getExecuteType().equals(MANUAL_CONFIRM.getValue())) { continue; @@ -763,54 +875,11 @@ private Map> getHostBindActions(List stepIns return hostBindActionsMap; } - - private void checkHosts(StepInstanceDTO stepInstance, boolean ignoreInvalidHost) throws ServiceException { - long appId = stepInstance.getAppId(); - ServersDTO targetServers = stepInstance.getTargetServers(); - if (targetServers == null || targetServers.getIpList() == null || targetServers.getIpList().isEmpty()) { - log.warn("Empty target server"); - throw new FailedPreconditionException(ErrorCode.SERVER_EMPTY); - } - - List ipList = targetServers.getIpList(); - Collection notInAppHosts = checkHostsNotInApp(appId, ipList); - if (notInAppHosts.isEmpty()) { - return; - } - - // 检查是否在白名单配置 - List invalidHosts = checkHostsNotAllowedInWhiteIpConfig(appId, Lists.newArrayList(stepInstance), - notInAppHosts); - if (!invalidHosts.isEmpty()) { - log.warn("Contains invalid host, invalidHost: {}", JsonUtils.toJson(invalidHosts)); - // 如果不允许忽略非法主机，或者全部主机都非法，那么直接拒绝 - if (!ignoreInvalidHost || (invalidHosts.size() == ipList.size())) { - throwHostInvalidException(invalidHosts, appId); - } - // 包含部分非法主机，需要继续走完流程，下发任务到合法的主机 - targetServers.setInvalidIpList(invalidHosts); - } - } - - private boolean shouldIgnoreInvalidHost(TaskInstanceDTO taskInstance) { - // 定时任务才支持自动忽略非法主机 - return (taskInstance.getStartupMode() != null - && taskInstance.getStartupMode().equals(TaskStartupModeEnum.CRON.getValue())); - } - - private void throwHostInvalidException(Collection unavailableHosts, long appId) { - String ipListStr = StringUtils.join(unavailableHosts.stream().map(IpDTO::getIp).collect(Collectors.toList()), + private void throwHostInvalidException(Collection unavailableHosts, long appId) { + String ipListStr = StringUtils.join(unavailableHosts.stream().map(HostDTO::getIp).collect(Collectors.toList()), ","); - log.warn("The following hosts are not registered, appId:{}, ips={}", appId, ipListStr); - throw new FailedPreconditionException(ErrorCode.SERVER_UNREGISTERED, new Object[]{ipListStr}); - } - - private Collection checkHostsNotInApp(Long appId, Collection hosts) { - List notInAppHosts = hostService.checkAppHosts(appId, hosts); - if (CollectionUtils.isNotEmpty(notInAppHosts)) { - log.info("Check host, appId:{}, not in current app hosts:{}", appId, notInAppHosts); - } - return notInAppHosts; + log.warn("The following hosts are invalid, appId:{}, ips={}", appId, ipListStr); + throw new FailedPreconditionException(ErrorCode.HOST_INVALID, new Object[]{ipListStr}); } private void checkStepInstanceConstraint(TaskInstanceDTO taskInstance, List stepInstanceList) { @@ -864,8 +933,9 @@ private void checkStepInstanceConstraint(TaskInstanceDTO taskInstance, List hostList = targetServers.getHostList(); if (hostList != null && !hostList.isEmpty()) { - List staticIpList = new ArrayList<>(); + List staticIpList = new ArrayList<>(); for (ServiceHostInfoDTO hostInfo : hostList) { - staticIpList.add(new IpDTO(hostInfo.getCloudAreaId(), hostInfo.getIp())); + staticIpList.add(new HostDTO(hostInfo.getCloudAreaId(), hostInfo.getIp())); } servers.setStaticIpList(staticIpList); } @@ -1711,15 +1782,15 @@ private ServersDTO convertToServersDTO(ServiceTaskTargetDTO taskTarget) { } private void acquireStaticIp(long appId, ServersDTO servers) throws ServiceException { - Set ipSet = new HashSet<>(); - List staticIps = servers.getStaticIpList(); + Set ipSet = new HashSet<>(); + List staticIps = servers.getStaticIpList(); if (staticIps != null) { ipSet.addAll(staticIps); } List dynamicServerGroups = servers.getDynamicServerGroups(); if (dynamicServerGroups != null) { for (DynamicServerGroupDTO group : dynamicServerGroups) { - List groupIps = hostService.getIpByDynamicGroupId(appId, group.getGroupId()); + List groupIps = hostService.getIpByDynamicGroupId(appId, group.getGroupId()); if (CollectionUtils.isEmpty(groupIps)) { servers.addInvalidDynamicServerGroup(group); } else { @@ -1732,7 +1803,7 @@ private void acquireStaticIp(long appId, ServersDTO servers) throws ServiceExcep if (topoNodes != null && !topoNodes.isEmpty()) { if (topoNodes.size() < 10) { for (DynamicServerTopoNodeDTO topoNode : topoNodes) { - List topoIps = hostService.getIpByTopoNodes(appId, + List topoIps = hostService.getIpByTopoNodes(appId, Collections.singletonList(new CcInstanceDTO(topoNode.getNodeType(), topoNode.getTopoNodeId()))); if (CollectionUtils.isEmpty(topoIps)) { servers.addInvalidTopoNodeDTO(topoNode); @@ -1744,24 +1815,24 @@ private void acquireStaticIp(long appId, ServersDTO servers) throws ServiceExcep getTopoHostsConcurrent(appId, topoNodes, servers, ipSet); } } - List ipList = new ArrayList<>(ipSet.size()); + List ipList = new ArrayList<>(ipSet.size()); ipList.addAll(ipSet); servers.setIpList(ipList); } private void getTopoHostsConcurrent(long appId, List topoNodes, ServersDTO servers, - Set ipSet) { + Set ipSet) { log.info("Get topo hosts concurrent, topoNodes: {}", topoNodes); CountDownLatch latch = new CountDownLatch(topoNodes.size()); - List>>> futures = new ArrayList<>(topoNodes.size()); + List>>> futures = new ArrayList<>(topoNodes.size()); for (DynamicServerTopoNodeDTO topoNode : topoNodes) { futures.add(GET_HOSTS_BY_TOPO_EXECUTOR.submit(new GetTopoHostTask(appId, topoNode, latch))); } try { - for (Future>> future : futures) { - Pair> topoAndHosts = future.get(); + for (Future>> future : futures) { + Pair> topoAndHosts = future.get(); if (CollectionUtils.isEmpty(topoAndHosts.getRight())) { servers.addInvalidTopoNodeDTO(topoAndHosts.getLeft()); } else { @@ -1779,18 +1850,18 @@ private void getTopoHostsConcurrent(long appId, List t log.info("Get topo hosts success, servers: {}", servers); } - private void setAgentStatus(List ips) { + private void setAgentStatus(List ips) { if (ips == null || ips.isEmpty()) { return; } List ipList = new ArrayList<>(ips.size()); - for (IpDTO ip : ips) { - String fullIp = ip.convertToStrIp(); + for (HostDTO ip : ips) { + String fullIp = ip.toCloudIp(); ipList.add(fullIp); } Map statusMap = queryAgentStatusClient.batchGetAgentStatus(ipList); - for (IpDTO ip : ips) { - String fullIp = ip.convertToStrIp(); + for (HostDTO ip : ips) { + String fullIp = ip.toCloudIp(); ip.setAlive(statusMap.get(fullIp) == null ? AgentStatusEnum.UNKNOWN.getValue() : statusMap.get(fullIp).status); } @@ -1840,6 +1911,9 @@ public Integer doStepOperation(Long appId, String operator, case SKIP: skipStep(stepInstance, operator); break; + case ROLLING_CONTINUE: + continueRolling(stepInstance); + break; default: log.warn("Undefined step operation!"); break; @@ -1847,12 +1921,31 @@ public Integer doStepOperation(Long appId, String operator, return executeCount; } + private void continueRolling(StepInstanceDTO stepInstance) { + // 只有“等待用户”的滚动步骤可以继续滚动 + if (stepInstance.getStatus() != RunStatusEnum.WAITING_USER) { + log.warn("Unsupported operation, stepInstanceId: {}, operation: {}, stepStatus: {}", + stepInstance.getId(), StepOperationEnum.ROLLING_CONTINUE.name(), stepInstance.getStatus().name()); + throw new FailedPreconditionException(ErrorCode.UNSUPPORTED_OPERATION); + } + if (!(stepInstance.isRollingStep())) { + log.warn("StepInstance:{} is not rolling step, Unsupported Operation:{}", stepInstance.getId(), + "rolling-continue"); + throw new FailedPreconditionException(ErrorCode.UNSUPPORTED_OPERATION); + } + + // 需要同步设置任务状态为RUNNING，保证客户端可以在操作完之后立马获取到运行状态，开启同步刷新 + taskInstanceService.updateTaskStatus(stepInstance.getTaskInstanceId(), RunStatusEnum.RUNNING.getValue()); + taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.startStep(stepInstance.getId(), + stepInstance.getBatch() + 1)); + } + private void confirmTerminate(StepInstanceDTO stepInstance, String operator, String reason) { TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(stepInstance.getTaskInstanceId()); // 只有人工确认等待中的任务，可以进行“终止流程”操作 - if (!RunStatusEnum.WAITING.getValue().equals(stepInstance.getStatus())) { - log.warn("StepInstance:{} status is not waiting, Unsupported Operation:{}", stepInstance.getId(), - "confirm-terminate"); + if (RunStatusEnum.WAITING_USER != stepInstance.getStatus()) { + log.warn("Unsupported operation, stepInstanceId: {}, operation: {}, stepStatus: {}", + stepInstance.getId(), StepOperationEnum.CONFIRM_TERMINATE.name(), stepInstance.getStatus().name()); throw new FailedPreconditionException(ErrorCode.UNSUPPORTED_OPERATION); } if (!stepInstance.getExecuteType().equals(MANUAL_CONFIRM.getValue())) { @@ -1870,14 +1963,14 @@ private void confirmTerminate(StepInstanceDTO stepInstance, String operator, Str taskInstanceService.updateConfirmReason(stepInstance.getId(), reason); taskInstanceService.updateStepOperator(stepInstance.getId(), operator); - controlMsgSender.confirmStepTerminate(stepInstance.getId()); + taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.confirmStepTerminate(stepInstance.getId())); } private void confirmRestart(StepInstanceDTO stepInstance, String operator) { // 只有“确认终止”状态的任务，可以进行“重新发起确认”操作 - if (!RunStatusEnum.CONFIRM_TERMINATED.getValue().equals(stepInstance.getStatus())) { - log.warn("StepInstance:{} status is not confirm_terminated, Unsupported Operation:{}", - stepInstance.getId(), "confirm-restart"); + if (RunStatusEnum.CONFIRM_TERMINATED != stepInstance.getStatus()) { + log.warn("Unsupported operation, stepInstanceId: {}, operation: {}, stepStatus: {}", + stepInstance.getId(), StepOperationEnum.CONFIRM_RESTART.name(), stepInstance.getStatus().name()); throw new FailedPreconditionException(ErrorCode.UNSUPPORTED_OPERATION); } if (!stepInstance.getExecuteType().equals(MANUAL_CONFIRM.getValue())) { @@ -1887,7 +1980,7 @@ private void confirmRestart(StepInstanceDTO stepInstance, String operator) { } taskOperationLogService.saveOperationLog(buildCommonStepOperationLog(stepInstance, operator, UserOperationEnum.CONFIRM_RESTART)); - controlMsgSender.confirmStepRestart(stepInstance.getId()); + taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.confirmStepRestart(stepInstance.getId())); } private void checkConfirmUser(TaskInstanceDTO taskInstance, StepInstanceDTO stepInstance, @@ -1925,9 +2018,9 @@ private void confirmContinue(StepInstanceDTO stepInstance, String operator, Stri "-continue"); throw new FailedPreconditionException(ErrorCode.UNSUPPORTED_OPERATION); } - if (!(RunStatusEnum.WAITING.getValue().equals(stepInstance.getStatus()))) { - log.warn("StepInstance:{} status is not waiting, Unsupported Operation:{}", stepInstance.getId(), - "confirm-continue"); + if (RunStatusEnum.WAITING_USER != stepInstance.getStatus()) { + log.warn("Unsupported operation, stepInstanceId: {}, operation: {}, stepStatus: {}", + stepInstance.getId(), StepOperationEnum.CONFIRM_CONTINUE.name(), stepInstance.getStatus().name()); throw new FailedPreconditionException(ErrorCode.UNSUPPORTED_OPERATION); } // 人工确认继续，需要判断操作者 @@ -1945,15 +2038,15 @@ private void confirmContinue(StepInstanceDTO stepInstance, String operator, Stri // 需要同步设置任务状态为RUNNING，保证客户端可以在操作完之后立马获取到运行状态，开启同步刷新 taskInstanceService.updateTaskStatus(taskInstance.getId(), RunStatusEnum.RUNNING.getValue()); - controlMsgSender.confirmStepContinue(stepInstance.getId()); + taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.confirmStepContinue(stepInstance.getId())); } private void nextStep(StepInstanceDTO stepInstance, String operator) { TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(stepInstance.getTaskInstanceId()); // 只有"终止成功"状态的任务，可以直接进入下一步 - if (!RunStatusEnum.STOP_SUCCESS.getValue().equals(stepInstance.getStatus())) { - log.warn("StepInstance:{} status is not stop-success, Unsupported Operation:{}", stepInstance.getId(), - "next-step"); + if (RunStatusEnum.STOP_SUCCESS != stepInstance.getStatus()) { + log.warn("Unsupported operation, stepInstanceId: {}, operation: {}, stepStatus: {}", + stepInstance.getId(), StepOperationEnum.NEXT_STEP.name(), stepInstance.getStatus().name()); throw new FailedPreconditionException(ErrorCode.UNSUPPORTED_OPERATION); } taskOperationLogService.saveOperationLog(buildCommonStepOperationLog(stepInstance, operator, @@ -1961,52 +2054,57 @@ private void nextStep(StepInstanceDTO stepInstance, String operator) { // 需要同步设置任务状态为RUNNING，保证客户端可以在操作完之后立马获取到运行状态，开启同步刷新 taskInstanceService.updateTaskStatus(taskInstance.getId(), RunStatusEnum.RUNNING.getValue()); - controlMsgSender.nextStep(stepInstance.getId()); + taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.nextStep(stepInstance.getId())); } private void retryStepFail(StepInstanceDTO stepInstance, String operator) { - // 只有“执行失败”的作业可以失败重试 - if (!stepInstance.getStatus().equals(RunStatusEnum.FAIL.getValue())) { - log.warn("StepInstance:{} status is not fail, Unsupported Operation:{}", stepInstance.getId(), "retry" + - "-fail"); + if (!isStepRetryable(stepInstance.getStatus())) { + log.warn("Unsupported operation, stepInstanceId: {}, operation: {}, stepStatus: {}", + stepInstance.getId(), StepOperationEnum.RETRY_FAIL_IP.name(), stepInstance.getStatus().name()); throw new FailedPreconditionException(ErrorCode.UNSUPPORTED_OPERATION); } + // 需要同步设置任务状态为RUNNING，保证客户端可以在操作完之后立马获取到运行状态，开启同步刷新 taskInstanceService.updateTaskStatus(stepInstance.getTaskInstanceId(), RunStatusEnum.RUNNING.getValue()); - taskInstanceService.updateStepStatus(stepInstance.getId(), RunStatusEnum.RUNNING.getValue()); -// taskInstanceService.addStepExecuteCount(stepInstance.getId()); - controlMsgSender.retryStepFail(stepInstance.getId()); + taskInstanceService.addStepInstanceExecuteCount(stepInstance.getId()); + taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.retryStepFail(stepInstance.getId())); OperationLogDTO operationLog = buildCommonStepOperationLog(stepInstance, operator, UserOperationEnum.RETRY_STEP_FAIL); taskOperationLogService.saveOperationLog(operationLog); } private void retryStepAll(StepInstanceDTO stepInstance, String operator) { - // 只有“执行失败”,"终止成功"的作业可以全部重试 - if (!(stepInstance.getStatus().equals(RunStatusEnum.FAIL.getValue()) - || stepInstance.getStatus().equals(RunStatusEnum.STOP_SUCCESS.getValue()))) { - log.warn("StepInstance:{} status is not fail, Unsupported Operation:{}", stepInstance.getId(), "retry-all"); + if (!isStepRetryable(stepInstance.getStatus())) { + log.warn("Unsupported operation, stepInstanceId: {}, operation: {}, stepStatus: {}", + stepInstance.getId(), StepOperationEnum.RETRY_ALL_IP.name(), stepInstance.getStatus().name()); throw new FailedPreconditionException(ErrorCode.UNSUPPORTED_OPERATION); } + // 需要同步设置任务状态为RUNNING，保证客户端可以在操作完之后立马获取到运行状态，开启同步刷新 taskInstanceService.updateTaskStatus(stepInstance.getTaskInstanceId(), RunStatusEnum.RUNNING.getValue()); - taskInstanceService.updateStepStatus(stepInstance.getId(), RunStatusEnum.RUNNING.getValue()); -// taskInstanceService.addStepExecuteCount(stepInstance.getId()); - controlMsgSender.retryStepAll(stepInstance.getId()); + taskInstanceService.addStepInstanceExecuteCount(stepInstance.getId()); + taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.retryStepAll(stepInstance.getId())); OperationLogDTO operationLog = buildCommonStepOperationLog(stepInstance, operator, UserOperationEnum.RETRY_STEP_ALL); taskOperationLogService.saveOperationLog(operationLog); } + private boolean isStepRetryable(RunStatusEnum stepStatus) { + // 只有“执行失败”,"状态异常","终止成功"的作业可以重试 + return stepStatus == RunStatusEnum.FAIL + || stepStatus == RunStatusEnum.ABNORMAL_STATE + || stepStatus == RunStatusEnum.STOP_SUCCESS; + } + private void ignoreError(StepInstanceDTO stepInstance, String operator) { // 只有“执行失败”的作业可以忽略错误进入下一步 - if (!stepInstance.getStatus().equals(RunStatusEnum.FAIL.getValue()) && - !stepInstance.getStatus().equals(RunStatusEnum.ABNORMAL_STATE.getValue())) { - log.warn("StepInstance:{} status is {}, Unsupported Operation:{}", stepInstance.getId(), - stepInstance.getStatus(), "ignore-error"); + if (stepInstance.getStatus() != RunStatusEnum.FAIL && + stepInstance.getStatus() != RunStatusEnum.ABNORMAL_STATE) { + log.warn("Unsupported operation, stepInstanceId: {}, operation: {}, stepStatus: {}", + stepInstance.getId(), StepOperationEnum.IGNORE_ERROR.name(), stepInstance.getStatus().name()); throw new FailedPreconditionException(ErrorCode.UNSUPPORTED_OPERATION); } // 需要同步设置任务状态为RUNNING，保证客户端可以在操作完之后立马获取到运行状态，开启同步刷新 taskInstanceService.updateTaskStatus(stepInstance.getTaskInstanceId(), RunStatusEnum.RUNNING.getValue()); - controlMsgSender.ignoreStepError(stepInstance.getId()); + taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.ignoreError(stepInstance.getId())); OperationLogDTO operationLog = buildCommonStepOperationLog(stepInstance, operator, UserOperationEnum.IGNORE_ERROR); taskOperationLogService.saveOperationLog(operationLog); @@ -2014,11 +2112,12 @@ private void ignoreError(StepInstanceDTO stepInstance, String operator) { private void skipStep(StepInstanceDTO stepInstance, String operator) { // 只有“强制终止中”的作业可以跳过 - if (!stepInstance.getStatus().equals(RunStatusEnum.STOPPING.getValue())) { - log.warn("StepInstance:{} status is not stopping, Unsupported Operation:{}", stepInstance.getId(), "skip"); + if (stepInstance.getStatus() != RunStatusEnum.STOPPING) { + log.warn("Unsupported operation, stepInstanceId: {}, operation: {}, stepStatus: {}", + stepInstance.getId(), StepOperationEnum.SKIP.name(), stepInstance.getStatus().name()); throw new FailedPreconditionException(ErrorCode.UNSUPPORTED_OPERATION); } - controlMsgSender.skipStep(stepInstance.getId()); + taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.skipStep(stepInstance.getId())); OperationLogDTO operationLog = buildCommonStepOperationLog(stepInstance, operator, UserOperationEnum.SKIP_STEP); taskOperationLogService.saveOperationLog(operationLog); } @@ -2035,6 +2134,7 @@ private OperationLogDTO buildCommonStepOperationLog(StepInstanceDTO stepInstance taskDetail.setStepInstanceId(stepInstance.getId()); taskDetail.setStepName(stepInstance.getName()); taskDetail.setExecuteCount(stepInstance.getExecuteCount()); + taskDetail.setBatch(stepInstance.getBatch()); operationLog.setDetail(taskDetail); return operationLog; } @@ -2062,15 +2162,16 @@ public void terminateJob(String username, Long appId, Long taskInstanceId) throw log.warn("Task instance is not exist, appId:{}, taskInstanceId:{}", appId, taskInstance); throw new NotFoundException(ErrorCode.TASK_INSTANCE_NOT_EXIST); } - if (!RunStatusEnum.RUNNING.getValue().equals(taskInstance.getStatus())) { - log.warn("TaskInstance:{} status is not running, should not terminate it!", taskInstance.getId()); + if (RunStatusEnum.RUNNING != taskInstance.getStatus() + && RunStatusEnum.WAITING_USER != taskInstance.getStatus()) { + log.warn("TaskInstance:{} status is not running/waiting, should not terminate it!", taskInstance.getId()); throw new FailedPreconditionException(ErrorCode.UNSUPPORTED_OPERATION); } - if (RunStatusEnum.STOPPING.getValue().equals(taskInstance.getStatus())) { + if (RunStatusEnum.STOPPING == taskInstance.getStatus()) { log.warn("TaskInstance:{} status is stopping now, should not terminate it!", taskInstance.getId()); throw new FailedPreconditionException(ErrorCode.TASK_STOPPING_DO_NOT_REPEAT); } - controlMsgSender.stopTask(taskInstanceId); + taskExecuteMQEventDispatcher.dispatchJobEvent(JobEvent.stopJob(taskInstanceId)); OperationLogDTO operationLog = buildTaskOperationLog(taskInstance, username, UserOperationEnum.TERMINATE_JOB); taskOperationLogService.saveOperationLog(operationLog); } @@ -2097,7 +2198,7 @@ public void authExecuteJobPlan(TaskExecuteParam executeParam) throws ServiceExce // 检查主机合法性 watch.start("checkHost"); - checkHosts(taskInfo.getStepInstances(), shouldIgnoreInvalidHost(taskInfo.getTaskInstance())); + checkAndSetHosts(taskInfo.getStepInstances()); watch.stop(); watch.start("auth-execute-job"); @@ -2110,7 +2211,7 @@ public void authExecuteJobPlan(TaskExecuteParam executeParam) throws ServiceExce } } - private class GetTopoHostTask implements Callable>> { + private class GetTopoHostTask implements Callable>> { private final long appId; private final DynamicServerTopoNodeDTO topoNode; private final CountDownLatch latch; @@ -2122,9 +2223,9 @@ private GetTopoHostTask(long appId, DynamicServerTopoNodeDTO topoNode, CountDown } @Override - public Pair> call() { + public Pair> call() { try { - List topoIps = hostService.getIpByTopoNodes(appId, + List topoIps = hostService.getIpByTopoNodes(appId, Collections.singletonList(new CcInstanceDTO(topoNode.getNodeType(), topoNode.getTopoNodeId()))); return new ImmutablePair<>(topoNode, topoIps); } catch (Throwable e) { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceServiceImpl.java index a8ac9e6d3f..3cd2f79f55 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceServiceImpl.java @@ -31,7 +31,6 @@ import com.tencent.bk.job.execute.common.constants.TaskStartupModeEnum; import com.tencent.bk.job.execute.common.constants.TaskTypeEnum; import com.tencent.bk.job.execute.common.converter.StepTypeExecuteTypeConverter; -import com.tencent.bk.job.execute.config.JobExecuteConfig; import com.tencent.bk.job.execute.dao.StepInstanceDAO; import com.tencent.bk.job.execute.dao.TaskInstanceDAO; import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; @@ -65,17 +64,16 @@ public class TaskInstanceServiceImpl implements TaskInstanceService { private final ApplicationService applicationService; private final StepInstanceDAO stepInstanceDAO; - private final JobExecuteConfig jobExecuteConfig; private final TaskInstanceDAO taskInstanceDAO; private final TaskInstanceVariableService taskInstanceVariableService; @Autowired - public TaskInstanceServiceImpl(ApplicationService applicationService, StepInstanceDAO stepInstanceDAO, - JobExecuteConfig jobExecuteConfig, TaskInstanceDAO taskInstanceDAO, + public TaskInstanceServiceImpl(ApplicationService applicationService, + StepInstanceDAO stepInstanceDAO, + TaskInstanceDAO taskInstanceDAO, TaskInstanceVariableService taskInstanceVariableService) { this.applicationService = applicationService; this.stepInstanceDAO = stepInstanceDAO; - this.jobExecuteConfig = jobExecuteConfig; this.taskInstanceDAO = taskInstanceDAO; this.taskInstanceVariableService = taskInstanceVariableService; } @@ -162,6 +160,11 @@ public StepInstanceDTO getStepInstanceDetail(long stepInstanceId) { return stepInstance; } + @Override + public StepInstanceBaseDTO getFirstStepInstance(long taskInstanceId) { + return stepInstanceDAO.getFirstStepInstanceBase(taskInstanceId); + } + @Override public void updateTaskStatus(long taskInstanceId, int status) { taskInstanceDAO.updateTaskStatus(taskInstanceId, status); @@ -193,9 +196,8 @@ public void resetStepExecuteInfoForRetry(long stepInstanceId) { } @Override - public void resetTaskExecuteInfoForResume(long taskInstanceId) { - taskInstanceDAO.resetTaskExecuteInfoForResume(taskInstanceId); - + public void resetTaskExecuteInfoForRetry(long taskInstanceId) { + taskInstanceDAO.resetTaskExecuteInfoForRetry(taskInstanceId); } @Override @@ -219,8 +221,8 @@ public void updateStepEndTime(long stepInstanceId, Long endTime) { } @Override - public void addTaskExecuteCount(long taskInstanceId) { - stepInstanceDAO.addTaskExecuteCount(taskInstanceId); + public void addStepInstanceExecuteCount(long stepInstanceId) { + stepInstanceDAO.addStepInstanceExecuteCount(stepInstanceId); } @Override @@ -228,11 +230,6 @@ public void updateStepTotalTime(long stepInstanceId, long totalTime) { stepInstanceDAO.updateStepTotalTime(stepInstanceId, totalTime); } - @Override - public void updateStepStatInfo(long stepInstanceId, int runIPNum, int successIPNum, int failIPNum) { - stepInstanceDAO.updateStepStatInfo(stepInstanceId, runIPNum, successIPNum, failIPNum); - } - @Override public void updateTaskExecutionInfo(long taskInstanceId, RunStatusEnum status, Long currentStepId, Long startTime , Long endTime, Long totalTime) { @@ -245,13 +242,6 @@ public void updateStepExecutionInfo(long stepInstanceId, RunStatusEnum status, L stepInstanceDAO.updateStepExecutionInfo(stepInstanceId, status, startTime, endTime, totalTime); } - @Override - public void updateStepExecutionInfo(long stepInstanceId, RunStatusEnum status, Long startTime, Long endTime, - Long totalTime, Integer runIPNum, Integer successIPNum, Integer failIPNum) { - stepInstanceDAO.updateStepExecutionInfo(stepInstanceId, status, startTime, endTime, - totalTime, runIPNum, successIPNum, failIPNum); - } - @Override public void updateResolvedScriptParam(long stepInstanceId, String resolvedScriptParam) { stepInstanceDAO.updateResolvedScriptParam(stepInstanceId, resolvedScriptParam); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java index e64989cdd0..5b81aba940 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java @@ -25,6 +25,7 @@ package com.tencent.bk.job.execute.service.impl; import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.Order; import com.tencent.bk.job.common.exception.FailedPreconditionException; import com.tencent.bk.job.common.exception.NotFoundException; import com.tencent.bk.job.common.exception.ServiceException; @@ -33,40 +34,45 @@ import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.model.dto.AppResourceScope; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.auth.ExecuteAuthService; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; +import com.tencent.bk.job.execute.common.constants.StepRunModeEnum; import com.tencent.bk.job.execute.common.converter.StepTypeExecuteTypeConverter; import com.tencent.bk.job.execute.common.util.TaskCostCalculator; import com.tencent.bk.job.execute.constants.UserOperationEnum; import com.tencent.bk.job.execute.dao.FileSourceTaskLogDAO; -import com.tencent.bk.job.execute.dao.GseTaskIpLogDAO; import com.tencent.bk.job.execute.dao.StepInstanceDAO; import com.tencent.bk.job.execute.dao.TaskInstanceDAO; -import com.tencent.bk.job.execute.engine.consts.IpStatus; -import com.tencent.bk.job.execute.model.AgentTaskExecutionDTO; +import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; +import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.model.AgentTaskDetailDTO; +import com.tencent.bk.job.execute.model.AgentTaskResultGroupBaseDTO; +import com.tencent.bk.job.execute.model.AgentTaskResultGroupDTO; import com.tencent.bk.job.execute.model.ConfirmStepInstanceDTO; -import com.tencent.bk.job.execute.model.ExecutionResultGroupDTO; import com.tencent.bk.job.execute.model.FileSourceTaskLogDTO; -import com.tencent.bk.job.execute.model.GseTaskIpLogDTO; -import com.tencent.bk.job.execute.model.GseTaskLogDTO; import com.tencent.bk.job.execute.model.OperationLogDTO; -import com.tencent.bk.job.execute.model.ResultGroupBaseDTO; +import com.tencent.bk.job.execute.model.RollingConfigDTO; import com.tencent.bk.job.execute.model.StepExecutionDTO; import com.tencent.bk.job.execute.model.StepExecutionDetailDTO; import com.tencent.bk.job.execute.model.StepExecutionRecordDTO; import com.tencent.bk.job.execute.model.StepExecutionResultQuery; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; +import com.tencent.bk.job.execute.model.StepInstanceRollingTaskDTO; import com.tencent.bk.job.execute.model.TaskExecuteResultDTO; import com.tencent.bk.job.execute.model.TaskExecutionDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.TaskInstanceQuery; import com.tencent.bk.job.execute.model.inner.CronTaskExecuteResult; import com.tencent.bk.job.execute.model.inner.ServiceCronTaskExecuteResultStatistics; -import com.tencent.bk.job.execute.service.GseTaskLogService; +import com.tencent.bk.job.execute.service.FileAgentTaskService; import com.tencent.bk.job.execute.service.HostService; import com.tencent.bk.job.execute.service.LogService; +import com.tencent.bk.job.execute.service.RollingConfigService; +import com.tencent.bk.job.execute.service.ScriptAgentTaskService; +import com.tencent.bk.job.execute.service.StepInstanceRollingTaskService; +import com.tencent.bk.job.execute.service.TaskInstanceService; import com.tencent.bk.job.execute.service.TaskOperationLogService; import com.tencent.bk.job.execute.service.TaskResultService; import lombok.extern.slf4j.Slf4j; @@ -96,33 +102,42 @@ public class TaskResultServiceImpl implements TaskResultService { private final TaskInstanceDAO taskInstanceDAO; private final StepInstanceDAO stepInstanceDAO; - private final GseTaskLogService gseTaskLogService; + private final TaskInstanceService taskInstanceService; private final FileSourceTaskLogDAO fileSourceTaskLogDAO; - private final GseTaskIpLogDAO gseTaskIpLogDAO; - private final HostService hostService; + private final ScriptAgentTaskService scriptAgentTaskService; + private final FileAgentTaskService fileAgentTaskService; private final LogService logService; private final ExecuteAuthService executeAuthService; private final TaskOperationLogService operationLogService; + private final RollingConfigService rollingConfigService; + private final StepInstanceRollingTaskService stepInstanceRollingTaskService; + private final HostService hostService; @Autowired public TaskResultServiceImpl(TaskInstanceDAO taskInstanceDAO, StepInstanceDAO stepInstanceDAO, - GseTaskLogService gseTaskLogService, + TaskInstanceService taskInstanceService, FileSourceTaskLogDAO fileSourceTaskLogDAO, - GseTaskIpLogDAO gseTaskIpLogDAO, - HostService hostService, + ScriptAgentTaskService scriptAgentTaskService, + FileAgentTaskService fileAgentTaskService, LogService logService, ExecuteAuthService executeAuthService, - TaskOperationLogService operationLogService) { + TaskOperationLogService operationLogService, + RollingConfigService rollingConfigService, + StepInstanceRollingTaskService stepInstanceRollingTaskService, + HostService hostService) { this.taskInstanceDAO = taskInstanceDAO; this.stepInstanceDAO = stepInstanceDAO; - this.gseTaskLogService = gseTaskLogService; + this.taskInstanceService = taskInstanceService; this.fileSourceTaskLogDAO = fileSourceTaskLogDAO; - this.gseTaskIpLogDAO = gseTaskIpLogDAO; - this.hostService = hostService; + this.scriptAgentTaskService = scriptAgentTaskService; + this.fileAgentTaskService = fileAgentTaskService; this.logService = logService; this.executeAuthService = executeAuthService; this.operationLogService = operationLogService; + this.rollingConfigService = rollingConfigService; + this.stepInstanceRollingTaskService = stepInstanceRollingTaskService; + this.hostService = hostService; } @Override @@ -132,9 +147,9 @@ public PageData listPageTaskInstance(TaskInstanceQuery taskQuer if (pageData != null && pageData.getData() != null && !pageData.getData().isEmpty()) { pageData.getData().forEach(taskInstanceDTO -> { if (taskInstanceDTO.getTotalTime() == null) { - if (taskInstanceDTO.getStatus().equals(RunStatusEnum.RUNNING.getValue()) - || taskInstanceDTO.getStatus().equals(RunStatusEnum.WAITING.getValue()) - || taskInstanceDTO.getStatus().equals(RunStatusEnum.STOPPING.getValue())) { + RunStatusEnum status = taskInstanceDTO.getStatus(); + if (status == RunStatusEnum.RUNNING || status == RunStatusEnum.WAITING_USER + || status == RunStatusEnum.STOPPING) { taskInstanceDTO.setTotalTime((TaskCostCalculator.calculate(taskInstanceDTO.getStartTime(), taskInstanceDTO.getEndTime(), taskInstanceDTO.getTotalTime()))); } @@ -169,10 +184,7 @@ public TaskExecuteResultDTO getTaskExecutionResult(String username, Long appId, } TaskExecuteResultDTO taskExecuteResult = new TaskExecuteResultDTO(); - boolean isFinish = (!taskInstance.getStatus().equals(RunStatusEnum.BLANK.getValue()) - && !taskInstance.getStatus().equals(RunStatusEnum.RUNNING.getValue()) - && !taskInstance.getStatus().equals(RunStatusEnum.STOPPING.getValue())); - taskExecuteResult.setFinished(isFinish); + taskExecuteResult.setFinished(RunStatusEnum.isFinishedStatus(taskInstance.getStatus())); taskExecuteResult.setTaskInstanceExecutionResult(taskExecution); taskExecuteResult.setStepInstanceExecutionResults(stepExecutionList); @@ -184,7 +196,7 @@ private TaskExecutionDTO buildTaskExecutionDTO(TaskInstanceDTO taskInstance) { taskExecution.setTaskInstanceId(taskInstance.getId()); taskExecution.setName(taskInstance.getName()); taskExecution.setType(taskInstance.getType()); - taskExecution.setStatus(taskInstance.getStatus()); + taskExecution.setStatus(taskInstance.getStatus().getValue()); taskExecution.setTotalTime(TaskCostCalculator.calculate(taskInstance.getStartTime(), taskInstance.getEndTime(), taskInstance.getTotalTime())); taskExecution.setStartTime(taskInstance.getStartTime()); @@ -192,7 +204,7 @@ private TaskExecutionDTO buildTaskExecutionDTO(TaskInstanceDTO taskInstance) { taskExecution.setTaskId(taskInstance.getTaskId()); taskExecution.setTaskTemplateId(taskInstance.getTaskTemplateId()); taskExecution.setDebugTask(taskInstance.isDebugTask()); - taskExecution.setCurrentStepInstanceId(taskInstance.getCurrentStepId()); + taskExecution.setCurrentStepInstanceId(taskInstance.getCurrentStepInstanceId()); return taskExecution; } @@ -201,7 +213,7 @@ private StepExecutionDTO buildStepExecutionDTO(StepInstanceBaseDTO stepInstance) stepExecution.setStepInstanceId(stepInstance.getId()); stepExecution.setName(stepInstance.getName()); stepExecution.setExecuteCount(stepInstance.getExecuteCount()); - stepExecution.setStatus(stepInstance.getStatus()); + stepExecution.setStatus(stepInstance.getStatus().getValue()); stepExecution.setType(StepTypeExecuteTypeConverter.convertToStepType(stepInstance.getExecuteType())); stepExecution.setStartTime(stepInstance.getStartTime()); stepExecution.setEndTime(stepInstance.getEndTime()); @@ -246,29 +258,29 @@ username, new AppResourceScope(appId), } /** - * 融入文件源文件拉取所使用的时间 + * 加入文件源文件拉取所使用的时间 * - * @param stepExecutionDetailDTO - * @param fileSourceTaskLog + * @param stepExecutionDetail 步骤执行详情 + * @param fileSourceTaskLog 第三方文件源任务 */ - private void involveFileSourceTaskLog(StepExecutionDetailDTO stepExecutionDetailDTO, + private void involveFileSourceTaskLog(StepExecutionDetailDTO stepExecutionDetail, FileSourceTaskLogDTO fileSourceTaskLog) { - List resultGroups = stepExecutionDetailDTO.getResultGroups(); - for (ExecutionResultGroupDTO resultGroup : resultGroups) { + List resultGroups = stepExecutionDetail.getResultGroups(); + for (AgentTaskResultGroupDTO resultGroup : resultGroups) { if (resultGroup == null) { continue; } - List agentTaskExecutionDetailList = resultGroup.getAgentTaskExecutionDetail(); + List agentTaskExecutionDetailList = resultGroup.getAgentTasks(); if (agentTaskExecutionDetailList == null) { continue; } - for (AgentTaskExecutionDTO agentTaskDetail : agentTaskExecutionDetailList) { + for (AgentTaskDTO agentTaskDetail : agentTaskExecutionDetailList) { if (agentTaskDetail == null) { continue; } agentTaskDetail.setStartTime(fileSourceTaskLog.getStartTime()); if (agentTaskDetail.getEndTime() == null || agentTaskDetail.getEndTime() == 0) { - agentTaskDetail.setEndTime(stepExecutionDetailDTO.getEndTime()); + agentTaskDetail.setEndTime(stepExecutionDetail.getEndTime()); } agentTaskDetail.calculateTotalTime(); } @@ -276,7 +288,7 @@ private void involveFileSourceTaskLog(StepExecutionDetailDTO stepExecutionDetail } private StepInstanceBaseDTO checkGetStepExecutionDetail(String username, long appId, long stepInstanceId) { - StepInstanceBaseDTO stepInstance = stepInstanceDAO.getStepInstanceBase(stepInstanceId); + StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(stepInstanceId); if (stepInstance == null) { log.warn("Step instance is not exist, stepInstanceId={}", stepInstanceId); throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); @@ -291,9 +303,9 @@ private StepInstanceBaseDTO checkGetStepExecutionDetail(String username, long ap return stepInstance; } - private boolean isMatchResultGroup(ExecutionResultGroupDTO resultGroup, Integer resultType, String tag) { + private boolean isMatchResultGroup(AgentTaskResultGroupDTO resultGroup, Integer resultType, String tag) { String matchTag = tag == null ? "" : tag; - return resultType.equals(resultGroup.getResultType()) && matchTag.equals(resultGroup.getTag()); + return resultType.equals(resultGroup.getStatus()) && matchTag.equals(resultGroup.getTag()); } private List getLimitedSizedList(List list, Integer maxSize) { @@ -308,28 +320,6 @@ private List getLimitedSizedList(List list, Integer maxSize) { } } - private StepExecutionDetailDTO buildStepExecutionDetailDTO(StepInstanceBaseDTO stepInstance, - GseTaskLogDTO gseTaskLog, Integer executeCount) { - StepExecutionDetailDTO executeDetail = new StepExecutionDetailDTO(); - executeDetail.setStepInstanceId(stepInstance.getId()); - executeDetail.setExecuteCount(executeCount); - - executeDetail.setName(stepInstance.getName()); - - executeDetail.setFinished(!stepInstance.getStatus().equals(RunStatusEnum.BLANK.getValue()) - && !stepInstance.getStatus().equals(RunStatusEnum.RUNNING.getValue()) - && !stepInstance.getStatus().equals(RunStatusEnum.STOPPING.getValue())); - executeDetail.setStatus(stepInstance.getStatus()); - executeDetail.setStartTime(gseTaskLog.getStartTime()); - executeDetail.setEndTime(gseTaskLog.getEndTime()); - executeDetail.setTotalTime(TaskCostCalculator.calculate(gseTaskLog.getStartTime(), gseTaskLog.getEndTime(), - gseTaskLog.getTotalTime())); - executeDetail.setLastStep(stepInstance.isLastStep()); - executeDetail.setStepType(stepInstance.getStepType()); - executeDetail.setGseTaskId(gseTaskLog.getGseTaskId()); - return executeDetail; - } - @Override public StepExecutionDetailDTO getFastTaskStepExecutionResult(String username, Long appId, Long taskInstanceId, StepExecutionResultQuery query) { @@ -348,268 +338,232 @@ public StepExecutionDetailDTO getFastTaskStepExecutionResult(String username, Lo return getStepExecutionResult(username, appId, query); } - private void setAgentTasksForSpecifiedResultType(List resultGroups, - long appId, Integer resultType, String tag, - List ipLogsForResultType) { - for (ExecutionResultGroupDTO resultGroup : resultGroups) { - if (resultType.equals(resultGroup.getResultType()) && ( + private void setAgentTasksForSpecifiedResultType(List resultGroups, + Integer status, + String tag, + List agentTasksForResultType) { + for (AgentTaskResultGroupDTO resultGroup : resultGroups) { + if (status.equals(resultGroup.getStatus()) && ( (StringUtils.isEmpty(tag) ? StringUtils.isEmpty(resultGroup.getTag()) : tag.equals(resultGroup.getTag())))) { - fillAgentTasksForResultGroup(resultGroup, appId, ipLogsForResultType); + resultGroup.setAgentTasks(agentTasksForResultType); } } } - private List getExecutionResultGroupsFromDB(long stepInstanceId, int executeCount) { - List baseResultGroups = gseTaskIpLogDAO.getResultGroups(stepInstanceId, executeCount); - return baseResultGroups.stream().map(this::buildExecutionResultGroup) - .collect(Collectors.toList()); - } - - private void setAgentTasksForAnyResultType(List resultGroups, - long appId, long stepInstanceId, - int executeCount, Integer maxAgentTasksForResultGroup) { + private void setAgentTasksForAnyResultType(List resultGroups, + StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch, + Integer maxAgentTasksForResultGroup) { boolean isAgentTaskSet = false; - for (ExecutionResultGroupDTO resultGroup : resultGroups) { + for (AgentTaskResultGroupDTO resultGroup : resultGroups) { if (!isAgentTaskSet) { - isAgentTaskSet = fillAgentTasksForResultGroup(resultGroup, appId, stepInstanceId, executeCount, - resultGroup.getResultType(), resultGroup.getTag(), maxAgentTasksForResultGroup); + isAgentTaskSet = fillAgentTasksForResultGroup(resultGroup, stepInstance, executeCount, + batch, resultGroup.getStatus(), resultGroup.getTag(), maxAgentTasksForResultGroup); } else { return; } } } - private ExecutionResultGroupDTO buildExecutionResultGroup(ResultGroupBaseDTO baseResultGroup) { - ExecutionResultGroupDTO resultGroup = new ExecutionResultGroupDTO(); - resultGroup.setResultType(baseResultGroup.getResultType()); - resultGroup.setTag(baseResultGroup.getTag()); - resultGroup.setAgentTaskSize(baseResultGroup.getAgentTaskCount()); - return resultGroup; - } - - private boolean fillAgentTasksForResultGroup(ExecutionResultGroupDTO resultGroup, long appId, long stepInstanceId, + private boolean fillAgentTasksForResultGroup(AgentTaskResultGroupDTO resultGroup, + StepInstanceBaseDTO stepInstance, int executeCount, - Integer resultType, String tag, Integer maxAgentTasksForResultGroup) { - List ipLogGroupByResultType = gseTaskIpLogDAO.getIpLogByResultType(stepInstanceId, - executeCount, resultType, tag, maxAgentTasksForResultGroup, null, null); - return fillAgentTasksForResultGroup(resultGroup, appId, ipLogGroupByResultType); - } - - private boolean fillAgentTasksForResultGroup(ExecutionResultGroupDTO resultGroup, long appId, - List ipLogGroupByResultType) { - if (CollectionUtils.isEmpty(ipLogGroupByResultType)) { + Integer batch, + Integer status, + String tag, + Integer maxAgentTasksForResultGroup) { + List agentTasks = listAgentTaskByResultGroup(stepInstance, + executeCount, batch, status, tag, maxAgentTasksForResultGroup, null, null); + if (CollectionUtils.isEmpty(agentTasks)) { return false; } - List agentTaskExecutions = ipLogGroupByResultType.stream() - .map(gseTaskIpLog -> buildAgentTaskExecutionDTO(appId, gseTaskIpLog)).collect(Collectors.toList()); - resultGroup.setAgentTaskExecutionDetail(agentTaskExecutions); + resultGroup.setAgentTasks(agentTasks); return true; } - private AgentTaskExecutionDTO buildAgentTaskExecutionDTO(long appId, GseTaskIpLogDTO gseTaskIpLog) { - AgentTaskExecutionDTO agentTaskExecution = new AgentTaskExecutionDTO(); - agentTaskExecution.setCloudIp(gseTaskIpLog.getCloudAreaAndIp()); - long cloudAreaId = Long.parseLong(gseTaskIpLog.getCloudAreaAndIp().split(":")[0]); - agentTaskExecution.setCloudAreaId(cloudAreaId); - agentTaskExecution.setCloudAreaName(hostService.getCloudAreaName(cloudAreaId)); - agentTaskExecution.setDisplayIp(gseTaskIpLog.getDisplayIp()); - agentTaskExecution.setStatus(gseTaskIpLog.getStatus()); - agentTaskExecution.setTag(gseTaskIpLog.getTag()); - agentTaskExecution.setErrorCode(gseTaskIpLog.getErrCode()); - agentTaskExecution.setExitCode(gseTaskIpLog.getExitCode()); - agentTaskExecution.setStartTime(gseTaskIpLog.getStartTime()); - agentTaskExecution.setEndTime(gseTaskIpLog.getEndTime()); - agentTaskExecution.setTotalTime(gseTaskIpLog.getTotalTime()); - agentTaskExecution.setExecuteCount(gseTaskIpLog.getExecuteCount()); - return agentTaskExecution; - } - - private List buildAgentTaskExecutionDTOList(long appId, - List gseTaskIpLogs) { - return gseTaskIpLogs.stream().map(gseTaskIpLog -> buildAgentTaskExecutionDTO(appId, gseTaskIpLog)) - .collect(Collectors.toList()); - } - - /* - * 任务未下发到gse的情况下，构造步骤执行返回结果 + /** + * 步骤未启动，AgentTask数据还未在DB初始化，构造初始任务结果 + * + * @param stepInstance 步骤实例 + * @param queryExecuteCount 执行次数 + * @param batch 滚动批次 + * @param maxAgentTasksForResultGroup 任务分组下最大返回的AgentTask数量 + * @param fuzzySearchIp 模糊查询IP + * @return 步骤执行结果 */ - private StepExecutionDetailDTO buildNotStartStepExecutionResult(long appId, StepInstanceBaseDTO stepInstance, - Integer executeCount, + private StepExecutionDetailDTO buildNotStartStepExecutionResult(StepInstanceBaseDTO stepInstance, + Integer queryExecuteCount, + Integer batch, Integer maxAgentTasksForResultGroup, String fuzzySearchIp) { - long stepInstanceId = stepInstance.getId(); - int retry = executeCount == null ? stepInstance.getExecuteCount() : executeCount; - - StepExecutionDetailDTO executeDetail = new StepExecutionDetailDTO(); - executeDetail.setStepInstanceId(stepInstanceId); - executeDetail.setExecuteCount(retry); - - executeDetail.setName(stepInstance.getName()); - - executeDetail.setFinished(false); - executeDetail.setStatus(stepInstance.getStatus()); - executeDetail.setStartTime(stepInstance.getStartTime()); - executeDetail.setEndTime(stepInstance.getEndTime()); - executeDetail.setTotalTime(0L); - executeDetail.setLastStep(stepInstance.isLastStep()); - executeDetail.setStepType(stepInstance.getStepType()); - - List resultGroups = new ArrayList<>(); - ExecutionResultGroupDTO resultGroup = new ExecutionResultGroupDTO(); - resultGroup.setResultType(IpStatus.WAITING.getValue()); + + StepExecutionDetailDTO stepExecuteDetail = new StepExecutionDetailDTO(stepInstance); + stepExecuteDetail.setExecuteCount(queryExecuteCount); + + List resultGroups = new ArrayList<>(); + AgentTaskResultGroupDTO resultGroup = new AgentTaskResultGroupDTO(); + resultGroup.setStatus(AgentTaskStatusEnum.WAITING.getValue()); resultGroup.setTag(null); - List agentTaskExecutionDTOS = new ArrayList<>(); + List targetHosts = filterTargetHostsByBatch(stepInstance, batch); + + List agentTasks = new ArrayList<>(); + // 如果需要根据IP过滤，那么需要重新计算Agent任务总数 boolean fuzzyFilterByIp = StringUtils.isNotEmpty(fuzzySearchIp); - int agentTaskSize = stepInstance.getTargetServers().getIpList().size(); + int agentTaskSize = targetHosts.size(); if (fuzzyFilterByIp) { - agentTaskSize = (int) stepInstance.getTargetServers().getIpList().stream() + agentTaskSize = (int) targetHosts.stream() .filter(ipDTO -> ipDTO.getIp().contains(fuzzySearchIp)).count(); } - if (stepInstance.getTargetServers().getIpList() != null) { + resultGroup.setTotalAgentTasks(agentTaskSize); + + if (CollectionUtils.isNotEmpty(targetHosts)) { int maxAgentTasks = (maxAgentTasksForResultGroup != null ? - Math.min(maxAgentTasksForResultGroup, stepInstance.getTargetServers().getIpList().size()) - : stepInstance.getTargetServers().getIpList().size()); - for (IpDTO ipDTO : stepInstance.getTargetServers().getIpList()) { - String ip = ipDTO.getIp(); + Math.min(maxAgentTasksForResultGroup, targetHosts.size()) : targetHosts.size()); + for (HostDTO targetHost : targetHosts) { + String ip = targetHost.getIp(); if (fuzzyFilterByIp && !ip.contains(fuzzySearchIp)) { + // 如果需要根据IP过滤，那么过滤掉不匹配的任务 continue; } if (maxAgentTasks-- > 0) { - AgentTaskExecutionDTO agentTaskExecutionDTO = new AgentTaskExecutionDTO(); - agentTaskExecutionDTO.setCloudIp(ipDTO.getCloudAreaId() + ":" + ipDTO.getIp()); - Long cloudAreaId = ipDTO.getCloudAreaId(); - agentTaskExecutionDTO.setCloudAreaId(cloudAreaId); - agentTaskExecutionDTO.setCloudAreaName(hostService.getCloudAreaName(cloudAreaId)); - agentTaskExecutionDTO.setDisplayIp(ipDTO.getIp()); - agentTaskExecutionDTO.setStatus(IpStatus.WAITING.getValue()); - agentTaskExecutionDTO.setTag(null); - agentTaskExecutionDTO.setErrorCode(0); - agentTaskExecutionDTO.setExitCode(0); - agentTaskExecutionDTO.setTotalTime(0L); - agentTaskExecutionDTOS.add(agentTaskExecutionDTO); + AgentTaskDetailDTO agentTask = new AgentTaskDetailDTO(); + agentTask.setHostId(targetHost.getHostId()); + agentTask.setIp(targetHost.getIp()); + agentTask.setBkCloudId(targetHost.getBkCloudId()); + agentTask.setBkCloudName(hostService.getCloudAreaName(targetHost.getBkCloudId())); + agentTask.setStatus(AgentTaskStatusEnum.WAITING); + agentTask.setTag(null); + agentTask.setErrorCode(0); + agentTask.setExitCode(0); + agentTask.setTotalTime(0L); + agentTasks.add(agentTask); } } } - resultGroup.setAgentTaskExecutionDetail(agentTaskExecutionDTOS); - resultGroup.setAgentTaskSize(agentTaskSize); + resultGroup.setAgentTasks(agentTasks); resultGroups.add(resultGroup); + stepExecuteDetail.setResultGroups(resultGroups); - executeDetail.setResultGroups(resultGroups); - return executeDetail; + if (stepInstance.isRollingStep()) { + setRollingInfoForStep(stepInstance, stepExecuteDetail); + } else { + stepExecuteDetail.setRunMode(StepRunModeEnum.RUN_ALL); + } + + return stepExecuteDetail; + } + + private List filterTargetHostsByBatch(StepInstanceBaseDTO stepInstance, Integer batch) { + List targetServers; + if (stepInstance.isRollingStep()) { + targetServers = rollingConfigService.getRollingServers(stepInstance, batch); + } else { + targetServers = stepInstance.getTargetServers().getIpList(); + } + + return targetServers; } @Override public StepExecutionDetailDTO getStepExecutionResult(String username, Long appId, StepExecutionResultQuery query) throws ServiceException { - Long stepInstanceId = query.getStepInstanceId(); StopWatch watch = new StopWatch("getStepExecutionResult"); - watch.start("checkGetStepExecutionDetail"); - StepInstanceBaseDTO stepInstance = checkGetStepExecutionDetail(username, appId, stepInstanceId); - int finalExecuteCount = query.getExecuteCount() == null ? stepInstance.getExecuteCount() : - query.getExecuteCount(); - query.setExecuteCount(finalExecuteCount); - watch.stop(); + try { + Long stepInstanceId = query.getStepInstanceId(); + + watch.start("checkGetStepExecutionDetail"); + StepInstanceBaseDTO stepInstance = checkGetStepExecutionDetail(username, appId, stepInstanceId); + int queryExecuteCount = query.getExecuteCount() == null ? stepInstance.getExecuteCount() : + query.getExecuteCount(); + query.setExecuteCount(queryExecuteCount); + if (stepInstance.isRollingStep() && query.isFilterByLatestBatch()) { + query.setBatch(stepInstance.getBatch()); + } + watch.stop(); - GseTaskLogDTO gseTaskLog = gseTaskLogService.getGseTaskLog(stepInstance.getId(), finalExecuteCount); - if (gseTaskLog == null) { - return buildNotStartStepExecutionResult(appId, stepInstance, finalExecuteCount, - query.getMaxAgentTasksForResultGroup(), query.getSearchIp()); - } + if (stepInstance.getStatus() == RunStatusEnum.BLANK) { + // 步骤未启动，AgentTask数据还未在DB初始化，构造初始任务结果 + return buildNotStartStepExecutionResult(stepInstance, queryExecuteCount, query.getBatch(), + query.getMaxAgentTasksForResultGroup(), query.getSearchIp()); + } - StepExecutionDetailDTO stepExecutionDetail; - // 如果步骤的目标服务器数量<100,或者通过IP匹配的方式过滤agent任务，为了提升性能，直接全量从DB查询数据，在内存进行处理 - if ((stepInstance.getTargetServerTotalCount() <= 100) || query.hasIpCondition()) { - stepExecutionDetail = loadAllTasksFromDBAndBuildExecutionResultInMemory(watch, stepInstance, gseTaskLog, - query); - } else { - stepExecutionDetail = filterAndSortExecutionResultInDB(watch, stepInstance, gseTaskLog, query); - } + StepExecutionDetailDTO stepExecutionDetail; + // 如果步骤的目标服务器数量<100,或者通过IP匹配的方式过滤agent任务，为了提升性能，直接全量从DB查询数据，在内存进行处理 + if ((stepInstance.getTargetServerTotalCount() <= 100) || query.hasIpCondition()) { + stepExecutionDetail = loadAllTasksFromDBAndBuildExecutionResultInMemory(watch, stepInstance, query); + } else { + stepExecutionDetail = filterAndSortExecutionResultInDB(watch, stepInstance, query); + } - stepExecutionDetail.setStartTime(stepInstance.getStartTime()); - if (stepInstance.getEndTime() != null) { - stepExecutionDetail.setEndTime(stepInstance.getEndTime()); - stepExecutionDetail.calculateTotalTime(); - } + if (stepInstance.isRollingStep()) { + watch.start("setRollingTasksForStep"); + setRollingInfoForStep(stepInstance, stepExecutionDetail); + watch.stop(); + } else { + stepExecutionDetail.setRunMode(StepRunModeEnum.RUN_ALL); + } - watch.start("involveFileSourceTaskLog"); - FileSourceTaskLogDTO fileSourceTaskLog = fileSourceTaskLogDAO.getFileSourceTaskLog(stepInstance.getId(), - finalExecuteCount); - if (fileSourceTaskLog != null) { - involveFileSourceTaskLog(stepExecutionDetail, fileSourceTaskLog); - } - watch.stop(); + if (stepInstance.isFileStep()) { + watch.start("involveFileSourceTaskLog"); + FileSourceTaskLogDTO fileSourceTaskLog = fileSourceTaskLogDAO.getFileSourceTaskLog(stepInstance.getId(), + queryExecuteCount); + if (fileSourceTaskLog != null) { + involveFileSourceTaskLog(stepExecutionDetail, fileSourceTaskLog); + } + watch.stop(); + } - if (watch.getTotalTimeMillis() > 1000L) { - log.info("Get step execution detail is slow, watch: {}", watch.prettyPrint()); + return stepExecutionDetail; + } finally { + if (watch.isRunning()) { + watch.stop(); + } + if (watch.getTotalTimeMillis() > 1000L) { + log.info("Get step execution detail is slow, watch: {}", watch.prettyPrint()); + } } - return stepExecutionDetail; } private StepExecutionDetailDTO loadAllTasksFromDBAndBuildExecutionResultInMemory(StopWatch watch, StepInstanceBaseDTO stepInstance, - GseTaskLogDTO gseTaskLog, StepExecutionResultQuery query) { try { - long stepInstanceId = query.getStepInstanceId(); - int executeCount = query.getExecuteCount(); - long appId = stepInstance.getAppId(); - if (query.hasIpCondition()) { watch.start("getMatchIps"); - Set matchIps = getMatchIps(query); - if (CollectionUtils.isEmpty(matchIps)) { + Set matchHostIds = getMatchHostIdsByIp(stepInstance, query); + if (CollectionUtils.isEmpty(matchHostIds)) { watch.stop(); - return buildExecutionDetailWhenTaskAreEmpty(stepInstance, gseTaskLog); + return buildExecutionDetailWhenTaskAreEmpty(stepInstance, query.getBatch()); } else { - query.setMatchIps(matchIps); + query.setMatchHostIds(matchHostIds); } watch.stop(); } - watch.start("loadAllTasksFromDb"); - List tasks = gseTaskLogService.getIpLog(stepInstanceId, executeCount, true); - log.debug("tasks.size={}", tasks.size()); - watch.stop(); - watch.start("buildResultGroupsFromTasks"); - List resultGroups = buildResultGroupsFromTasks(appId, tasks); - watch.stop(); + watch.start("loadAllTasksFromDbAndGroup"); + List resultGroups = listAndGroupAgentTasks(stepInstance, + query.getExecuteCount(), query.getBatch()); - if (CollectionUtils.isNotEmpty(query.getMatchIps())) { - watch.start("filterExecutionResultByMatchIp"); - filterExecutionResultByMatchIp(resultGroups, query.getMatchIps()); - watch.stop(); + if (CollectionUtils.isNotEmpty(query.getMatchHostIds())) { + filterAgentTasksByMatchIp(resultGroups, query.getMatchHostIds()); } - watch.start("removeAgentTasksExecutionDetail"); - removeAgentTasksExecutionDetail(resultGroups, query.getResultType(), query.getTag()); + removeAgentTasksForNotSpecifiedResultGroup(resultGroups, query.getStatus(), query.getTag()); watch.stop(); - ExecutionResultGroupDTO resultGroupWithTasks = resultGroups.stream() - .filter(resultGroup -> CollectionUtils.isNotEmpty(resultGroup.getAgentTaskExecutionDetail())) - .findFirst().orElse(null); - if (resultGroupWithTasks == null) { - StepExecutionDetailDTO executeDetail = buildStepExecutionDetailDTO(stepInstance, gseTaskLog, - executeCount); - executeDetail.setResultGroups(resultGroups); - return executeDetail; - } watch.start("sortAndLimitTasks"); - sortAgentTasks(resultGroupWithTasks, query); - - if (query.getMaxAgentTasksForResultGroup() != null) { - resultGroupWithTasks.setAgentTaskExecutionDetail( - getLimitedSizedList(resultGroupWithTasks.getAgentTaskExecutionDetail(), - query.getMaxAgentTasksForResultGroup())); - } + sortAgentTasksAndLimitSize(resultGroups, query); + watch.stop(); - StepExecutionDetailDTO executeDetail = buildStepExecutionDetailDTO(stepInstance, gseTaskLog, executeCount); + StepExecutionDetailDTO executeDetail = new StepExecutionDetailDTO(stepInstance); + executeDetail.setExecuteCount(query.getExecuteCount()); executeDetail.setResultGroups(resultGroups); - watch.stop(); return executeDetail; } finally { if (watch.isRunning()) { @@ -619,191 +573,279 @@ private StepExecutionDetailDTO loadAllTasksFromDBAndBuildExecutionResultInMemory } - private void sortAgentTasks(ExecutionResultGroupDTO resultGroupWithTasks, StepExecutionResultQuery query) { - if (StringUtils.isNotEmpty(query.getOrderField())) { - List taskList = resultGroupWithTasks.getAgentTaskExecutionDetail(); - if (StepExecutionResultQuery.ORDER_FIELD_TOTAL_TIME.equals(query.getOrderField())) { - taskList.sort(Comparator.comparingLong(task -> task.getTotalTime() == null ? 0L : task.getTotalTime())); - if (query.getOrder() == DESCENDING) { - Collections.reverse(taskList); - } - } else if (StepExecutionResultQuery.ORDER_FIELD_EXIT_CODE.equals(query.getOrderField())) { - taskList.sort((o1, o2) -> { - if (o1.getExitCode() != null && o2.getExitCode() != null) { - if (o1.getExitCode().equals(o2.getExitCode())) { - return 0; - } else { - return o1.getExitCode() > o2.getExitCode() ? 1 : -1; + private List listAndGroupAgentTasks(StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch) { + List resultGroups = null; + if (stepInstance.isScriptStep()) { + resultGroups = scriptAgentTaskService.listAndGroupAgentTasks(stepInstance, executeCount, batch); + } else if (stepInstance.isFileStep()) { + resultGroups = fileAgentTaskService.listAndGroupAgentTasks(stepInstance, executeCount, batch); + } + return resultGroups; + + } + + private void sortAgentTasksAndLimitSize(List resultGroups, + StepExecutionResultQuery query) { + resultGroups.stream() + .filter(resultGroup -> CollectionUtils.isNotEmpty(resultGroup.getAgentTasks())) + .forEach(resultGroup -> { + // 排序 + if (StringUtils.isNotEmpty(query.getOrderField())) { + List agentTasks = resultGroup.getAgentTasks(); + if (StepExecutionResultQuery.ORDER_FIELD_TOTAL_TIME.equals(query.getOrderField())) { + agentTasks.sort(Comparator.comparingLong(task -> task.getTotalTime() == null ? 0L : + task.getTotalTime())); + if (query.getOrder() == DESCENDING) { + Collections.reverse(agentTasks); + } + } else if (StepExecutionResultQuery.ORDER_FIELD_EXIT_CODE.equals(query.getOrderField())) { + agentTasks.sort((o1, o2) -> { + if (o1.getExitCode() != null && o2.getExitCode() != null) { + if (o1.getExitCode().equals(o2.getExitCode())) { + return 0; + } else { + return o1.getExitCode() > o2.getExitCode() ? 1 : -1; + } + } else if (o1.getExitCode() == null) { + return -1; + } else if (o2.getExitCode() == null) { + return 1; + } else { + return 0; + } + }); + if (query.getOrder() == DESCENDING) { + Collections.reverse(agentTasks); } - } else if (o1.getExitCode() == null) { - return -1; - } else if (o2.getExitCode() == null) { - return 1; - } else { - return 0; } - }); - if (query.getOrder() == DESCENDING) { - Collections.reverse(taskList); } - } else if (StepExecutionResultQuery.ORDER_FIELD_CLOUD_AREA_ID.equals(query.getOrderField())) { - taskList.sort(Comparator.comparing(AgentTaskExecutionDTO::getCloudIp)); - if (query.getOrder() == DESCENDING) { - Collections.reverse(taskList); + + // 截断 + if (query.getMaxAgentTasksForResultGroup() != null) { + resultGroup.setAgentTasks( + getLimitedSizedList(resultGroup.getAgentTasks(), query.getMaxAgentTasksForResultGroup())); } - } - } + }); } - private Set getMatchIps(StepExecutionResultQuery query) { + private Set getMatchHostIdsByIp(StepInstanceBaseDTO stepInstance, StepExecutionResultQuery query) { long stepInstanceId = query.getStepInstanceId(); int executeCount = query.getExecuteCount(); - Set matchIpsByLogKeywordSearch = null; + Set matchHostIdsByLogKeywordSearch = null; if (StringUtils.isNotBlank(query.getLogKeyword())) { - List matchHosts = getHostsByLogContentKeyword(stepInstanceId, executeCount, query.getLogKeyword()); + List matchHosts = getHostsByLogContentKeyword(stepInstanceId, executeCount, + query.getBatch(), query.getLogKeyword()); if (CollectionUtils.isNotEmpty(matchHosts)) { - matchIpsByLogKeywordSearch = matchHosts.stream().map(IpDTO::convertToStrIp).collect(Collectors.toSet()); + matchHostIdsByLogKeywordSearch = + matchHosts.stream().map(HostDTO::getHostId).collect(Collectors.toSet()); } } - Set matchIpsByIpSearch = null; + Set matchHostIdsByIpSearch = null; if (StringUtils.isNotBlank(query.getSearchIp())) { - List matchHosts = fuzzySearchHostsByIp(stepInstanceId, executeCount, query.getSearchIp()); + List matchHosts = fuzzySearchHostsByIp(stepInstance, query.getSearchIp()); if (CollectionUtils.isNotEmpty(matchHosts)) { - matchIpsByIpSearch = matchHosts.stream().map(IpDTO::convertToStrIp).collect(Collectors.toSet()); + matchHostIdsByIpSearch = matchHosts.stream().map(HostDTO::getHostId).collect(Collectors.toSet()); } } - log.info("matchIpsByLogKeywordSearch: {}, matchIpsByIpSearch: {}", matchIpsByLogKeywordSearch, - matchIpsByIpSearch); - - if (matchIpsByLogKeywordSearch != null && matchIpsByIpSearch != null) { - return new HashSet<>(CollectionUtils.intersection(matchIpsByLogKeywordSearch, matchIpsByIpSearch)); - } else if (matchIpsByLogKeywordSearch != null) { - return matchIpsByLogKeywordSearch; - } else if (matchIpsByIpSearch != null) { - return matchIpsByIpSearch; + if (matchHostIdsByLogKeywordSearch != null && matchHostIdsByIpSearch != null) { + return new HashSet<>(CollectionUtils.intersection(matchHostIdsByLogKeywordSearch, matchHostIdsByIpSearch)); + } else if (matchHostIdsByLogKeywordSearch != null) { + return matchHostIdsByLogKeywordSearch; + } else if (matchHostIdsByIpSearch != null) { + return matchHostIdsByIpSearch; } else { return Collections.emptySet(); } } private StepExecutionDetailDTO buildExecutionDetailWhenTaskAreEmpty(StepInstanceBaseDTO stepInstance, - GseTaskLogDTO gseTaskLog) { - List baseResultGroups = gseTaskIpLogDAO - .getResultGroups(stepInstance.getId(), stepInstance.getExecuteCount()); - StepExecutionDetailDTO executeDetail = buildStepExecutionDetailDTO(stepInstance, - gseTaskLog, stepInstance.getExecuteCount()); - List resultGroups = new ArrayList<>(); - for (ResultGroupBaseDTO baseResultGroup : baseResultGroups) { - ExecutionResultGroupDTO resultGroup = buildExecutionResultGroup(baseResultGroup); - resultGroup.setAgentTaskSize(0); - resultGroups.add(resultGroup); - } + Integer batch) { + List resultGroups = listAndGroupAgentTasks(stepInstance, + stepInstance.getExecuteCount(), batch); + StepExecutionDetailDTO executeDetail = new StepExecutionDetailDTO(stepInstance); executeDetail.setResultGroups(resultGroups); return executeDetail; } - private void filterExecutionResultByMatchIp(List resultGroups, Set matchIps) { - for (ExecutionResultGroupDTO resultGroup : resultGroups) { - List agentTaskExecutions = resultGroup.getAgentTaskExecutionDetail(); - if (CollectionUtils.isNotEmpty(agentTaskExecutions)) { - agentTaskExecutions = agentTaskExecutions.stream().filter( - agentTaskExecution -> matchIps.contains(agentTaskExecution.getCloudIp())).collect( - Collectors.toList()); - resultGroup.setAgentTaskExecutionDetail(agentTaskExecutions); - resultGroup.setAgentTaskSize(agentTaskExecutions.size()); + private void filterAgentTasksByMatchIp(List resultGroups, Set matchHostIds) { + for (AgentTaskResultGroupDTO resultGroup : resultGroups) { + List agentTasks = resultGroup.getAgentTasks(); + if (CollectionUtils.isNotEmpty(agentTasks)) { + agentTasks = agentTasks.stream() + .filter(agentTask -> matchHostIds.contains(agentTask.getHostId())) + .collect(Collectors.toList()); + resultGroup.setAgentTasks(agentTasks); } } } - private void removeAgentTasksExecutionDetail(List resultGroups, Integer resultType, - String tag) { - if (resultType != null && resultGroups.contains(new ExecutionResultGroupDTO(resultType, tag))) { + private void removeAgentTasksForNotSpecifiedResultGroup(List resultGroups, Integer status, + String tag) { + if (status != null && resultGroups.contains(new AgentTaskResultGroupDTO(status, tag))) { resultGroups.forEach(resultGroup -> { - if (!isMatchResultGroup(resultGroup, resultType, tag)) { - resultGroup.setAgentTaskExecutionDetail(Collections.emptyList()); + if (!isMatchResultGroup(resultGroup, status, tag)) { + resultGroup.setAgentTasks(Collections.emptyList()); } }); } else { int i = 0; - for (ExecutionResultGroupDTO resultGroup : resultGroups) { + for (AgentTaskResultGroupDTO resultGroup : resultGroups) { i++; if (i != 1) { - resultGroup.setAgentTaskExecutionDetail(Collections.emptyList()); + resultGroup.setAgentTasks(Collections.emptyList()); } } } } - private List buildResultGroupsFromTasks(long appId, List tasks) { - Map> resultGroups = new HashMap<>(); - // 对执行结果按状态status与标签tag进行分组 - tasks.forEach(task -> resultGroups.computeIfAbsent(task.getExecutionResultGroup(), - resultGroup -> new ArrayList<>()).add(task)); - resultGroups.forEach((resultGroup, taskList) -> { - List agentTasks = buildAgentTaskExecutionDTOList(appId, taskList); - resultGroup.setAgentTaskExecutionDetail(agentTasks); - resultGroup.setAgentTaskSize(agentTasks.size()); - }); - return new ArrayList<>(resultGroups.keySet()); - } - private StepExecutionDetailDTO filterAndSortExecutionResultInDB(StopWatch watch, StepInstanceBaseDTO stepInstance, - GseTaskLogDTO gseTaskLog, StepExecutionResultQuery query) { query.transformOrderFieldToDbField(); - long stepInstanceId = query.getStepInstanceId(); - int executeCount = query.getExecuteCount(); - Integer resultType = query.getResultType(); + int queryExecuteCount = query.getExecuteCount(); + Integer status = query.getStatus(); String tag = query.getTag(); - long appId = stepInstance.getAppId(); - StepExecutionDetailDTO executeDetail = buildStepExecutionDetailDTO(stepInstance, gseTaskLog, executeCount); + StepExecutionDetailDTO executeDetail = new StepExecutionDetailDTO(stepInstance); + executeDetail.setExecuteCount(queryExecuteCount); watch.start("getBaseResultGroups"); - List resultGroups = getExecutionResultGroupsFromDB(stepInstanceId, executeCount); + List baseResultGroups = listBaseResultGroups(stepInstance, + queryExecuteCount, query.getBatch()); watch.stop(); - if (resultType != null) { - watch.start("loadTasksFromDbForResultType"); - List tasks = gseTaskIpLogDAO.getIpLogByResultType(stepInstanceId, executeCount, - resultType, tag, query.getMaxAgentTasksForResultGroup(), query.getOrderField(), query.getOrder()); - watch.stop(); + + watch.start("setAgentTasks"); + List resultGroups = baseResultGroups.stream() + .map(AgentTaskResultGroupDTO::new) + .collect(Collectors.toList()); + if (status != null) { + List tasks = listAgentTaskByResultGroup(stepInstance, queryExecuteCount, + query.getBatch(), status, tag, query.getMaxAgentTasksForResultGroup(), query.getOrderField(), + query.getOrder()); if (CollectionUtils.isNotEmpty(tasks)) { - watch.start("setAgentTasksForSpecifiedResultType"); - setAgentTasksForSpecifiedResultType(resultGroups, appId, resultType, tag, tasks); - watch.stop(); + setAgentTasksForSpecifiedResultType(resultGroups, status, tag, tasks); } else { - watch.start("setAgentTasksForAnyResultType"); - setAgentTasksForAnyResultType(resultGroups, appId, stepInstanceId, executeCount, - query.getMaxAgentTasksForResultGroup()); - watch.stop(); + setAgentTasksForAnyResultType(resultGroups, stepInstance, queryExecuteCount, + query.getBatch(), query.getMaxAgentTasksForResultGroup()); } } else { - watch.start("setAgentTasksForAnyResultType"); - setAgentTasksForAnyResultType(resultGroups, appId, stepInstanceId, executeCount, - query.getMaxAgentTasksForResultGroup()); - watch.stop(); + setAgentTasksForAnyResultType(resultGroups, stepInstance, queryExecuteCount, + query.getBatch(), query.getMaxAgentTasksForResultGroup()); } + watch.stop(); + executeDetail.setResultGroups(resultGroups); return executeDetail; } - private List getHostsByLogContentKeyword(long stepInstanceId, int executeCount, String keyword) { + private List listBaseResultGroups(StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch) { + List resultGroups = null; + if (stepInstance.isScriptStep()) { + resultGroups = scriptAgentTaskService.listResultGroups(stepInstance.getId(), executeCount, batch); + } else if (stepInstance.isFileStep()) { + resultGroups = fileAgentTaskService.listResultGroups(stepInstance.getId(), executeCount, batch); + } + return resultGroups; + } + + private List listAgentTaskByResultGroup(StepInstanceBaseDTO stepInstance, + int queryExecuteCount, + Integer batch, + Integer status, + String tag, + Integer maxAgentTasksForResultGroup, + String orderField, + Order order) { + List tasks = null; + if (stepInstance.isScriptStep()) { + tasks = scriptAgentTaskService.listAgentTaskDetailByResultGroup(stepInstance, queryExecuteCount, + batch, status, tag, maxAgentTasksForResultGroup, orderField, order); + } else if (stepInstance.isFileStep()) { + tasks = fileAgentTaskService.listAgentTaskDetailByResultGroup(stepInstance, queryExecuteCount, + batch, status, tag, maxAgentTasksForResultGroup, orderField, order); + } + return tasks; + } + + private List listAgentTaskByResultGroup(StepInstanceBaseDTO stepInstance, + int queryExecuteCount, + Integer batch, + Integer status, + String tag) { + List tasks = null; + if (stepInstance.isScriptStep()) { + tasks = scriptAgentTaskService.listAgentTaskDetailByResultGroup(stepInstance, queryExecuteCount, + batch, status, tag); + } else if (stepInstance.isFileStep()) { + tasks = fileAgentTaskService.listAgentTaskDetailByResultGroup(stepInstance, queryExecuteCount, + batch, status, tag); + } + return tasks; + } + + private void setRollingInfoForStep(StepInstanceBaseDTO stepInstance, + StepExecutionDetailDTO stepExecutionDetail) { + RollingConfigDTO rollingConfig = + rollingConfigService.getRollingConfig(stepInstance.getRollingConfigId()); + + Map latestStepInstanceRollingTasks = + stepInstanceRollingTaskService.listLatestRollingTasks( + stepExecutionDetail.getStepInstanceId(), + stepExecutionDetail.getExecuteCount()) + .stream() + .collect(Collectors.toMap(StepInstanceRollingTaskDTO::getBatch, + stepInstanceRollingTask -> stepInstanceRollingTask)); + + // 如果滚动任务还未调度，那么需要在结果中补充 + int totalBatch = rollingConfig.getConfigDetail().getTotalBatch(); + List stepInstanceRollingTasks = new ArrayList<>(); + for (int batch = 1; batch <= totalBatch; batch++) { + StepInstanceRollingTaskDTO stepInstanceRollingTask = latestStepInstanceRollingTasks.get(batch); + if (stepInstanceRollingTask == null) { + stepInstanceRollingTask = new StepInstanceRollingTaskDTO(); + stepInstanceRollingTask.setStepInstanceId(stepExecutionDetail.getStepInstanceId()); + stepInstanceRollingTask.setExecuteCount(stepInstance.getExecuteCount()); + stepInstanceRollingTask.setBatch(batch); + stepInstanceRollingTask.setStatus(RunStatusEnum.BLANK); + if (RunStatusEnum.WAITING_USER == stepInstance.getStatus() + && stepInstance.getBatch() + 1 == batch) { + // 如果当前步骤状态为"等待用户"，那么需要设置下一批次的滚动任务状态为WAITING_USER + stepInstanceRollingTask.setStatus(RunStatusEnum.WAITING_USER); + } else { + stepInstanceRollingTask.setStatus(RunStatusEnum.BLANK); + } + } + stepInstanceRollingTasks.add(stepInstanceRollingTask); + } + + stepExecutionDetail.setRollingTasks(stepInstanceRollingTasks); + stepExecutionDetail.setLatestBatch(stepInstance.getBatch()); + stepExecutionDetail.setRunMode(rollingConfig.isBatchRollingStep(stepInstance.getId()) ? + StepRunModeEnum.ROLLING_IN_BATCH : StepRunModeEnum.ROLLING_ALL); + } + + private List getHostsByLogContentKeyword(long stepInstanceId, int executeCount, Integer batch, + String keyword) { String searchKey = keyword.replaceAll("'", "").replaceAll("\\$", "") - .replaceAll("&", "").replaceAll("$", "") + .replaceAll("&", "").replaceAll("\\$", "") .replaceAll("\\|", "").replaceAll("`", "") .replaceAll(";", ""); - return logService.getIpsByContentKeyword(stepInstanceId, executeCount, searchKey); + return logService.getIpsByContentKeyword(stepInstanceId, executeCount, batch, searchKey); } - private List fuzzySearchHostsByIp(long stepInstanceId, int executeCount, String searchIp) { - return gseTaskIpLogDAO.fuzzySearchTargetIpsByIp(stepInstanceId, executeCount, searchIp) - .stream().map(IpDTO::fromCloudAreaIdAndIpStr).collect(Collectors.toList()); + private List fuzzySearchHostsByIp(StepInstanceBaseDTO stepInstance, String searchIp) { + return stepInstance.getTargetServers().getIpList().stream() + .filter(host -> host.getIp().contains(searchIp)) + .collect(Collectors.toList()); } @Override @@ -823,8 +865,8 @@ public Map getCronTaskExecuteResul // 已执行完成任务计数 int doneTaskCount = 0; for (TaskInstanceDTO taskInstance : last24HourTaskInstances) { - if (!taskInstance.getStatus().equals(RunStatusEnum.RUNNING.getValue()) - && !taskInstance.getStatus().equals(RunStatusEnum.STOPPING.getValue())) { + if (taskInstance.getStatus() != RunStatusEnum.RUNNING + && taskInstance.getStatus() != RunStatusEnum.STOPPING) { doneTaskCount++; } if (doneTaskCount >= 10) { @@ -863,15 +905,21 @@ private List convertToCronTaskExecuteResult(List getHostsByResultType(String username, Long appId, Long stepInstanceId, Integer executeCount, - Integer resultType, String tag, String keyword) { + public List getHostsByResultType(String username, + Long appId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + Integer resultType, + String tag, + String keyword) { StepInstanceBaseDTO stepInstance = checkGetStepExecutionDetail(username, appId, stepInstanceId); if (!stepInstance.getAppId().equals(appId)) { @@ -879,42 +927,39 @@ public List getHostsByResultType(String username, Long appId, Long stepIn throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); } - StepExecutionResultQuery query = new StepExecutionResultQuery(); - query.setStepInstanceId(stepInstanceId); - query.setExecuteCount(executeCount); - query.setLogKeyword(keyword); + StepExecutionResultQuery query = StepExecutionResultQuery.builder() + .stepInstanceId(stepInstanceId) + .executeCount(executeCount) + .batch(batch) + .logKeyword(keyword) + .build(); - Set matchIps = null; + Set matchHostIds = null; boolean filterByKeyword = StringUtils.isNotEmpty(keyword); if (filterByKeyword) { - matchIps = getMatchIps(query); - if (CollectionUtils.isEmpty(matchIps)) { + matchHostIds = getMatchHostIdsByIp(stepInstance, query); + if (CollectionUtils.isEmpty(matchHostIds)) { return Collections.emptyList(); } } - GseTaskLogDTO gseTaskLog = gseTaskLogService.getGseTaskLog(stepInstance.getId(), - stepInstance.getExecuteCount()); - if (gseTaskLog == null) { - if (stepInstance.getTargetServers().getIpList() != null) { - return stepInstance.getTargetServers().getIpList(); - } else { - return Collections.emptyList(); - } + if (stepInstance.getStatus() == RunStatusEnum.BLANK) { + // 步骤未启动，AgentTask数据还未在DB初始化，构造初始任务结果 + return filterTargetHostsByBatch(stepInstance, query.getBatch()); } - List ipLogGroupByResultType = gseTaskIpLogDAO.getIpLogByResultType(stepInstanceId, - executeCount, resultType, tag); - if (CollectionUtils.isEmpty(ipLogGroupByResultType)) { + List agentTaskGroupByResultType = listAgentTaskByResultGroup(stepInstance, + executeCount, batch, resultType, tag); + if (CollectionUtils.isEmpty(agentTaskGroupByResultType)) { return Collections.emptyList(); } - List hosts = ipLogGroupByResultType.stream() - .map(gseTaskIpLog -> new IpDTO(gseTaskIpLog.getCloudAreaId(), gseTaskIpLog.getIp())) + List hosts = agentTaskGroupByResultType.stream() + .map(AgentTaskDetailDTO::getHost) .collect(Collectors.toList()); - if (filterByKeyword && CollectionUtils.isNotEmpty(matchIps)) { - List finalHosts = new ArrayList<>(); - for (IpDTO host : hosts) { - if (matchIps.contains(host.convertToStrIp())) { + if (filterByKeyword && CollectionUtils.isNotEmpty(matchHostIds)) { + List finalHosts = new ArrayList<>(); + for (HostDTO host : hosts) { + if (matchHostIds.contains(host.getHostId())) { finalHosts.add(host); } } @@ -924,31 +969,46 @@ public List getHostsByResultType(String username, Long appId, Long stepIn } } + @Override - public List listStepExecutionHistory(String username, Long appId, Long stepInstanceId) { + public List listStepExecutionHistory(String username, + Long appId, + Long stepInstanceId, + Integer batch) { StepInstanceBaseDTO stepInstance = checkGetStepExecutionDetail(username, appId, stepInstanceId); - int latestExecuteCount = stepInstance.getExecuteCount(); - if (latestExecuteCount == 0) { + + // 步骤没有重试执行过 + if (stepInstance.getExecuteCount() == 0) { StepExecutionRecordDTO record = new StepExecutionRecordDTO(); record.setStepInstanceId(stepInstanceId); - record.setRetryCount(latestExecuteCount); + record.setRetryCount(0); record.setCreateTime(stepInstance.getCreateTime()); return Collections.singletonList(record); } - List operationLogs = operationLogService.listOperationLog(stepInstance.getTaskInstanceId()); - List records = new ArrayList<>(); - if (CollectionUtils.isEmpty(operationLogs)) { - for (int executeCount = latestExecuteCount; executeCount >= 0; executeCount--) { - StepExecutionRecordDTO record = new StepExecutionRecordDTO(); - record.setStepInstanceId(stepInstanceId); - record.setRetryCount(executeCount); - record.setCreateTime(stepInstance.getCreateTime()); - records.add(record); - } - return records; + List records; + if (batch == null || batch == 0) { + // 获取步骤维度的重试记录 + records = queryStepRetryRecords(stepInstance); + } else { + // 获取滚动任务维度的重试记录 + records = queryStepRollingTaskRetryRecords(stepInstanceId, batch); } + records.sort(Comparator.comparingInt(StepExecutionRecordDTO::getRetryCount).reversed()); + + return records; + } + + private List queryStepRetryRecords(StepInstanceBaseDTO stepInstance) { + Long stepInstanceId = stepInstance.getId(); + // 步骤最新的重试次数 + int latestExecuteCount = stepInstance.getExecuteCount(); + List operationLogs = + operationLogService.listOperationLog(stepInstance.getTaskInstanceId()); + + + List records = new ArrayList<>(); Map executeCountAndCreateTimeMap = new HashMap<>(); operationLogs.forEach(opLog -> { UserOperationEnum operation = opLog.getOperationEnum(); @@ -968,6 +1028,21 @@ public List listStepExecutionHistory(String username, Lo record.setCreateTime(executeCountAndCreateTimeMap.get(executeCount)); records.add(record); } + + return records; + } + + private List queryStepRollingTaskRetryRecords(long stepInstanceId, int batch) { + List records = new ArrayList<>(); + List rollingTasks = + stepInstanceRollingTaskService.listRollingTasksByBatch(stepInstanceId, batch); + rollingTasks.forEach(rollingTask -> { + StepExecutionRecordDTO record = new StepExecutionRecordDTO(); + record.setStepInstanceId(stepInstanceId); + record.setRetryCount(rollingTask.getExecuteCount()); + record.setCreateTime(rollingTask.getStartTime()); + records.add(record); + }); return records; } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/StatisticsServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/StatisticsServiceImpl.java index a1aed56c0c..b2e9c2117b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/StatisticsServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/StatisticsServiceImpl.java @@ -247,7 +247,7 @@ public void updateExecutedFastFileStatistics(TaskInstanceDTO taskInstanceDTO, public void updateFailedTaskCount(TaskInstanceDTO taskInstanceDTO, Map metricsMap) { // 累计执行失败次数统计 - if (RunStatusEnum.FAIL.getValue().equals(taskInstanceDTO.getStatus())) { + if (taskInstanceDTO.getStatus() == RunStatusEnum.FAIL) { StatisticsKey keyFailedTaskCount = new StatisticsKey(taskInstanceDTO.getAppId(), StatisticsConstants.RESOURCE_FAILED_TASK, StatisticsConstants.DIMENSION_TIME_UNIT, StatisticsConstants.DIMENSION_VALUE_TIME_UNIT_DAY); @@ -263,7 +263,8 @@ public void updateFastScriptCountByStatus(TaskInstanceDTO taskInstanceDTO, // 快速脚本按状态统计 Integer type = taskInstanceDTO.getType(); if (type != null && type.equals(TaskTypeEnum.SCRIPT.getValue())) { - if (RunStatusEnum.SUCCESS.getValue().equals(taskInstanceDTO.getStatus())) { + RunStatusEnum status = taskInstanceDTO.getStatus(); + if (status == RunStatusEnum.SUCCESS) { // 执行成功的快速脚本统计 StatisticsKey keySuccessFastScriptCount = new StatisticsKey(taskInstanceDTO.getAppId(), StatisticsConstants.RESOURCE_EXECUTED_FAST_SCRIPT, StatisticsConstants.DIMENSION_STEP_RUN_STATUS, @@ -272,7 +273,7 @@ public void updateFastScriptCountByStatus(TaskInstanceDTO taskInstanceDTO, statisticsKey -> new AtomicInteger(0)); int successFastScriptCountValue = successFastScriptCount.incrementAndGet(); log.debug("successFastScriptCount={}", successFastScriptCountValue); - } else if (RunStatusEnum.FAIL.getValue().equals(taskInstanceDTO.getStatus())) { + } else if (status == RunStatusEnum.FAIL) { // 执行失败的快速脚本统计 StatisticsKey keyFailedFastScriptCount = new StatisticsKey(taskInstanceDTO.getAppId(), StatisticsConstants.RESOURCE_EXECUTED_FAST_SCRIPT, StatisticsConstants.DIMENSION_STEP_RUN_STATUS, @@ -281,7 +282,7 @@ public void updateFastScriptCountByStatus(TaskInstanceDTO taskInstanceDTO, statisticsKey -> new AtomicInteger(0)); int failedFastScriptCountValue = failedFastScriptCount.incrementAndGet(); log.debug("failedFastScriptCount={}", failedFastScriptCountValue); - } else if (RunStatusEnum.ABNORMAL_STATE.getValue().equals(taskInstanceDTO.getStatus())) { + } else if (status == RunStatusEnum.ABNORMAL_STATE) { // 状态异常的快速脚本统计 StatisticsKey keyExceptionFastScriptCount = new StatisticsKey(taskInstanceDTO.getAppId(), StatisticsConstants.RESOURCE_EXECUTED_FAST_SCRIPT, StatisticsConstants.DIMENSION_STEP_RUN_STATUS, @@ -299,7 +300,8 @@ public void updateFastFileCountByStatus(TaskInstanceDTO taskInstanceDTO, // 快速文件按状态统计 Integer type = taskInstanceDTO.getType(); if (type != null && type.equals(TaskTypeEnum.FILE.getValue())) { - if (RunStatusEnum.SUCCESS.getValue().equals(taskInstanceDTO.getStatus())) { + RunStatusEnum status = taskInstanceDTO.getStatus(); + if (status == RunStatusEnum.SUCCESS) { // 执行成功的快速文件统计 StatisticsKey keySuccessFastFileCount = new StatisticsKey(taskInstanceDTO.getAppId(), StatisticsConstants.RESOURCE_EXECUTED_FAST_FILE, StatisticsConstants.DIMENSION_STEP_RUN_STATUS, @@ -308,7 +310,7 @@ public void updateFastFileCountByStatus(TaskInstanceDTO taskInstanceDTO, statisticsKey -> new AtomicInteger(0)); int successFastFileCountValue = successFastFileCount.incrementAndGet(); log.debug("successFastFileCount={}", successFastFileCountValue); - } else if (RunStatusEnum.FAIL.getValue().equals(taskInstanceDTO.getStatus())) { + } else if (status == RunStatusEnum.FAIL) { // 执行失败的快速文件统计 StatisticsKey keyFailedFastFileCount = new StatisticsKey(taskInstanceDTO.getAppId(), StatisticsConstants.RESOURCE_EXECUTED_FAST_FILE, StatisticsConstants.DIMENSION_STEP_RUN_STATUS, @@ -317,7 +319,7 @@ public void updateFastFileCountByStatus(TaskInstanceDTO taskInstanceDTO, statisticsKey -> new AtomicInteger(0)); int failedFastFileCountValue = failedFastFileCount.incrementAndGet(); log.debug("failedFastFileCount={}", failedFastFileCountValue); - } else if (RunStatusEnum.ABNORMAL_STATE.getValue().equals(taskInstanceDTO.getStatus())) { + } else if (status == RunStatusEnum.ABNORMAL_STATE) { // 执行失败的快速文件统计 StatisticsKey keyExceptionFastFileCount = new StatisticsKey(taskInstanceDTO.getAppId(), StatisticsConstants.RESOURCE_EXECUTED_FAST_FILE, StatisticsConstants.DIMENSION_STEP_RUN_STATUS, diff --git a/src/backend/job-execute/service-job-execute/src/main/resources/sqltask/mysql_exec_template.sh b/src/backend/job-execute/service-job-execute/src/main/resources/sqltask/mysql_exec_template.sh index 5daefebccd..29b40e936e 100644 --- a/src/backend/job-execute/service-job-execute/src/main/resources/sqltask/mysql_exec_template.sh +++ b/src/backend/job-execute/service-job-execute/src/main/resources/sqltask/mysql_exec_template.sh @@ -38,7 +38,7 @@ timeSec=`date +'%s'` LOG=/tmp/result_${SCRIPT_NAME}_${timeSec}.log if [ ${PORT} -gt 0 ]; then - HOST_AND_PORT=`netstat -ntl | grep ${PORT} | awk '{print $4}'` + HOST_AND_PORT=`netstat -ntl | grep ":${PORT} " | awk '{print $4}'` HOST=${HOST_AND_PORT%:${PORT}} if [ ${HOST} = '0.0.0.0' -o ${HOST} = '::' ];then HOST='127.0.0.1' @@ -81,4 +81,4 @@ egrep -qi "ERROR [0-9]+.* Can't connect to" ${LOG} && job_fail "DB error or unkn if [ ${RET_CODE} -ne 0 ]; then job_fail fi -job_success "success" \ No newline at end of file +job_success "success" diff --git a/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/ExponentIncrementRollingExprPartTest.java b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/ExponentIncrementRollingExprPartTest.java new file mode 100644 index 0000000000..2541733187 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/ExponentIncrementRollingExprPartTest.java @@ -0,0 +1,104 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.rolling; + +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.common.exception.RollingExprParseException; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +@DisplayName("PlusIncrementRollingExprPartTest") +class ExponentIncrementRollingExprPartTest { + private static final ExponentIncrementRollingExprPart ROLLING_EXPR_PART = new ExponentIncrementRollingExprPart(); + + @Test + @DisplayName("解析非法滚动策略") + void parseInvalidExpr() { + RollingExprPart rollingExprPart = ROLLING_EXPR_PART.parseExpr("1%"); + assertThat(rollingExprPart).isNull(); + + rollingExprPart = ROLLING_EXPR_PART.parseExpr("test"); + assertThat(rollingExprPart).isNull(); + + rollingExprPart = ROLLING_EXPR_PART.parseExpr("-1"); + assertThat(rollingExprPart).isNull(); + + rollingExprPart = ROLLING_EXPR_PART.parseExpr("+2"); + assertThat(rollingExprPart).isNull(); + + assertThrows(RollingExprParseException.class, () -> ROLLING_EXPR_PART.parseExpr("*0")); + } + + @Test + @DisplayName("解析滚动策略") + void parseValidExpr() { + RollingExprPart rollingExprPart = ROLLING_EXPR_PART.parseExpr("*2"); + assertThat(rollingExprPart).isNotNull(); + assertThat(rollingExprPart).isInstanceOf(ExponentIncrementRollingExprPart.class); + ExponentIncrementRollingExprPart exponentIncrementRollingExprPart = (ExponentIncrementRollingExprPart) rollingExprPart; + assertThat(exponentIncrementRollingExprPart.getExpr()).isEqualTo("*2"); + assertThat(exponentIncrementRollingExprPart.getExponent()).isEqualTo(2); + } + + @Test + void compute() { + List rollingServers = new ArrayList<>(); + rollingServers.add(new HostDTO(0L, "127.0.0.1")); + rollingServers.add(new HostDTO(0L, "127.0.0.2")); + rollingServers.add(new HostDTO(0L, "127.0.0.3")); + rollingServers.add(new HostDTO(0L, "127.0.0.4")); + rollingServers.add(new HostDTO(0L, "127.0.0.5")); + RollingServerBatchContext context = new RollingServerBatchContext(rollingServers); + + List remainingServers = new ArrayList<>(); + remainingServers.add(new HostDTO(0L, "127.0.0.2")); + remainingServers.add(new HostDTO(0L, "127.0.0.3")); + remainingServers.add(new HostDTO(0L, "127.0.0.4")); + remainingServers.add(new HostDTO(0L, "127.0.0.5")); + context.setRemainedServers(remainingServers); + + RollingServerBatch preRollingServerBatch = new RollingServerBatch(); + preRollingServerBatch.setBatch(1); + preRollingServerBatch.setServers(Collections.singletonList(new HostDTO(0L, "127.0.0.1"))); + context.addServerBatch(preRollingServerBatch); + context.setBatchCount(1); + + ExponentIncrementRollingExprPart exponentIncrementRollingExprPart = + (ExponentIncrementRollingExprPart) ROLLING_EXPR_PART.parseExpr("*3"); + List serversOnBatch = exponentIncrementRollingExprPart.compute(context); + assertThat(serversOnBatch).containsSequence( + new HostDTO(0L, "127.0.0.2"), + new HostDTO(0L, "127.0.0.3"), + new HostDTO(0L, "127.0.0.4") + ); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/PercentRollingExprPartTest.java b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/PercentRollingExprPartTest.java new file mode 100644 index 0000000000..5070866360 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/PercentRollingExprPartTest.java @@ -0,0 +1,136 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.rolling; + +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.common.exception.RollingExprParseException; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class PercentRollingExprPartTest { + private static final PercentRollingExprPart PERCENT_ROLLING_EXPR_PART = new PercentRollingExprPart(); + + @Test + void parseInvalidExpr() { + RollingExprPart rollingExprPart = PERCENT_ROLLING_EXPR_PART.parseExpr("1"); + assertThat(rollingExprPart).isNull(); + + rollingExprPart = PERCENT_ROLLING_EXPR_PART.parseExpr("test"); + assertThat(rollingExprPart).isNull(); + + rollingExprPart = PERCENT_ROLLING_EXPR_PART.parseExpr("2.0%"); + assertThat(rollingExprPart).isNull(); + + assertThrows(RollingExprParseException.class, () -> PERCENT_ROLLING_EXPR_PART.parseExpr("0%")); + assertThrows(RollingExprParseException.class, () -> PERCENT_ROLLING_EXPR_PART.parseExpr("101%")); + } + + @Test + void parseValidExpr() { + RollingExprPart rollingExprPart = PERCENT_ROLLING_EXPR_PART.parseExpr("1%"); + assertThat(rollingExprPart).isNotNull(); + assertThat(rollingExprPart).isInstanceOf(PercentRollingExprPart.class); + PercentRollingExprPart PercentRollingExprPart = (PercentRollingExprPart) rollingExprPart; + assertThat(PercentRollingExprPart.getExpr()).isEqualTo("1%"); + assertThat(PercentRollingExprPart.getPercent()).isEqualTo(1); + + rollingExprPart = PERCENT_ROLLING_EXPR_PART.parseExpr("100%"); + assertThat(rollingExprPart).isNotNull(); + assertThat(rollingExprPart).isInstanceOf(PercentRollingExprPart.class); + PercentRollingExprPart = (PercentRollingExprPart) rollingExprPart; + assertThat(PercentRollingExprPart.getExpr()).isEqualTo("100%"); + assertThat(PercentRollingExprPart.getPercent()).isEqualTo(100); + } + + @Nested + @DisplayName("验证根据百分比计算服务器分批大小") + class ComputeTest { + + @Test + @DisplayName("验证100%表达式") + void compute() { + List rollingServers = new ArrayList<>(); + rollingServers.add(new HostDTO(0L, "127.0.0.1")); + rollingServers.add(new HostDTO(0L, "127.0.0.2")); + rollingServers.add(new HostDTO(0L, "127.0.0.3")); + + PercentRollingExprPart percentRollingExprPart = + (PercentRollingExprPart) PERCENT_ROLLING_EXPR_PART.parseExpr("100%"); + RollingServerBatchContext context = new RollingServerBatchContext(rollingServers); + List serversOnBatch = percentRollingExprPart.compute(context); + assertThat(serversOnBatch).hasSize(3); + assertThat(serversOnBatch).containsSequence( + new HostDTO(0L, "127.0.0.1"), + new HostDTO(0L, "127.0.0.2"), + new HostDTO(0L, "127.0.0.3") + ); + } + + @Test + @DisplayName("验证批次计算向上取整") + void testCeil() { + List rollingServers = new ArrayList<>(); + rollingServers.add(new HostDTO(0L, "127.0.0.1")); + rollingServers.add(new HostDTO(0L, "127.0.0.2")); + rollingServers.add(new HostDTO(0L, "127.0.0.3")); + rollingServers.add(new HostDTO(0L, "127.0.0.4")); + rollingServers.add(new HostDTO(0L, "127.0.0.5")); + RollingServerBatchContext context = new RollingServerBatchContext(rollingServers); + + PercentRollingExprPart percentRollingExprPart = + (PercentRollingExprPart) PERCENT_ROLLING_EXPR_PART.parseExpr("10%"); + List serversOnBatch = percentRollingExprPart.compute(context); + assertThat(serversOnBatch).hasSize(1); + assertThat(serversOnBatch).containsSequence( + new HostDTO(0L, "127.0.0.1") + ); + + percentRollingExprPart = + (PercentRollingExprPart) PERCENT_ROLLING_EXPR_PART.parseExpr("20%"); + serversOnBatch = percentRollingExprPart.compute(context); + assertThat(serversOnBatch).hasSize(1); + assertThat(serversOnBatch).containsSequence( + new HostDTO(0L, "127.0.0.1") + ); + + percentRollingExprPart = + (PercentRollingExprPart) PERCENT_ROLLING_EXPR_PART.parseExpr("30%"); + serversOnBatch = percentRollingExprPart.compute(context); + assertThat(serversOnBatch).hasSize(2); + assertThat(serversOnBatch).containsSequence( + new HostDTO(0L, "127.0.0.1"), + new HostDTO(0L, "127.0.0.2") + ); + } + } + +} diff --git a/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/PlusIncrementRollingExprPartTest.java b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/PlusIncrementRollingExprPartTest.java new file mode 100644 index 0000000000..54cf205608 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/PlusIncrementRollingExprPartTest.java @@ -0,0 +1,112 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.rolling; + +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.common.exception.RollingExprParseException; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +@DisplayName("PlusIncrementRollingExprPartTest") +class PlusIncrementRollingExprPartTest { + private static final PlusIncrementRollingExprPart ROLLING_EXPR_PART = new PlusIncrementRollingExprPart(); + + @Test + @DisplayName("解析非法滚动策略") + void parseInvalidExpr() { + RollingExprPart rollingExprPart = ROLLING_EXPR_PART.parseExpr("1%"); + assertThat(rollingExprPart).isNull(); + + rollingExprPart = ROLLING_EXPR_PART.parseExpr("test"); + assertThat(rollingExprPart).isNull(); + + rollingExprPart = ROLLING_EXPR_PART.parseExpr("-1"); + assertThat(rollingExprPart).isNull(); + + assertThrows(RollingExprParseException.class, () -> ROLLING_EXPR_PART.parseExpr("+0")); + + rollingExprPart = ROLLING_EXPR_PART.parseExpr("*2"); + assertThat(rollingExprPart).isNull(); + } + + @Test + @DisplayName("解析滚动策略") + void parseValidExpr() { + RollingExprPart rollingExprPart = ROLLING_EXPR_PART.parseExpr("+10"); + assertThat(rollingExprPart).isNotNull(); + assertThat(rollingExprPart).isInstanceOf(PlusIncrementRollingExprPart.class); + PlusIncrementRollingExprPart plusIncrementRollingExprPart = (PlusIncrementRollingExprPart) rollingExprPart; + assertThat(plusIncrementRollingExprPart.getExpr()).isEqualTo("+10"); + assertThat(plusIncrementRollingExprPart.getAddend()).isEqualTo(10); + + rollingExprPart = ROLLING_EXPR_PART.parseExpr("+100"); + assertThat(rollingExprPart).isNotNull(); + assertThat(rollingExprPart).isInstanceOf(PlusIncrementRollingExprPart.class); + plusIncrementRollingExprPart = (PlusIncrementRollingExprPart) rollingExprPart; + assertThat(plusIncrementRollingExprPart.getExpr()).isEqualTo("+100"); + assertThat(plusIncrementRollingExprPart.getAddend()).isEqualTo(100); + } + + @Test + void compute() { + List rollingServers = new ArrayList<>(); + rollingServers.add(new HostDTO(0L, "127.0.0.1")); + rollingServers.add(new HostDTO(0L, "127.0.0.2")); + rollingServers.add(new HostDTO(0L, "127.0.0.3")); + rollingServers.add(new HostDTO(0L, "127.0.0.4")); + rollingServers.add(new HostDTO(0L, "127.0.0.5")); + RollingServerBatchContext context = new RollingServerBatchContext(rollingServers); + + List remainingServers = new ArrayList<>(); + remainingServers.add(new HostDTO(0L, "127.0.0.2")); + remainingServers.add(new HostDTO(0L, "127.0.0.3")); + remainingServers.add(new HostDTO(0L, "127.0.0.4")); + remainingServers.add(new HostDTO(0L, "127.0.0.5")); + context.setRemainedServers(remainingServers); + + RollingServerBatch preRollingServerBatch = new RollingServerBatch(); + preRollingServerBatch.setBatch(1); + preRollingServerBatch.setServers(Collections.singletonList(new HostDTO(0L, "127.0.0.1"))); + context.addServerBatch(preRollingServerBatch); + context.setBatchCount(1); + + PlusIncrementRollingExprPart plusIncrementRollingExprPart = + (PlusIncrementRollingExprPart) ROLLING_EXPR_PART.parseExpr("+3"); + List serversOnBatch = plusIncrementRollingExprPart.compute(context); + assertThat(serversOnBatch).containsSequence( + new HostDTO(0L, "127.0.0.2"), + new HostDTO(0L, "127.0.0.3"), + new HostDTO(0L, "127.0.0.4"), + new HostDTO(0L, "127.0.0.5") + ); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/QuantityRollingExprPartTest.java b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/QuantityRollingExprPartTest.java new file mode 100644 index 0000000000..b88a2113b2 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/QuantityRollingExprPartTest.java @@ -0,0 +1,102 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.rolling; + +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.common.exception.RollingExprParseException; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class QuantityRollingExprPartTest { + private static final QuantityRollingExprPart QUANTITY_ROLLING_EXPR_PART = new QuantityRollingExprPart(); + + @Test + void parseInvalidExpr() { + RollingExprPart rollingExprPart = QUANTITY_ROLLING_EXPR_PART.parseExpr("1%"); + assertThat(rollingExprPart).isNull(); + + rollingExprPart = QUANTITY_ROLLING_EXPR_PART.parseExpr("test"); + assertThat(rollingExprPart).isNull(); + + rollingExprPart = QUANTITY_ROLLING_EXPR_PART.parseExpr("-1"); + assertThat(rollingExprPart).isNull(); + + assertThrows(RollingExprParseException.class, () -> QUANTITY_ROLLING_EXPR_PART.parseExpr("0")); + } + + @Test + void parseValidExpr() { + RollingExprPart rollingExprPart = QUANTITY_ROLLING_EXPR_PART.parseExpr("1"); + assertThat(rollingExprPart).isNotNull(); + assertThat(rollingExprPart).isInstanceOf(QuantityRollingExprPart.class); + QuantityRollingExprPart quantityRollingExprPart = (QuantityRollingExprPart) rollingExprPart; + assertThat(quantityRollingExprPart.getExpr()).isEqualTo("1"); + assertThat(quantityRollingExprPart.getQuantity()).isEqualTo(1); + + rollingExprPart = QUANTITY_ROLLING_EXPR_PART.parseExpr("100"); + assertThat(rollingExprPart).isNotNull(); + assertThat(rollingExprPart).isInstanceOf(QuantityRollingExprPart.class); + quantityRollingExprPart = (QuantityRollingExprPart) rollingExprPart; + assertThat(quantityRollingExprPart.getExpr()).isEqualTo("100"); + assertThat(quantityRollingExprPart.getQuantity()).isEqualTo(100); + } + + @Test + void compute() { + List rollingServers = new ArrayList<>(); + rollingServers.add(new HostDTO(0L, "127.0.0.1")); + rollingServers.add(new HostDTO(0L, "127.0.0.2")); + rollingServers.add(new HostDTO(0L, "127.0.0.3")); + rollingServers.add(new HostDTO(0L, "127.0.0.4")); + rollingServers.add(new HostDTO(0L, "127.0.0.5")); + RollingServerBatchContext context = new RollingServerBatchContext(rollingServers); + + QuantityRollingExprPart quantityRollingExprPart = + (QuantityRollingExprPart) QUANTITY_ROLLING_EXPR_PART.parseExpr("2"); + List serversOnBatch = quantityRollingExprPart.compute(context); + assertThat(serversOnBatch).hasSize(2); + assertThat(serversOnBatch).containsSequence( + new HostDTO(0L, "127.0.0.1"), + new HostDTO(0L, "127.0.0.2") + ); + + quantityRollingExprPart = + (QuantityRollingExprPart) QUANTITY_ROLLING_EXPR_PART.parseExpr("10"); + serversOnBatch = quantityRollingExprPart.compute(context); + assertThat(serversOnBatch).hasSize(5); + assertThat(serversOnBatch).containsSequence( + new HostDTO(0L, "127.0.0.1"), + new HostDTO(0L, "127.0.0.2"), + new HostDTO(0L, "127.0.0.3"), + new HostDTO(0L, "127.0.0.4"), + new HostDTO(0L, "127.0.0.5") + ); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/RollingExprTest.java b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/RollingExprTest.java new file mode 100644 index 0000000000..740d5dcf00 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/RollingExprTest.java @@ -0,0 +1,181 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.rolling; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class RollingExprTest { + + @Test + @DisplayName("解析滚动表达式 - 一个子表达式，按数量分批") + void nextRollingExprPartForSingleQuantityExpr() { + RollingExpr rollingExpr = new RollingExpr("1"); + RollingExprPart rollingExprPart = rollingExpr.nextRollingExprPart(1); + assertThat(rollingExprPart).isNotNull(); + assertThat(rollingExprPart).isInstanceOf(QuantityRollingExprPart.class); + QuantityRollingExprPart quantityRollingExprPart = (QuantityRollingExprPart) rollingExprPart; + assertThat(quantityRollingExprPart.getQuantity()).isEqualTo(1); + assertThat(quantityRollingExprPart.getExpr()).isEqualTo("1"); + + rollingExprPart = rollingExpr.nextRollingExprPart(2); + assertThat(rollingExprPart).isNotNull(); + assertThat(rollingExprPart).isInstanceOf(QuantityRollingExprPart.class); + quantityRollingExprPart = (QuantityRollingExprPart) rollingExprPart; + assertThat(quantityRollingExprPart.getQuantity()).isEqualTo(1); + } + + @Test + @DisplayName("解析滚动表达式 - 多个子表达式，按数量分批") + void nextRollingExprPartForMultiQuantityExpr() { + RollingExpr rollingExpr = new RollingExpr("1 2 3"); + + // 第一批次 + RollingExprPart rollingExprPart = rollingExpr.nextRollingExprPart(1); + assertThat(rollingExprPart).isNotNull(); + assertThat(rollingExprPart).isInstanceOf(QuantityRollingExprPart.class); + QuantityRollingExprPart quantityRollingExprPart = (QuantityRollingExprPart) rollingExprPart; + assertThat(quantityRollingExprPart.getQuantity()).isEqualTo(1); + assertThat(quantityRollingExprPart.getExpr()).isEqualTo("1"); + + // 第二批次 + rollingExprPart = rollingExpr.nextRollingExprPart(2); + assertThat(rollingExprPart).isNotNull(); + assertThat(rollingExprPart).isInstanceOf(QuantityRollingExprPart.class); + quantityRollingExprPart = (QuantityRollingExprPart) rollingExprPart; + assertThat(quantityRollingExprPart.getQuantity()).isEqualTo(2); + assertThat(quantityRollingExprPart.getExpr()).isEqualTo("2"); + + // 第三批次 + rollingExprPart = rollingExpr.nextRollingExprPart(3); + assertThat(rollingExprPart).isNotNull(); + assertThat(rollingExprPart).isInstanceOf(QuantityRollingExprPart.class); + quantityRollingExprPart = (QuantityRollingExprPart) rollingExprPart; + assertThat(quantityRollingExprPart.getQuantity()).isEqualTo(3); + assertThat(quantityRollingExprPart.getExpr()).isEqualTo("3"); + + // 第四批次 + rollingExprPart = rollingExpr.nextRollingExprPart(4); + assertThat(rollingExprPart).isNotNull(); + assertThat(rollingExprPart).isInstanceOf(QuantityRollingExprPart.class); + quantityRollingExprPart = (QuantityRollingExprPart) rollingExprPart; + assertThat(quantityRollingExprPart.getQuantity()).isEqualTo(3); + assertThat(quantityRollingExprPart.getExpr()).isEqualTo("3"); + } + + @Test + @DisplayName("解析滚动表达式 - 一个子表达式，按百分比分批") + void nextRollingExprPartForSinglePercentExpr() { + RollingExpr rollingExpr = new RollingExpr("10%"); + + RollingExprPart rollingExprPart = rollingExpr.nextRollingExprPart(1); + assertThat(rollingExprPart).isNotNull(); + assertThat(rollingExprPart).isInstanceOf(PercentRollingExprPart.class); + PercentRollingExprPart percentRollingExprPart = (PercentRollingExprPart) rollingExprPart; + assertThat(percentRollingExprPart.getPercent()).isEqualTo(10); + assertThat(percentRollingExprPart.getExpr()).isEqualTo("10%"); + + rollingExprPart = rollingExpr.nextRollingExprPart(2); + assertThat(rollingExprPart).isNotNull(); + assertThat(rollingExprPart).isInstanceOf(PercentRollingExprPart.class); + percentRollingExprPart = (PercentRollingExprPart) rollingExprPart; + assertThat(percentRollingExprPart.getPercent()).isEqualTo(10); + assertThat(percentRollingExprPart.getExpr()).isEqualTo("10%"); + } + + @Test + @DisplayName("解析滚动表达式 - 多个子表达式，按百分比分批") + void nextRollingExprPartForMultiPercentExpr() { + RollingExpr rollingExpr = new RollingExpr("10% 20%"); + + // 第一批次 + RollingExprPart rollingExprPart = rollingExpr.nextRollingExprPart(1); + assertThat(rollingExprPart).isNotNull(); + assertThat(rollingExprPart).isInstanceOf(PercentRollingExprPart.class); + PercentRollingExprPart percentRollingExprPart = (PercentRollingExprPart) rollingExprPart; + assertThat(percentRollingExprPart.getPercent()).isEqualTo(10); + assertThat(percentRollingExprPart.getExpr()).isEqualTo("10%"); + + // 第二批次 + rollingExprPart = rollingExpr.nextRollingExprPart(2); + assertThat(rollingExprPart).isNotNull(); + assertThat(rollingExprPart).isInstanceOf(PercentRollingExprPart.class); + percentRollingExprPart = (PercentRollingExprPart) rollingExprPart; + assertThat(percentRollingExprPart.getPercent()).isEqualTo(20); + assertThat(percentRollingExprPart.getExpr()).isEqualTo("20%"); + + // 第三批次 + rollingExprPart = rollingExpr.nextRollingExprPart(3); + assertThat(rollingExprPart).isNotNull(); + assertThat(rollingExprPart).isInstanceOf(PercentRollingExprPart.class); + percentRollingExprPart = (PercentRollingExprPart) rollingExprPart; + assertThat(percentRollingExprPart.getPercent()).isEqualTo(20); + assertThat(percentRollingExprPart.getExpr()).isEqualTo("20%"); + } + + @Test + @DisplayName("解析滚动表达式 - 混合数量与百分比") + void nextRollingExprPartForMixedExpr() { + RollingExpr rollingExpr = new RollingExpr("1 10%"); + RollingExprPart rollingExprPart = rollingExpr.nextRollingExprPart(1); + assertThat(rollingExprPart).isNotNull(); + assertThat(rollingExprPart).isInstanceOf(QuantityRollingExprPart.class); + QuantityRollingExprPart quantityRollingExprPart = (QuantityRollingExprPart) rollingExprPart; + assertThat(quantityRollingExprPart.getQuantity()).isEqualTo(1); + assertThat(quantityRollingExprPart.getExpr()).isEqualTo("1"); + + rollingExprPart = rollingExpr.nextRollingExprPart(2); + assertThat(rollingExprPart).isNotNull(); + assertThat(rollingExprPart).isInstanceOf(PercentRollingExprPart.class); + PercentRollingExprPart percentRollingExprPart = (PercentRollingExprPart) rollingExprPart; + assertThat(percentRollingExprPart.getPercent()).isEqualTo(10); + + rollingExprPart = rollingExpr.nextRollingExprPart(3); + assertThat(rollingExprPart).isNotNull(); + assertThat(rollingExprPart).isInstanceOf(PercentRollingExprPart.class); + percentRollingExprPart = (PercentRollingExprPart) rollingExprPart; + assertThat(percentRollingExprPart.getPercent()).isEqualTo(10); + } + + @Test + @DisplayName("解析滚动表达式 - 剩下所有表达式解析") + void nextRollingExprPartForAllRemainedExpr() { + RollingExpr rollingExpr = new RollingExpr("1 100%"); + RollingExprPart rollingExprPart = rollingExpr.nextRollingExprPart(1); + assertThat(rollingExprPart).isNotNull(); + assertThat(rollingExprPart).isInstanceOf(QuantityRollingExprPart.class); + QuantityRollingExprPart quantityRollingExprPart = (QuantityRollingExprPart) rollingExprPart; + assertThat(quantityRollingExprPart.getQuantity()).isEqualTo(1); + assertThat(quantityRollingExprPart.getExpr()).isEqualTo("1"); + + rollingExprPart = rollingExpr.nextRollingExprPart(2); + assertThat(rollingExprPart).isNotNull(); + assertThat(rollingExprPart).isInstanceOf(PercentRollingExprPart.class); + PercentRollingExprPart percentRollingExprPart = (PercentRollingExprPart) rollingExprPart; + assertThat(percentRollingExprPart.getPercent()).isEqualTo(100); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/RollingServerBatchResolverTest.java b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/RollingServerBatchResolverTest.java new file mode 100644 index 0000000000..7ff654e1a7 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/RollingServerBatchResolverTest.java @@ -0,0 +1,388 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.rolling; + +import com.tencent.bk.job.common.model.dto.HostDTO; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class RollingServerBatchResolverTest { + + @Test + @DisplayName("计算滚动批次 - 一个子表达式，按百分比分批") + void resolveForSinglePercentExpr() { + List servers = new ArrayList<>(); + servers.add(new HostDTO(0L, "127.0.0.1")); + servers.add(new HostDTO(0L, "127.0.0.2")); + servers.add(new HostDTO(0L, "127.0.0.3")); + servers.add(new HostDTO(0L, "127.0.0.4")); + servers.add(new HostDTO(0L, "127.0.0.5")); + servers.add(new HostDTO(0L, "127.0.0.6")); + servers.add(new HostDTO(0L, "127.0.0.7")); + servers.add(new HostDTO(0L, "127.0.0.8")); + servers.add(new HostDTO(0L, "127.0.0.9")); + servers.add(new HostDTO(0L, "127.0.0.10")); + servers.add(new HostDTO(0L, "127.0.0.11")); + servers.add(new HostDTO(0L, "127.0.0.12")); + servers.add(new HostDTO(0L, "127.0.0.13")); + servers.add(new HostDTO(0L, "127.0.0.14")); + servers.add(new HostDTO(0L, "127.0.0.15")); + servers.add(new HostDTO(0L, "127.0.0.16")); + RollingBatchServersResolver context = new RollingBatchServersResolver(servers, "25%"); + List serverBatchList = context.resolve(); + assertThat(serverBatchList).hasSize(4); + + assertThat(serverBatchList.get(0).getBatch()).isEqualTo(1); + assertThat(serverBatchList.get(0).getServers()).containsSequence( + new HostDTO(0L, "127.0.0.1"), + new HostDTO(0L, "127.0.0.2"), + new HostDTO(0L, "127.0.0.3"), + new HostDTO(0L, "127.0.0.4") + ); + assertThat(serverBatchList.get(0).getRollingExprPart().getExpr()).isEqualTo("25%"); + + assertThat(serverBatchList.get(1).getBatch()).isEqualTo(2); + assertThat(serverBatchList.get(1).getServers()).containsSequence( + new HostDTO(0L, "127.0.0.5"), + new HostDTO(0L, "127.0.0.6"), + new HostDTO(0L, "127.0.0.7"), + new HostDTO(0L, "127.0.0.8") + ); + assertThat(serverBatchList.get(1).getRollingExprPart().getExpr()).isEqualTo("25%"); + + assertThat(serverBatchList.get(2).getBatch()).isEqualTo(3); + assertThat(serverBatchList.get(2).getServers()).containsSequence( + new HostDTO(0L, "127.0.0.9"), + new HostDTO(0L, "127.0.0.10"), + new HostDTO(0L, "127.0.0.11"), + new HostDTO(0L, "127.0.0.12") + ); + assertThat(serverBatchList.get(2).getRollingExprPart().getExpr()).isEqualTo("25%"); + + assertThat(serverBatchList.get(3).getBatch()).isEqualTo(4); + assertThat(serverBatchList.get(3).getServers()).containsSequence( + new HostDTO(0L, "127.0.0.13"), + new HostDTO(0L, "127.0.0.14"), + new HostDTO(0L, "127.0.0.15"), + new HostDTO(0L, "127.0.0.16") + ); + assertThat(serverBatchList.get(3).getRollingExprPart().getExpr()).isEqualTo("25%"); + } + + @Test + @DisplayName("计算滚动批次 - 多个子表达式，按百分比分批") + void resolveForMultiPercentExpr() { + List servers = new ArrayList<>(); + servers.add(new HostDTO(0L, "127.0.0.1")); + servers.add(new HostDTO(0L, "127.0.0.2")); + servers.add(new HostDTO(0L, "127.0.0.3")); + servers.add(new HostDTO(0L, "127.0.0.4")); + servers.add(new HostDTO(0L, "127.0.0.5")); + servers.add(new HostDTO(0L, "127.0.0.6")); + servers.add(new HostDTO(0L, "127.0.0.7")); + servers.add(new HostDTO(0L, "127.0.0.8")); + servers.add(new HostDTO(0L, "127.0.0.9")); + servers.add(new HostDTO(0L, "127.0.0.10")); + servers.add(new HostDTO(0L, "127.0.0.11")); + servers.add(new HostDTO(0L, "127.0.0.12")); + servers.add(new HostDTO(0L, "127.0.0.13")); + servers.add(new HostDTO(0L, "127.0.0.14")); + servers.add(new HostDTO(0L, "127.0.0.15")); + servers.add(new HostDTO(0L, "127.0.0.16")); + RollingBatchServersResolver context = new RollingBatchServersResolver(servers, "10% 30%"); + List serverBatchList = context.resolve(); + assertThat(serverBatchList).hasSize(4); + + assertThat(serverBatchList.get(0).getBatch()).isEqualTo(1); + assertThat(serverBatchList.get(0).getServers()).containsSequence( + new HostDTO(0L, "127.0.0.1"), + new HostDTO(0L, "127.0.0.2") + ); + assertThat(serverBatchList.get(0).getRollingExprPart().getExpr()).isEqualTo("10%"); + + assertThat(serverBatchList.get(1).getBatch()).isEqualTo(2); + assertThat(serverBatchList.get(1).getServers()).containsSequence( + new HostDTO(0L, "127.0.0.3"), + new HostDTO(0L, "127.0.0.4"), + new HostDTO(0L, "127.0.0.5"), + new HostDTO(0L, "127.0.0.6"), + new HostDTO(0L, "127.0.0.7") + ); + assertThat(serverBatchList.get(1).getRollingExprPart().getExpr()).isEqualTo("30%"); + + assertThat(serverBatchList.get(2).getBatch()).isEqualTo(3); + assertThat(serverBatchList.get(2).getServers()).containsSequence( + new HostDTO(0L, "127.0.0.8"), + new HostDTO(0L, "127.0.0.9"), + new HostDTO(0L, "127.0.0.10"), + new HostDTO(0L, "127.0.0.11"), + new HostDTO(0L, "127.0.0.12") + ); + assertThat(serverBatchList.get(2).getRollingExprPart().getExpr()).isEqualTo("30%"); + + assertThat(serverBatchList.get(3).getBatch()).isEqualTo(4); + assertThat(serverBatchList.get(3).getServers()).containsSequence( + new HostDTO(0L, "127.0.0.13"), + new HostDTO(0L, "127.0.0.14"), + new HostDTO(0L, "127.0.0.15"), + new HostDTO(0L, "127.0.0.16") + ); + assertThat(serverBatchList.get(3).getRollingExprPart().getExpr()).isEqualTo("30%"); + } + + @Test + @DisplayName("计算滚动批次 - 一个子表达式，按数量分批") + void resolveForSingleQuantityExpr() { + List servers = new ArrayList<>(); + servers.add(new HostDTO(0L, "127.0.0.1")); + servers.add(new HostDTO(0L, "127.0.0.2")); + servers.add(new HostDTO(0L, "127.0.0.3")); + servers.add(new HostDTO(0L, "127.0.0.4")); + servers.add(new HostDTO(0L, "127.0.0.5")); + servers.add(new HostDTO(0L, "127.0.0.6")); + servers.add(new HostDTO(0L, "127.0.0.7")); + servers.add(new HostDTO(0L, "127.0.0.8")); + servers.add(new HostDTO(0L, "127.0.0.9")); + servers.add(new HostDTO(0L, "127.0.0.10")); + servers.add(new HostDTO(0L, "127.0.0.11")); + servers.add(new HostDTO(0L, "127.0.0.12")); + servers.add(new HostDTO(0L, "127.0.0.13")); + servers.add(new HostDTO(0L, "127.0.0.14")); + servers.add(new HostDTO(0L, "127.0.0.15")); + servers.add(new HostDTO(0L, "127.0.0.16")); + RollingBatchServersResolver context = new RollingBatchServersResolver(servers, "10"); + List serverBatchList = context.resolve(); + assertThat(serverBatchList).hasSize(2); + + assertThat(serverBatchList.get(0).getBatch()).isEqualTo(1); + assertThat(serverBatchList.get(0).getServers()).containsSequence( + new HostDTO(0L, "127.0.0.1"), + new HostDTO(0L, "127.0.0.2"), + new HostDTO(0L, "127.0.0.3"), + new HostDTO(0L, "127.0.0.4"), + new HostDTO(0L, "127.0.0.5"), + new HostDTO(0L, "127.0.0.6"), + new HostDTO(0L, "127.0.0.7"), + new HostDTO(0L, "127.0.0.8"), + new HostDTO(0L, "127.0.0.9"), + new HostDTO(0L, "127.0.0.10") + ); + assertThat(serverBatchList.get(0).getRollingExprPart().getExpr()).isEqualTo("10"); + + assertThat(serverBatchList.get(1).getBatch()).isEqualTo(2); + assertThat(serverBatchList.get(1).getServers()).containsSequence( + new HostDTO(0L, "127.0.0.11"), + new HostDTO(0L, "127.0.0.12"), + new HostDTO(0L, "127.0.0.13"), + new HostDTO(0L, "127.0.0.14"), + new HostDTO(0L, "127.0.0.15"), + new HostDTO(0L, "127.0.0.16") + ); + assertThat(serverBatchList.get(1).getRollingExprPart().getExpr()).isEqualTo("10"); + } + + @Test + @DisplayName("计算滚动批次 - 多个子表达式，按数量分批") + void resolveForMultiQuantityExpr() { + List servers = new ArrayList<>(); + servers.add(new HostDTO(0L, "127.0.0.1")); + servers.add(new HostDTO(0L, "127.0.0.2")); + servers.add(new HostDTO(0L, "127.0.0.3")); + servers.add(new HostDTO(0L, "127.0.0.4")); + servers.add(new HostDTO(0L, "127.0.0.5")); + servers.add(new HostDTO(0L, "127.0.0.6")); + servers.add(new HostDTO(0L, "127.0.0.7")); + servers.add(new HostDTO(0L, "127.0.0.8")); + servers.add(new HostDTO(0L, "127.0.0.9")); + servers.add(new HostDTO(0L, "127.0.0.10")); + servers.add(new HostDTO(0L, "127.0.0.11")); + servers.add(new HostDTO(0L, "127.0.0.12")); + servers.add(new HostDTO(0L, "127.0.0.13")); + servers.add(new HostDTO(0L, "127.0.0.14")); + servers.add(new HostDTO(0L, "127.0.0.15")); + servers.add(new HostDTO(0L, "127.0.0.16")); + RollingBatchServersResolver context = new RollingBatchServersResolver(servers, "1 5"); + List serverBatchList = context.resolve(); + assertThat(serverBatchList).hasSize(4); + + assertThat(serverBatchList.get(0).getBatch()).isEqualTo(1); + assertThat(serverBatchList.get(0).getServers()).containsSequence( + new HostDTO(0L, "127.0.0.1") + ); + assertThat(serverBatchList.get(0).getRollingExprPart().getExpr()).isEqualTo("1"); + + assertThat(serverBatchList.get(1).getBatch()).isEqualTo(2); + assertThat(serverBatchList.get(1).getServers()).containsSequence( + new HostDTO(0L, "127.0.0.2"), + new HostDTO(0L, "127.0.0.3"), + new HostDTO(0L, "127.0.0.4"), + new HostDTO(0L, "127.0.0.5"), + new HostDTO(0L, "127.0.0.6") + ); + assertThat(serverBatchList.get(1).getRollingExprPart().getExpr()).isEqualTo("5"); + + assertThat(serverBatchList.get(2).getBatch()).isEqualTo(3); + assertThat(serverBatchList.get(2).getServers()).containsSequence( + new HostDTO(0L, "127.0.0.7"), + new HostDTO(0L, "127.0.0.8"), + new HostDTO(0L, "127.0.0.9"), + new HostDTO(0L, "127.0.0.10"), + new HostDTO(0L, "127.0.0.11") + ); + assertThat(serverBatchList.get(2).getRollingExprPart().getExpr()).isEqualTo("5"); + + assertThat(serverBatchList.get(3).getBatch()).isEqualTo(4); + assertThat(serverBatchList.get(3).getServers()).containsSequence( + new HostDTO(0L, "127.0.0.12"), + new HostDTO(0L, "127.0.0.13"), + new HostDTO(0L, "127.0.0.14"), + new HostDTO(0L, "127.0.0.15"), + new HostDTO(0L, "127.0.0.16") + ); + assertThat(serverBatchList.get(3).getRollingExprPart().getExpr()).isEqualTo("5"); + } + + @Test + @DisplayName("计算滚动批次 - 混合表达式，按数量和百分比分批") + void resolveForMixedExpr() { + List servers = new ArrayList<>(); + servers.add(new HostDTO(0L, "127.0.0.1")); + servers.add(new HostDTO(0L, "127.0.0.2")); + servers.add(new HostDTO(0L, "127.0.0.3")); + servers.add(new HostDTO(0L, "127.0.0.4")); + servers.add(new HostDTO(0L, "127.0.0.5")); + servers.add(new HostDTO(0L, "127.0.0.6")); + servers.add(new HostDTO(0L, "127.0.0.7")); + servers.add(new HostDTO(0L, "127.0.0.8")); + servers.add(new HostDTO(0L, "127.0.0.9")); + servers.add(new HostDTO(0L, "127.0.0.10")); + servers.add(new HostDTO(0L, "127.0.0.11")); + servers.add(new HostDTO(0L, "127.0.0.12")); + servers.add(new HostDTO(0L, "127.0.0.13")); + servers.add(new HostDTO(0L, "127.0.0.14")); + servers.add(new HostDTO(0L, "127.0.0.15")); + RollingBatchServersResolver context = new RollingBatchServersResolver(servers, "1 30%"); + List serverBatchList = context.resolve(); + assertThat(serverBatchList).hasSize(4); + + assertThat(serverBatchList.get(0).getBatch()).isEqualTo(1); + assertThat(serverBatchList.get(0).getServers()).containsSequence( + new HostDTO(0L, "127.0.0.1") + ); + assertThat(serverBatchList.get(0).getRollingExprPart().getExpr()).isEqualTo("1"); + + assertThat(serverBatchList.get(1).getBatch()).isEqualTo(2); + assertThat(serverBatchList.get(1).getServers()).containsSequence( + new HostDTO(0L, "127.0.0.2"), + new HostDTO(0L, "127.0.0.3"), + new HostDTO(0L, "127.0.0.4"), + new HostDTO(0L, "127.0.0.5"), + new HostDTO(0L, "127.0.0.6") + ); + assertThat(serverBatchList.get(1).getRollingExprPart().getExpr()).isEqualTo("30%"); + + assertThat(serverBatchList.get(2).getBatch()).isEqualTo(3); + assertThat(serverBatchList.get(2).getServers()).containsSequence( + new HostDTO(0L, "127.0.0.7"), + new HostDTO(0L, "127.0.0.8"), + new HostDTO(0L, "127.0.0.9"), + new HostDTO(0L, "127.0.0.10"), + new HostDTO(0L, "127.0.0.11") + ); + assertThat(serverBatchList.get(2).getRollingExprPart().getExpr()).isEqualTo("30%"); + + assertThat(serverBatchList.get(3).getBatch()).isEqualTo(4); + assertThat(serverBatchList.get(3).getServers()).containsSequence( + new HostDTO(0L, "127.0.0.12"), + new HostDTO(0L, "127.0.0.13"), + new HostDTO(0L, "127.0.0.14"), + new HostDTO(0L, "127.0.0.15") + + ); + assertThat(serverBatchList.get(3).getRollingExprPart().getExpr()).isEqualTo("30%"); + } + + + @Test + @DisplayName("计算滚动批次 - 最后一批包含所有") + void resolveForAllRemainedExpr() { + List servers = new ArrayList<>(); + servers.add(new HostDTO(0L, "127.0.0.1")); + servers.add(new HostDTO(0L, "127.0.0.2")); + servers.add(new HostDTO(0L, "127.0.0.3")); + servers.add(new HostDTO(0L, "127.0.0.4")); + servers.add(new HostDTO(0L, "127.0.0.5")); + servers.add(new HostDTO(0L, "127.0.0.6")); + servers.add(new HostDTO(0L, "127.0.0.7")); + servers.add(new HostDTO(0L, "127.0.0.8")); + servers.add(new HostDTO(0L, "127.0.0.9")); + servers.add(new HostDTO(0L, "127.0.0.10")); + servers.add(new HostDTO(0L, "127.0.0.11")); + servers.add(new HostDTO(0L, "127.0.0.12")); + servers.add(new HostDTO(0L, "127.0.0.13")); + servers.add(new HostDTO(0L, "127.0.0.14")); + servers.add(new HostDTO(0L, "127.0.0.15")); + servers.add(new HostDTO(0L, "127.0.0.16")); + RollingBatchServersResolver context = new RollingBatchServersResolver(servers, "1 30% 100%"); + List serverBatchList = context.resolve(); + assertThat(serverBatchList).hasSize(3); + + assertThat(serverBatchList.get(0).getBatch()).isEqualTo(1); + assertThat(serverBatchList.get(0).getServers()).containsSequence( + new HostDTO(0L, "127.0.0.1") + ); + assertThat(serverBatchList.get(0).getRollingExprPart().getExpr()).isEqualTo("1"); + + assertThat(serverBatchList.get(1).getBatch()).isEqualTo(2); + assertThat(serverBatchList.get(1).getServers()).containsSequence( + new HostDTO(0L, "127.0.0.2"), + new HostDTO(0L, "127.0.0.3"), + new HostDTO(0L, "127.0.0.4"), + new HostDTO(0L, "127.0.0.5"), + new HostDTO(0L, "127.0.0.6") + ); + assertThat(serverBatchList.get(1).getRollingExprPart().getExpr()).isEqualTo("30%"); + + assertThat(serverBatchList.get(2).getBatch()).isEqualTo(3); + assertThat(serverBatchList.get(2).getServers()).containsSequence( + new HostDTO(0L, "127.0.0.7"), + new HostDTO(0L, "127.0.0.8"), + new HostDTO(0L, "127.0.0.9"), + new HostDTO(0L, "127.0.0.10"), + new HostDTO(0L, "127.0.0.11"), + new HostDTO(0L, "127.0.0.12"), + new HostDTO(0L, "127.0.0.13"), + new HostDTO(0L, "127.0.0.14"), + new HostDTO(0L, "127.0.0.15"), + new HostDTO(0L, "127.0.0.16") + ); + assertThat(serverBatchList.get(2).getRollingExprPart().getExpr()).isEqualTo("100%"); + } +} diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/esb/v3/EsbCreateOrUpdateFileSourceV3Req.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/esb/v3/EsbCreateOrUpdateFileSourceV3Req.java index aafd741a38..1e0a94f7ad 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/esb/v3/EsbCreateOrUpdateFileSourceV3Req.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/esb/v3/EsbCreateOrUpdateFileSourceV3Req.java @@ -6,6 +6,7 @@ import lombok.Data; import lombok.EqualsAndHashCode; +import java.util.HashMap; import java.util.Map; @EqualsAndHashCode(callSuper = true) @@ -35,7 +36,7 @@ public class EsbCreateOrUpdateFileSourceV3Req extends EsbAppScopeReq { */ @ApiModelProperty(value = "文件源信息Map") @JsonProperty(value = "access_params") - private Map accessParams; + private Map accessParams = new HashMap<>(); /** * 文件源凭证Id */ @@ -47,5 +48,5 @@ public class EsbCreateOrUpdateFileSourceV3Req extends EsbAppScopeReq { */ @ApiModelProperty(value = "文件前缀：后台自动生成UUID传${UUID}，自定义字符串直接传") @JsonProperty(value = "file_prefix") - private String filePrefix; + private String filePrefix = ""; } diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/FileSourceTaskStatusDTO.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/FileSourceTaskStatusDTO.java index 16c2c82ed7..2b03b436ea 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/FileSourceTaskStatusDTO.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/FileSourceTaskStatusDTO.java @@ -25,7 +25,7 @@ package com.tencent.bk.job.file_gateway.model.resp.inner; import com.tencent.bk.job.file_gateway.consts.TaskStatusEnum; -import com.tencent.bk.job.logsvr.model.service.ServiceIpLogDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceHostLogDTO; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -47,7 +47,7 @@ public class FileSourceTaskStatusDTO { * 文件源文件路径->文件在机器上的真实路径 */ Map filePathMap; - List logList; + List logList; Boolean logEnd; public boolean isDone() { diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/web/FileSourceVO.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/web/FileSourceVO.java index 463271d2f0..3ad0e193cb 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/web/FileSourceVO.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/web/FileSourceVO.java @@ -156,4 +156,3 @@ public class FileSourceVO { @ApiModelProperty("是否可以查看") private Boolean canView; } - diff --git a/src/backend/job-file-gateway/boot-job-file-gateway/src/main/resources/application.yml b/src/backend/job-file-gateway/boot-job-file-gateway/src/main/resources/application.yml index 0281e36c71..3e3c4fe8d6 100644 --- a/src/backend/job-file-gateway/boot-job-file-gateway/src/main/resources/application.yml +++ b/src/backend/job-file-gateway/boot-job-file-gateway/src/main/resources/application.yml @@ -1,6 +1,4 @@ spring: - application: - name: ${SPRING_APPLICATION_NAME:job-file-gateway} profiles: active: prod ribbon: diff --git a/src/backend/job-file-gateway/boot-job-file-gateway/src/main/resources/bootstrap.yml b/src/backend/job-file-gateway/boot-job-file-gateway/src/main/resources/bootstrap.yml index 77bd21fa13..809540ddb0 100644 --- a/src/backend/job-file-gateway/boot-job-file-gateway/src/main/resources/bootstrap.yml +++ b/src/backend/job-file-gateway/boot-job-file-gateway/src/main/resources/bootstrap.yml @@ -1,4 +1,6 @@ spring: + application: + name: ${SPRING_APPLICATION_NAME:job-file-gateway} cloud: kubernetes: config: diff --git a/src/backend/job-file-gateway/boot-job-file-gateway/src/main/resources/logback/logback-app-props.xml b/src/backend/job-file-gateway/boot-job-file-gateway/src/main/resources/logback/logback-app-props.xml index b8f5a5170a..3782587094 100644 --- a/src/backend/job-file-gateway/boot-job-file-gateway/src/main/resources/logback/logback-app-props.xml +++ b/src/backend/job-file-gateway/boot-job-file-gateway/src/main/resources/logback/logback-app-props.xml @@ -1,4 +1,5 @@ + diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/esb/EsbFileSourceV3ResourceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/esb/EsbFileSourceV3ResourceImpl.java index 7538caa202..297cb8a8a8 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/esb/EsbFileSourceV3ResourceImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/esb/EsbFileSourceV3ResourceImpl.java @@ -2,6 +2,7 @@ import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.common.exception.FailedPreconditionException; import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; import com.tencent.bk.job.common.iam.model.AuthResult; @@ -64,32 +65,33 @@ public EsbResp updateFileSource(EsbCreateOrUpdateF Integer id = req.getId(); checkManageFileSourcePermission(username, req.getAppResourceScope(), id); FileSourceDTO fileSourceDTO = buildFileSourceDTO(req.getUserName(), appId, req); - Integer fileSourceId = fileSourceService.updateFileSourceById(appId, fileSourceDTO); - return EsbResp.buildSuccessResp(new EsbFileSourceSimpleInfoV3DTO(fileSourceId)); + int affectedNum = fileSourceService.updateFileSourceById(appId, fileSourceDTO); + log.info("{} fileSource updated", affectedNum); + return EsbResp.buildSuccessResp(new EsbFileSourceSimpleInfoV3DTO(id)); } - private void checkCreateParam(EsbCreateOrUpdateFileSourceV3Req req) { - String code = req.getCode(); - if (StringUtils.isBlank(code)) { - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME_AND_REASON, - new String[]{"code", "code cannot be null or blank"}); - } - if (fileSourceService.existsCode(code)) { - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME_AND_REASON, - new String[]{"code", String.format("code [%s] already exists", code)}); - } + private void checkCommonParam(EsbCreateOrUpdateFileSourceV3Req req) { if (StringUtils.isBlank(req.getAlias())) { - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME_AND_REASON, - new String[]{"alias", "alias cannot be null or blank"}); + throw new InvalidParamException(ErrorCode.MISSING_PARAM_WITH_PARAM_NAME, new String[]{"alias"}); } if (StringUtils.isBlank(req.getType())) { - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME_AND_REASON, - new String[]{"type", "type cannot be null or blank"}); + throw new InvalidParamException(ErrorCode.MISSING_PARAM_WITH_PARAM_NAME, new String[]{"type"}); } if (StringUtils.isBlank(req.getCredentialId())) { - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME_AND_REASON, - new String[]{"credential_id", "credential_id cannot be null or blank"}); + throw new InvalidParamException(ErrorCode.MISSING_PARAM_WITH_PARAM_NAME, new String[]{"credential_id"}); + } + } + + private void checkCreateParam(EsbCreateOrUpdateFileSourceV3Req req) { + String code = req.getCode(); + if (StringUtils.isBlank(code)) { + throw new InvalidParamException(ErrorCode.MISSING_PARAM_WITH_PARAM_NAME, + new String[]{"code"}); + } + if (fileSourceService.existsCode(req.getAppId(), code)) { + throw new FailedPreconditionException(ErrorCode.FILE_SOURCE_CODE_ALREADY_EXISTS, new String[]{code}); } + checkCommonParam(req); } private void checkUpdateParam(EsbCreateOrUpdateFileSourceV3Req req) { @@ -97,23 +99,19 @@ private void checkUpdateParam(EsbCreateOrUpdateFileSourceV3Req req) { Integer id = req.getId(); String code = req.getCode(); if (id == null && StringUtils.isBlank(code)) { - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME_AND_REASON, - new String[]{"id/code", "id and code cannot be null/blank simultaneously"}); + throw new InvalidParamException(ErrorCode.ID_AND_CODE_AT_LEAST_ONE); } if (id == null) { id = fileSourceService.getFileSourceIdByCode(appId, code); if (id == null) { - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME_AND_REASON, - new String[]{"code", String.format("cannot find fileSource by code [%s]", code)}); + throw new FailedPreconditionException(ErrorCode.FAIL_TO_FIND_FILE_SOURCE_BY_CODE, new String[]{code}); } } req.setId(id); if (!fileSourceService.existsFileSource(appId, id)) { - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME_AND_REASON, - new String[]{"bk_biz_id/id", - String.format("fileSource [%s] not exists in biz [%s]", id, appId)} - ); + throw new FailedPreconditionException(ErrorCode.FILE_SOURCE_ID_NOT_IN_BIZ, new String[]{id.toString()}); } + checkCommonParam(req); } private FileSourceDTO buildFileSourceDTO(String username, Long appId, diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileSourceResourceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileSourceResourceImpl.java index a4d324c97c..f1f2878fc6 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileSourceResourceImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileSourceResourceImpl.java @@ -25,7 +25,7 @@ package com.tencent.bk.job.file_gateway.api.web; import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.common.exception.FailedPreconditionException; import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.exception.ServiceException; import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; @@ -72,9 +72,42 @@ public WebFileSourceResourceImpl( this.appScopeMappingService = appScopeMappingService; } - private void checkParam(FileSourceCreateUpdateReq fileSourceCreateUpdateReq) { + private void checkCodeBlank(String code) { + if (StringUtils.isBlank(code)) { + throw new InvalidParamException(ErrorCode.MISSING_PARAM_WITH_PARAM_NAME, new String[]{"code"}); + } + } + + private void confirmIdExists(Integer id) { + if (id == null || id <= 0) { + throw new InvalidParamException(ErrorCode.MISSING_OR_ILLEGAL_PARAM_WITH_PARAM_NAME, new String[]{"id"}); + } + } + + private void checkParam(Long appId, FileSourceCreateUpdateReq fileSourceCreateUpdateReq, boolean forCreate) { + Integer id = fileSourceCreateUpdateReq.getId(); + String code = fileSourceCreateUpdateReq.getCode(); + checkCodeBlank(code); + if (forCreate) { + // 创建 + if (fileSourceService.existsCode(appId, code)) { + throw new FailedPreconditionException( + ErrorCode.FILE_SOURCE_CODE_ALREADY_EXISTS, + new String[]{code} + ); + } + } else { + // 更新 + confirmIdExists(id); + if (fileSourceService.existsCodeExceptId(appId, code, id)) { + throw new FailedPreconditionException( + ErrorCode.FILE_SOURCE_CODE_ALREADY_EXISTS, + new String[]{code} + ); + } + } if (StringUtils.isBlank(fileSourceCreateUpdateReq.getCredentialId())) { - throw new InternalException(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME, new String[]{"credentialId"}); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME, new String[]{"credentialId"}); } } @@ -102,7 +135,7 @@ public Response saveFileSource(String username, throw new PermissionDeniedException(authResult); } - checkParam(fileSourceCreateUpdateReq); + checkParam(appId, fileSourceCreateUpdateReq, true); FileSourceDTO fileSourceDTO = buildFileSourceDTO(username, appId, fileSourceCreateUpdateReq); Integer fileSourceId = fileSourceService.saveFileSource(appId, fileSourceDTO); boolean registerResult = fileSourceAuthService.registerFileSource( @@ -129,6 +162,7 @@ public Response updateFileSource(String username, if (!authResult.isPass()) { throw new PermissionDeniedException(authResult); } + checkParam(appId, fileSourceCreateUpdateReq, false); return Response.buildSuccessResp(fileSourceService.updateFileSourceById(appId, fileSourceDTO)); } diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileSourceDAO.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileSourceDAO.java index ffcb68dfaf..d120c51052 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileSourceDAO.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileSourceDAO.java @@ -67,7 +67,9 @@ List listWorkTableFileSource(DSLContext dslContext, Long appId, S List listWorkTableFileSource(DSLContext dslContext, List appIdList, List idList, Integer start, Integer pageSize); - boolean existsCode(String code); + boolean existsCode(Long appId, String code); + + boolean existsCodeExceptId(Long appId, String code, Integer exceptId); boolean existsFileSource(Long appId, Integer id); diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileSourceDAOImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileSourceDAOImpl.java index 8ec6e31bf9..b512d5b8f9 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileSourceDAOImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileSourceDAOImpl.java @@ -426,9 +426,21 @@ public List listWorkTableFileSource(DSLContext dslContext, List 0; + } + + @Override + public boolean existsCodeExceptId(Long appId, String code, Integer exceptId) { + val query = defaultContext.selectZero().from(defaultTable) + .where(defaultTable.APP_ID.eq(appId)) + .and(defaultTable.CODE.eq(code)) + .and(defaultTable.ID.notEqual(exceptId)) + .limit(1); return query.fetch().size() > 0; } diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/FileSourceService.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/FileSourceService.java index 87abdb71d2..a42e50ad2c 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/FileSourceService.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/FileSourceService.java @@ -70,7 +70,7 @@ List listWorkTableFileSource( Integer saveFileSource(Long appId, FileSourceDTO fileSourceDTO); - Integer updateFileSourceById(Long appId, FileSourceDTO fileSourceDTO); + int updateFileSourceById(Long appId, FileSourceDTO fileSourceDTO); int updateFileSourceStatus(Integer fileSourceId, Integer status); @@ -94,7 +94,9 @@ List listWorkTableFileSource( Boolean checkFileSourceAlias(Long appId, String alias, Integer fileSourceId); - boolean existsCode(String code); + boolean existsCode(Long appId, String code); + + boolean existsCodeExceptId(Long appId, String code, Integer exceptId); boolean existsFileSource(Long appId, Integer id); diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileSourceServiceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileSourceServiceImpl.java index 39dbef2592..4b1033c915 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileSourceServiceImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileSourceServiceImpl.java @@ -126,7 +126,7 @@ public Integer saveFileSource(Long appId, FileSourceDTO fileSourceDTO) { } @Override - public Integer updateFileSourceById(Long appId, FileSourceDTO fileSourceDTO) { + public int updateFileSourceById(Long appId, FileSourceDTO fileSourceDTO) { return fileSourceDAO.updateFileSource(dslContext, fileSourceDTO); } @@ -217,8 +217,13 @@ public Boolean checkFileSourceAlias(Long appId, String alias, Integer fileSource } @Override - public boolean existsCode(String code) { - return fileSourceDAO.existsCode(code); + public boolean existsCode(Long appId, String code) { + return fileSourceDAO.existsCode(appId, code); + } + + @Override + public boolean existsCodeExceptId(Long appId, String code, Integer exceptId) { + return fileSourceDAO.existsCodeExceptId(appId, code, exceptId); } @Override diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileSourceTaskServiceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileSourceTaskServiceImpl.java index 5e3af5569a..193bc2b8b1 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileSourceTaskServiceImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileSourceTaskServiceImpl.java @@ -57,7 +57,7 @@ import com.tencent.bk.job.file_gateway.service.listener.FileTaskStatusChangeListener; import com.tencent.bk.job.file_gateway.service.remote.FileSourceTaskReqGenService; import com.tencent.bk.job.logsvr.model.service.ServiceFileTaskLogDTO; -import com.tencent.bk.job.logsvr.model.service.ServiceIpLogDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceHostLogDTO; import lombok.extern.slf4j.Slf4j; import org.jooq.DSLContext; import org.springframework.beans.factory.annotation.Autowired; @@ -189,11 +189,11 @@ private void writeLog(FileSourceTaskDTO fileSourceTaskDTO, FileWorkerDTO fileWor String downloadPath, Long fileSize, String speed, Integer progress, String content) { String taskId = fileSourceTaskDTO.getId(); String fileSizeStr = FileSizeUtil.getFileSizeStr(fileSize); - ServiceIpLogDTO serviceIpLogDTO = new ServiceIpLogDTO(); - serviceIpLogDTO.setStepInstanceId(fileSourceTaskDTO.getStepInstanceId()); - serviceIpLogDTO.setExecuteCount(fileSourceTaskDTO.getExecuteCount()); + ServiceHostLogDTO serviceHostLogDTO = new ServiceHostLogDTO(); + serviceHostLogDTO.setStepInstanceId(fileSourceTaskDTO.getStepInstanceId()); + serviceHostLogDTO.setExecuteCount(fileSourceTaskDTO.getExecuteCount()); String sourceIp = fileWorkerDTO.getCloudAreaId().toString() + ":" + fileWorkerDTO.getInnerIp(); - serviceIpLogDTO.setIp(sourceIp); + serviceHostLogDTO.setIp(sourceIp); // 追加文件源名称 // 日志定位坐标：（文件源，文件路径），需要区分不同文件源下相同文件路径的日志 FileSourceDTO fileSourceDTO = fileSourceDAO.getFileSourceById(dslContext, fileSourceTaskDTO.getFileSourceId()); @@ -214,9 +214,9 @@ private void writeLog(FileSourceTaskDTO fileSourceTaskDTO, FileWorkerDTO fileWor serviceFileTaskLogDTO.setStatus(FileDistStatusEnum.PULLING.getValue()); serviceFileTaskLogDTO.setStatusDesc(FileDistStatusEnum.PULLING.getName()); fileTaskLogs.add(serviceFileTaskLogDTO); - serviceIpLogDTO.setFileTaskLogs(fileTaskLogs); + serviceHostLogDTO.setFileTaskLogs(fileTaskLogs); // 写入Redis - redisTemplate.opsForList().rightPush(PREFIX_REDIS_TASK_LOG + taskId, serviceIpLogDTO); + redisTemplate.opsForList().rightPush(PREFIX_REDIS_TASK_LOG + taskId, serviceHostLogDTO); // 一小时后过期 redisTemplate.expireAt(PREFIX_REDIS_TASK_LOG + taskId, new Date(System.currentTimeMillis() + 3600 * 1000)); } @@ -340,7 +340,7 @@ public FileSourceTaskStatusDTO getFileSourceTaskStatusAndLogs(String taskId, Lon logEnd = logSize; } log.debug("logStart={},logEnd={}", logStart, logEnd); - List logDTOList = redisTemplate.opsForList().range(PREFIX_REDIS_TASK_LOG + taskId, logStart, + List logDTOList = redisTemplate.opsForList().range(PREFIX_REDIS_TASK_LOG + taskId, logStart, logEnd); log.debug("logDTOList={}", logDTOList); fileSourceTaskStatusDTO.setLogList(logDTOList); diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/build.gradle b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/build.gradle index 1cac2ee25a..c192ee89be 100644 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/build.gradle +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/build.gradle @@ -25,18 +25,22 @@ dependencies { api project(":commons:common-i18n") api project(":job-file-worker-sdk:api-job-file-worker-sdk") - implementation "org.springframework.boot:spring-boot-starter-web" - implementation "ch.qos.logback:logback-core" - implementation "ch.qos.logback:logback-classic" - implementation "org.slf4j:slf4j-api" - implementation "org.apache.commons:commons-collections4" - implementation 'org.apache.httpcomponents:httpclient' - implementation group: 'org.apache.thrift', name: 'libthrift' - implementation "commons-io:commons-io" - implementation "commons-codec:commons-codec" - implementation 'io.springfox:springfox-swagger2' - implementation 'io.springfox:springfox-swagger-ui' - implementation 'net.coobird:thumbnailator:0.4.14' + api "org.springframework.boot:spring-boot-starter-web" + api "ch.qos.logback:logback-core" + api "ch.qos.logback:logback-classic" + api "org.slf4j:slf4j-api" + api "org.apache.commons:commons-collections4" + api 'org.apache.httpcomponents:httpclient' + api group: 'org.apache.thrift', name: 'libthrift' + api "commons-io:commons-io" + api "commons-codec:commons-codec" + api 'io.springfox:springfox-swagger2' + api 'io.springfox:springfox-swagger-ui' + api 'net.coobird:thumbnailator:0.4.14' + api "org.springframework.cloud:spring-cloud-sleuth-api" + api "org.springframework.cloud:spring-cloud-sleuth-otel" + api "org.springframework.cloud:spring-cloud-sleuth-otel-autoconfigure" + api "io.opentelemetry:opentelemetry-exporter-otlp-trace" testImplementation 'org.springframework.boot:spring-boot-starter-test' testImplementation 'org.apache.commons:commons-lang3' } diff --git a/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/application.yml b/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/application.yml index 11a1f0ee8d..f323171ad8 100644 --- a/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/application.yml +++ b/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/application.yml @@ -1,6 +1,4 @@ spring: - application: - name: ${SPRING_APPLICATION_NAME:job-file-worker} profiles: active: prod servlet: diff --git a/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/bootstrap.yml b/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/bootstrap.yml index 77bd21fa13..cb75cb699c 100644 --- a/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/bootstrap.yml +++ b/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/bootstrap.yml @@ -1,4 +1,6 @@ spring: + application: + name: ${SPRING_APPLICATION_NAME:job-file-worker} cloud: kubernetes: config: @@ -9,3 +11,6 @@ spring: name: ${JOB_APPLICATION_CONFIGMAP_NAME} - namespace: ${KUBERNETES_NAMESPACE} name: ${JOB_COMMON_CONFIGMAP_NAME} +logging: + pattern: + level: "%5p [${spring.application.name:},%X{trace_id:-},%X{span_id:-}]" diff --git a/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/logback-spring.xml b/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/logback-spring.xml index 9420526bf0..22debb699b 100644 --- a/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/logback-spring.xml +++ b/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/logback-spring.xml @@ -1,12 +1,14 @@ - + logback + + + - logback ${FILE_WORKER_LOG_FILE} @@ -17,7 +19,7 @@ 10GB - ${LOG_PATTERN} + ${FILE_LOG_PATTERN} UTF-8 @@ -30,13 +32,13 @@ 10GB - ${LOG_PATTERN} + ${FILE_LOG_PATTERN} UTF-8 - ${LOG_PATTERN} + ${FILE_LOG_PATTERN} utf8 diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/global/AddTraceResponseHeaderGlobalFilter.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/global/AddTraceResponseHeaderGlobalFilter.java index b2fe64675b..ed28ea100f 100644 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/global/AddTraceResponseHeaderGlobalFilter.java +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/global/AddTraceResponseHeaderGlobalFilter.java @@ -24,12 +24,13 @@ package com.tencent.bk.job.gateway.filter.global; -import brave.Tracer; import com.tencent.bk.job.common.constant.JobCommonHeaders; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cloud.gateway.filter.GatewayFilterChain; import org.springframework.cloud.gateway.filter.GlobalFilter; +import org.springframework.cloud.sleuth.Span; +import org.springframework.cloud.sleuth.Tracer; import org.springframework.core.Ordered; import org.springframework.http.server.reactive.ServerHttpRequest; import org.springframework.http.server.reactive.ServerHttpResponse; @@ -49,11 +50,23 @@ public AddTraceResponseHeaderGlobalFilter(Tracer tracer) { @Override public Mono filter(ServerWebExchange exchange, GatewayFilterChain chain) { - String traceId = tracer.currentSpan().context().traceIdString(); - ServerHttpRequest request = exchange.getRequest(); - ServerHttpResponse response = exchange.getResponse(); - response.getHeaders().add(JobCommonHeaders.REQUEST_ID, traceId); - return chain.filter(exchange.mutate().request(request).build()); + Span span = tracer.currentSpan(); + if (span == null) { + span = tracer.nextSpan(); + span.start(); + } + try (Tracer.SpanInScope ignore = tracer.withSpan(span)) { + String traceId = span.context().traceId(); + ServerHttpRequest request = exchange.getRequest(); + ServerHttpResponse response = exchange.getResponse(); + response.getHeaders().add(JobCommonHeaders.REQUEST_ID, traceId); + return chain.filter(exchange.mutate().request(request).build()); + } catch (Exception e) { + span.error(e); + throw e; + } finally { + span.end(); + } } @Override diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/web/service/impl/LoginServiceImpl.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/web/service/impl/LoginServiceImpl.java index d3adcdf199..116db99e7a 100644 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/web/service/impl/LoginServiceImpl.java +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/web/service/impl/LoginServiceImpl.java @@ -106,7 +106,7 @@ public BkUserDTO getUser(String bkToken) { return userDto.orElse(null); } catch (ExecutionException e) { log.warn("Error occur when get user from paas!"); - throw new InternalException("Query userinfo from paas fail", e, ErrorCode.PAAS_API_DATA_ERROR); + throw new InternalException("Query userinfo from paas fail", e, ErrorCode.USER_MANAGE_API_ACCESS_ERROR); } } diff --git a/src/backend/job-gateway/src/main/resources/application.yml b/src/backend/job-gateway/src/main/resources/application.yml index 89f337b5ad..54e8455799 100644 --- a/src/backend/job-gateway/src/main/resources/application.yml +++ b/src/backend/job-gateway/src/main/resources/application.yml @@ -1,6 +1,4 @@ spring: - application: - name: job-gateway main: allow-bean-definition-overriding: true profiles: diff --git a/src/backend/job-gateway/src/main/resources/bootstrap.yml b/src/backend/job-gateway/src/main/resources/bootstrap.yml index 77bd21fa13..55ccc70423 100644 --- a/src/backend/job-gateway/src/main/resources/bootstrap.yml +++ b/src/backend/job-gateway/src/main/resources/bootstrap.yml @@ -1,4 +1,6 @@ spring: + application: + name: job-gateway cloud: kubernetes: config: diff --git a/src/backend/job-gateway/src/main/resources/logback/logback-app-props.xml b/src/backend/job-gateway/src/main/resources/logback/logback-app-props.xml index dd69ac4f0c..e890aed4ec 100644 --- a/src/backend/job-gateway/src/main/resources/logback/logback-app-props.xml +++ b/src/backend/job-gateway/src/main/resources/logback/logback-app-props.xml @@ -1,4 +1,5 @@ + diff --git a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/api/ServiceLogResource.java b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/api/ServiceLogResource.java index c9ed065c9d..e5c2eb128c 100644 --- a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/api/ServiceLogResource.java +++ b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/api/ServiceLogResource.java @@ -24,20 +24,20 @@ package com.tencent.bk.job.logsvr.api; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; import com.tencent.bk.job.common.annotation.InternalAPI; import com.tencent.bk.job.common.model.InternalResponse; -import com.tencent.bk.job.common.model.dto.IpDTO; -import com.tencent.bk.job.logsvr.model.service.BatchSaveLogRequest; -import com.tencent.bk.job.logsvr.model.service.FileLogQueryRequest; -import com.tencent.bk.job.logsvr.model.service.SaveLogRequest; -import com.tencent.bk.job.logsvr.model.service.ScriptLogQueryRequest; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceBatchSaveLogRequest; +import com.tencent.bk.job.logsvr.model.service.ServiceFileLogQueryRequest; import com.tencent.bk.job.logsvr.model.service.ServiceFileTaskLogDTO; -import com.tencent.bk.job.logsvr.model.service.ServiceIpLogDTO; -import com.tencent.bk.job.logsvr.model.service.ServiceIpLogsDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceHostLogDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceHostLogsDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceSaveLogRequest; +import com.tencent.bk.job.logsvr.model.service.ServiceScriptLogQueryRequest; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; -import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; @@ -57,129 +57,169 @@ @InternalAPI public interface ServiceLogResource { + /** + * 保存执行日志 + * + * @param request 保存日志请求 + */ @ApiOperation("保存执行日志") @PostMapping InternalResponse saveLog( @ApiParam("保存日志请求报文") - @RequestBody SaveLogRequest request); + @RequestBody ServiceSaveLogRequest request + ); - @ApiOperation("保存执行日志") + /** + * 批量保存执行日志 + * + * @param request 保存日志请求 + */ + @ApiOperation("批量保存执行日志") @PostMapping("/batch") InternalResponse saveLogs( @ApiParam("批量保存日志请求报文") - @RequestBody BatchSaveLogRequest request); + @RequestBody ServiceBatchSaveLogRequest request + ); - @ApiOperation("根据服务器IP获取对应的执行日志") - @GetMapping("/jobCreateDate/{jobCreateDate}/step/{stepInstanceId}/retry/{executeCount}/ip/{ip}") - InternalResponse getIpLogContent( - @ApiParam("步骤ID") + @CompatibleImplementation(name = "rolling_execute", explain = "兼容API,后续使用hostId查询", version = "3.7.x") + @ApiOperation("根据目标服务器IP获取脚本任务对应的执行日志") + @GetMapping(value = {"/script/jobCreateDate/{jobCreateDate}/step/{stepInstanceId}/retry/{executeCount}/ip/{ip}"}) + InternalResponse getScriptHostLogByIp( + @ApiParam("作业创建时间") + @PathVariable("jobCreateDate") String jobCreateDate, + @ApiParam("步骤实例ID") @PathVariable("stepInstanceId") Long stepInstanceId, @ApiParam("执行次数") @PathVariable("executeCount") Integer executeCount, @ApiParam("ip") @PathVariable("ip") String ip, + @ApiParam("滚动执行批次，非滚动任务传入null") + @RequestParam(value = "batch", required = false) Integer batch); + + @ApiOperation("根据目标主机ID获取脚本任务对应的执行日志") + @GetMapping(value = { + "/script/jobCreateDate/{jobCreateDate}/step/{stepInstanceId}/retry/{executeCount}/hostId/{hostId}" + }) + InternalResponse getScriptHostLogByHostId( @ApiParam("作业创建时间") @PathVariable("jobCreateDate") String jobCreateDate, - @ApiParam("日志类型") - @RequestParam("logType") Integer logType); - - @ApiOperation("根据目标服务器IP获取脚本任务对应的执行日志") - @GetMapping("/script/jobCreateDate/{jobCreateDate}/step/{stepInstanceId}/retry/{executeCount}/ip/{ip}") - InternalResponse getScriptIpLogContent( - @ApiParam("步骤ID") + @ApiParam("步骤实例ID") @PathVariable("stepInstanceId") Long stepInstanceId, @ApiParam("执行次数") @PathVariable("executeCount") Integer executeCount, - @ApiParam("ip") - @PathVariable("ip") String ip, - @ApiParam("作业创建时间") - @PathVariable("jobCreateDate") String jobCreateDate); + @ApiParam("hostId") + @PathVariable("hostId") Long hostId, + @ApiParam("滚动执行批次，非滚动任务传入null") + @RequestParam(value = "batch", required = false) Integer batch); @ApiOperation("批量获取脚本任务对应的执行日志") - @PostMapping("/script/jobCreateDate/{jobCreateDate}/step/{stepInstanceId}/retry/{executeCount}") - InternalResponse> batchGetScriptLogContent( - @ApiParam("步骤ID") + @PostMapping(value = {"/script/jobCreateDate/{jobCreateDate}/step/{stepInstanceId}/retry/{executeCount}"}) + InternalResponse> listScriptLogs( + @ApiParam("作业创建时间") + @PathVariable("jobCreateDate") String jobCreateDate, + @ApiParam("步骤实例ID") @PathVariable("stepInstanceId") Long stepInstanceId, @ApiParam("执行次数") @PathVariable("executeCount") Integer executeCount, - @ApiParam("作业创建时间") - @PathVariable("jobCreateDate") String jobCreateDate, @ApiParam("查询请求") - @RequestBody ScriptLogQueryRequest query); + @RequestBody ServiceScriptLogQueryRequest query + ); + @CompatibleImplementation(name = "rolling_execute", explain = "兼容API,后续使用hostId查询", version = "3.7.x") @ApiOperation("按照IP获取文件任务对应的执行日志") - @GetMapping("/file/jobCreateDate/{jobCreateDate}/step/{stepInstanceId}/retry/{executeCount}/ip/{ip}") - InternalResponse getFileIpLogContent( - @ApiParam("步骤ID") + @GetMapping(value = {"/file/jobCreateDate/{jobCreateDate}/step/{stepInstanceId}/retry/{executeCount}/ip/{ip}"}) + InternalResponse getFileHostLogByIp( + @ApiParam("作业创建时间") + @PathVariable("jobCreateDate") String jobCreateDate, + @ApiParam("步骤实例ID") @PathVariable("stepInstanceId") Long stepInstanceId, @ApiParam("执行次数") @PathVariable("executeCount") Integer executeCount, @ApiParam("ip") @PathVariable("ip") String ip, + @ApiParam("分发模式,0:upload,1:download") + @RequestParam(value = "mode", required = false) Integer mode, + @ApiParam("滚动执行批次，非滚动任务传入null") + @RequestParam(value = "batch", required = false) Integer batch); + + @ApiOperation("按照hostId获取文件任务对应的执行日志") + @GetMapping(value = { + "/file/jobCreateDate/{jobCreateDate}/step/{stepInstanceId}/retry/{executeCount}/hostId/{hostId}" + }) + InternalResponse getFileHostLogByHostId( @ApiParam("作业创建时间") @PathVariable("jobCreateDate") String jobCreateDate, + @ApiParam("步骤实例ID") + @PathVariable("stepInstanceId") Long stepInstanceId, + @ApiParam("执行次数") + @PathVariable("executeCount") Integer executeCount, + @ApiParam("hostId") + @PathVariable("hostId") Long hostId, @ApiParam("分发模式,0:upload,1:download") - @RequestParam(value = "mode", required = false) Integer mode); + @RequestParam(value = "mode", required = false) Integer mode, + @ApiParam("滚动执行批次，非滚动任务传入null") + @RequestParam(value = "batch", required = false) Integer batch); @ApiOperation("获取文件任务对应的执行日志") @GetMapping("/file/jobCreateDate/{jobCreateDate}/step/{stepInstanceId}/retry/{executeCount}") - InternalResponse> getFileLogContent( - @ApiParam("步骤ID") + InternalResponse> listFileHostLogs( + @ApiParam("作业创建时间") + @PathVariable("jobCreateDate") String jobCreateDate, + @ApiParam("步骤实例ID") @PathVariable("stepInstanceId") Long stepInstanceId, @ApiParam("执行次数") @PathVariable("executeCount") Integer executeCount, - @ApiParam("作业创建时间") - @PathVariable("jobCreateDate") String jobCreateDate, + @ApiParam("滚动执行批次，非滚动任务传入null") + @RequestParam(value = "batch", required = false) Integer batch, @ApiParam("分发模式,0:upload,1:download") @RequestParam(value = "mode", required = false) Integer mode, @ApiParam("ip") - @RequestParam(value = "ip", required = false) String ip); + @RequestParam(value = "ip", required = false) String ip, + @ApiParam("hostId") + @RequestParam(value = "hostId", required = false) Long hostId + ); @ApiOperation("获取文件任务agent对应的执行日志") @PostMapping("/file/jobCreateDate/{jobCreateDate}/step/{stepInstanceId}/retry/{executeCount}/queryByTaskIds") - InternalResponse getFileLogContentListByTaskIds( - @ApiParam("步骤ID") + InternalResponse listFileHostLogsByTaskIds( + @ApiParam("作业创建时间") + @PathVariable("jobCreateDate") String jobCreateDate, + @ApiParam("步骤实例ID") @PathVariable("stepInstanceId") Long stepInstanceId, @ApiParam("执行次数") @PathVariable("executeCount") Integer executeCount, - @ApiParam("作业创建时间") - @PathVariable("jobCreateDate") String jobCreateDate, + @ApiParam("滚动执行批次，非滚动任务传入null") + @RequestParam(value = "batch", required = false) Integer batch, @ApiParam("文件任务ID列表，多个任务ID以;分隔") - @RequestBody List taskIds); + @RequestBody List taskIds + ); @ApiOperation("获取文件任务对应的执行日志") @PostMapping("/file") - InternalResponse getFileLogContent(@RequestBody FileLogQueryRequest request); - - @ApiOperation("删除执行日志") - @DeleteMapping("/jobCreateDate/{jobCreateDate}/step/{stepInstanceId}/retry/{executeCount}") - InternalResponse deleteStepContent( - @ApiParam("步骤ID") - @PathVariable("stepInstanceId") Long stepInstanceId, - @ApiParam("执行次数") - @PathVariable("executeCount") Integer executeCount, - @ApiParam("作业创建时间") - @PathVariable("jobCreateDate") String jobCreateDate); + InternalResponse listFileHostLogs(@RequestBody ServiceFileLogQueryRequest request); /** * 返回日志内容包含关键字的主机ip * - * @param stepInstanceId 步骤ID - * @param executeCount 执行次数 * @param jobCreateDate 创建时间 + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 * @param keyword 查询关键字 * @return ip */ @ApiOperation("根据脚本任务日志关键字获取对应的ip") @GetMapping("/keywordMatch/jobCreateDate/{jobCreateDate}/step/{stepInstanceId}/retry/{executeCount}") - InternalResponse> getIpsByKeyword( - @ApiParam("步骤ID") + InternalResponse> getIpsByKeyword( + @ApiParam("作业创建时间") + @PathVariable("jobCreateDate") String jobCreateDate, + @ApiParam("步骤实例ID") @PathVariable("stepInstanceId") Long stepInstanceId, @ApiParam("执行次数") @PathVariable("executeCount") Integer executeCount, - @ApiParam("作业创建时间") - @PathVariable("jobCreateDate") String jobCreateDate, + @ApiParam("滚动执行批次，非滚动任务传入null") + @RequestParam(value = "batch", required = false) Integer batch, @ApiParam("关键字") - @RequestParam("keyword") String keyword); + @RequestParam("keyword") String keyword + ); } diff --git a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/consts/FileTaskModeEnum.java b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/consts/FileTaskModeEnum.java index c3ad85af06..2abbcbdae3 100644 --- a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/consts/FileTaskModeEnum.java +++ b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/consts/FileTaskModeEnum.java @@ -28,14 +28,12 @@ * 文件分发任务模式 */ public enum FileTaskModeEnum { - UPLOAD(0, "upload"), DOWNLOAD(1, "download"); + UPLOAD(0), DOWNLOAD(1); private final Integer value; - private final String name; - FileTaskModeEnum(Integer val, String name) { + FileTaskModeEnum(Integer val) { this.value = val; - this.name = name; } public static FileTaskModeEnum getFileTaskMode(Integer mode) { diff --git a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/consts/LogTypeEnum.java b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/consts/LogTypeEnum.java index 487133e1fd..f1115560a8 100644 --- a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/consts/LogTypeEnum.java +++ b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/consts/LogTypeEnum.java @@ -28,33 +28,34 @@ * 日志类型 */ public enum LogTypeEnum { - SCRIPT(1, "script"), FILE(2, "file"); + /** + * 脚本执行任务日志 + */ + SCRIPT(1), + /** + * 文件分发任务日志 + */ + FILE(2); private final Integer value; - private final String name; - LogTypeEnum(Integer val, String name) { + LogTypeEnum(Integer val) { this.value = val; - this.name = name; } public static LogTypeEnum getLogType(Integer logType) { if (logType == null) { - return null; + throw new IllegalArgumentException("Empty logType value!"); } for (LogTypeEnum logTypeEnum : values()) { if (logTypeEnum.getValue().equals(logType)) { return logTypeEnum; } } - return null; + throw new IllegalArgumentException("Illegal logType: " + logType); } public Integer getValue() { return value; } - - public String getName() { - return this.name; - } } diff --git a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/BatchSaveLogRequest.java b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceBatchSaveLogRequest.java similarity index 81% rename from src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/BatchSaveLogRequest.java rename to src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceBatchSaveLogRequest.java index ae7dbb476a..8dc19c19a9 100644 --- a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/BatchSaveLogRequest.java +++ b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceBatchSaveLogRequest.java @@ -26,41 +26,30 @@ import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; -import lombok.Getter; -import lombok.Setter; +import lombok.Data; import java.util.List; -import java.util.StringJoiner; /** * 批量保存执行日志请求 */ @ApiModel("批量保存执行日志请求") -@Getter -@Setter -public class BatchSaveLogRequest { +@Data +public class ServiceBatchSaveLogRequest { /** * 作业实例创建时间 */ @ApiModelProperty(value = "作业实例创建时间，格式为yyyy_MM_dd", required = true) private String jobCreateDate; + /** * 执行日志 */ @ApiModelProperty(value = "执行日志", required = true) - private List logs; + private List logs; /** * 日志类型 */ private Integer logType; - - @Override - public String toString() { - return new StringJoiner(", ", BatchSaveLogRequest.class.getSimpleName() + "[", "]") - .add("jobCreateDate='" + jobCreateDate + "'") - .add("logType='" + logType + "'") - .add("logs=" + logs) - .toString(); - } } diff --git a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/FileLogQueryRequest.java b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceFileLogQueryRequest.java similarity index 78% rename from src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/FileLogQueryRequest.java rename to src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceFileLogQueryRequest.java index d6c6481de6..e4b1faca67 100644 --- a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/FileLogQueryRequest.java +++ b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceFileLogQueryRequest.java @@ -24,20 +24,20 @@ package com.tencent.bk.job.logsvr.model.service; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; +import lombok.Data; import java.util.List; /** * 文件任务执行日志查询请求 */ -@Getter -@Setter -@ToString -public class FileLogQueryRequest { +@Data +@ApiModel("文件任务执行日志查询请求") +public class ServiceFileLogQueryRequest { + @ApiModelProperty(value = "作业实例创建时间，格式为yyyy_MM_dd", required = true) private String jobCreateDate; @@ -47,11 +47,15 @@ public class FileLogQueryRequest { @ApiModelProperty(value = "执行次数", required = true) private Integer executeCount; - @ApiModelProperty("服务器IP列表;如果ips参数不为空，那么忽略ip参数") - private List ips; + @ApiModelProperty(value = "滚动执行批次") + private Integer batch; - @ApiModelProperty("服务器IP") - private String ip; + @ApiModelProperty("主机ID列表") + private List hostIds; + + @ApiModelProperty("IP列表;如果hostIds参数不为空，那么忽略ips参数") + @CompatibleImplementation(name = "rolling_execute", explain = "兼容字段，后续用hostIds替换", version = "3.7.x") + private List ips; /** * @see com.tencent.bk.job.logsvr.consts.FileTaskModeEnum diff --git a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceFileTaskLogDTO.java b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceFileTaskLogDTO.java index afe69518df..945a09ecdd 100644 --- a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceFileTaskLogDTO.java +++ b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceFileTaskLogDTO.java @@ -26,6 +26,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.annotations.ApiModel; import lombok.Data; import lombok.NoArgsConstructor; import org.apache.commons.lang3.StringUtils; @@ -33,6 +34,7 @@ /** * 文件分发执行日志 */ +@ApiModel("文件分发执行日志") @Data @NoArgsConstructor @JsonInclude(JsonInclude.Include.NON_EMPTY) @@ -42,78 +44,123 @@ public class ServiceFileTaskLogDTO { */ @JsonProperty("taskId") private String taskId; + /** * 文件分发类型，mode: upload-0/download-1 */ @JsonProperty("mode") private Integer mode; + /** - * 目标IP - 真实 + * 目标IP */ @JsonProperty("destIp") private String destIp; + + /** + * 用于展示的目标IP + */ + @JsonProperty("displayDestIp") + private String displayDestIp; + + /** + * 目标主机ID + */ + @JsonProperty("destHostId") + private Long destHostId; + /** * 目标文件路径 */ @JsonProperty("destFile") private String destFile; + /** - * 文件源IP - 真实 + * 文件源IP */ @JsonProperty("srcIp") private String srcIp; + + /** + * 文件源主机ID + */ + @JsonProperty("srcHostId") + private Long srcHostId; + /** * 文件源IP - 显示 */ @JsonProperty("displaySrcIp") private String displaySrcIp; + /** * 源文件路径 - 真实路径 */ @JsonProperty("srcFile") private String srcFile; + /** * 源文件路径 - 用于显示 */ @JsonProperty("displaySrcFile") private String displaySrcFile; + /** * 文件大小 */ @JsonProperty("size") private String size; + /** * 文件任务状态 */ @JsonProperty("status") private Integer status; + /** * 文件任务状态描述 */ @JsonProperty("statusDesc") private String statusDesc; + /** * 速度 */ @JsonProperty("speed") private String speed; + /** * 进度 */ @JsonProperty("process") private String process; + /** * 日志内容 */ @JsonProperty("content") private String content; - public ServiceFileTaskLogDTO(Integer mode, String destIp, String destFile, String srcIp, String displaySrcIp, - String srcFile, String displaySrcFile, String size, Integer status, - String statusDesc, String speed, String process, String content) { + public ServiceFileTaskLogDTO(Integer mode, + Long destHostId, + String destIp, + String destFile, + Long srcHostId, + String srcIp, + String displaySrcIp, + String srcFile, + String displaySrcFile, + String size, + Integer status, + String statusDesc, + String speed, + String process, + String content) { this.mode = mode; + this.destHostId = destHostId; this.destIp = destIp; this.destFile = destFile; + this.srcHostId = srcHostId; this.srcIp = srcIp; this.displaySrcIp = displaySrcIp; this.srcFile = srcFile; @@ -134,3 +181,5 @@ public String getDisplaySrcIp() { } } } + + diff --git a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceIpLogDTO.java b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceHostLogDTO.java similarity index 79% rename from src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceIpLogDTO.java rename to src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceHostLogDTO.java index 5fd399d1b4..ccfc492889 100644 --- a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceIpLogDTO.java +++ b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceHostLogDTO.java @@ -26,38 +26,53 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; import java.util.ArrayList; import java.util.List; -import java.util.StringJoiner; @ApiModel("主机执行日志") @Data @JsonInclude(JsonInclude.Include.NON_EMPTY) -public class ServiceIpLogDTO { +public class ServiceHostLogDTO { /** * 作业步骤实例ID */ @ApiModelProperty("步骤实例ID") private Long stepInstanceId; + + /** + * 执行次数 + */ + @ApiModelProperty("执行次数") + private Integer executeCount; + + /** + * 滚动执行批次 + */ + @ApiModelProperty("滚动执行批次") + private Integer batch; + /** * ip */ - @ApiModelProperty("ip") + @CompatibleImplementation(name = "rolling_execute", explain = "兼容字段，后续用hostId替换", version = "3.7.x") + @ApiModelProperty(value = "ip, 兼容参数, 如果存在hostId那么忽略ip参数") private String ip; + /** - * 执行次数 + * 主机ID */ - @ApiModelProperty("执行次数") - private Integer executeCount; + @ApiModelProperty(value = "主机ID") + private Long hostId; /** * 脚本日志内容 */ - @ApiModelProperty(value = "脚本日志内容,新协议，保存日志的时候需要传入,查询日志的时候该字段无效") + @ApiModelProperty(value = "脚本日志内容，保存日志的时候需要传入,查询日志的时候该字段无效") @JsonProperty("scriptLog") private ServiceScriptLogDTO scriptLog; @@ -72,15 +87,4 @@ public void addFileTaskLog(ServiceFileTaskLogDTO fileTaskDetailLog) { } fileTaskLogs.add(fileTaskDetailLog); } - - @Override - public String toString() { - StringJoiner joiner = new StringJoiner(", ", ServiceIpLogDTO.class.getSimpleName() + "[", "]") - .add("stepInstanceId=" + stepInstanceId) - .add("ip='" + ip + "'") - .add("executeCount=" + executeCount) - .add("scriptLog=" + scriptLog) - .add("fileTaskLogSize=" + (fileTaskLogs == null ? 0 : fileTaskLogs.size())); - return joiner.toString(); - } } diff --git a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceIpLogsDTO.java b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceHostLogsDTO.java similarity index 82% rename from src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceIpLogsDTO.java rename to src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceHostLogsDTO.java index 7903217911..c6c5501956 100644 --- a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceIpLogsDTO.java +++ b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceHostLogsDTO.java @@ -28,17 +28,14 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; -import lombok.Getter; -import lombok.Setter; +import lombok.Data; import java.util.List; -import java.util.StringJoiner; @ApiModel("主机执行日志-批量") -@Getter -@Setter +@Data @JsonInclude(JsonInclude.Include.NON_EMPTY) -public class ServiceIpLogsDTO { +public class ServiceHostLogsDTO { /** * 作业步骤实例ID */ @@ -56,14 +53,6 @@ public class ServiceIpLogsDTO { */ @ApiModelProperty(value = "主机执行日志") @JsonProperty("ipLogs") - private List ipLogs; + private List ipLogs; - @Override - public String toString() { - return new StringJoiner(", ", ServiceIpLogsDTO.class.getSimpleName() + "[", "]") - .add("stepInstanceId=" + stepInstanceId) - .add("executeCount=" + executeCount) - .add("ipLogs=" + ipLogs) - .toString(); - } } diff --git a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/SaveLogRequest.java b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceSaveLogRequest.java similarity index 73% rename from src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/SaveLogRequest.java rename to src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceSaveLogRequest.java index d1e9e949a1..7d791ff0bf 100644 --- a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/SaveLogRequest.java +++ b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceSaveLogRequest.java @@ -25,62 +25,72 @@ package com.tencent.bk.job.logsvr.model.service; import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; -import lombok.Getter; -import lombok.Setter; +import lombok.Data; import java.util.List; -import java.util.StringJoiner; @ApiModel("执行日志保存请求") -@Getter -@Setter -public class SaveLogRequest { +@Data +public class ServiceSaveLogRequest { /** * 作业实例创建时间 */ @ApiModelProperty(value = "作业实例创建时间，格式为yyyy_MM_dd", required = true) private String jobCreateDate; + /** * 作业步骤实例ID */ @ApiModelProperty(value = "步骤实例ID", required = true) private Long stepInstanceId; + /** * ip */ - @ApiModelProperty(value = "ip", required = true) + @CompatibleImplementation(name = "rolling_execute", explain = "兼容参数,由于IP不再唯一，后续使用hostId参数替换", + version = "3.7.x") + @ApiModelProperty(value = "ip, 兼容参数, 如果存在hostId那么忽略ip参数") private String ip; + + /** + * 主机ID + */ + @ApiModelProperty(value = "主机ID") + private Long hostId; + /** * 执行次数 */ @ApiModelProperty(value = "执行次数", required = true) private Integer executeCount; + /** + * 滚动执行批次 + */ + @ApiModelProperty(value = "滚动批次") + private Integer batch; + /** * 脚本日志内容 */ - @ApiModelProperty(value = "脚本日志内容,新协议") + @ApiModelProperty(value = "脚本日志内容") @JsonProperty("scriptLog") private ServiceScriptLogDTO scriptLog; /** * 文件日志 */ - @ApiModelProperty + @ApiModelProperty(value = "文件任务日志") private List fileTaskLogs; + /** + * 日志类型 + * + * @see com.tencent.bk.job.logsvr.consts.LogTypeEnum + */ + @ApiModelProperty(value = "日志类型,1-script;2-file") private Integer logType; - - @Override - public String toString() { - return new StringJoiner(", ", SaveLogRequest.class.getSimpleName() + "[", "]") - .add("jobCreateDate='" + jobCreateDate + "'") - .add("stepInstanceId=" + stepInstanceId) - .add("ip='" + ip + "'") - .add("executeCount=" + executeCount) - .add("logType=" + logType) - .toString(); - } } diff --git a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceScriptLogDTO.java b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceScriptLogDTO.java index 8f0d83e844..d00f0e0620 100644 --- a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceScriptLogDTO.java +++ b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceScriptLogDTO.java @@ -25,49 +25,58 @@ package com.tencent.bk.job.logsvr.model.service; import com.fasterxml.jackson.annotation.JsonInclude; -import lombok.Getter; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.model.dto.HostDTO; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; import lombok.NoArgsConstructor; -import lombok.Setter; - -import java.util.StringJoiner; /** * 脚本日志 */ -@Getter -@Setter +@Data @JsonInclude(JsonInclude.Include.NON_NULL) @NoArgsConstructor +@ApiModel("脚本日志") public class ServiceScriptLogDTO { /** - * 云IP + * 主机ID */ + @ApiModelProperty("主机ID") + private Long hostId; + + /** + * 云区域+IP + */ + @CompatibleImplementation(name = "rolling_execute", explain = "兼容字段，后续使用hostId替换", version = "3.7.x") + @ApiModelProperty("云区域+IP") private String cloudIp; + /** * 日志偏移 - 字节 */ + @ApiModelProperty("日志偏移 - 字节") private Integer offset; + /** * 日志内容 */ + @ApiModelProperty("日志内容") private String content; - public ServiceScriptLogDTO(String cloudIp, Integer offset, String content) { - this.cloudIp = cloudIp; + public ServiceScriptLogDTO(HostDTO host, Integer offset, String content) { + this.hostId = host.getHostId(); + if (host.getIp() != null && host.getBkCloudId() != null) { + this.cloudIp = host.toCloudIp(); + } this.offset = offset; this.content = content; } - public ServiceScriptLogDTO(String content) { + public ServiceScriptLogDTO(Long hostId, String cloudIp, String content) { + this.hostId = hostId; + this.cloudIp = cloudIp; this.content = content; } - - @Override - public String toString() { - return new StringJoiner(", ", ServiceScriptLogDTO.class.getSimpleName() + "[", "]") - .add("cloudIp='" + cloudIp + "'") - .add("offset=" + offset) - .add("contentLength='" + (content == null ? 0 : content.length()) + "'") - .toString(); - } } diff --git a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceScriptLogQueryRequest.java b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceScriptLogQueryRequest.java new file mode 100644 index 0000000000..e4c477c7d5 --- /dev/null +++ b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceScriptLogQueryRequest.java @@ -0,0 +1,67 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.logsvr.model.service; + +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.util.List; + +/** + * 脚本日志查询请求 + */ +@Data +@ApiModel("脚本日志查询请求") +public class ServiceScriptLogQueryRequest { + /** + * 步骤实例ID + */ + @ApiModelProperty(value = "步骤实例ID", required = true) + private Long stepInstanceId; + + /** + * 执行次数 + */ + @ApiModelProperty(value = "执行次数", required = true) + private Integer executeCount; + + /** + * 滚动执行批次，可能为null + */ + @ApiModelProperty(value = "滚动执行批次，可能为null") + private Integer batch; + + @ApiModelProperty("主机IP列表;兼容参数,如果hostIds参数不为空，那么忽略ips参数") + @CompatibleImplementation(name = "rolling_execute", explain = "兼容字段，后续用hostIds替换", version = "3.7.x") + private List ips; + + /** + * 查询的主机ID列表 + */ + @ApiModelProperty("查询的主机ID列表") + private List hostIds; +} diff --git a/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/application.yml b/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/application.yml index d23af86ba8..39fcad002a 100644 --- a/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/application.yml +++ b/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/application.yml @@ -1,6 +1,4 @@ spring: - application: - name: job-logsvr profiles: active: prod data: diff --git a/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/bootstrap.yml b/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/bootstrap.yml index 77bd21fa13..ddeac107d9 100644 --- a/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/bootstrap.yml +++ b/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/bootstrap.yml @@ -1,4 +1,6 @@ spring: + application: + name: job-logsvr cloud: kubernetes: config: diff --git a/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/logback/logback-app-props.xml b/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/logback/logback-app-props.xml index 1a923f6578..352f606720 100644 --- a/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/logback/logback-app-props.xml +++ b/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/logback/logback-app-props.xml @@ -1,4 +1,5 @@ + diff --git a/src/backend/job-logsvr/boot-job-logsvr/src/test/java/com/tencent/bk/job/logsvr/service/impl/LogServiceImplIntegrationTest.java b/src/backend/job-logsvr/boot-job-logsvr/src/test/java/com/tencent/bk/job/logsvr/service/impl/LogServiceImplIntegrationTest.java index 11941b5945..0731fa11f3 100644 --- a/src/backend/job-logsvr/boot-job-logsvr/src/test/java/com/tencent/bk/job/logsvr/service/impl/LogServiceImplIntegrationTest.java +++ b/src/backend/job-logsvr/boot-job-logsvr/src/test/java/com/tencent/bk/job/logsvr/service/impl/LogServiceImplIntegrationTest.java @@ -24,15 +24,20 @@ package com.tencent.bk.job.logsvr.service.impl; +import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; import com.tencent.bk.job.logsvr.consts.LogTypeEnum; +import com.tencent.bk.job.logsvr.model.FileLogQuery; +import com.tencent.bk.job.logsvr.model.FileTaskLogDoc; import com.tencent.bk.job.logsvr.model.ScriptLogQuery; -import com.tencent.bk.job.logsvr.model.ScriptTaskLog; -import com.tencent.bk.job.logsvr.model.TaskIpLog; +import com.tencent.bk.job.logsvr.model.ScriptTaskLogDoc; +import com.tencent.bk.job.logsvr.model.TaskHostLog; import com.tencent.bk.job.logsvr.mongo.FileLogsCollectionLoader; import com.tencent.bk.job.logsvr.mongo.LogCollectionFactory; import com.tencent.bk.job.logsvr.mongo.LogCollectionLoaderFactory; import com.tencent.bk.job.logsvr.mongo.ScriptLogsCollectionLoader; import com.tencent.bk.job.logsvr.service.LogService; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.data.mongo.DataMongoTest; @@ -40,7 +45,9 @@ import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.TestPropertySource; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; import static org.assertj.core.api.Assertions.assertThat; @@ -53,148 +60,358 @@ public class LogServiceImplIntegrationTest { @Autowired private LogService logService; - @Test - void testSaveScriptLog() { - TaskIpLog taskIpLog1 = new TaskIpLog(); - taskIpLog1.setStepInstanceId(1L); - taskIpLog1.setExecuteCount(0); - taskIpLog1.setJobCreateDate("2020_07_29"); - taskIpLog1.setIp("0:127.0.0.1"); - ScriptTaskLog scriptTaskLog1 = new ScriptTaskLog(1L, "0:127.0.0.1", 0, "hello", 10); - taskIpLog1.setScriptTaskLog(scriptTaskLog1); - taskIpLog1.setLogType(LogTypeEnum.SCRIPT.getValue()); - logService.saveLog(taskIpLog1); - - TaskIpLog taskIpLog2 = new TaskIpLog(); - taskIpLog2.setStepInstanceId(1L); - taskIpLog2.setExecuteCount(0); - taskIpLog2.setJobCreateDate("2020_07_29"); - ScriptTaskLog scriptTaskLog2 = new ScriptTaskLog(1L, "0:127.0.0.1", 0, "world", 20); - taskIpLog2.setScriptTaskLog(scriptTaskLog2); - taskIpLog2.setLogType(LogTypeEnum.SCRIPT.getValue()); - logService.saveLog(taskIpLog2); - - ScriptLogQuery searchRequest = new ScriptLogQuery(); - searchRequest.setStepInstanceId(1L); - searchRequest.setExecuteCount(0); - searchRequest.setJobCreateDate("2020_07_29"); - searchRequest.setIps(Collections.singletonList("0:127.0.0.1")); - TaskIpLog result = logService.getScriptLogByIp(searchRequest); - assertThat(result.getStepInstanceId()).isEqualTo(1L); - assertThat(result.getExecuteCount()).isEqualTo(0); - assertThat(result.getIp()).isEqualTo("0:127.0.0.1"); - assertThat(result.getScriptContent()).isEqualTo("helloworld"); + @Nested + @DisplayName("测试保存脚本日志") + class SaveScriptLogTest { + @Test + @DisplayName("测试保存脚本日志 - 使用hostId") + void testSaveScriptLogV2() { + TaskHostLog taskHostLog1 = new TaskHostLog(); + taskHostLog1.setStepInstanceId(1L); + taskHostLog1.setExecuteCount(0); + taskHostLog1.setBatch(1); + taskHostLog1.setJobCreateDate("2020_07_29"); + ScriptTaskLogDoc scriptTaskLog1 = new ScriptTaskLogDoc(1L, 0, 1, 101L, null, "hello", 10); + taskHostLog1.setScriptTaskLog(scriptTaskLog1); + taskHostLog1.setLogType(LogTypeEnum.SCRIPT.getValue()); + logService.saveLog(taskHostLog1); + + TaskHostLog taskHostLog2 = new TaskHostLog(); + taskHostLog2.setStepInstanceId(1L); + taskHostLog2.setExecuteCount(0); + taskHostLog2.setBatch(1); + taskHostLog2.setJobCreateDate("2020_07_29"); + ScriptTaskLogDoc scriptTaskLog2 = new ScriptTaskLogDoc(1L, 0, 1, 101L, null, "world", 20); + taskHostLog2.setScriptTaskLog(scriptTaskLog2); + taskHostLog2.setLogType(LogTypeEnum.SCRIPT.getValue()); + logService.saveLog(taskHostLog2); + + ScriptLogQuery searchRequest = new ScriptLogQuery(); + searchRequest.setStepInstanceId(1L); + searchRequest.setExecuteCount(0); + searchRequest.setBatch(1); + searchRequest.setJobCreateDate("2020_07_29"); + searchRequest.setHostIds(Collections.singletonList(101L)); + List result = logService.listScriptLogs(searchRequest); + assertThat(result).hasSize(1); + TaskHostLog hostLog = result.get(0); + assertThat(hostLog.getStepInstanceId()).isEqualTo(1L); + assertThat(hostLog.getExecuteCount()).isEqualTo(0); + assertThat(hostLog.getBatch()).isEqualTo(1); + assertThat(hostLog.getHostId()).isEqualTo(101L); + assertThat(hostLog.getScriptContent()).isEqualTo("helloworld"); + + + TaskHostLog taskHostLog3 = new TaskHostLog(); + taskHostLog3.setStepInstanceId(2L); + taskHostLog3.setExecuteCount(0); + taskHostLog3.setBatch(null); + taskHostLog3.setJobCreateDate("2020_07_29"); + ScriptTaskLogDoc scriptTaskLog3 = new ScriptTaskLogDoc(2L, 0, null, 101L, null, "abc", 20); + taskHostLog3.setScriptTaskLog(scriptTaskLog3); + taskHostLog3.setLogType(LogTypeEnum.SCRIPT.getValue()); + logService.saveLog(taskHostLog3); + + searchRequest = new ScriptLogQuery(); + searchRequest.setStepInstanceId(2L); + searchRequest.setExecuteCount(0); + searchRequest.setBatch(null); + searchRequest.setJobCreateDate("2020_07_29"); + searchRequest.setIps(Collections.singletonList(null)); + searchRequest.setHostIds(Collections.singletonList(101L)); + result = logService.listScriptLogs(searchRequest); + assertThat(result).hasSize(1); + hostLog = result.get(0); + assertThat(hostLog.getStepInstanceId()).isEqualTo(2L); + assertThat(hostLog.getExecuteCount()).isEqualTo(0); + assertThat(hostLog.getBatch()).isEqualTo(null); + assertThat(hostLog.getHostId()).isEqualTo(101L); + assertThat(hostLog.getScriptContent()).isEqualTo("abc"); + } + + @Test + @DisplayName("测试保存脚本日志 - 使用ip") + void testSaveScriptLogV1() { + TaskHostLog taskHostLog1 = new TaskHostLog(); + taskHostLog1.setStepInstanceId(1L); + taskHostLog1.setExecuteCount(0); + taskHostLog1.setBatch(1); + taskHostLog1.setJobCreateDate("2020_07_29"); + taskHostLog1.setIp("0:127.0.0.1"); + ScriptTaskLogDoc scriptTaskLog1 = new ScriptTaskLogDoc(1L, 0, 1, null, "0:127.0.0.1", "hello", 10); + taskHostLog1.setScriptTaskLog(scriptTaskLog1); + taskHostLog1.setLogType(LogTypeEnum.SCRIPT.getValue()); + logService.saveLog(taskHostLog1); + + TaskHostLog taskHostLog2 = new TaskHostLog(); + taskHostLog2.setStepInstanceId(1L); + taskHostLog2.setExecuteCount(0); + taskHostLog2.setBatch(1); + taskHostLog2.setJobCreateDate("2020_07_29"); + ScriptTaskLogDoc scriptTaskLog2 = new ScriptTaskLogDoc(1L, 0, 1, null, "0:127.0.0.1", "world", 20); + taskHostLog2.setScriptTaskLog(scriptTaskLog2); + taskHostLog2.setLogType(LogTypeEnum.SCRIPT.getValue()); + logService.saveLog(taskHostLog2); + + ScriptLogQuery searchRequest = new ScriptLogQuery(); + searchRequest.setStepInstanceId(1L); + searchRequest.setExecuteCount(0); + searchRequest.setBatch(1); + searchRequest.setJobCreateDate("2020_07_29"); + searchRequest.setIps(Collections.singletonList("0:127.0.0.1")); + List result = logService.listScriptLogs(searchRequest); + assertThat(result).hasSize(1); + TaskHostLog hostLog = result.get(0); + assertThat(hostLog.getStepInstanceId()).isEqualTo(1L); + assertThat(hostLog.getExecuteCount()).isEqualTo(0); + assertThat(hostLog.getBatch()).isEqualTo(1); + assertThat(hostLog.getIp()).isEqualTo("0:127.0.0.1"); + assertThat(hostLog.getScriptContent()).isEqualTo("helloworld"); + + + TaskHostLog taskHostLog3 = new TaskHostLog(); + taskHostLog3.setStepInstanceId(2L); + taskHostLog3.setExecuteCount(0); + taskHostLog3.setBatch(null); + taskHostLog3.setJobCreateDate("2020_07_29"); + ScriptTaskLogDoc scriptTaskLog3 = new ScriptTaskLogDoc(2L, 0, null, null, "0:127.0.0.1", "abc", 20); + taskHostLog3.setScriptTaskLog(scriptTaskLog3); + taskHostLog3.setLogType(LogTypeEnum.SCRIPT.getValue()); + logService.saveLog(taskHostLog3); + + searchRequest = new ScriptLogQuery(); + searchRequest.setStepInstanceId(2L); + searchRequest.setExecuteCount(0); + searchRequest.setBatch(null); + searchRequest.setJobCreateDate("2020_07_29"); + searchRequest.setIps(Collections.singletonList("0:127.0.0.1")); + result = logService.listScriptLogs(searchRequest); + assertThat(result).hasSize(1); + hostLog = result.get(0); + assertThat(hostLog.getStepInstanceId()).isEqualTo(2L); + assertThat(hostLog.getExecuteCount()).isEqualTo(0); + assertThat(hostLog.getBatch()).isEqualTo(null); + assertThat(hostLog.getIp()).isEqualTo("0:127.0.0.1"); + assertThat(hostLog.getScriptContent()).isEqualTo("abc"); + } } -// @Test -// void testSaveFileLog1() { -// FileTaskLog fileTaskLog1 = buildFileTaskDetailLog(3, "Downloading", "100KB/S", "0:1.1.1.1", "100MB", "50%", -// "/tmp/1.log", 1, "[2020-07-30 11:00:00] Downloading...\n"); -// FileTaskLog fileTaskLog2 = buildFileTaskDetailLog(3, "Downloading", "100KB/S", "0:1.1.1.1", "100MB", "50%", -// "/tmp/2.log", 1, "[2020-07-30 11:00:00] Downloading...\n"); -// List fileTaskLogList = new ArrayList<>(); -// fileTaskLogList.add(fileTaskLog1); -// fileTaskLogList.add(fileTaskLog2); -// TaskLog executionLog1 = buildFileTaskLog(1L, 0, "2020_07_29", "0:127.0.0.1", -// fileTaskLogList); -// logService.saveLog(executionLog1); -// -// TaskLog searchRequest = new TaskLog(); -// searchRequest.setStepInstanceId(1L); -// searchRequest.setExecuteCount(0); -// searchRequest.setJobCreateDate("2020_07_29"); -// searchRequest.setIp("0:127.0.0.1"); -// searchRequest.setLogType(LogTypeEnum.FILE.getValue()); -// TaskLog result = logService.getLogByIp(searchRequest); -// assertThat(result.getStepInstanceId()).isEqualTo(1L); -// assertThat(result.getExecuteCount()).isEqualTo(0); -// assertThat(result.getIp()).isEqualTo("0:127.0.0.1"); -// FileTaskLog resultFileTaskLog1 = result.getFileTaskLogs().get(0); -// assertThat(resultFileTaskLog1.getStatus()).isEqualTo(3); -// assertThat(resultFileTaskLog1.getStatusDesc()).isEqualTo("Downloading"); -// assertThat(resultFileTaskLog1.getMode()).isEqualTo(1); -// assertThat(resultFileTaskLog1.getFile()).isEqualTo("/tmp/1.log"); -// assertThat(resultFileTaskLog1.getSourceIp()).isEqualTo("0:1.1.1.1"); -// assertThat(resultFileTaskLog1.getSize()).isEqualTo("100MB"); -// assertThat(resultFileTaskLog1.getSpeed()).isEqualTo("100KB/S"); -// assertThat(resultFileTaskLog1.getProcess()).isEqualTo("50%"); -// assertThat(resultFileTaskLog1.getContent()).isEqualTo("[2020-07-30 11:00:00] Downloading...\n"); -// FileTaskLog resultFileTaskLog2 = result.getFileTaskLogs().get(1); -// assertThat(resultFileTaskLog2.getStatus()).isEqualTo(3); -// assertThat(resultFileTaskLog2.getStatusDesc()).isEqualTo("Downloading"); -// assertThat(resultFileTaskLog2.getMode()).isEqualTo(1); -// assertThat(resultFileTaskLog2.getFile()).isEqualTo("/tmp/2.log"); -// assertThat(resultFileTaskLog2.getSourceIp()).isEqualTo("0:1.1.1.1"); -// assertThat(resultFileTaskLog2.getSize()).isEqualTo("100MB"); -// assertThat(resultFileTaskLog2.getSpeed()).isEqualTo("100KB/S"); -// assertThat(resultFileTaskLog2.getProcess()).isEqualTo("50%"); -// assertThat(resultFileTaskLog2.getContent()).isEqualTo("[2020-07-30 11:00:00] Downloading...\n"); -// -// // 再次插入日志，验证日志更新场景 -// fileTaskLog1 = buildFileTaskDetailLog(4, "Finished", "0KB/S", "0:1.1.1.1", "100MB", "100%", -// "/tmp/1.log", 1, "[2020-07-30 11:00:00] Download success\n"); -// fileTaskLog2 = buildFileTaskDetailLog(4, "Finished", "0KB/S", "0:1.1.1.1", "100MB", "100%", -// "/tmp/2.log", 1, "[2020-07-30 11:00:00] Download success\n"); -// fileTaskLogList = new ArrayList<>(); -// fileTaskLogList.add(fileTaskLog1); -// fileTaskLogList.add(fileTaskLog2); -// executionLog1 = buildFileTaskLog(1L, 0, "2020_07_29", "0:127.0.0.1", -// fileTaskLogList); -// logService.saveLog(executionLog1); -// -// result = logService.getLogByIp(searchRequest); -// assertThat(result.getStepInstanceId()).isEqualTo(1L); -// assertThat(result.getExecuteCount()).isEqualTo(0); -// assertThat(result.getIp()).isEqualTo("0:127.0.0.1"); -// fileTaskLog1 = result.getFileTaskLogs().get(0); -// assertThat(fileTaskLog1.getStatus()).isEqualTo(4); -// assertThat(fileTaskLog1.getStatusDesc()).isEqualTo("Finished"); -// assertThat(fileTaskLog1.getMode()).isEqualTo(1); -// assertThat(fileTaskLog1.getFile()).isEqualTo("/tmp/1.log"); -// assertThat(fileTaskLog1.getSourceIp()).isEqualTo("0:1.1.1.1"); -// assertThat(fileTaskLog1.getSize()).isEqualTo("100MB"); -// assertThat(fileTaskLog1.getSpeed()).isEqualTo("0KB/S"); -// assertThat(fileTaskLog1.getProcess()).isEqualTo("100%"); -// assertThat(fileTaskLog1.getContent()).isEqualTo("[2020-07-30 11:00:00] Downloading...\n[2020-07-30 -// 11:00:00] Download success\n"); -// fileTaskLog2 = result.getFileTaskLogs().get(1); -// assertThat(fileTaskLog2.getStatus()).isEqualTo(4); -// assertThat(fileTaskLog2.getStatusDesc()).isEqualTo("Finished"); -// assertThat(fileTaskLog2.getMode()).isEqualTo(1); -// assertThat(fileTaskLog2.getFile()).isEqualTo("/tmp/2.log"); -// assertThat(fileTaskLog2.getSourceIp()).isEqualTo("0:1.1.1.1"); -// assertThat(fileTaskLog2.getSize()).isEqualTo("100MB"); -// assertThat(fileTaskLog2.getSpeed()).isEqualTo("0KB/S"); -// assertThat(fileTaskLog2.getProcess()).isEqualTo("100%"); -// assertThat(fileTaskLog2.getContent()).isEqualTo("[2020-07-30 11:00:00] Downloading...\n[2020-07-30 -// 11:00:00] Download success\n"); -// } -// -// private TaskIpLog buildFileTaskLog(long stepInstanceId, int executeCount, String jobCreateDate, String ip, -// List fileTaskLogs) { -// TaskIpLog executionLog = new TaskIpLog(); -// executionLog.setStepInstanceId(stepInstanceId); -// executionLog.setExecuteCount(executeCount); -// executionLog.setJobCreateDate(jobCreateDate); -// executionLog.setIp(ip); -// executionLog.setFileTaskLogs(fileTaskLogs); -// executionLog.setLogType(LogTypeEnum.FILE.getValue()); -// return executionLog; -// } -// -// private FileTaskLog buildFileTaskDetailLog(Integer status, String statusDesc, String speed, String sourceIp, -// String size, String process, String name, -// Integer mode, String content) { -// FileTaskLog fileTaskLog1 = new FileTaskLog(); -// fileTaskLog1.setStatus(status); -// fileTaskLog1.setStatusDesc(statusDesc); -// fileTaskLog1.setSpeed(speed); -// fileTaskLog1.setSourceIp(sourceIp); -// fileTaskLog1.setSize(size); -// fileTaskLog1.setProcess(process); -// fileTaskLog1.setDisplayFile(name); -// fileTaskLog1.setMode(mode); -// fileTaskLog1.setContent(content); -// return fileTaskLog1; -// } + @Nested + @DisplayName("测试保存文件分发日志") + class SaveFileLogTest { + @Test + @DisplayName("测试保存文件分发日志 - 使用IP") + void testSaveFileLog1V1() { + FileTaskLogDoc fileTaskLog1 = buildFileTaskDetailLog( + FileTaskModeEnum.DOWNLOAD.getValue(), + null, + "0:127.0.0.1", + "/tmp/1.log", + "/tmp/1.log", + null, + "0:127.0.0.2", + "/tmp/2.log", + 3, + "Downloading", + "100KB/S", + "100MB", + "50%", + "[2020-07-30 11:00:00] Downloading...\n"); + + List fileTaskLogList = new ArrayList<>(); + fileTaskLogList.add(fileTaskLog1); + TaskHostLog taskHostLog = buildFileTaskHostLog(1L, 0, "2020_07_29", null, + "0:127.0.0.2", fileTaskLogList); + logService.saveLog(taskHostLog); + + FileLogQuery searchRequest = FileLogQuery.builder() + .stepInstanceId(1L) + .executeCount(0) + .jobCreateDate("2020_07_29") + .ips(Collections.singletonList("0:127.0.0.2")) + .build(); + List fileLogDocs = logService.listFileLogs(searchRequest); + assertThat(fileLogDocs).hasSize(1); + FileTaskLogDoc resultFileTaskLog1 = fileLogDocs.get(0); + assertThat(resultFileTaskLog1.getMode()).isEqualTo(FileTaskModeEnum.DOWNLOAD.getValue()); + assertThat(resultFileTaskLog1.getSrcIp()).isEqualTo("0:127.0.0.1"); + assertThat(resultFileTaskLog1.getSrcFile()).isEqualTo("/tmp/1.log"); + assertThat(resultFileTaskLog1.getIp()).isEqualTo("0:127.0.0.2"); + assertThat(resultFileTaskLog1.getDestFile()).isEqualTo("/tmp/2.log"); + assertThat(resultFileTaskLog1.getStatus()).isEqualTo(3); + assertThat(resultFileTaskLog1.getStatusDesc()).isEqualTo("Downloading"); + assertThat(resultFileTaskLog1.getSize()).isEqualTo("100MB"); + assertThat(resultFileTaskLog1.getSpeed()).isEqualTo("100KB/S"); + assertThat(resultFileTaskLog1.getProcess()).isEqualTo("50%"); + assertThat(resultFileTaskLog1.getContent()).isEqualTo("[2020-07-30 11:00:00] Downloading...\n"); + + // 再次插入日志，验证日志更新场景 + fileTaskLog1 = buildFileTaskDetailLog( + FileTaskModeEnum.DOWNLOAD.getValue(), + null, + "0:127.0.0.1", + "/tmp/1.log", + "/tmp/1.log", + null, + "0:127.0.0.2", + "/tmp/2.log", + 4, + "Finished", + "0KB/S", + "100MB", + "100%", + "[2020-07-30 11:00:00] Download success\n"); + fileTaskLogList.clear(); + fileTaskLogList.add(fileTaskLog1); + taskHostLog = buildFileTaskHostLog(1L, 0, "2020_07_29", null, "0:127.0.0.2", + fileTaskLogList); + logService.saveLog(taskHostLog); + + fileLogDocs = logService.listFileLogs(searchRequest); + assertThat(fileLogDocs).hasSize(1); + resultFileTaskLog1 = fileLogDocs.get(0); + assertThat(resultFileTaskLog1.getMode()).isEqualTo(FileTaskModeEnum.DOWNLOAD.getValue()); + assertThat(resultFileTaskLog1.getSrcIp()).isEqualTo("0:127.0.0.1"); + assertThat(resultFileTaskLog1.getSrcFile()).isEqualTo("/tmp/1.log"); + assertThat(resultFileTaskLog1.getIp()).isEqualTo("0:127.0.0.2"); + assertThat(resultFileTaskLog1.getDestFile()).isEqualTo("/tmp/2.log"); + assertThat(resultFileTaskLog1.getStatus()).isEqualTo(4); + assertThat(resultFileTaskLog1.getStatusDesc()).isEqualTo("Finished"); + assertThat(resultFileTaskLog1.getSize()).isEqualTo("100MB"); + assertThat(resultFileTaskLog1.getSpeed()).isEqualTo("0KB/S"); + assertThat(resultFileTaskLog1.getProcess()).isEqualTo("100%"); + assertThat(resultFileTaskLog1.getContent()).isEqualTo("[2020-07-30 11:00:00] Downloading...\n[2020-07-30 11:00:00] Download success\n"); + } + + @Test + @DisplayName("测试保存文件分发日志 - 使用hostId") + void testSaveFileLog1V2() { + FileTaskLogDoc fileTaskLog1 = buildFileTaskDetailLog( + FileTaskModeEnum.DOWNLOAD.getValue(), + 101L, + null, + "/tmp/1.log", + "/tmp/1.log", + 102L, + null, + "/tmp/2.log", + 3, + "Downloading", + "100KB/S", + "100MB", + "50%", + "[2020-07-30 11:00:00] Downloading...\n"); + + List fileTaskLogList = new ArrayList<>(); + fileTaskLogList.add(fileTaskLog1); + TaskHostLog taskHostLog = buildFileTaskHostLog(1L, 0, "2020_07_29", 102L, + null, fileTaskLogList); + logService.saveLog(taskHostLog); + + FileLogQuery searchRequest = FileLogQuery.builder() + .stepInstanceId(1L) + .executeCount(0) + .jobCreateDate("2020_07_29") + .hostIds(Collections.singletonList(102L)) + .build(); + List fileLogDocs = logService.listFileLogs(searchRequest); + assertThat(fileLogDocs).hasSize(1); + FileTaskLogDoc resultFileTaskLog1 = fileLogDocs.get(0); + assertThat(resultFileTaskLog1.getMode()).isEqualTo(FileTaskModeEnum.DOWNLOAD.getValue()); + assertThat(resultFileTaskLog1.getSrcHostId()).isEqualTo(101L); + assertThat(resultFileTaskLog1.getSrcFile()).isEqualTo("/tmp/1.log"); + assertThat(resultFileTaskLog1.getHostId()).isEqualTo(102L); + assertThat(resultFileTaskLog1.getDestFile()).isEqualTo("/tmp/2.log"); + assertThat(resultFileTaskLog1.getStatus()).isEqualTo(3); + assertThat(resultFileTaskLog1.getStatusDesc()).isEqualTo("Downloading"); + assertThat(resultFileTaskLog1.getSize()).isEqualTo("100MB"); + assertThat(resultFileTaskLog1.getSpeed()).isEqualTo("100KB/S"); + assertThat(resultFileTaskLog1.getProcess()).isEqualTo("50%"); + assertThat(resultFileTaskLog1.getContent()).isEqualTo("[2020-07-30 11:00:00] Downloading...\n"); + + // 再次插入日志，验证日志更新场景 + fileTaskLog1 = buildFileTaskDetailLog( + FileTaskModeEnum.DOWNLOAD.getValue(), + 101L, + null, + "/tmp/1.log", + "/tmp/1.log", + 102L, + null, + "/tmp/2.log", + 4, + "Finished", + "0KB/S", + "100MB", + "100%", + "[2020-07-30 11:00:00] Download success\n"); + fileTaskLogList.clear(); + fileTaskLogList.add(fileTaskLog1); + taskHostLog = buildFileTaskHostLog(1L, 0, "2020_07_29", 102L, null, + fileTaskLogList); + logService.saveLog(taskHostLog); + + fileLogDocs = logService.listFileLogs(searchRequest); + assertThat(fileLogDocs).hasSize(1); + resultFileTaskLog1 = fileLogDocs.get(0); + assertThat(resultFileTaskLog1.getMode()).isEqualTo(FileTaskModeEnum.DOWNLOAD.getValue()); + assertThat(resultFileTaskLog1.getSrcHostId()).isEqualTo(101L); + assertThat(resultFileTaskLog1.getSrcFile()).isEqualTo("/tmp/1.log"); + assertThat(resultFileTaskLog1.getHostId()).isEqualTo(102L); + assertThat(resultFileTaskLog1.getDestFile()).isEqualTo("/tmp/2.log"); + assertThat(resultFileTaskLog1.getStatus()).isEqualTo(4); + assertThat(resultFileTaskLog1.getStatusDesc()).isEqualTo("Finished"); + assertThat(resultFileTaskLog1.getSize()).isEqualTo("100MB"); + assertThat(resultFileTaskLog1.getSpeed()).isEqualTo("0KB/S"); + assertThat(resultFileTaskLog1.getProcess()).isEqualTo("100%"); + assertThat(resultFileTaskLog1.getContent()).isEqualTo("[2020-07-30 11:00:00] Downloading...\n[2020-07-30 11:00:00] Download success\n"); + } + } + + + + private TaskHostLog buildFileTaskHostLog(long stepInstanceId, int executeCount, String jobCreateDate, Long hostId, + String ip, List fileTaskLogs) { + TaskHostLog taskHostLog = new TaskHostLog(); + taskHostLog.setStepInstanceId(stepInstanceId); + taskHostLog.setExecuteCount(executeCount); + taskHostLog.setJobCreateDate(jobCreateDate); + taskHostLog.setHostId(hostId); + taskHostLog.setIp(ip); + taskHostLog.setFileTaskLogs(fileTaskLogs); + taskHostLog.setLogType(LogTypeEnum.FILE.getValue()); + return taskHostLog; + } + + FileTaskLogDoc buildFileTaskDetailLog(Integer mode, + Long srcHostId, + String srcIp, + String srcFileName, + String srcFileDisplayName, + Long destHostId, + String destIp, + String destFileName, + Integer status, + String statusDesc, + String speed, + String size, + String process, + String content) { + FileTaskLogDoc fileTaskLogDoc = new FileTaskLogDoc(); + fileTaskLogDoc.setMode(mode); + fileTaskLogDoc.setSrcHostId(srcHostId); + fileTaskLogDoc.setSrcIp(srcIp); + fileTaskLogDoc.setSrcFile(srcFileName); + fileTaskLogDoc.setDisplaySrcFile(srcFileDisplayName); + fileTaskLogDoc.setHostId(destHostId); + fileTaskLogDoc.setIp(destIp); + fileTaskLogDoc.setDestFile(destFileName); + fileTaskLogDoc.setStatus(status); + fileTaskLogDoc.setStatusDesc(statusDesc); + fileTaskLogDoc.setSpeed(speed); + fileTaskLogDoc.setSize(size); + fileTaskLogDoc.setProcess(process); + fileTaskLogDoc.setContent(content); + return fileTaskLogDoc; + } } diff --git a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/api/service/impl/ServiceLogResourceImpl.java b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/api/service/impl/ServiceLogResourceImpl.java index 6a467c031b..622b5e3842 100644 --- a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/api/service/impl/ServiceLogResourceImpl.java +++ b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/api/service/impl/ServiceLogResourceImpl.java @@ -24,28 +24,28 @@ package com.tencent.bk.job.logsvr.api.service.impl; -import com.tencent.bk.job.common.i18n.service.MessageI18nService; import com.tencent.bk.job.common.model.InternalResponse; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.logsvr.api.ServiceLogResource; import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; import com.tencent.bk.job.logsvr.consts.LogTypeEnum; import com.tencent.bk.job.logsvr.model.FileLogQuery; -import com.tencent.bk.job.logsvr.model.FileTaskLog; +import com.tencent.bk.job.logsvr.model.FileTaskLogDoc; import com.tencent.bk.job.logsvr.model.ScriptLogQuery; -import com.tencent.bk.job.logsvr.model.ScriptTaskLog; -import com.tencent.bk.job.logsvr.model.TaskIpLog; -import com.tencent.bk.job.logsvr.model.service.BatchSaveLogRequest; -import com.tencent.bk.job.logsvr.model.service.FileLogQueryRequest; -import com.tencent.bk.job.logsvr.model.service.SaveLogRequest; -import com.tencent.bk.job.logsvr.model.service.ScriptLogQueryRequest; +import com.tencent.bk.job.logsvr.model.ScriptTaskLogDoc; +import com.tencent.bk.job.logsvr.model.TaskHostLog; +import com.tencent.bk.job.logsvr.model.service.ServiceBatchSaveLogRequest; +import com.tencent.bk.job.logsvr.model.service.ServiceFileLogQueryRequest; import com.tencent.bk.job.logsvr.model.service.ServiceFileTaskLogDTO; -import com.tencent.bk.job.logsvr.model.service.ServiceIpLogDTO; -import com.tencent.bk.job.logsvr.model.service.ServiceIpLogsDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceHostLogDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceHostLogsDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceSaveLogRequest; import com.tencent.bk.job.logsvr.model.service.ServiceScriptLogDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceScriptLogQueryRequest; import com.tencent.bk.job.logsvr.service.LogService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; @@ -60,174 +60,264 @@ @Slf4j public class ServiceLogResourceImpl implements ServiceLogResource { private final LogService logService; - private final MessageI18nService i18nService; @Autowired - public ServiceLogResourceImpl(LogService logService, MessageI18nService i18nService) { + public ServiceLogResourceImpl(LogService logService) { this.logService = logService; - this.i18nService = i18nService; } @Override - public InternalResponse saveLog(SaveLogRequest request) { - TaskIpLog taskIpLog = convertToTaskLog(request.getLogType(), request.getJobCreateDate(), - request.getStepInstanceId(), - request.getExecuteCount(), request.getIp(), request.getScriptLog(), request.getFileTaskLogs()); - logService.saveLog(taskIpLog); + public InternalResponse saveLog(ServiceSaveLogRequest request) { + TaskHostLog taskHostLog = convertToTaskLog(request.getLogType(), request.getJobCreateDate(), + request.getStepInstanceId(), request.getExecuteCount(), request.getBatch(), request.getHostId(), + request.getIp(), request.getScriptLog(), request.getFileTaskLogs()); + logService.saveLog(taskHostLog); return InternalResponse.buildSuccessResp(null); } - private TaskIpLog convertToTaskLog(Integer logType, String jobCreateDate, long stepInstanceId, int executeCount, - String ip, ServiceScriptLogDTO scriptLog, - List serviceFileTaskLogs) { - TaskIpLog taskIpLog = new TaskIpLog(); - taskIpLog.setLogType(logType); - taskIpLog.setStepInstanceId(stepInstanceId); - taskIpLog.setExecuteCount(executeCount); - taskIpLog.setIp(ip); - taskIpLog.setJobCreateDate(jobCreateDate); + private TaskHostLog convertToTaskLog(Integer logType, + String jobCreateDate, + long stepInstanceId, + int executeCount, + Integer batch, + Long hostId, + String ip, + ServiceScriptLogDTO scriptLog, + List serviceFileTaskLogs) { + TaskHostLog taskHostLog = new TaskHostLog(); + taskHostLog.setLogType(logType); + taskHostLog.setStepInstanceId(stepInstanceId); + taskHostLog.setExecuteCount(executeCount); + taskHostLog.setBatch(batch); + taskHostLog.setHostId(hostId); + taskHostLog.setIp(ip); + taskHostLog.setJobCreateDate(jobCreateDate); if (scriptLog != null) { - taskIpLog.setScriptTaskLog(new ScriptTaskLog(stepInstanceId, ip, executeCount, scriptLog.getContent(), - scriptLog.getOffset())); + taskHostLog.setScriptTaskLog(new ScriptTaskLogDoc(stepInstanceId, executeCount, batch, hostId, ip, + scriptLog.getContent(), scriptLog.getOffset())); } if (CollectionUtils.isNotEmpty(serviceFileTaskLogs)) { - List fileTaskLogs = serviceFileTaskLogs.parallelStream() - .map(FileTaskLog::convert).collect(Collectors.toList()); - taskIpLog.setFileTaskLogs(fileTaskLogs); + List fileTaskLogs = serviceFileTaskLogs.stream() + .map(FileTaskLogDoc::convert).collect(Collectors.toList()); + taskHostLog.setFileTaskLogs(fileTaskLogs); } - return taskIpLog; + return taskHostLog; } @Override - public InternalResponse saveLogs(BatchSaveLogRequest request) { - List taskIpLogs = - request.getLogs().parallelStream().map(log -> convertToTaskLog(request.getLogType(), - request.getJobCreateDate(), log.getStepInstanceId(), - log.getExecuteCount(), log.getIp(), log.getScriptLog(), log.getFileTaskLogs())) + public InternalResponse saveLogs(ServiceBatchSaveLogRequest request) { + if (CollectionUtils.isEmpty(request.getLogs())) { + return InternalResponse.buildSuccessResp(null); + } + + List taskHostLogs = + request.getLogs().stream() + .map(log -> convertToTaskLog(request.getLogType(), request.getJobCreateDate(), log.getStepInstanceId(), + log.getExecuteCount(), log.getBatch(), log.getHostId(), log.getIp(), log.getScriptLog(), + log.getFileTaskLogs())) .collect(Collectors.toList()); LogTypeEnum logType = LogTypeEnum.getLogType(request.getLogType()); - logService.saveLogs(logType, taskIpLogs); + logService.saveLogs(logType, taskHostLogs); return InternalResponse.buildSuccessResp(null); } @Override - public InternalResponse getIpLogContent(Long stepInstanceId, Integer executeCount, String ip, - String jobCreateDate, Integer logType) { - if (LogTypeEnum.SCRIPT.getValue().equals(logType)) { - return getScriptIpLogContent(stepInstanceId, executeCount, ip, jobCreateDate); - } else if (LogTypeEnum.FILE.getValue().equals(logType)) { - return getFileIpLogContent(stepInstanceId, executeCount, ip, jobCreateDate, null); - } else { - return InternalResponse.buildSuccessResp(null); + public InternalResponse getScriptHostLogByIp(String jobCreateDate, + Long stepInstanceId, + Integer executeCount, + String ip, + Integer batch) { + return InternalResponse.buildSuccessResp( + getScriptHostLog(jobCreateDate, stepInstanceId, executeCount, null, batch, ip)); + } + + private ServiceHostLogDTO toServiceLogDTO(TaskHostLog taskHostLog) { + if (taskHostLog == null) { + return null; + } + ServiceHostLogDTO result = new ServiceHostLogDTO(); + result.setStepInstanceId(taskHostLog.getStepInstanceId()); + result.setExecuteCount(taskHostLog.getExecuteCount()); + result.setBatch(taskHostLog.getBatch()); + result.setHostId(taskHostLog.getHostId()); + result.setIp(taskHostLog.getIp()); + if (StringUtils.isNotEmpty(taskHostLog.getScriptContent())) { + result.setScriptLog(new ServiceScriptLogDTO(taskHostLog.getHostId(), taskHostLog.getIp(), + taskHostLog.getScriptContent())); + } + if (CollectionUtils.isNotEmpty(taskHostLog.getFileTaskLogs())) { + result.setFileTaskLogs(taskHostLog.getFileTaskLogs().stream() + .map(FileTaskLogDoc::toServiceFileTaskLogDTO) + .collect(Collectors.toList())); } + return result; } @Override - public InternalResponse getScriptIpLogContent(Long stepInstanceId, Integer executeCount, String ip, - String jobCreateDate) { - ScriptLogQuery query = new ScriptLogQuery(jobCreateDate, stepInstanceId, ip, executeCount); - TaskIpLog taskIpLog = logService.getScriptLogByIp(query); - ServiceIpLogDTO result = toServiceLogDTO(taskIpLog); - return InternalResponse.buildSuccessResp(result); + public InternalResponse getScriptHostLogByHostId(String jobCreateDate, + Long stepInstanceId, + Integer executeCount, + Long hostId, + Integer batch) { + + return InternalResponse.buildSuccessResp( + getScriptHostLog(jobCreateDate, stepInstanceId, executeCount, hostId, batch, null)); } - private ServiceIpLogDTO toServiceLogDTO(TaskIpLog taskIpLog) { - ServiceIpLogDTO result = new ServiceIpLogDTO(); - if (taskIpLog != null) { - result.setStepInstanceId(taskIpLog.getStepInstanceId()); - result.setExecuteCount(taskIpLog.getExecuteCount()); - result.setIp(taskIpLog.getIp()); - result.setScriptLog(new ServiceScriptLogDTO(taskIpLog.getScriptContent())); + private ServiceHostLogDTO getScriptHostLog(String jobCreateDate, + Long stepInstanceId, + Integer executeCount, + Long hostId, + Integer batch, + String ip) { + ScriptLogQuery query = new ScriptLogQuery(jobCreateDate, stepInstanceId, executeCount, batch, + hostId == null ? null : Collections.singletonList(hostId), + StringUtils.isEmpty(ip) ? null : Collections.singletonList(ip)); + List taskHostLogs = logService.listScriptLogs(query); + if (CollectionUtils.isEmpty(taskHostLogs)) { + return null; } - return result; + + return toServiceLogDTO(taskHostLogs.get(0)); } @Override - public InternalResponse> batchGetScriptLogContent(Long stepInstanceId, Integer executeCount, - String jobCreateDate, - ScriptLogQueryRequest query) { - ScriptLogQuery scriptLogQuery = new ScriptLogQuery(jobCreateDate, stepInstanceId, query.getIps(), executeCount); - List taskIpLogs = logService.batchGetScriptLogByIps(scriptLogQuery); - List scriptLogs = taskIpLogs.stream().map(this::toServiceLogDTO).collect(Collectors.toList()); + public InternalResponse> listScriptLogs(String jobCreateDate, + Long stepInstanceId, + Integer executeCount, + ServiceScriptLogQueryRequest query) { + ScriptLogQuery scriptLogQuery = new ScriptLogQuery(jobCreateDate, stepInstanceId, executeCount, + query.getBatch(), query.getHostIds(), query.getIps()); + List taskHostLogs = logService.listScriptLogs(scriptLogQuery); + List scriptLogs = + taskHostLogs.stream().map(this::toServiceLogDTO).collect(Collectors.toList()); return InternalResponse.buildSuccessResp(scriptLogs); } @Override - public InternalResponse getFileIpLogContent(Long stepInstanceId, Integer executeCount, String ip, - String jobCreateDate, Integer mode) { - FileLogQuery query = FileLogQuery.builder().stepInstanceId(stepInstanceId) - .executeCount(executeCount).jobCreateDate(jobCreateDate).mode(mode).ip(ip).build(); - TaskIpLog taskIpLog = logService.getFileLogByIp(query); - ServiceIpLogDTO result = new ServiceIpLogDTO(); + public InternalResponse getFileHostLogByIp(String jobCreateDate, + Long stepInstanceId, + Integer executeCount, + String ip, + Integer mode, + Integer batch) { + FileLogQuery query = FileLogQuery.builder() + .jobCreateDate(jobCreateDate) + .stepInstanceId(stepInstanceId) + .executeCount(executeCount) + .mode(mode) + .batch(batch) + .ips(Collections.singletonList(ip)) + .build(); + List fileTaskLogs = logService.listFileLogs(query); + ServiceHostLogDTO result = new ServiceHostLogDTO(); result.setStepInstanceId(stepInstanceId); result.setExecuteCount(executeCount); + result.setBatch(batch); result.setIp(ip); - if (taskIpLog != null) { - if (taskIpLog.getFileTaskLogs() != null && !taskIpLog.getFileTaskLogs().isEmpty()) { - result.setFileTaskLogs(taskIpLog.getFileTaskLogs().parallelStream() - .map(FileTaskLog::toServiceFileTaskLogDTO).collect(Collectors.toList())); - } + if (CollectionUtils.isNotEmpty(fileTaskLogs)) { + result.setFileTaskLogs(fileTaskLogs.stream() + .map(FileTaskLogDoc::toServiceFileTaskLogDTO) + .collect(Collectors.toList())); } return InternalResponse.buildSuccessResp(result); } @Override - public InternalResponse> getFileLogContent(Long stepInstanceId, - Integer executeCount, - String jobCreateDate, - Integer mode, - String ip) { - FileLogQuery query = FileLogQuery.builder().stepInstanceId(stepInstanceId) - .executeCount(executeCount).jobCreateDate(jobCreateDate).mode(mode).ip(ip).build(); - List fileTaskLogs = logService.getFileLogs(query); + public InternalResponse getFileHostLogByHostId(String jobCreateDate, Long stepInstanceId, + Integer executeCount, Long hostId, Integer mode, + Integer batch) { + FileLogQuery query = FileLogQuery.builder() + .jobCreateDate(jobCreateDate) + .stepInstanceId(stepInstanceId) + .executeCount(executeCount) + .mode(mode) + .batch(batch) + .hostIds(Collections.singletonList(hostId)) + .build(); + List fileTaskLogs = logService.listFileLogs(query); + ServiceHostLogDTO result = new ServiceHostLogDTO(); + result.setStepInstanceId(stepInstanceId); + result.setExecuteCount(executeCount); + result.setBatch(batch); + result.setHostId(hostId); + if (CollectionUtils.isNotEmpty(fileTaskLogs)) { + result.setFileTaskLogs(fileTaskLogs.stream() + .map(FileTaskLogDoc::toServiceFileTaskLogDTO) + .collect(Collectors.toList())); + } + return InternalResponse.buildSuccessResp(result); + } + + public InternalResponse> listFileHostLogs(String jobCreateDate, + Long stepInstanceId, + Integer executeCount, + Integer batch, + Integer mode, + String ip, + Long hostId) { + FileLogQuery query = FileLogQuery.builder() + .jobCreateDate(jobCreateDate) + .stepInstanceId(stepInstanceId) + .executeCount(executeCount) + .batch(batch) + .mode(mode) + .ips(StringUtils.isEmpty(ip) ? null : Collections.singletonList(ip)) + .hostIds(hostId == null ? null : Collections.singletonList(hostId)) + .build(); + + List fileTaskLogs = logService.listFileLogs(query); if (CollectionUtils.isEmpty(fileTaskLogs)) { return InternalResponse.buildSuccessResp(Collections.emptyList()); } - List results = fileTaskLogs.stream().map(FileTaskLog::toServiceFileTaskLogDTO) + List results = fileTaskLogs.stream().map(FileTaskLogDoc::toServiceFileTaskLogDTO) .collect(Collectors.toList()); return InternalResponse.buildSuccessResp(results); } @Override - public InternalResponse getFileLogContentListByTaskIds(Long stepInstanceId, Integer executeCount, - String jobCreateDate, - List taskIds) { - ServiceIpLogDTO result = new ServiceIpLogDTO(); + public InternalResponse listFileHostLogsByTaskIds(String jobCreateDate, + Long stepInstanceId, + Integer executeCount, + Integer batch, + List taskIds) { + ServiceHostLogDTO result = new ServiceHostLogDTO(); result.setStepInstanceId(stepInstanceId); result.setExecuteCount(executeCount); if (CollectionUtils.isEmpty(taskIds)) { return InternalResponse.buildSuccessResp(result); } - List fileTaskLogs = logService.getFileLogsByTaskIds(jobCreateDate, stepInstanceId, executeCount, - taskIds); + List fileTaskLogs = logService.getFileLogsByTaskIds(jobCreateDate, stepInstanceId, executeCount, + batch, taskIds); if (CollectionUtils.isNotEmpty(fileTaskLogs)) { - result.setFileTaskLogs(fileTaskLogs.stream().map(FileTaskLog::toServiceFileTaskLogDTO) + result.setFileTaskLogs(fileTaskLogs.stream().map(FileTaskLogDoc::toServiceFileTaskLogDTO) .collect(Collectors.toList())); } return InternalResponse.buildSuccessResp(result); } @Override - public InternalResponse getFileLogContent(FileLogQueryRequest request) { + public InternalResponse listFileHostLogs(ServiceFileLogQueryRequest request) { FileLogQuery query = FileLogQuery.builder() .stepInstanceId(request.getStepInstanceId()) .executeCount(request.getExecuteCount()) .jobCreateDate(request.getJobCreateDate()) + .batch(request.getBatch()) .mode(request.getMode()) - .ip(request.getIp()) .ips(request.getIps()) + .hostIds(request.getHostIds()) .build(); - ServiceIpLogsDTO ipLogsResult = new ServiceIpLogsDTO(); + ServiceHostLogsDTO ipLogsResult = new ServiceHostLogsDTO(); ipLogsResult.setStepInstanceId(request.getStepInstanceId()); ipLogsResult.setExecuteCount(request.getExecuteCount()); - List fileTaskLogs = logService.getFileLogs(query); + List fileTaskLogs = logService.listFileLogs(query); if (CollectionUtils.isEmpty(fileTaskLogs)) { return InternalResponse.buildSuccessResp(ipLogsResult); } - List fileLogs = fileTaskLogs.stream().map(FileTaskLog::toServiceFileTaskLogDTO) + List fileLogs = fileTaskLogs.stream().map(FileTaskLogDoc::toServiceFileTaskLogDTO) .collect(Collectors.toList()); Map> cloudIpAndLogs = new HashMap<>(); @@ -243,10 +333,10 @@ public InternalResponse getFileLogContent(FileLogQueryRequest }); }); - List ipLogs = new ArrayList<>(); + List ipLogs = new ArrayList<>(); ipLogsResult.setIpLogs(ipLogs); cloudIpAndLogs.forEach((cloudIp, logs) -> { - ServiceIpLogDTO ipLog = new ServiceIpLogDTO(); + ServiceHostLogDTO ipLog = new ServiceHostLogDTO(); ipLog.setIp(cloudIp); ipLog.setFileTaskLogs(logs); ipLogs.add(ipLog); @@ -256,16 +346,13 @@ public InternalResponse getFileLogContent(FileLogQueryRequest } @Override - public InternalResponse deleteStepContent(Long stepInstanceId, Integer executeCount, String jobCreateDate) { - long deleteCount = logService.deleteStepContent(stepInstanceId, executeCount, jobCreateDate); - return InternalResponse.buildSuccessResp(deleteCount); - } - - @Override - public InternalResponse> getIpsByKeyword(Long stepInstanceId, Integer executeCount, - String jobCreateDate, - String keyword) { - List ips = logService.getIpsByKeyword(stepInstanceId, executeCount, jobCreateDate, keyword); + public InternalResponse> getIpsByKeyword(String jobCreateDate, + Long stepInstanceId, + Integer executeCount, + Integer batch, + String keyword) { + List ips = logService.getIpsByKeyword(jobCreateDate, stepInstanceId, executeCount, batch, keyword); return InternalResponse.buildSuccessResp(ips); } + } diff --git a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/FileLogQuery.java b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/FileLogQuery.java index 6eccd2ddac..323f9781c6 100644 --- a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/FileLogQuery.java +++ b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/FileLogQuery.java @@ -24,6 +24,7 @@ package com.tencent.bk.job.logsvr.model; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; import lombok.Builder; import lombok.Getter; import lombok.Setter; @@ -43,18 +44,23 @@ public class FileLogQuery { * 作业步骤实例ID */ private Long stepInstanceId; - /** - * 执行任务的主机ip - */ - private String ip; /** * 执行任务的主机ip列表 */ + @CompatibleImplementation(name = "rolling_execute", explain = "兼容字段，后续用hostIds替换", version = "3.7.x") private List ips; + /** + * 执行任务的主机ID列表 + */ + private List hostIds; /** * 执行次数 */ private Integer executeCount; + /** + * 滚动执行批次 + */ + private Integer batch; /** * 分发模式 * @@ -67,7 +73,6 @@ public String toString() { return new StringJoiner(", ", FileLogQuery.class.getSimpleName() + "[", "]") .add("jobCreateDate='" + jobCreateDate + "'") .add("stepInstanceId=" + stepInstanceId) - .add("ip='" + ip + "'") .add("ips='" + ips + "'") .add("executeCount=" + executeCount) .add("mode=" + mode) diff --git a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/FileTaskLog.java b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/FileTaskLogDoc.java similarity index 78% rename from src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/FileTaskLog.java rename to src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/FileTaskLogDoc.java index 3fe8ee78c6..1f952727d2 100644 --- a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/FileTaskLog.java +++ b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/FileTaskLogDoc.java @@ -26,128 +26,159 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; import com.tencent.bk.job.logsvr.model.service.ServiceFileTaskLogDTO; -import lombok.Getter; -import lombok.Setter; +import lombok.Data; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.data.mongodb.core.mapping.Document; import org.springframework.data.mongodb.core.mapping.Field; import java.util.List; -import java.util.StringJoiner; /** - * 文件任务执行日志 + * 文件任务执行日志 - MongoDB Doc */ -@Getter -@Setter +@Data @JsonInclude(JsonInclude.Include.NON_EMPTY) @Document -public class FileTaskLog { +public class FileTaskLogDoc { /** * 任务ID */ @JsonProperty("taskId") @Field("taskId") private String taskId; + /** * 文件任务模式，mode: 0-upload;1-download */ @JsonProperty("mode") @Field("mode") private Integer mode; + /** * ip - 真实IP。当mode=0时,ip=上传源IP;mode=1时,ip=下载目标IP */ + @CompatibleImplementation(name = "rolling_execute", explain = "兼容字段，后续用hostId替换", version = "3.7.x") @JsonProperty("ip") @Field("ip") private String ip; + + /** + * hostId。当mode=0时,hostId=上传源hostId;mode=1时,hostId=下载目标hostId + */ + @JsonProperty("hostId") + @Field("hostId") + private Long hostId; + /** - * 文件源IP - 真实 + * 文件源IP */ + @CompatibleImplementation(name = "rolling_execute", explain = "兼容字段，后续用srcHostId替换", version = "3.7.x") @JsonProperty("srcIp") @Field("srcIp") private String srcIp; + + /** + * 文件源主机ID + */ + @JsonProperty("srcHostId") + @Field("srcHostId") + private Long srcHostId; + /** * 文件源IP - 显示 */ @JsonProperty("displaySrcIp") @Field("displaySrcIp") private String displaySrcIp; + /** * 目标文件路径 */ @JsonProperty("destFile") @Field("destFile") private String destFile; + /** * 源文件路径 - 用于显示 */ @JsonProperty("displaySrcFile") @Field("displaySrcFile") private String displaySrcFile; + /** * 源文件路径 - 真实路径 */ @JsonProperty("srcFile") @Field("srcFile") private String srcFile; + /** * 文件大小 */ @JsonProperty("size") @Field("size") private String size; + /** * 文件任务状态 */ @JsonProperty("status") @Field("status") private Integer status; + /** * 文件任务状态描述 */ @JsonProperty("statusDesc") @Field("statusDesc") private String statusDesc; + /** * 速度 */ @JsonProperty("speed") @Field("speed") private String speed; + /** * 进度 */ @JsonProperty("process") @Field("process") private String process; + /** * 日志内容在mongodb中按照list的方式存储，与mongodb中的字段contentList对应 */ @JsonProperty("contentList") @Field("contentList") private List contentList; + /** * 日志内容 */ private String content; - public static FileTaskLog convert(ServiceFileTaskLogDTO serviceFileLog) { - FileTaskLog fileLog = new FileTaskLog(); + public static FileTaskLogDoc convert(ServiceFileTaskLogDTO serviceFileLog) { + FileTaskLogDoc fileLog = new FileTaskLogDoc(); fileLog.setContent(serviceFileLog.getContent()); fileLog.setMode(serviceFileLog.getMode()); if (FileTaskModeEnum.UPLOAD.getValue().equals(serviceFileLog.getMode())) { fileLog.setSrcIp(serviceFileLog.getSrcIp()); fileLog.setIp(serviceFileLog.getSrcIp()); + fileLog.setHostId(serviceFileLog.getSrcHostId()); fileLog.setSrcFile(serviceFileLog.getSrcFile()); fileLog.setDisplaySrcFile(serviceFileLog.getDisplaySrcFile()); fileLog.setDisplaySrcIp(serviceFileLog.getDisplaySrcIp()); } else if (FileTaskModeEnum.DOWNLOAD.getValue().equals(serviceFileLog.getMode())) { + fileLog.setHostId(serviceFileLog.getDestHostId()); fileLog.setIp(serviceFileLog.getDestIp()); fileLog.setSrcIp(serviceFileLog.getSrcIp()); + fileLog.setSrcHostId(serviceFileLog.getSrcHostId()); fileLog.setDisplaySrcIp(serviceFileLog.getDisplaySrcIp()); fileLog.setSrcFile(serviceFileLog.getSrcFile()); fileLog.setDisplaySrcFile(serviceFileLog.getDisplaySrcFile()); @@ -172,10 +203,12 @@ public ServiceFileTaskLogDTO toServiceFileTaskLogDTO() { fileLog.setContent(content); } fileLog.setMode(mode); + fileLog.setSrcHostId(srcHostId); fileLog.setSrcIp(srcIp); fileLog.setDisplaySrcIp(displaySrcIp); fileLog.setDisplaySrcFile(displaySrcFile); fileLog.setSrcFile(srcFile); + fileLog.setDestHostId(hostId); fileLog.setDestIp(ip); fileLog.setDestFile(destFile); fileLog.setProcess(process); @@ -186,41 +219,36 @@ public ServiceFileTaskLogDTO toServiceFileTaskLogDTO() { return fileLog; } + @CompatibleImplementation(name = "rolling_execute", explain = "兼容方法，后续不再使用ip相关参数，发布完成后删除", + version = "3.6.x") public String buildTaskId() { StringBuilder sb = new StringBuilder(); sb.append(mode).append("_"); if (FileTaskModeEnum.UPLOAD.getValue().equals(mode)) { - sb.append(displaySrcIp).append("_"); + if (hostId != null) { + sb.append(hostId).append("_"); + } else { + sb.append(displaySrcIp).append("_"); + } sb.append(displaySrcFile); } else { - sb.append(displaySrcIp).append("_"); + if (srcHostId != null) { + sb.append(srcHostId).append("_"); + } else { + sb.append(displaySrcIp).append("_"); + } // 暂时不加入源文件，GSE暂不支持 // sb.append(displaySrcFile).append("_"); - sb.append(ip).append("_"); + if (hostId != null) { + sb.append(hostId).append("_"); + } else { + sb.append(ip).append("_"); + } sb.append(destFile); } return sb.toString(); } - @Override - public String toString() { - return new StringJoiner(", ", FileTaskLog.class.getSimpleName() + "[", "]") - .add("taskId='" + taskId + "'") - .add("mode=" + mode) - .add("ip='" + ip + "'") - .add("srcIp='" + srcIp + "'") - .add("displaySrcIp='" + displaySrcIp + "'") - .add("destFile='" + destFile + "'") - .add("displaySrcFile='" + displaySrcFile + "'") - .add("srcFile='" + srcFile + "'") - .add("size='" + size + "'") - .add("status=" + status) - .add("statusDesc='" + statusDesc + "'") - .add("speed='" + speed + "'") - .add("process='" + process + "'") - .toString(); - } - public String getTaskId() { return this.taskId == null ? buildTaskId() : this.taskId; } diff --git a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/ScriptLogQuery.java b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/ScriptLogQuery.java index 05f931607a..9ba53e9614 100644 --- a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/ScriptLogQuery.java +++ b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/ScriptLogQuery.java @@ -24,10 +24,10 @@ package com.tencent.bk.job.logsvr.model; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; import lombok.Data; import lombok.NoArgsConstructor; -import java.util.Collections; import java.util.List; @Data @@ -42,25 +42,34 @@ public class ScriptLogQuery { */ private Long stepInstanceId; /** - * 执行任务的主机ip + * 执行任务的主机ip列表 */ + @CompatibleImplementation(name = "rolling_execute", explain = "兼容字段，后续用hostIds替换", version = "3.7.x") private List ips; + /** + * 执行任务的主机ID列表 + */ + private List hostIds; /** * 执行次数 */ private Integer executeCount; + /** + * 滚动执行批次 + */ + private Integer batch; - public ScriptLogQuery(String jobCreateDate, Long stepInstanceId, String ip, Integer executeCount) { + public ScriptLogQuery(String jobCreateDate, + Long stepInstanceId, + Integer executeCount, + Integer batch, + List hostIds, + List ips) { this.jobCreateDate = jobCreateDate; this.stepInstanceId = stepInstanceId; - this.ips = Collections.singletonList(ip); this.executeCount = executeCount; - } - - public ScriptLogQuery(String jobCreateDate, Long stepInstanceId, List ips, Integer executeCount) { - this.jobCreateDate = jobCreateDate; - this.stepInstanceId = stepInstanceId; + this.batch = batch; + this.hostIds = hostIds; this.ips = ips; - this.executeCount = executeCount; } } diff --git a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/ScriptTaskLog.java b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/ScriptTaskLogDoc.java similarity index 76% rename from src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/ScriptTaskLog.java rename to src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/ScriptTaskLogDoc.java index d09ba0f3ff..99d6bfd926 100644 --- a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/ScriptTaskLog.java +++ b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/ScriptTaskLogDoc.java @@ -26,6 +26,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; import lombok.Data; import lombok.NoArgsConstructor; import org.springframework.data.mongodb.core.mapping.Document; @@ -40,7 +41,7 @@ @JsonInclude(JsonInclude.Include.NON_EMPTY) @NoArgsConstructor @Document -public class ScriptTaskLog { +public class ScriptTaskLogDoc { public static final ScriptTaskLogOffsetComparator LOG_OFFSET_COMPARATOR = new ScriptTaskLogOffsetComparator(); /** * 作业步骤实例ID @@ -48,18 +49,31 @@ public class ScriptTaskLog { @JsonProperty("stepId") @Field("stepId") private Long stepInstanceId; + /** + * 执行次数 + */ + @JsonProperty("executeCount") + @Field("executeCount") + private Integer executeCount; + /** + * 滚动执行批次 + */ + @JsonProperty("batch") + @Field("batch") + private Integer batch; /** * 执行任务的主机ip */ + @CompatibleImplementation(name = "rolling_execute", explain = "兼容字段，后续用hostId替换", version = "3.7.x") @JsonProperty("ip") @Field("ip") private String ip; /** - * 执行次数 + * 执行任务的主机hostId */ - @JsonProperty("executeCount") - @Field("executeCount") - private Integer executeCount; + @JsonProperty("hostId") + @Field("hostId") + private Long hostId; /** * 日志内容 */ @@ -73,18 +87,26 @@ public class ScriptTaskLog { @Field("offset") private Integer offset; - public ScriptTaskLog(Long stepInstanceId, String ip, Integer executeCount, String content, Integer offset) { + public ScriptTaskLogDoc(Long stepInstanceId, + Integer executeCount, + Integer batch, + Long hostId, + String ip, + String content, + Integer offset) { this.stepInstanceId = stepInstanceId; - this.ip = ip; this.executeCount = executeCount; + this.batch = batch; + this.hostId = hostId; + this.ip = ip; this.content = content; this.offset = offset; } - private static class ScriptTaskLogOffsetComparator implements Comparator { + private static class ScriptTaskLogOffsetComparator implements Comparator { @Override - public int compare(ScriptTaskLog log1, ScriptTaskLog log2) { + public int compare(ScriptTaskLogDoc log1, ScriptTaskLogDoc log2) { if (log1 == null || log2 == null) { return 0; } diff --git a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/TaskIpLog.java b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/TaskHostLog.java similarity index 76% rename from src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/TaskIpLog.java rename to src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/TaskHostLog.java index 3cb1dc5d14..f030b4af36 100644 --- a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/TaskIpLog.java +++ b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/TaskHostLog.java @@ -24,8 +24,10 @@ package com.tencent.bk.job.logsvr.model; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; import lombok.Getter; import lombok.Setter; +import org.apache.commons.lang3.StringUtils; import java.util.List; import java.util.StringJoiner; @@ -35,11 +37,11 @@ */ @Getter @Setter -public class TaskIpLog { +public class TaskHostLog { /** * 文件任务执行日志 */ - List fileTaskLogs; + List fileTaskLogs; /** * 作业实例创建时间,格式yyyy_MM_dd */ @@ -51,15 +53,24 @@ public class TaskIpLog { /** * 执行任务的主机ip */ + @CompatibleImplementation(explain = "兼容参数,由于IP不再唯一，后续使用hostId参数替换", version = "3.7.x") private String ip; + /** + * 主机ID + */ + private Long hostId; /** * 执行次数 */ private Integer executeCount; + /** + * 滚动批次 + */ + private Integer batch; /** * 脚本任务执行日志 */ - private ScriptTaskLog scriptTaskLog; + private ScriptTaskLogDoc scriptTaskLog; /** * 脚本任务执行日志内容 */ @@ -73,13 +84,15 @@ public class TaskIpLog { @Override public String toString() { - return new StringJoiner(", ", TaskIpLog.class.getSimpleName() + "[", "]") + return new StringJoiner(", ", TaskHostLog.class.getSimpleName() + "[", "]") .add("jobCreateDate='" + jobCreateDate + "'") .add("stepInstanceId=" + stepInstanceId) + .add("hostId='" + hostId + "'") .add("ip='" + ip + "'") .add("executeCount=" + executeCount) - .add("scriptTaskLog=" + scriptTaskLog) - .add("fileTaskLogs=" + fileTaskLogs) + .add("batch=" + batch) + .add("scriptContentLength=" + (StringUtils.isEmpty(scriptContent) ? 0 : scriptContent.length())) + .add("logType=" + logType) .toString(); } } diff --git a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/FileLogsCollectionLoader.java b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/FileLogsCollectionLoader.java index aeadea6a05..c41ee042b3 100644 --- a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/FileLogsCollectionLoader.java +++ b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/FileLogsCollectionLoader.java @@ -40,8 +40,8 @@ @Slf4j public class FileLogsCollectionLoader extends CollectionLoaderBase { - private static final String IDX_STEP_COUNT_MODE_IP = "stepId_executeCount_mode_ip"; - private static final String IDX_STEP_COUNT_TASK_ID = "stepId_executeCount_taskId"; + private static final String IDX_STEP_EXECUTE_COUNT_MODE_HOST_ID = "stepId_executeCount_mode_hostId"; + private static final String IDX_STEP_EXECUTE_COUNT_TASK_ID = "stepId_executeCount_taskId"; private static final String IDX_STEP_ID_HASHED = "stepId_hashed"; @Override @@ -57,33 +57,37 @@ private void createIndexIfUnavailable(MongoCollection collection, List String collectionName) { log.info("Create index for collection: {} start...", collectionName); if (!indexes.contains(IDX_STEP_ID_HASHED)) { - log.info("Create index stepId_hashed for collection: {} start...", collectionName); + log.info("Create index {} for collection: {} start...", IDX_STEP_ID_HASHED, collectionName); IndexOptions indexOptions = new IndexOptions(); indexOptions.background(false); indexOptions.name(IDX_STEP_ID_HASHED); collection.createIndex(Document.parse("{\"stepId\":\"hashed\"}"), indexOptions); - log.info("Create index stepId_hashed for collection: {} successfully!", collectionName); + log.info("Create index {} for collection: {} successfully!", IDX_STEP_ID_HASHED, collectionName); } - if (!indexes.contains(IDX_STEP_COUNT_MODE_IP)) { - log.info("Create index stepId_executeCount_mode_ip for collection: {} start...", collectionName); + if (!indexes.contains(IDX_STEP_EXECUTE_COUNT_MODE_HOST_ID)) { + log.info("Create index {} for collection: {} start...", IDX_STEP_EXECUTE_COUNT_MODE_HOST_ID, + collectionName); IndexOptions indexOptions = new IndexOptions(); indexOptions.background(false); - indexOptions.name(IDX_STEP_COUNT_MODE_IP); - collection.createIndex(Document.parse("{\"stepId\":1,\"executeCount\":1,\"mode\":1,\"ip\":1}"), + indexOptions.name(IDX_STEP_EXECUTE_COUNT_MODE_HOST_ID); + collection.createIndex(Document.parse("{\"stepId\":1,\"executeCount\":1,\"mode\":1,\"hostId\":1}"), indexOptions); - log.info("Create index stepId_executeCount_mode_ip for collection: {} successfully!", collectionName); + log.info("Create index {} for collection: {} successfully!", IDX_STEP_EXECUTE_COUNT_MODE_HOST_ID, + collectionName); } - if (!indexes.contains(IDX_STEP_COUNT_TASK_ID)) { - log.info("Create index stepId_executeCount_taskId for collection: {} start...", collectionName); + if (!indexes.contains(IDX_STEP_EXECUTE_COUNT_TASK_ID)) { + log.info("Create index {} for collection: {} start...", IDX_STEP_EXECUTE_COUNT_TASK_ID, collectionName); IndexOptions indexOptions = new IndexOptions(); indexOptions.background(false); - indexOptions.name(IDX_STEP_COUNT_TASK_ID); + indexOptions.name(IDX_STEP_EXECUTE_COUNT_TASK_ID); collection.createIndex(Document.parse("{\"stepId\":1,\"executeCount\":1,\"taskId\":1}"), indexOptions); - log.info("Create index stepId_executeCount_taskId for collection: {} successfully!", collectionName); + log.info("Create index {} for collection: {} successfully!", IDX_STEP_EXECUTE_COUNT_TASK_ID, + collectionName); } + log.info("Create index for collection : {} successfully!", collectionName); } } diff --git a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/ScriptLogsCollectionLoader.java b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/ScriptLogsCollectionLoader.java index d6f8c7baf2..2f4c7d3347 100644 --- a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/ScriptLogsCollectionLoader.java +++ b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/ScriptLogsCollectionLoader.java @@ -40,7 +40,7 @@ @Slf4j public class ScriptLogsCollectionLoader extends CollectionLoaderBase { - private static final String IDX_STEP_EXECUTE_COUNT_IP = "stepId_1_executeCount_1_ip_1"; + private static final String IDX_STEP_ID_EXECUTE_COUNT_HOST_ID = "stepId_executeCount_hostId"; private static final String IDX_STEP_ID_HASHED = "stepId_hashed"; @Override @@ -52,27 +52,28 @@ public MongoCollection load(MongoTemplate mongoTemplate, String collec return collection; } - private void createIndexIfUnavailable(MongoCollection collection, List indexes, String collectionName) { + private void createIndexIfUnavailable(MongoCollection collection, List indexes, + String collectionName) { log.info("Create index for collection: {} start...", collectionName); if (!indexes.contains(IDX_STEP_ID_HASHED)) { - log.info("Create index stepId_hashed for collection: {}start...", collectionName); - IndexOptions indexOptions1 = new IndexOptions(); - indexOptions1.background(false); - indexOptions1.name(IDX_STEP_ID_HASHED); - collection.createIndex(Document.parse("{\"stepId\":\"hashed\"}"), indexOptions1); - log.info("Create index stepId_hashed for collection: {} successfully!", collectionName); + log.info("Create index {} for collection: {}start...", IDX_STEP_ID_HASHED, collectionName); + IndexOptions indexOptions = new IndexOptions(); + indexOptions.background(false); + indexOptions.name(IDX_STEP_ID_HASHED); + collection.createIndex(Document.parse("{\"stepId\":\"hashed\"}"), indexOptions); + log.info("Create index {} for collection: {} successfully!", IDX_STEP_ID_HASHED, collectionName); } - if (!indexes.contains(IDX_STEP_EXECUTE_COUNT_IP)) { - log.info("Create index stepId_1_executeCount_1_ip_1 for collection: {} start...", collectionName); - IndexOptions indexOption2 = new IndexOptions(); - indexOption2.background(false); - indexOption2.name(IDX_STEP_EXECUTE_COUNT_IP); - collection.createIndex(Document.parse("{\"stepId\":1,\"executeCount\":1,\"ip\":1}"), indexOption2); - log.info("Create index stepId_1_executeCount_1_ip_1 for collection: {} successfully!", collectionName); + if (!indexes.contains(IDX_STEP_ID_EXECUTE_COUNT_HOST_ID)) { + log.info("Create index {} for collection: {} start...", IDX_STEP_ID_EXECUTE_COUNT_HOST_ID, collectionName); + IndexOptions indexOptions = new IndexOptions(); + indexOptions.background(false); + indexOptions.name(IDX_STEP_ID_EXECUTE_COUNT_HOST_ID); + collection.createIndex(Document.parse("{\"stepId\":1,\"executeCount\":1,\"hostId\":1}"), + indexOptions); + log.info("Create index {} for collection: {} successfully!", IDX_STEP_ID_EXECUTE_COUNT_HOST_ID, + collectionName); } - log.info("Create index for collection : {} successfully!", collectionName); - } } diff --git a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/service/LogService.java b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/service/LogService.java index a44f8f9c84..e134d3ddf1 100644 --- a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/service/LogService.java +++ b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/service/LogService.java @@ -25,12 +25,12 @@ package com.tencent.bk.job.logsvr.service; import com.tencent.bk.job.common.exception.ServiceException; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.logsvr.consts.LogTypeEnum; import com.tencent.bk.job.logsvr.model.FileLogQuery; -import com.tencent.bk.job.logsvr.model.FileTaskLog; +import com.tencent.bk.job.logsvr.model.FileTaskLogDoc; import com.tencent.bk.job.logsvr.model.ScriptLogQuery; -import com.tencent.bk.job.logsvr.model.TaskIpLog; +import com.tencent.bk.job.logsvr.model.TaskHostLog; import java.util.List; @@ -41,50 +41,33 @@ public interface LogService { /** * 保存执行日志 * - * @param taskIpLog 执行日志 + * @param taskHostLog 执行日志 */ - void saveLog(TaskIpLog taskIpLog); + void saveLog(TaskHostLog taskHostLog); /** * 保存执行日志 * - * @param logType 日志类型 - * @param taskIpLogs 执行日志 + * @param logType 日志类型 + * @param taskHostLogs 执行日志 */ - void saveLogs(LogTypeEnum logType, List taskIpLogs); - - /** - * 获取ip对应的脚本任务日志 - * - * @param query 获取执行日志请求 - * @return 执行日志 - */ - TaskIpLog getScriptLogByIp(ScriptLogQuery query); + void saveLogs(LogTypeEnum logType, List taskHostLogs); /** * 批量获取脚本执行日志 * - * @param query 获取执行日志请求 + * @param scriptLogQuery 获取执行日志请求 * @return 日志内容 * @throws ServiceException 异常 */ - List batchGetScriptLogByIps(ScriptLogQuery query) throws ServiceException; - - /** - * 获取ip对应的文件任务日志 - * - * @param query 获取执行日志请求 - * @return 执行日志 - */ - TaskIpLog getFileLogByIp(FileLogQuery query); + List listScriptLogs(ScriptLogQuery scriptLogQuery); /** - * 根据分发模式获取 + * 查询文件任务执行日志 * - * @param query 获取执行日志请求 - * @return + * @param query 查询文件任务执行日志 */ - List getFileLogs(FileLogQuery query); + List listFileLogs(FileLogQuery query); /** * 根据文件任务ID获取日志 @@ -92,31 +75,31 @@ public interface LogService { * @param jobCreateDate 创建时间 * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 + * @param batch 滚动执行批次 * @param taskIds 任务ID列表 - * @return + * @return 文件任务执行日志 */ - List getFileLogsByTaskIds(String jobCreateDate, long stepInstanceId, int executeCount, - List taskIds); + List getFileLogsByTaskIds(String jobCreateDate, + long stepInstanceId, + int executeCount, + Integer batch, + List taskIds); - /** - * 删除步骤日志 - * - * @param stepInstanceId 步骤实例ID - * @param executeCount 执行次数 - * @param jobCreateDate 任务创建时间,yyyy_MM_dd - */ - long deleteStepContent(Long stepInstanceId, Integer executeCount, String jobCreateDate); - /** * 返回日志内容包含关键字的任务对应的主机ip * + * @param jobCreateDate 创建时间 * @param stepInstanceId 步骤ID * @param executeCount 执行次数 - * @param jobCreateDate 创建时间 + * @param batch 滚动执行批次 * @param keyword 查询关键字 * @return ip */ - List getIpsByKeyword(long stepInstanceId, Integer executeCount, String jobCreateDate, String keyword); + List getIpsByKeyword(String jobCreateDate, + long stepInstanceId, + int executeCount, + Integer batch, + String keyword); } diff --git a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/service/impl/LogServiceImpl.java b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/service/impl/LogServiceImpl.java index 4873d6cca3..900c16c75c 100644 --- a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/service/impl/LogServiceImpl.java +++ b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/service/impl/LogServiceImpl.java @@ -30,16 +30,15 @@ import com.mongodb.client.model.UpdateOneModel; import com.mongodb.client.model.UpdateOptions; import com.mongodb.client.model.WriteModel; -import com.mongodb.client.result.DeleteResult; import com.tencent.bk.job.common.exception.ServiceException; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.util.BatchUtil; import com.tencent.bk.job.logsvr.consts.LogTypeEnum; import com.tencent.bk.job.logsvr.model.FileLogQuery; -import com.tencent.bk.job.logsvr.model.FileTaskLog; +import com.tencent.bk.job.logsvr.model.FileTaskLogDoc; import com.tencent.bk.job.logsvr.model.ScriptLogQuery; -import com.tencent.bk.job.logsvr.model.ScriptTaskLog; -import com.tencent.bk.job.logsvr.model.TaskIpLog; +import com.tencent.bk.job.logsvr.model.ScriptTaskLogDoc; +import com.tencent.bk.job.logsvr.model.TaskHostLog; import com.tencent.bk.job.logsvr.mongo.LogCollectionFactory; import com.tencent.bk.job.logsvr.service.LogService; import lombok.extern.slf4j.Slf4j; @@ -74,67 +73,69 @@ public LogServiceImpl(MongoTemplate mongoTemplate, LogCollectionFactory logColle } @Override - public void saveLog(TaskIpLog taskIpLog) { - if (taskIpLog.getLogType().equals(LogTypeEnum.SCRIPT.getValue())) { - writeScriptLog(taskIpLog); - } else if (taskIpLog.getLogType().equals(LogTypeEnum.FILE.getValue())) { - writeFileLog(taskIpLog); + public void saveLog(TaskHostLog taskHostLog) { + if (taskHostLog.getLogType().equals(LogTypeEnum.SCRIPT.getValue())) { + writeScriptLog(taskHostLog); + } else if (taskHostLog.getLogType().equals(LogTypeEnum.FILE.getValue())) { + writeFileLog(taskHostLog); } } + @Override - public void saveLogs(LogTypeEnum logType, List taskIpLogs) { + public void saveLogs(LogTypeEnum logType, List taskHostLogs) { if (logType == LogTypeEnum.SCRIPT) { - batchWriteScriptLogs(taskIpLogs); + batchWriteScriptLogs(taskHostLogs); } else if (logType == LogTypeEnum.FILE) { - batchWriteFileLogs(taskIpLogs); + batchWriteFileLogs(taskHostLogs); } } - private void batchWriteScriptLogs(List taskIpLogs) { - String jobCreateDate = taskIpLogs.get(0).getJobCreateDate(); + private void batchWriteScriptLogs(List taskHostLogs) { + String jobCreateDate = taskHostLogs.get(0).getJobCreateDate(); String collectionName = buildLogCollectionName(jobCreateDate, LogTypeEnum.SCRIPT); - List scriptLogDocList = taskIpLogs.stream() + List scriptLogDocList = taskHostLogs.stream() .map(taskIpLog -> buildScriptLogDoc(taskIpLog.getScriptTaskLog())).collect(Collectors.toList()); List> batchDocList = BatchUtil.buildBatchList(scriptLogDocList, BATCH_SIZE); long start = System.currentTimeMillis(); - batchDocList.parallelStream().forEach(docs -> { - logCollectionFactory.getCollection(collectionName).insertMany(docs, new InsertManyOptions().ordered(false)); - }); + batchDocList.parallelStream().forEach(docs -> + logCollectionFactory.getCollection(collectionName) + .insertMany(docs, new InsertManyOptions().ordered(false))); long end = System.currentTimeMillis(); log.info("Batch write script logs, docSize: {}, cost: {} ms", scriptLogDocList.size(), end - start); } - private void batchWriteFileLogs(List taskIpLogs) { - String jobCreateDate = taskIpLogs.get(0).getJobCreateDate(); + private void batchWriteFileLogs(List taskHostLogs) { + String jobCreateDate = taskHostLogs.get(0).getJobCreateDate(); String collectionName = buildLogCollectionName(jobCreateDate, LogTypeEnum.FILE); - List> updateOps = buildUpdateOpsFileTask(taskIpLogs); + List> updateOps = buildUpdateOpsFileTask(taskHostLogs); List>> batchList = BatchUtil.buildBatchList(updateOps, BATCH_SIZE); long start = System.currentTimeMillis(); - batchList.parallelStream().forEach(batchOps -> { - logCollectionFactory.getCollection(collectionName) - .bulkWrite(batchOps, new BulkWriteOptions().ordered(false)); - }); + batchList.parallelStream().forEach(batchOps -> logCollectionFactory.getCollection(collectionName) + .bulkWrite(batchOps, new BulkWriteOptions().ordered(false))); long end = System.currentTimeMillis(); log.warn("Batch write file logs, stepInstanceId: {}, opSize: {}, cost: {} ms", - taskIpLogs.get(0).getStepInstanceId(), updateOps.size(), end - start); + taskHostLogs.get(0).getStepInstanceId(), updateOps.size(), end - start); } - private List> buildUpdateOpsFileTask(List taskIpLogs) { + private List> buildUpdateOpsFileTask(List taskHostLogs) { List> updateOps = new ArrayList<>(); - taskIpLogs.forEach(taskIpLog -> { - long stepInstanceId = taskIpLog.getStepInstanceId(); - String ip = taskIpLog.getIp(); - int executeCount = taskIpLog.getExecuteCount(); - List fileTaskLogs = taskIpLog.getFileTaskLogs(); - - if (CollectionUtils.isNotEmpty(taskIpLog.getFileTaskLogs())) { + taskHostLogs.forEach(taskHostLog -> { + long stepInstanceId = taskHostLog.getStepInstanceId(); + String ip = taskHostLog.getIp(); + int executeCount = taskHostLog.getExecuteCount(); + Integer batch = taskHostLog.getBatch(); + List fileTaskLogs = taskHostLog.getFileTaskLogs(); + + if (CollectionUtils.isNotEmpty(taskHostLog.getFileTaskLogs())) { fileTaskLogs.forEach(fileTaskLog -> { - BasicDBObject filter = buildQueryDocForFileTaskLog(stepInstanceId, executeCount, fileTaskLog); - BasicDBObject update = buildUpdateDocForFileTaskLog(stepInstanceId, executeCount, ip, fileTaskLog); + BasicDBObject filter = buildQueryDocForFileTaskLog(stepInstanceId, executeCount, batch, + fileTaskLog); + BasicDBObject update = buildUpdateDocForFileTaskLog(stepInstanceId, executeCount, batch, + taskHostLog.getHostId(), ip, fileTaskLog); UpdateOneModel updateOp = new UpdateOneModel<>(filter, update, new UpdateOptions().upsert(true)); updateOps.add(updateOp); @@ -144,101 +145,116 @@ private List> buildUpdateOpsFileTask(List taskIp return updateOps; } - private void writeScriptLog(TaskIpLog taskIpLog) { - if (taskIpLog == null || taskIpLog.getScriptTaskLog() == null) { - return; - } - LogTypeEnum logType = LogTypeEnum.getLogType(taskIpLog.getLogType()); - if (logType == null) { + private void writeScriptLog(TaskHostLog taskHostLog) { + if (taskHostLog == null || taskHostLog.getScriptTaskLog() == null) { return; } + LogTypeEnum logType = LogTypeEnum.getLogType(taskHostLog.getLogType()); long start = System.currentTimeMillis(); - long stepInstanceId = taskIpLog.getStepInstanceId(); - int executeCount = taskIpLog.getExecuteCount(); - String ip = taskIpLog.getIp(); - if (log.isDebugEnabled()) { - log.debug("Save script log, stepInstanceId: {}, executeCount: {}, ip: {}", stepInstanceId, executeCount, - ip); - } - String collectionName = buildLogCollectionName(taskIpLog.getJobCreateDate(), logType); + long stepInstanceId = taskHostLog.getStepInstanceId(); + String collectionName = buildLogCollectionName(taskHostLog.getJobCreateDate(), logType); try { - Document scriptLogDoc = buildScriptLogDoc(taskIpLog.getScriptTaskLog()); + Document scriptLogDoc = buildScriptLogDoc(taskHostLog.getScriptTaskLog()); logCollectionFactory.getCollection(collectionName).insertOne(scriptLogDoc); } finally { long cost = (System.currentTimeMillis() - start); - if (cost > 10L) { - log.warn("Save log slow, stepInstanceId: {}, executeCount: {}, ip: {}, cost: {} ms", stepInstanceId, - executeCount, ip, cost); + if (cost > 20L) { + log.warn("Save script task log slow, stepInstanceId: {}, cost: {} ms", stepInstanceId, cost); } } } - private void writeFileLog(TaskIpLog taskIpLog) { - long stepInstanceId = taskIpLog.getStepInstanceId(); - int executeCount = taskIpLog.getExecuteCount(); - String ip = taskIpLog.getIp(); - if (taskIpLog.getFileTaskLogs().size() == 1) { - if (log.isDebugEnabled()) { - log.debug("Save file log, stepInstanceId: {}, executeCount: {}, ip: {}", stepInstanceId, executeCount - , ip); - } - taskIpLog.getFileTaskLogs().parallelStream() - .forEach(fileTaskLog -> writeFileLog(taskIpLog.getJobCreateDate(), - taskIpLog.getStepInstanceId(), taskIpLog.getExecuteCount(), taskIpLog.getIp(), fileTaskLog)); + private void writeFileLog(TaskHostLog taskHostLog) { + if (taskHostLog.getFileTaskLogs().size() == 1) { + taskHostLog.getFileTaskLogs().forEach( + fileTaskLog -> writeFileLog(taskHostLog.getJobCreateDate(), taskHostLog.getStepInstanceId(), + taskHostLog.getExecuteCount(), taskHostLog.getBatch(), taskHostLog.getHostId(), taskHostLog.getIp(), + fileTaskLog)); } else { - batchWriteFileLogs(Collections.singletonList(taskIpLog)); + batchWriteFileLogs(Collections.singletonList(taskHostLog)); } } - private Document buildScriptLogDoc(ScriptTaskLog scriptTaskLog) { + private Document buildScriptLogDoc(ScriptTaskLogDoc scriptTaskLog) { Document doc = new Document(); doc.put("stepId", scriptTaskLog.getStepInstanceId()); doc.put("executeCount", scriptTaskLog.getExecuteCount()); - doc.put("ip", scriptTaskLog.getIp()); + if (scriptTaskLog.getBatch() != null && scriptTaskLog.getBatch() > 0) { + doc.put("batch", scriptTaskLog.getBatch()); + } + if (scriptTaskLog.getHostId() != null) { + doc.put("hostId", scriptTaskLog.getHostId()); + } + if (StringUtils.isNotEmpty(scriptTaskLog.getIp())) { + doc.put("ip", scriptTaskLog.getIp()); + } doc.put("content", scriptTaskLog.getContent()); doc.put("offset", scriptTaskLog.getOffset()); return doc; } - private void writeFileLog(String jobCreateDate, long stepInstanceId, int executeCount, String ip, - FileTaskLog fileTaskLog) { + private void writeFileLog(String jobCreateDate, + long stepInstanceId, + int executeCount, + Integer batch, + Long hostId, + String ip, + FileTaskLogDoc fileTaskLog) { long start = System.currentTimeMillis(); String collectionName = buildLogCollectionName(jobCreateDate, LogTypeEnum.FILE); try { - BasicDBObject filter = buildQueryDocForFileTaskLog(stepInstanceId, executeCount, fileTaskLog); - BasicDBObject update = buildUpdateDocForFileTaskLog(stepInstanceId, executeCount, ip, fileTaskLog); + BasicDBObject filter = buildQueryDocForFileTaskLog(stepInstanceId, executeCount, batch, fileTaskLog); + BasicDBObject update = buildUpdateDocForFileTaskLog(stepInstanceId, executeCount, batch, hostId, ip, + fileTaskLog); logCollectionFactory.getCollection(collectionName) .updateOne(filter, update, new UpdateOptions().upsert(true)); } finally { long cost = (System.currentTimeMillis() - start); if (cost > 10L) { - log.warn("Save log slow, stepInstanceId:{}, executeCount: {}, ip: {}, cost: {} ms", stepInstanceId, - executeCount, ip, cost); + log.warn("Save log slow, stepInstanceId:{}, cost: {} ms", stepInstanceId, cost); } } } - private BasicDBObject buildQueryDocForFileTaskLog(long stepInstanceId, int executeCount, - FileTaskLog fileTaskLog) { + private BasicDBObject buildQueryDocForFileTaskLog(long stepInstanceId, int executeCount, Integer batch, + FileTaskLogDoc fileTaskLog) { BasicDBObject filter = new BasicDBObject(); filter.append("stepId", stepInstanceId); filter.append("executeCount", executeCount); + if (batch != null && batch > 0) { + filter.append("batch", batch); + } filter.append("taskId", fileTaskLog.getTaskId()); return filter; } - private BasicDBObject buildUpdateDocForFileTaskLog(long stepInstanceId, int executeCount, String ip, - FileTaskLog fileTaskLog) { + private BasicDBObject buildUpdateDocForFileTaskLog(long stepInstanceId, + int executeCount, + Integer batch, + Long hostId, + String ip, + FileTaskLogDoc fileTaskLog) { BasicDBObject update = new BasicDBObject(); BasicDBObject setDBObject = new BasicDBObject(); BasicDBObject pushDBObject = new BasicDBObject(); setDBObject.append("stepId", stepInstanceId) .append("executeCount", executeCount) .append("mode", fileTaskLog.getMode()) - .append("ip", ip) .append("taskId", fileTaskLog.getTaskId()); + if (hostId != null) { + setDBObject.append("hostId", hostId); + } + if (StringUtils.isNotEmpty(ip)) { + setDBObject.append("ip", ip); + } + if (batch != null && batch > 0) { + setDBObject.append("batch", batch); + } + if (fileTaskLog.getSrcHostId() != null) { + setDBObject.append("srcHostId", fileTaskLog.getSrcHostId()); + } if (StringUtils.isNotEmpty(fileTaskLog.getSrcIp())) { setDBObject.append("srcIp", fileTaskLog.getSrcIp()); } @@ -281,62 +297,82 @@ private BasicDBObject buildUpdateDocForFileTaskLog(long stepInstanceId, int exec } private String buildLogCollectionName(String jobCreateDate, LogTypeEnum logType) { - return "job_log_" + logType.getName() + "_" + jobCreateDate; + return "job_log_" + getLogTypeName(logType) + "_" + jobCreateDate; } - @Override - public TaskIpLog getScriptLogByIp(ScriptLogQuery query) { - return getScriptTaskLogByIp(query); + private String getLogTypeName(LogTypeEnum logType) { + if (logType == LogTypeEnum.SCRIPT) { + return "script"; + } else if (logType == LogTypeEnum.FILE) { + return "file"; + } else { + throw new IllegalArgumentException("Invalid logType"); + } } @Override - public List batchGetScriptLogByIps(ScriptLogQuery query) throws ServiceException { - return getScriptTaskLogByIps(query); - } + public List listScriptLogs(ScriptLogQuery scriptLogQuery) throws ServiceException { + long start = System.currentTimeMillis(); + String collectionName = buildLogCollectionName(scriptLogQuery.getJobCreateDate(), LogTypeEnum.SCRIPT); - @Override - public TaskIpLog getFileLogByIp(FileLogQuery query) { - return getFileTaskLog(query); + try { + Query query = buildScriptLogMongoQuery(scriptLogQuery); + + List scriptLogs = mongoTemplate.find(query, ScriptTaskLogDoc.class, collectionName); + + if (CollectionUtils.isEmpty(scriptLogs)) { + return Collections.emptyList(); + } + + return groupScriptTaskLogsByHost(scriptLogQuery.getStepInstanceId(), scriptLogQuery.getExecuteCount(), + scriptLogQuery.getBatch(), scriptLogs); + } finally { + long cost = (System.currentTimeMillis() - start); + if (cost > 100L) { + log.warn("Get script log by hosts slow, req: {}, cost: {} ms", scriptLogQuery, cost); + } + } } - private TaskIpLog getFileTaskLog(FileLogQuery getLogRequest) { - long stepInstanceId = getLogRequest.getStepInstanceId(); - int executeCount = getLogRequest.getExecuteCount(); - String ip = getLogRequest.getIp(); - List fileTaskLogs = getFileLogs(getLogRequest); - TaskIpLog taskIpLog = new TaskIpLog(); - taskIpLog.setStepInstanceId(stepInstanceId); - taskIpLog.setExecuteCount(executeCount); - taskIpLog.setIp(ip); - taskIpLog.setFileTaskLogs(fileTaskLogs); - if (CollectionUtils.isNotEmpty(fileTaskLogs)) { - taskIpLog.getFileTaskLogs().forEach(fileTaskDetailLog -> - fileTaskDetailLog.setContent(StringUtils.join(fileTaskDetailLog.getContentList(), null))); - } - return taskIpLog; + private Query buildScriptLogMongoQuery(ScriptLogQuery scriptLogQuery) { + long stepInstanceId = scriptLogQuery.getStepInstanceId(); + int executeCount = scriptLogQuery.getExecuteCount(); + Integer batch = scriptLogQuery.getBatch(); + List ips = scriptLogQuery.getIps(); + List hostIds = scriptLogQuery.getHostIds(); + + Query query = new Query(); + query.addCriteria(Criteria.where("stepId").is(stepInstanceId)); + query.addCriteria(Criteria.where("executeCount").is(executeCount)); + if (batch != null && batch > 0) { + query.addCriteria(Criteria.where("batch").is(batch)); + } + if (CollectionUtils.isNotEmpty(hostIds)) { + if (hostIds.size() == 1) { + query.addCriteria(Criteria.where("hostId").is(hostIds.get(0))); + } else { + query.addCriteria(Criteria.where("hostId").in(hostIds)); + } + } else if (CollectionUtils.isNotEmpty(ips)) { + if (ips.size() == 1) { + query.addCriteria(Criteria.where("ip").is(ips.get(0))); + } else { + query.addCriteria(Criteria.where("ip").in(ips)); + } + } + + return query; } @Override - public List getFileLogs(FileLogQuery getLogRequest) { + public List listFileLogs(FileLogQuery getLogRequest) { String collectionName = buildLogCollectionName(getLogRequest.getJobCreateDate(), LogTypeEnum.FILE); - long stepInstanceId = getLogRequest.getStepInstanceId(); - int executeCount = getLogRequest.getExecuteCount(); long start = System.currentTimeMillis(); try { - Query query = new Query(); - query.addCriteria(Criteria.where("stepId").is(stepInstanceId)); - query.addCriteria(Criteria.where("executeCount").is(executeCount)); - if (getLogRequest.getMode() != null) { - query.addCriteria(Criteria.where("mode").is(getLogRequest.getMode())); - } - if (CollectionUtils.isNotEmpty(getLogRequest.getIps())) { - query.addCriteria(Criteria.where("ip").in(getLogRequest.getIps())); - } else if (StringUtils.isNotEmpty(getLogRequest.getIp())) { - query.addCriteria(Criteria.where("ip").is(getLogRequest.getIp())); - } + Query query = buildFileLogMongoQuery(getLogRequest); - List fileTaskLogs = mongoTemplate.find(query, FileTaskLog.class, collectionName); + List fileTaskLogs = mongoTemplate.find(query, FileTaskLogDoc.class, collectionName); if (CollectionUtils.isNotEmpty(fileTaskLogs)) { fileTaskLogs.forEach(fileTaskLog -> fileTaskLog.setContent(StringUtils.join(fileTaskLog.getContentList(), null))); @@ -350,98 +386,84 @@ public List getFileLogs(FileLogQuery getLogRequest) { } } - private TaskIpLog getScriptTaskLogByIp(ScriptLogQuery getLogRequest) { - long start = System.currentTimeMillis(); - String collectionName = buildLogCollectionName(getLogRequest.getJobCreateDate(), LogTypeEnum.SCRIPT); - long stepInstanceId = getLogRequest.getStepInstanceId(); - int executeCount = getLogRequest.getExecuteCount(); - String ip = getLogRequest.getIps().get(0); - - try { - Query query = new Query(); - query.addCriteria(Criteria.where("stepId").is(stepInstanceId)); - query.addCriteria(Criteria.where("executeCount").is(executeCount)); - query.addCriteria(Criteria.where("ip").is(ip)); - List scriptLogs = mongoTemplate.find(query, ScriptTaskLog.class, collectionName); - - TaskIpLog taskIpLog = buildTaskIpLog(stepInstanceId, executeCount, ip, scriptLogs); - - if (log.isDebugEnabled()) { - log.debug("Get log by ip, stepInstanceId: {}, executeCount: {}, ip: {}, scriptLogs: {}", - stepInstanceId, executeCount, - ip, scriptLogs); + private Query buildFileLogMongoQuery(FileLogQuery getLogRequest) { + Query query = new Query(); + query.addCriteria(Criteria.where("stepId").is(getLogRequest.getStepInstanceId())); + query.addCriteria(Criteria.where("executeCount").is(getLogRequest.getExecuteCount())); + if (getLogRequest.getMode() != null) { + query.addCriteria(Criteria.where("mode").is(getLogRequest.getMode())); + } + if (CollectionUtils.isNotEmpty(getLogRequest.getHostIds())) { + if (getLogRequest.getHostIds().size() > 1) { + query.addCriteria(Criteria.where("hostId").in(getLogRequest.getHostIds())); + } else { + query.addCriteria(Criteria.where("hostId").is(getLogRequest.getHostIds().get(0))); } - taskIpLog.setScriptContent(scriptLogs.stream().map(ScriptTaskLog::getContent).collect(Collectors.joining( - ""))); - return taskIpLog; - } finally { - long cost = (System.currentTimeMillis() - start); - if (cost > 10L) { - log.warn("Get script log by ip slow, stepInstanceId: {}, ip: {}, cost: {} ms", stepInstanceId, ip, - cost); + } else if (CollectionUtils.isNotEmpty(getLogRequest.getIps())) { + if (getLogRequest.getIps().size() > 1) { + query.addCriteria(Criteria.where("ip").in(getLogRequest.getIps())); + } else { + query.addCriteria(Criteria.where("ip").is(getLogRequest.getIps().get(0))); } } - } - - private List getScriptTaskLogByIps(ScriptLogQuery getLogRequest) { - long start = System.currentTimeMillis(); - String collectionName = buildLogCollectionName(getLogRequest.getJobCreateDate(), LogTypeEnum.SCRIPT); - long stepInstanceId = getLogRequest.getStepInstanceId(); - int executeCount = getLogRequest.getExecuteCount(); - List ips = getLogRequest.getIps(); - - try { - Query query = new Query(); - query.addCriteria(Criteria.where("stepId").is(stepInstanceId)); - query.addCriteria(Criteria.where("executeCount").is(executeCount)); - query.addCriteria(Criteria.where("ip").in(ips)); - List scriptLogs = mongoTemplate.find(query, ScriptTaskLog.class, collectionName); - - if (CollectionUtils.isEmpty(scriptLogs)) { - return Collections.emptyList(); - } + if (getLogRequest.getBatch() != null && getLogRequest.getBatch() > 0) { + query.addCriteria(Criteria.where("batch").is(getLogRequest.getBatch())); + } - Map> scriptTaskLogGroups = groupScriptTaskLogsByIp(scriptLogs); + return query; + } - List taskIpLogs = new ArrayList<>(scriptTaskLogGroups.size()); - scriptTaskLogGroups.forEach((ip, scriptLogGroup) -> { - taskIpLogs.add(buildTaskIpLog(stepInstanceId, executeCount, ip, scriptLogGroup)); + private List groupScriptTaskLogsByHost(long stepInstanceId, + int executeCount, + Integer batch, + List scriptTaskLogs) { + List taskHostLogs = new ArrayList<>(); + boolean existHostIdField = scriptTaskLogs.get(0).getHostId() != null; + if (existHostIdField) { + Map> scriptLogsGroups = new HashMap<>(); + scriptTaskLogs.forEach(scriptTaskLog -> { + List scriptLogGroup = scriptLogsGroups.computeIfAbsent(scriptTaskLog.getHostId(), + k -> new ArrayList<>()); + scriptLogGroup.add(scriptTaskLog); }); - return taskIpLogs; - } finally { - long cost = (System.currentTimeMillis() - start); - if (cost > 100L) { - log.warn("Get script log by ips slow, stepInstanceId: {}, ip: {}, cost: {} ms", stepInstanceId, ips, - cost); - } + scriptLogsGroups.forEach( + (hostId, scriptLogGroup) -> + taskHostLogs.add( + buildTaskHostLog(stepInstanceId, executeCount, batch, scriptLogGroup))); + } else { + Map> scriptLogsGroups = new HashMap<>(); + scriptTaskLogs.forEach(scriptTaskLog -> { + List scriptLogGroup = scriptLogsGroups.computeIfAbsent(scriptTaskLog.getIp(), + k -> new ArrayList<>()); + scriptLogGroup.add(scriptTaskLog); + }); + scriptLogsGroups.forEach( + (ip, scriptLogGroup) -> + taskHostLogs.add( + buildTaskHostLog(stepInstanceId, executeCount, batch,scriptLogGroup))); } - } - private Map> groupScriptTaskLogsByIp(List scriptTaskLogs) { - Map> scriptLogsGroups = new HashMap<>(); - scriptTaskLogs.forEach(scriptTaskLog -> { - List scriptLogGroup = scriptLogsGroups.computeIfAbsent(scriptTaskLog.getIp(), - k -> new ArrayList<>()); - scriptLogGroup.add(scriptTaskLog); - }); - return scriptLogsGroups; + return taskHostLogs; } - private TaskIpLog buildTaskIpLog(long stepInstanceId, int executeCount, String ip, List scriptLogs) { - TaskIpLog taskIpLog = new TaskIpLog(); - taskIpLog.setStepInstanceId(stepInstanceId); - taskIpLog.setExecuteCount(executeCount); - taskIpLog.setIp(ip); + private TaskHostLog buildTaskHostLog(long stepInstanceId, int executeCount, Integer batch, + List scriptLogs) { + TaskHostLog taskHostLog = new TaskHostLog(); + taskHostLog.setStepInstanceId(stepInstanceId); + taskHostLog.setExecuteCount(executeCount); + taskHostLog.setBatch(batch); + taskHostLog.setHostId(scriptLogs.get(0).getHostId()); + taskHostLog.setIp(scriptLogs.get(0).getIp()); - scriptLogs.sort(ScriptTaskLog.LOG_OFFSET_COMPARATOR); - taskIpLog.setScriptContent(scriptLogs.stream().map(ScriptTaskLog::getContent).collect(Collectors.joining(""))); + scriptLogs.sort(ScriptTaskLogDoc.LOG_OFFSET_COMPARATOR); + taskHostLog.setScriptContent(scriptLogs.stream().map(ScriptTaskLogDoc::getContent).collect(Collectors.joining(""))); - return taskIpLog; + return taskHostLog; } @Override - public List getFileLogsByTaskIds(String jobCreateDate, long stepInstanceId, int executeCount, - List taskIds) { + public List getFileLogsByTaskIds(String jobCreateDate, long stepInstanceId, int executeCount, + Integer batch, List taskIds) { long start = System.currentTimeMillis(); String collectionName = buildLogCollectionName(jobCreateDate, LogTypeEnum.FILE); try { @@ -449,7 +471,10 @@ public List getFileLogsByTaskIds(String jobCreateDate, long stepIns query.addCriteria(Criteria.where("stepId").is(stepInstanceId)); query.addCriteria(Criteria.where("executeCount").is(executeCount)); query.addCriteria(Criteria.where("taskId").in(taskIds)); - List fileTaskLogs = mongoTemplate.find(query, FileTaskLog.class, collectionName); + if (batch != null && batch > 0) { + query.addCriteria(Criteria.where("batch").is(batch)); + } + List fileTaskLogs = mongoTemplate.find(query, FileTaskLogDoc.class, collectionName); if (CollectionUtils.isNotEmpty(fileTaskLogs)) { fileTaskLogs.forEach(taskTaskLog -> taskTaskLog.setContent(StringUtils.join(taskTaskLog.getContentList(), null))); @@ -457,7 +482,7 @@ public List getFileLogsByTaskIds(String jobCreateDate, long stepIns return fileTaskLogs; } finally { long cost = (System.currentTimeMillis() - start); - if (cost > 10L) { + if (cost > 20L) { log.warn("Get file log by task ids slow, stepInstanceId: {}, executeCount: {}, taskIds: {} cost: {} ms", stepInstanceId, executeCount, taskIds, cost); } @@ -465,33 +490,33 @@ public List getFileLogsByTaskIds(String jobCreateDate, long stepIns } @Override - public long deleteStepContent(Long stepInstanceId, Integer executeCount, String jobCreateDate) { - Query query = new Query(); - query.addCriteria(Criteria.where("stepId").is(stepInstanceId)); - query.addCriteria(Criteria.where("executeCount").is(executeCount)); - DeleteResult deleteResult = mongoTemplate.remove(query, buildLogCollectionName(jobCreateDate, - LogTypeEnum.SCRIPT)); - return deleteResult.getDeletedCount(); - } - - @Override - public List getIpsByKeyword(long stepInstanceId, Integer executeCount, String jobCreateDate, - String keyword) { + public List getIpsByKeyword(String jobCreateDate, long stepInstanceId, int executeCount, + Integer batch, String keyword) { String collectionName = buildLogCollectionName(jobCreateDate, LogTypeEnum.SCRIPT); - Query query = buildQueryForKeywordSearch(stepInstanceId, executeCount, keyword); - query.fields().include("ip"); - List logs = mongoTemplate.find(query, ScriptTaskLog.class, collectionName); + Query query = buildQueryForKeywordSearch(stepInstanceId, executeCount, batch, keyword); + query.fields().include("ip", "hostId"); + List logs = mongoTemplate.find(query, ScriptTaskLogDoc.class, collectionName); if (logs.isEmpty()) { return Collections.emptyList(); } return logs.stream() - .map(ScriptTaskLog::getIp) + .map(this::extractHostInfo) .distinct() - .map(cloudIp -> new IpDTO(Long.valueOf(cloudIp.split(":")[0]), cloudIp.split(":")[1])) .collect(Collectors.toList()); } - private Query buildQueryForKeywordSearch(long stepInstanceId, int executeCount, String keyword) { + private HostDTO extractHostInfo(ScriptTaskLogDoc log) { + HostDTO host; + if (log.getHostId() != null) { + host = new HostDTO(); + host.setHostId(log.getHostId()); + } else { + host = HostDTO.fromCloudIp(log.getIp()); + } + return host; + } + + private Query buildQueryForKeywordSearch(long stepInstanceId, int executeCount, Integer batch, String keyword) { Query query = new Query(); query.addCriteria(Criteria.where("stepId").is(stepInstanceId)); if (executeCount == 0) { @@ -499,6 +524,9 @@ private Query buildQueryForKeywordSearch(long stepInstanceId, int executeCount, } else { query.addCriteria(Criteria.where("executeCount").lte(executeCount)); } + if (batch != null && batch > 0) { + query.addCriteria(Criteria.where("batch").is(batch)); + } Pattern pattern = Pattern.compile(keyword.replaceAll("['$&|`;#]", ""), Pattern.LITERAL | Pattern.CASE_INSENSITIVE); query.addCriteria(Criteria.where("content").regex(pattern)); diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceApplicationResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceApplicationResource.java index 66b3c876e5..a44508eaf1 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceApplicationResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceApplicationResource.java @@ -97,10 +97,6 @@ ServiceApplicationDTO queryAppByScope(@ApiParam(value = "资源范围类型", al @ApiParam(value = "资源范围ID", required = true) @PathVariable("scopeId") String scopeId); - @GetMapping("/app/{appId}/permission") - InternalResponse checkAppPermission(@PathVariable("appId") Long appId, - @RequestParam("username") String username); - @ApiOperation(value = "获取业务列表", produces = "application/json") @GetMapping("/app/list") InternalResponse> listApps( diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceHostResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceHostResource.java index c1cfbf58c2..eb527ba3e2 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceHostResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceHostResource.java @@ -26,9 +26,9 @@ import com.tencent.bk.job.common.annotation.InternalAPI; import com.tencent.bk.job.common.model.InternalResponse; -import com.tencent.bk.job.common.model.dto.IpDTO; import com.tencent.bk.job.manage.model.inner.ServiceHostDTO; import com.tencent.bk.job.manage.model.inner.ServiceHostStatusDTO; +import com.tencent.bk.job.manage.model.inner.ServiceListAppHostResultDTO; import com.tencent.bk.job.manage.model.inner.request.ServiceBatchGetHostsReq; import com.tencent.bk.job.manage.model.inner.request.ServiceCheckAppHostsReq; import com.tencent.bk.job.manage.model.inner.request.ServiceGetHostStatusByDynamicGroupReq; @@ -72,15 +72,14 @@ InternalResponse> getHostStatusByIp( ); /** - * 检查主机是否在业务下 + * 获取业务下的主机并返回主机详情 * * @param appId Job业务ID * @param req 请求 - * @return 非法的主机 */ @ApiOperation(value = "检查主机是否在业务下", produces = "application/json") - @PostMapping("/app/{appId}/host/checkAppHosts") - InternalResponse> checkAppHosts( + @PostMapping("/app/{appId}/host/batchGet") + InternalResponse batchGetAppHosts( @PathVariable("appId") Long appId, @RequestBody ServiceCheckAppHostsReq req ); diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebGlobalSettingsResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebGlobalSettingsResource.java index 4c4aabd8ff..19105e77e2 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebGlobalSettingsResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebGlobalSettingsResource.java @@ -110,10 +110,10 @@ Response getChannelTemplateDetail( @RequestHeader("username") String username, @ApiParam("渠道Code") - @RequestParam(value = "channelCode", required = true) + @RequestParam(value = "channelCode") String channelCode, @ApiParam("消息类型Code") - @RequestParam(value = "messageTypeCode", required = true) + @RequestParam(value = "messageTypeCode") String messageTypeCode ); diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/notify/ExecuteStatusEnum.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/notify/ExecuteStatusEnum.java index 61f8748d9e..043bf5292a 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/notify/ExecuteStatusEnum.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/notify/ExecuteStatusEnum.java @@ -82,14 +82,4 @@ public static List getVOList() { } return resultList; } - - public static ExecuteStatusVO getVO(ExecuteStatusEnum executeStatus) { - return new ExecuteStatusVO(executeStatus.name(), I18nUtil.getI18nMessage(executeStatus.getI18nCode())); - } - - public static ExecuteStatusVO getVO(int status) { - ExecuteStatusEnum executeStatus = ExecuteStatusEnum.get(status); - if (executeStatus == null) return null; - return new ExecuteStatusVO(executeStatus.name(), I18nUtil.getI18nMessage(executeStatus.getI18nCode())); - } } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/task/TaskFileTypeEnum.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/task/TaskFileTypeEnum.java index 7c02a39bc1..74120ea65a 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/task/TaskFileTypeEnum.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/task/TaskFileTypeEnum.java @@ -53,7 +53,7 @@ public enum TaskFileTypeEnum { */ BASE64_FILE(4); - private int type; + private final int type; public static TaskFileTypeEnum valueOf(int type) { for (TaskFileTypeEnum fileType : values()) { @@ -61,6 +61,18 @@ public static TaskFileTypeEnum valueOf(int type) { return fileType; } } - return null; + throw new IllegalArgumentException("No TaskFileTypeEnum constant: " + type); + } + + public static boolean isValid(Integer type) { + if (type == null) { + return false; + } + for (TaskFileTypeEnum fileType : values()) { + if (fileType.type == type) { + return true; + } + } + return false; } } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/task/TaskStepTypeEnum.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/task/TaskStepTypeEnum.java index 830a8e5bdd..59e06a7b67 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/task/TaskStepTypeEnum.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/task/TaskStepTypeEnum.java @@ -24,14 +24,9 @@ package com.tencent.bk.job.manage.common.consts.task; -import lombok.AllArgsConstructor; -import lombok.Getter; - /** - * @since 2/10/2019 20:55 + * 作业步骤类型 */ -@Getter -@AllArgsConstructor public enum TaskStepTypeEnum { /** @@ -49,7 +44,11 @@ public enum TaskStepTypeEnum { */ APPROVAL(3); - private int type; + private final int type; + + TaskStepTypeEnum(int type) { + this.type = type; + } public static TaskStepTypeEnum valueOf(int type) { for (TaskStepTypeEnum stepType : values()) { @@ -57,7 +56,7 @@ public static TaskStepTypeEnum valueOf(int type) { return stepType; } } - return null; + throw new IllegalArgumentException("No TaskStepTypeEnum constant: " + type); } public int getValue() { diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetScriptVersionListV3Req.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetScriptVersionListV3Req.java index 2a230ec3d3..86b0741a7b 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetScriptVersionListV3Req.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetScriptVersionListV3Req.java @@ -29,8 +29,6 @@ import lombok.Data; import lombok.EqualsAndHashCode; -import javax.validation.constraints.NotBlank; - /** * 查询脚本版本列表请求 */ @@ -41,9 +39,7 @@ public class EsbGetScriptVersionListV3Req extends EsbAppScopeReq { * 脚本ID */ @JsonProperty("script_id") - @NotBlank(message = "{validation.constraints.InvalidScriptId_empty.message}") private String scriptId; - /** * 是否需要返回脚本内容。true:返回脚本内容；false：不返回脚本内容。默认为false。 */ diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceHostDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceHostDTO.java index 3081bfb650..0b604f1179 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceHostDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceHostDTO.java @@ -24,10 +24,15 @@ package com.tencent.bk.job.manage.model.inner; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; import lombok.ToString; +import org.apache.commons.lang3.StringUtils; /** * 主机 @@ -36,12 +41,19 @@ @Setter @ToString @NoArgsConstructor +@AllArgsConstructor +@Builder public class ServiceHostDTO { /** * 主机ID */ private Long hostId; + /** + * AgentID + */ + private String agentId; + /** * 云区域ID */ @@ -62,11 +74,27 @@ public class ServiceHostDTO { */ private Long bizId; - public ServiceHostDTO(Long hostId, Long cloudAreaId, String ip, Long appId, Long bizId) { - this.hostId = hostId; - this.cloudAreaId = cloudAreaId; - this.ip = ip; - this.appId = appId; - this.bizId = bizId; + @JsonIgnore + public String getCloudIp() { + return cloudAreaId + ":" + ip; + } + + @JsonIgnore + public String getFinalAgentId() { + if (StringUtils.isNotBlank(agentId)) { + return agentId; + } + return getCloudIp(); + } + + public static ServiceHostDTO fromApplicationHostDTO(ApplicationHostDTO host) { + return ServiceHostDTO.builder() + .bizId(host.getBizId()) + .appId(host.getAppId()) + .hostId(host.getHostId()) + .cloudAreaId(host.getCloudAreaId()) + .ip(host.getIp()) + .agentId(host.getAgentId()) + .build(); } } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceListAppHostResultDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceListAppHostResultDTO.java new file mode 100644 index 0000000000..51ec865b96 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceListAppHostResultDTO.java @@ -0,0 +1,30 @@ +package com.tencent.bk.job.manage.model.inner; + +import com.tencent.bk.job.common.model.dto.HostDTO; +import lombok.Data; + +import java.util.List; + +/** + * 主机查询结果 + */ +@Data +public class ServiceListAppHostResultDTO { + /** + * 合法的主机（在当前业务下) + */ + private List validHosts; + /** + * 不存在的主机 + */ + private List notExistHosts; + /** + * 在其他业务下的主机 + */ + private List notInAppHosts; +// /** +// * 主机白名单 +// */ +// private Map> whiteHosts; + +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceBatchGetHostsReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceBatchGetHostsReq.java index ca6481b2db..d58e8a643a 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceBatchGetHostsReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceBatchGetHostsReq.java @@ -24,7 +24,7 @@ package com.tencent.bk.job.manage.model.inner.request; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; @@ -40,9 +40,9 @@ @ToString @NoArgsConstructor public class ServiceBatchGetHostsReq { - List hosts; + List hosts; - public ServiceBatchGetHostsReq(List hosts) { + public ServiceBatchGetHostsReq(List hosts) { this.hosts = hosts; } } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceCheckAppHostsReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceCheckAppHostsReq.java index c2be6d1849..6a77b216fa 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceCheckAppHostsReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceCheckAppHostsReq.java @@ -24,7 +24,7 @@ package com.tencent.bk.job.manage.model.inner.request; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; @@ -40,9 +40,9 @@ @ToString @NoArgsConstructor public class ServiceCheckAppHostsReq { - List hosts; + List hosts; - public ServiceCheckAppHostsReq(List hosts) { + public ServiceCheckAppHostsReq(List hosts) { this.hosts = hosts; } } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/resp/ServiceApplicationDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/resp/ServiceApplicationDTO.java index 18b8c8abf5..b51aad50cb 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/resp/ServiceApplicationDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/resp/ServiceApplicationDTO.java @@ -24,18 +24,12 @@ package com.tencent.bk.job.manage.model.inner.resp; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.tencent.bk.job.common.annotation.CompatibleImplementation; import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; -import com.tencent.bk.job.common.model.dto.ApplicationDTO; -import com.tencent.bk.job.common.model.dto.ResourceScope; import com.tencent.bk.job.manage.model.inner.ServiceApplicationAttrsDTO; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; -import java.util.List; - /** * 业务 */ @@ -62,35 +56,14 @@ public class ServiceApplicationDTO { @ApiModelProperty("业务名称") private String name; - /** - * 业务类型 - */ - @ApiModelProperty("业务类型") - @CompatibleImplementation(explain = "兼容字段，等发布完成之后可以删除", version = "3.5.x") - private Integer appType; - /** * 运维 */ private String maintainers; - /** - * 子业务 - */ - @ApiModelProperty("子业务ID") - @CompatibleImplementation(explain = "兼容字段，等发布完成之后可以删除", version = "3.5.x") - @JsonProperty("subAppIds") - private List subBizIds; - @ApiModelProperty("开发商") private String owner; - /** - * 初始运维部门Id - */ - @CompatibleImplementation(explain = "兼容字段，等发布完成之后可以删除", version = "3.5.x") - private Long operateDeptId; - /** * 时区 */ @@ -106,47 +79,6 @@ public class ServiceApplicationDTO { */ private ServiceApplicationAttrsDTO attrs; - /** - * 将服务间调用的业务对象转为通用Job业务对象 - * - * @param serviceAppDTO 服务间调用业务对象 - * @return 通用Job业务对象 - */ - public static ApplicationDTO toApplicationInfoDTO(ServiceApplicationDTO serviceAppDTO) { - ApplicationDTO applicationInfo = new ApplicationDTO(); - applicationInfo.setId(serviceAppDTO.getId()); - applicationInfo.setName(serviceAppDTO.getName()); - applicationInfo.setScope(new ResourceScope(serviceAppDTO.getScopeType(), serviceAppDTO.getScopeId())); - applicationInfo.setSubBizIds(serviceAppDTO.getSubBizIds()); - applicationInfo.setLanguage(serviceAppDTO.getLanguage()); - return applicationInfo; - } - - /** - * 将通用Job业务对象转为服务间调用的业务对象 - * - * @param appDTO 通用Job业务对象 - * @return 服务间调用业务对象 - */ - public static ServiceApplicationDTO fromApplicationDTO(ApplicationDTO appDTO) { - if (appDTO == null) { - return null; - } - ServiceApplicationDTO app = new ServiceApplicationDTO(); - app.setId(appDTO.getId()); - app.setSubBizIds(appDTO.getSubBizIds()); - app.setName(appDTO.getName()); - app.setScopeType(appDTO.getScope().getType().getValue()); - app.setScopeId(appDTO.getScope().getId()); - app.setOwner(appDTO.getBkSupplierAccount()); - app.setTimeZone(appDTO.getTimeZone()); - // 暂时保留appType/maintainers/operateDeptId三个参数 - app.setAppType(appDTO.getAppType().getValue()); - app.setMaintainers(appDTO.getMaintainers()); - app.setOperateDeptId(appDTO.getOperateDeptId()); - return app; - } - public boolean isBiz() { return ResourceScopeTypeEnum.BIZ.getValue().equals(scopeType); } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/AddOrUpdateDangerousRuleReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/AddOrUpdateDangerousRuleReq.java index 9260150ecc..02e2f2b790 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/AddOrUpdateDangerousRuleReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/AddOrUpdateDangerousRuleReq.java @@ -61,10 +61,12 @@ public class AddOrUpdateDangerousRuleReq { private String description; @ApiModelProperty("处理动作,1:扫描,2:拦截") @NotNull(message = "{validation.constraints.InvalidHighRiskGrammarHandleAction.message}") - @CheckEnum(enumClass = HighRiskGrammarActionEnum.class, enumMethod = "isValid", message = "{validation.constraints.InvalidHighRiskGrammarHandleAction.message}") + @CheckEnum(enumClass = HighRiskGrammarActionEnum.class, enumMethod = "isValid", + message = "{validation.constraints.InvalidHighRiskGrammarHandleAction.message}") private Integer action; @ApiModelProperty("规则启停状态，1:启用,0:停止") @NotNull(message = "{validation.constraints.InvalidHighRiskRegularStatus.message}", groups = {Update.class}) - @CheckEnum(enumClass = HighRiskGrammarRuleStatusEnum.class, enumMethod = "isValid", message = "{validation.constraints.InvalidHighRiskRegularStatus.message}", groups = {Update.class}) + @CheckEnum(enumClass = HighRiskGrammarRuleStatusEnum.class, enumMethod = "isValid", + message = "{validation.constraints.InvalidHighRiskRegularStatus.message}", groups = {Update.class}) private Integer status; } diff --git a/src/backend/job-manage/boot-job-manage/src/main/resources/application.yml b/src/backend/job-manage/boot-job-manage/src/main/resources/application.yml index fe4bd96089..171be41030 100644 --- a/src/backend/job-manage/boot-job-manage/src/main/resources/application.yml +++ b/src/backend/job-manage/boot-job-manage/src/main/resources/application.yml @@ -1,6 +1,4 @@ spring: - application: - name: job-manage profiles: active: prod diff --git a/src/backend/job-manage/boot-job-manage/src/main/resources/bootstrap.yml b/src/backend/job-manage/boot-job-manage/src/main/resources/bootstrap.yml index 77bd21fa13..665d6a09be 100644 --- a/src/backend/job-manage/boot-job-manage/src/main/resources/bootstrap.yml +++ b/src/backend/job-manage/boot-job-manage/src/main/resources/bootstrap.yml @@ -1,4 +1,6 @@ spring: + application: + name: job-manage cloud: kubernetes: config: diff --git a/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message.properties b/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message.properties index 8ab2db278d..ddcc596a9f 100644 --- a/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message.properties +++ b/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message.properties @@ -22,6 +22,10 @@ # IN THE SOFTWARE. # +job.resource.status.online=已上线 +job.resource.status.draft=未上线 +job.resource.status.offline=已下线 +job.resource.status.disabled=禁用 job.account.category.system=系统账号 job.account.category.db=数据库账号 diff --git a/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message_en.properties b/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message_en.properties index 37c6202fd1..02edd8fbf0 100644 --- a/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message_en.properties +++ b/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message_en.properties @@ -22,6 +22,10 @@ # IN THE SOFTWARE. # +job.resource.status.online=Online +job.resource.status.draft=Stand-by +job.resource.status.offline=Offline +job.resource.status.disabled=Banned job.account.category.system=System Account job.account.category.db=Database Account diff --git a/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message_en_US.properties b/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message_en_US.properties index 37c6202fd1..02edd8fbf0 100644 --- a/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message_en_US.properties +++ b/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message_en_US.properties @@ -22,6 +22,10 @@ # IN THE SOFTWARE. # +job.resource.status.online=Online +job.resource.status.draft=Stand-by +job.resource.status.offline=Offline +job.resource.status.disabled=Banned job.account.category.system=System Account job.account.category.db=Database Account diff --git a/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message_zh.properties b/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message_zh.properties index 8ab2db278d..ddcc596a9f 100644 --- a/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message_zh.properties +++ b/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message_zh.properties @@ -22,6 +22,10 @@ # IN THE SOFTWARE. # +job.resource.status.online=已上线 +job.resource.status.draft=未上线 +job.resource.status.offline=已下线 +job.resource.status.disabled=禁用 job.account.category.system=系统账号 job.account.category.db=数据库账号 diff --git a/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message_zh_CN.properties b/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message_zh_CN.properties index 8ab2db278d..ddcc596a9f 100644 --- a/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message_zh_CN.properties +++ b/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message_zh_CN.properties @@ -22,6 +22,10 @@ # IN THE SOFTWARE. # +job.resource.status.online=已上线 +job.resource.status.draft=未上线 +job.resource.status.offline=已下线 +job.resource.status.disabled=禁用 job.account.category.system=系统账号 job.account.category.db=数据库账号 diff --git a/src/backend/job-manage/boot-job-manage/src/main/resources/logback/logback-app-props.xml b/src/backend/job-manage/boot-job-manage/src/main/resources/logback/logback-app-props.xml index 7121861857..fbd1f1b92e 100644 --- a/src/backend/job-manage/boot-job-manage/src/main/resources/logback/logback-app-props.xml +++ b/src/backend/job-manage/boot-job-manage/src/main/resources/logback/logback-app-props.xml @@ -1,4 +1,5 @@ + diff --git a/src/backend/job-manage/boot-job-manage/src/test/resources/init_script_relate_task_plan_data.sql b/src/backend/job-manage/boot-job-manage/src/test/resources/init_script_relate_task_plan_data.sql index a3511963eb..6b4954f7ef 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/resources/init_script_relate_task_plan_data.sql +++ b/src/backend/job-manage/boot-job-manage/src/test/resources/init_script_relate_task_plan_data.sql @@ -43,4 +43,4 @@ VALUES (1, 2, 1, 'plan1', 'admin', 0, 'admin'), INSERT INTO `job_manage`.`script_version`(`id`, `row_create_time`, `row_update_time`, `script_id`, `content`, `creator`, `create_time`, `last_modify_user`, `last_modify_time`, `version`, `is_deleted`, `status`, `version_desc`) VALUES (1000, '2019-12-27 12:05:15', '2020-05-06 14:20:03', '1000', '', 'admin', 1577390716, 'admin', 1577390716, 'v1.0.0', 0, 1, NULL); INSERT INTO `job_manage`.`script_version`(`id`, `row_create_time`, `row_update_time`, `script_id`, `content`, `creator`, `create_time`, `last_modify_user`, `last_modify_time`, `version`, `is_deleted`, `status`, `version_desc`) VALUES (1001, '2019-12-27 12:05:15', '2020-05-06 14:20:03', '1000', '', 'admin', 1577390716, 'admin', 1577390716, 'v1.0.0', 0, 1, NULL); -INSERT INTO `job_manage`.`script_version`(`id`, `row_create_time`, `row_update_time`, `script_id`, `content`, `creator`, `create_time`, `last_modify_user`, `last_modify_time`, `version`, `is_deleted`, `status`, `version_desc`) VALUES (2001, '2019-12-27 12:10:06', '2020-04-14 10:17:10', '2000', '', 'admin', 1577391007, 'admin', 1577391007, '10.0.0.0.1', 1, 2, NULL); +INSERT INTO `job_manage`.`script_version`(`id`, `row_create_time`, `row_update_time`, `script_id`, `content`, `creator`, `create_time`, `last_modify_user`, `last_modify_time`, `version`, `is_deleted`, `status`, `version_desc`) VALUES (2001, '2019-12-27 12:10:06', '2020-04-14 10:17:10', '2000', '', 'admin', 1577391007, 'admin', 1577391007, '127.0.0.0.1', 1, 2, NULL); diff --git a/src/backend/job-manage/service-job-manage/build.gradle b/src/backend/job-manage/service-job-manage/build.gradle index 6b7e21f646..7a285b0511 100644 --- a/src/backend/job-manage/service-job-manage/build.gradle +++ b/src/backend/job-manage/service-job-manage/build.gradle @@ -52,6 +52,9 @@ dependencies { implementation "commons-io:commons-io" implementation "commons-codec:commons-codec" implementation 'joda-time:joda-time' + implementation "org.springframework.cloud:spring-cloud-sleuth-api" + implementation "org.springframework.cloud:spring-cloud-sleuth-otel-autoconfigure" + implementation "io.opentelemetry:opentelemetry-exporter-otlp-trace" testImplementation 'org.springframework.boot:spring-boot-starter-test' testImplementation 'org.apache.commons:commons-lang3' } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbScriptResourceV3Impl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbScriptResourceV3Impl.java index 858125868f..1da572e7e4 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbScriptResourceV3Impl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbScriptResourceV3Impl.java @@ -29,7 +29,6 @@ import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.common.esb.model.job.v3.EsbPageDataV3; import com.tencent.bk.job.common.exception.InvalidParamException; -import com.tencent.bk.job.common.exception.NotFoundException; import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; import com.tencent.bk.job.common.iam.model.AuthResult; import com.tencent.bk.job.common.metrics.CommonMetricNames; @@ -57,6 +56,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @RestController @Slf4j @@ -203,6 +203,7 @@ private void setOnlineScriptVersionInfo(List scripts) { public EsbResp> getScriptVersionListUsingPost( EsbGetScriptVersionListV3Req request) { request.fillAppResourceScope(appScopeMappingService); + checkEsbGetScriptVersionListV3Req(request); long appId = request.getAppId(); ScriptQuery scriptQuery = new ScriptQuery(); @@ -210,13 +211,13 @@ public EsbResp> getScriptVersionListU scriptQuery.setPublicScript(false); scriptQuery.setId(request.getScriptId()); - checkScriptAvailable(request.getScriptId(), appId); - authViewScript(request.getUserName(), request.getAppResourceScope(), request.getScriptId()); - BaseSearchCondition baseSearchCondition = BaseSearchCondition.pageCondition(request.getStart(), request.getLength()); + PageData pageScriptVersions = scriptService.listPageScriptVersion(scriptQuery, baseSearchCondition); + batchAuthViewScript(request.getUserName(), request.getAppResourceScope(), pageScriptVersions.getData()); + EsbPageDataV3 result = EsbPageDataV3.from(pageScriptVersions, ScriptDTO::toEsbScriptVersionDetailV3DTO); if (request.getReturnScriptContent() == null || !request.getReturnScriptContent()) { @@ -227,22 +228,25 @@ public EsbResp> getScriptVersionListU return EsbResp.buildSuccessResp(result); } - private void checkScriptAvailable(String scriptId, long appId) { - ScriptDTO script = scriptService.getScriptByScriptId(scriptId); - if (script == null) { - log.warn("Script is not exist, scriptId: {}", scriptId); - throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST); - } - if (script.getAppId() != appId) { - log.warn("Script is not in app, scriptId: {}, scriptAppId: {}", scriptId, script.getAppId()); - throw new NotFoundException(ErrorCode.SCRIPT_NOT_IN_APP); + private void checkEsbGetScriptVersionListV3Req(EsbGetScriptVersionListV3Req request) { + if (StringUtils.isBlank(request.getScriptId())) { + log.warn("Param [script_id] is empty!"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME, "script_id"); } } - private void authViewScript(String username, AppResourceScope appResourceScope, String scriptId) { - AuthResult authResult = scriptAuthService.authViewScript(username, appResourceScope, scriptId, null); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); + private void batchAuthViewScript(String username, AppResourceScope appResourceScope, List scripts) { + if (CollectionUtils.isNotEmpty(scripts)) { + // 鉴权 + List resourceIds = + scripts.stream().map(ScriptDTO::getId).distinct().collect(Collectors.toList()); + if (!resourceIds.isEmpty()) { + AuthResult authResult = scriptAuthService.batchAuthResultViewScript(username, + appResourceScope, resourceIds); + if (!authResult.isPass()) { + throw new PermissionDeniedException(authResult); + } + } } } @@ -251,9 +255,7 @@ private void authViewScript(String username, AppResourceScope appResourceScope, public EsbResp getScriptVersionDetailUsingPost( EsbGetScriptVersionDetailV3Req request) { request.fillAppResourceScope(appScopeMappingService); - checkEsbGetScriptVersionDetailV3Req(request); - authViewScript(request.getUserName(), request.getAppResourceScope(), request.getScriptId()); long appId = request.getAppId(); String scriptId = request.getScriptId(); @@ -266,6 +268,13 @@ public EsbResp getScriptVersionDetailUsingPost( scriptVersion = scriptService.getByScriptIdAndVersion(null, appId, scriptId, version); } + if (scriptVersion != null) { + AuthResult authResult = scriptAuthService.authViewScript(request.getUserName(), request.getAppResourceScope(), + scriptVersion.getId(), null); + if (!authResult.isPass()) { + throw new PermissionDeniedException(authResult); + } + } EsbScriptVersionDetailV3DTO result = null; if (scriptVersion != null) { diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceApplicationResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceApplicationResourceImpl.java index b67449ffac..3320570261 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceApplicationResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceApplicationResourceImpl.java @@ -91,11 +91,7 @@ private ServiceApplicationDTO convertToServiceApp(ApplicationDTO appInfo) { app.setScopeType(appInfo.getScope().getType().getValue()); app.setScopeId(appInfo.getScope().getId()); app.setName(appInfo.getName()); - app.setAppType(appInfo.getAppType().getValue()); - app.setSubBizIds(appInfo.getSubBizIds()); - app.setMaintainers(appInfo.getMaintainers()); app.setOwner(appInfo.getBkSupplierAccount()); - app.setOperateDeptId(appInfo.getOperateDeptId()); app.setTimeZone(appInfo.getTimeZone()); app.setLanguage(appInfo.getLanguage()); if (appInfo.getAttrs() != null) { @@ -136,14 +132,6 @@ public List listAppsByAppIds(String appIds) { return applications.stream().map(this::convertToServiceApp).collect(Collectors.toList()); } - @Override - public InternalResponse checkAppPermission(Long appId, String username) { - if (appId == null || appId < 0) { - return InternalResponse.buildSuccessResp(false); - } - return InternalResponse.buildSuccessResp(applicationService.checkAppPermission(appId, username)); - } - @Override public InternalResponse> listApps(String scopeType) { List appList = applicationService.listAllApps(); @@ -152,9 +140,9 @@ public InternalResponse> listApps(String scopeType) } if (scopeType != null) { - appList = appList.stream().filter( - app -> app.getScope().getType() == ResourceScopeTypeEnum.from(scopeType) - ).collect(Collectors.toList()); + appList = appList.stream() + .filter(app -> app.getScope().getType() == ResourceScopeTypeEnum.from(scopeType)) + .collect(Collectors.toList()); } List resultList = diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceHostResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceHostResourceImpl.java index 6794eabc44..981bfdf060 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceHostResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceHostResourceImpl.java @@ -31,11 +31,12 @@ import com.tencent.bk.job.common.model.dto.ApplicationDTO; import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; import com.tencent.bk.job.common.model.dto.DynamicGroupInfoDTO; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.model.vo.HostInfoVO; import com.tencent.bk.job.manage.api.inner.ServiceHostResource; import com.tencent.bk.job.manage.model.inner.ServiceHostDTO; import com.tencent.bk.job.manage.model.inner.ServiceHostStatusDTO; +import com.tencent.bk.job.manage.model.inner.ServiceListAppHostResultDTO; import com.tencent.bk.job.manage.model.inner.request.ServiceBatchGetHostsReq; import com.tencent.bk.job.manage.model.inner.request.ServiceCheckAppHostsReq; import com.tencent.bk.job.manage.model.inner.request.ServiceGetHostStatusByDynamicGroupReq; @@ -148,14 +149,14 @@ public InternalResponse> getHostStatusByIp(Long appId } @Override - public InternalResponse> checkAppHosts(Long appId, - ServiceCheckAppHostsReq req) { - return InternalResponse.buildSuccessResp(hostService.checkAppHosts(appId, req.getHosts())); + public InternalResponse batchGetAppHosts(Long appId, + ServiceCheckAppHostsReq req) { + return InternalResponse.buildSuccessResp(hostService.listAppHosts(appId, req.getHosts())); } @Override public InternalResponse> batchGetHosts(ServiceBatchGetHostsReq req) { - List hostIps = req.getHosts(); + List hostIps = req.getHosts(); List hosts = hostService.listHosts(hostIps); if (CollectionUtils.isEmpty(hosts)) { return InternalResponse.buildSuccessResp(null); @@ -163,8 +164,7 @@ public InternalResponse> batchGetHosts(ServiceBatchGetHosts return InternalResponse.buildSuccessResp( hosts.stream() - .map(host -> new ServiceHostDTO(host.getHostId(), host.getCloudAreaId(), host.getIp(), - host.getAppId(), host.getBizId())) + .map(ServiceHostDTO::fromApplicationHostDTO) .collect(Collectors.toList())); } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceNotificationResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceNotificationResourceImpl.java index adebc1a633..c026f1597e 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceNotificationResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceNotificationResourceImpl.java @@ -58,7 +58,7 @@ public ServiceNotificationResourceImpl(NotifyService notifyService) { @Override public InternalResponse sendNotificationsToUsers(ServiceUserNotificationDTO serviceUserNotificationDTO) { log.info(String.format("Input:%s", JsonUtils.toJson(serviceUserNotificationDTO))); - return InternalResponse.buildSuccessResp(notifyService.sendNotificationsToUsers(serviceUserNotificationDTO)); + return InternalResponse.buildSuccessResp(notifyService.asyncSendNotificationsToUsers(serviceUserNotificationDTO)); } @Override @@ -67,7 +67,7 @@ public InternalResponse sendNotificationsToAdministrators( ) { log.info(String.format("Input:%s", JsonUtils.toJson(serviceNotificationMessage))); return InternalResponse.buildSuccessResp( - notifyService.sendNotificationsToAdministrators(serviceNotificationMessage) + notifyService.asyncSendNotificationsToAdministrators(serviceNotificationMessage) ); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebGlobalSettingsResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebGlobalSettingsResourceImpl.java index ee5fb6764b..488f9e1546 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebGlobalSettingsResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebGlobalSettingsResourceImpl.java @@ -53,7 +53,7 @@ @Slf4j public class WebGlobalSettingsResourceImpl implements WebGlobalSettingsResource { - private GlobalSettingsService globalSettingsService; + private final GlobalSettingsService globalSettingsService; @Autowired public WebGlobalSettingsResourceImpl(GlobalSettingsService globalSettingsService) { diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/common/TopologyHelper.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/common/TopologyHelper.java index a0fe0830ad..51ce2551f2 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/common/TopologyHelper.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/common/TopologyHelper.java @@ -254,8 +254,6 @@ public static DynamicGroupInfoVO convertToDynamicGroupInfoVO(DynamicGroupInfoDTO DynamicGroupInfoVO dynamicGroupInfoVO = new DynamicGroupInfoVO(); dynamicGroupInfoVO.setScopeType(ResourceScopeTypeEnum.BIZ.getValue()); dynamicGroupInfoVO.setScopeId(dynamicGroupInfoDTO.getBizId().toString()); - // TODO:发布后去除 - dynamicGroupInfoVO.setAppName(dynamicGroupInfoDTO.getBizName()); dynamicGroupInfoVO.setScopeName(dynamicGroupInfoDTO.getBizName()); dynamicGroupInfoVO.setId(dynamicGroupInfoDTO.getId()); dynamicGroupInfoVO.setOwner(dynamicGroupInfoDTO.getOwner()); @@ -332,21 +330,7 @@ public List getBizSetSubBizIds(ApplicationDTO appInfo) { if (appInfo.isAllBizSet()) { return applicationDAO.listAllBizAppBizIds(); } else if (appInfo.isBizSet()) { - List subAppIds = appInfo.getSubBizIds(); - // 兼容发布过程中未完成子业务字段同步的部门型业务集 - Long optDeptId = appInfo.getOperateDeptId(); - if (CollectionUtils.isEmpty(subAppIds) && optDeptId != null) { - // 使用OperateDeptId - subAppIds = applicationDAO.getBizIdsByOptDeptId(optDeptId); - } else { - // subAppIds与OperateDeptId同时生效 - if (optDeptId != null) { - subAppIds.addAll(applicationDAO.getBizIdsByOptDeptId(optDeptId)); - } - } - // 去重 - subAppIds = new ArrayList<>(new HashSet<>(subAppIds)); - return subAppIds; + return appInfo.getSubBizIds(); } else { FormattingTuple msg = MessageFormatter.format( "app {} is not bizSet app, please check, detail:{}", diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ApplicationDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ApplicationDAO.java index 6c7ff60aef..ff36c50c7c 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ApplicationDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ApplicationDAO.java @@ -24,8 +24,6 @@ package com.tencent.bk.job.manage.dao; -import com.tencent.bk.job.common.annotation.CompatibleImplementation; -import com.tencent.bk.job.common.constant.AppTypeEnum; import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; import com.tencent.bk.job.common.model.dto.ApplicationDTO; import com.tencent.bk.job.common.model.dto.ResourceScope; @@ -43,8 +41,6 @@ public interface ApplicationDAO { ApplicationDTO getAppById(long appId); - List getBizIdsByOptDeptId(Long optDeptId); - List listAppsByAppIds(List appIdList); List listBizAppsByBizIds(Collection bizIdList); @@ -59,15 +55,10 @@ public interface ApplicationDAO { List listAllBizSetAppsWithDeleted(); - @CompatibleImplementation(explain = "兼容方法，等业务集全部迁移到cmdb之后可以删除", version = "3.6.x") - List listAppsByType(AppTypeEnum appType); - List listAppsByScopeType(ResourceScopeTypeEnum scopeType); Long insertApp(DSLContext dslContext, ApplicationDTO applicationDTO); - Long insertAppWithSpecifiedAppId(DSLContext dslContext, ApplicationDTO applicationDTO); - int updateApp(DSLContext dslContext, ApplicationDTO applicationDTO); /** @@ -88,10 +79,6 @@ public interface ApplicationDAO { */ int deleteAppByIdSoftly(DSLContext dslContext, long appId); - int updateMaintainers(long appId, String maintainers); - - int updateSubBizIds(long appId, String subBizIds); - Integer countApps(); Integer countBizSetAppsWithDeleted(); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ApplicationHostDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ApplicationHostDAO.java index 63bd84514b..a44d818696 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ApplicationHostDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ApplicationHostDAO.java @@ -28,7 +28,6 @@ import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; -import com.tencent.bk.job.common.model.dto.IpDTO; import org.jooq.DSLContext; import java.util.Collection; @@ -133,5 +132,10 @@ int updateBizHostInfoByHostId(DSLContext dslContext, int syncHostTopo(DSLContext dslContext, Long hostId); - List listHosts(Collection hostIps); + /** + * 根据ip查询主机 + * + * @param cloudIps 主机ip(云区域+ip)列表 + */ + List listHostsByIps(Collection cloudIps); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ApplicationDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ApplicationDAOImpl.java index 2cc1112441..c6c694517f 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ApplicationDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ApplicationDAOImpl.java @@ -24,7 +24,6 @@ package com.tencent.bk.job.manage.dao.impl; -import com.tencent.bk.job.common.constant.AppTypeEnum; import com.tencent.bk.job.common.constant.Bool; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; @@ -33,12 +32,10 @@ import com.tencent.bk.job.common.model.dto.ApplicationDTO; import com.tencent.bk.job.common.model.dto.ResourceScope; import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.manage.common.util.JooqDataTypeUtil; import com.tencent.bk.job.manage.dao.ApplicationDAO; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; import org.jooq.Condition; import org.jooq.DSLContext; import org.jooq.Record; @@ -55,8 +52,6 @@ import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; -import java.util.LinkedList; import java.util.List; import java.util.stream.Collectors; @@ -72,12 +67,8 @@ public class ApplicationDAOImpl implements ApplicationDAO { T_APP.BK_SCOPE_TYPE, T_APP.BK_SCOPE_ID, T_APP.APP_NAME, - T_APP.MAINTAINERS, T_APP.BK_SUPPLIER_ACCOUNT, - T_APP.APP_TYPE, - T_APP.SUB_APP_IDS, T_APP.TIMEZONE, - T_APP.BK_OPERATE_DEPT_ID, T_APP.LANGUAGE, T_APP.IS_DELETED, T_APP.ATTRS @@ -125,54 +116,20 @@ public static ApplicationDTO extract(Record record) { String scopeId = record.get(T_APP.BK_SCOPE_ID); applicationDTO.setScope(new ResourceScope(scopeType, scopeId)); applicationDTO.setName(record.get(T_APP.APP_NAME)); - applicationDTO.setMaintainers(record.get(T_APP.MAINTAINERS)); applicationDTO.setBkSupplierAccount(record.get(T_APP.BK_SUPPLIER_ACCOUNT)); - applicationDTO.setAppType(AppTypeEnum.valueOf(record.get(T_APP.APP_TYPE))); - applicationDTO.setSubBizIds(splitSubBizIds(record.get(T_APP.SUB_APP_IDS))); applicationDTO.setTimeZone(record.get(T_APP.TIMEZONE)); - applicationDTO.setOperateDeptId(record.get(T_APP.BK_OPERATE_DEPT_ID)); applicationDTO.setLanguage(record.get(T_APP.LANGUAGE)); - applicationDTO.setDeleted(Bool.isTrue(record.get(T_APP.IS_DELETED).intValue())); applicationDTO.setAttrs(JsonUtils.fromJson(record.get(T_APP.ATTRS), ApplicationAttrsDO.class)); + applicationDTO.setDeleted(Bool.isTrue(record.get(T_APP.IS_DELETED).byteValue())); return applicationDTO; } - private static List splitSubBizIds(String bizIdsStr) { - List bizIdList = new LinkedList<>(); - if (StringUtils.isNotBlank(bizIdsStr)) { - for (String bizIdStr : bizIdsStr.split("[,;]")) { - if (StringUtils.isNotBlank(bizIdStr)) { - bizIdList.add(Long.valueOf(bizIdStr)); - } - } - - } - return bizIdList; - } - private List getBasicNotDeletedConditions() { List conditions = new ArrayList<>(); conditions.add(T_APP.IS_DELETED.eq(UByte.valueOf(0))); return conditions; } - @Override - public List getBizIdsByOptDeptId(Long optDeptId) { - List conditions = getBasicNotDeletedConditions(); - conditions.add(T_APP.BK_SCOPE_TYPE.eq(ResourceScopeTypeEnum.BIZ.getValue())); - if (optDeptId == null) { - conditions.add(T_APP.BK_OPERATE_DEPT_ID.isNull()); - } else { - conditions.add(T_APP.BK_OPERATE_DEPT_ID.eq(optDeptId)); - } - val records = context.select(T_APP.BK_SCOPE_ID).from(T_APP).where(conditions).fetch(); - if (records.isEmpty()) { - return Collections.emptyList(); - } else { - return records.map(it -> Long.parseLong(it.component1())); - } - } - private List listAppsByConditions(List conditions) { if (conditions == null) { conditions = new ArrayList<>(); @@ -246,13 +203,6 @@ public List listAllBizSetAppsWithDeleted() { return listAppsByConditions(conditions); } - @Override - public List listAppsByType(AppTypeEnum appType) { - List conditions = getBasicNotDeletedConditions(); - conditions.add(T_APP.APP_TYPE.eq((byte) appType.getValue())); - return listAppsByConditions(conditions); - } - @Override public List listAppsByScopeType(ResourceScopeTypeEnum scopeType) { List conditions = getBasicNotDeletedConditions(); @@ -262,20 +212,11 @@ public List listAppsByScopeType(ResourceScopeTypeEnum scopeType) @Override public Long insertApp(DSLContext dslContext, ApplicationDTO applicationDTO) { - val subBizIds = applicationDTO.getSubBizIds(); - String subBizIdsStr = null; - if (subBizIds != null) { - subBizIdsStr = subBizIds.stream().map(Object::toString).collect(Collectors.joining(";")); - } ResourceScope scope = applicationDTO.getScope(); val query = dslContext.insertInto(T_APP, T_APP.APP_NAME, - T_APP.APP_TYPE, T_APP.BK_SUPPLIER_ACCOUNT, - T_APP.MAINTAINERS, - T_APP.SUB_APP_IDS, T_APP.TIMEZONE, - T_APP.BK_OPERATE_DEPT_ID, T_APP.LANGUAGE, T_APP.BK_SCOPE_TYPE, T_APP.BK_SCOPE_ID, @@ -283,12 +224,8 @@ public Long insertApp(DSLContext dslContext, ApplicationDTO applicationDTO) { T_APP.IS_DELETED ).values( applicationDTO.getName(), - (byte) (applicationDTO.getAppType().getValue()), applicationDTO.getBkSupplierAccount(), - applicationDTO.getMaintainers(), - subBizIdsStr, applicationDTO.getTimeZone(), - applicationDTO.getOperateDeptId(), applicationDTO.getLanguage(), scope == null ? null : scope.getType().getValue(), scope == null ? null : scope.getId(), @@ -311,69 +248,14 @@ val record = query.returning(T_APP.APP_ID).fetchOne(); } } - @Override - public Long insertAppWithSpecifiedAppId(DSLContext dslContext, - ApplicationDTO applicationDTO) { - val subBizIds = applicationDTO.getSubBizIds(); - String subBizIdsStr = null; - if (subBizIds != null) { - subBizIdsStr = subBizIds.stream().map(Object::toString).collect(Collectors.joining(";")); - } - ResourceScope scope = applicationDTO.getScope(); - val query = dslContext.insertInto(T_APP, - T_APP.APP_ID, - T_APP.APP_NAME, - T_APP.APP_TYPE, - T_APP.BK_SUPPLIER_ACCOUNT, - T_APP.MAINTAINERS, - T_APP.SUB_APP_IDS, - T_APP.TIMEZONE, - T_APP.BK_OPERATE_DEPT_ID, - T_APP.LANGUAGE, - T_APP.BK_SCOPE_TYPE, - T_APP.BK_SCOPE_ID, - T_APP.ATTRS, - T_APP.IS_DELETED - ).values( - JooqDataTypeUtil.buildULong(applicationDTO.getId()), - applicationDTO.getName(), - (byte) (applicationDTO.getAppType().getValue()), - applicationDTO.getBkSupplierAccount(), - applicationDTO.getMaintainers(), - subBizIdsStr, - applicationDTO.getTimeZone(), - applicationDTO.getOperateDeptId(), - applicationDTO.getLanguage(), - scope == null ? null : scope.getType().getValue(), - scope == null ? null : scope.getId(), - applicationDTO.getAttrs() == null ? null : JsonUtils.toJson(applicationDTO.getAttrs()), - UByte.valueOf(Bool.FALSE.getValue()) - ); - try { - query.execute(); - } catch (Exception e) { - log.info("Fail to insertAppInfo:SQL={}", query.getSQL(ParamType.INLINED), e); - } - return applicationDTO.getId(); - } - @Override public int updateApp(DSLContext dslContext, ApplicationDTO applicationDTO) { - List subBizIds = applicationDTO.getSubBizIds(); - String subBizIdsStr = null; - if (subBizIds != null) { - subBizIdsStr = subBizIds.stream().map(Object::toString).collect(Collectors.joining(",")); - } val query = dslContext.update(T_APP) .set(T_APP.APP_NAME, applicationDTO.getName()) .set(T_APP.BK_SUPPLIER_ACCOUNT, applicationDTO.getBkSupplierAccount()) - .set(T_APP.MAINTAINERS, applicationDTO.getMaintainers()) - .set(T_APP.SUB_APP_IDS, subBizIdsStr) .set(T_APP.TIMEZONE, applicationDTO.getTimeZone()) - .set(T_APP.BK_OPERATE_DEPT_ID, applicationDTO.getOperateDeptId()) .set(T_APP.LANGUAGE, applicationDTO.getLanguage()) .set(T_APP.ATTRS, applicationDTO.getAttrs() == null ? null : JsonUtils.toJson(applicationDTO.getAttrs())) - .set(T_APP.APP_TYPE, (byte) applicationDTO.getAppType().getValue()) .where(T_APP.APP_ID.eq(ULong.valueOf(applicationDTO.getId()))); return query.execute(); } @@ -402,22 +284,6 @@ public int deleteAppByIdSoftly(DSLContext dslContext, long appId) { return affectedNum; } - @Override - public int updateMaintainers(long appId, String maintainers) { - return context.update(T_APP) - .set(T_APP.MAINTAINERS, maintainers) - .where(T_APP.APP_ID.eq(ULong.valueOf(appId))) - .execute(); - } - - @Override - public int updateSubBizIds(long appId, String subBizIds) { - return context.update(T_APP) - .set(T_APP.SUB_APP_IDS, subBizIds) - .where(T_APP.APP_ID.eq(ULong.valueOf(appId))) - .execute(); - } - @Override public Integer countApps() { return context.selectCount() diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ApplicationHostDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ApplicationHostDAOImpl.java index 59abdbcec1..9fe8ed1a5f 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ApplicationHostDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ApplicationHostDAOImpl.java @@ -33,7 +33,6 @@ import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.model.dto.ApplicationDTO; import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; -import com.tencent.bk.job.common.model.dto.IpDTO; import com.tencent.bk.job.common.model.dto.ResourceScope; import com.tencent.bk.job.common.util.StringUtil; import com.tencent.bk.job.common.util.TagUtils; @@ -965,10 +964,9 @@ private List buildCondition(ApplicationHostDTO applicationHostInfoCon } @Override - public List listHosts(Collection hostIps) { + public List listHostsByIps(Collection cloudIps) { List conditions = new ArrayList<>(); - List cloudIpList = hostIps.stream().map(IpDTO::convertToStrIp).collect(Collectors.toList()); - conditions.add(TABLE.CLOUD_IP.in(cloudIpList)); + conditions.add(TABLE.CLOUD_IP.in(cloudIps)); return queryHostsByCondition(conditions); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/NotifyEsbChannelDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/NotifyEsbChannelDAOImpl.java index aa69b80ee0..bb41e22346 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/NotifyEsbChannelDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/NotifyEsbChannelDAOImpl.java @@ -96,7 +96,7 @@ public List listNotifyEsbChannel(DSLContext dslContext) { } catch (ExecutionException e) { String errorMsg = "Fail to load EsbChannel from cache"; logger.error(errorMsg, e); - throw new InternalException(errorMsg, e, ErrorCode.PAAS_MSG_CHANNEL_DATA_ERROR); + throw new InternalException(errorMsg, e, ErrorCode.CMSI_MSG_CHANNEL_DATA_ERROR); } } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanStepDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanStepDAOImpl.java index 88c789e4ee..9e2cd5b9c2 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanStepDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanStepDAOImpl.java @@ -31,7 +31,11 @@ import com.tencent.bk.job.manage.model.dto.task.TaskStepDTO; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; -import org.jooq.*; +import org.jooq.Condition; +import org.jooq.DSLContext; +import org.jooq.Record11; +import org.jooq.Result; +import org.jooq.UpdateSetMoreStep; import org.jooq.generated.tables.TaskPlanStep; import org.jooq.generated.tables.records.TaskPlanStepRecord; import org.jooq.types.UByte; @@ -109,7 +113,7 @@ public long insertStep(TaskStepDTO taskStep) { .columns(TABLE.PLAN_ID, TABLE.NAME, TABLE.TYPE, TABLE.PREVIOUS_STEP_ID, TABLE.NEXT_STEP_ID, TABLE.TEMPLATE_STEP_ID, TABLE.IS_ENABLE) .values(ULong.valueOf(taskStep.getPlanId()), taskStep.getName(), - UByte.valueOf(taskStep.getType().getType()), ULong.valueOf(taskStep.getPreviousStepId()), + UByte.valueOf(taskStep.getType().getValue()), ULong.valueOf(taskStep.getPreviousStepId()), ULong.valueOf(taskStep.getNextStepId()), ULong.valueOf(taskStep.getTemplateStepId()), UByte.valueOf(taskStep.getEnable())) .returning(TABLE.ID).fetchOne(); @@ -154,7 +158,7 @@ public boolean updateStepById(TaskStepDTO taskStep) { List conditions = new ArrayList<>(); conditions.add(TABLE.ID.eq(ULong.valueOf(taskStep.getId()))); conditions.add(TABLE.PLAN_ID.eq(ULong.valueOf(taskStep.getPlanId()))); - conditions.add(TABLE.TYPE.eq(UByte.valueOf(taskStep.getType().getType()))); + conditions.add(TABLE.TYPE.eq(UByte.valueOf(taskStep.getType().getValue()))); return 1 == updateStep.where(conditions).limit(1).execute(); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanVariableDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanVariableDAOImpl.java index 56267ce0af..27623b97b6 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanVariableDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanVariableDAOImpl.java @@ -92,8 +92,8 @@ private TaskVariableDTO extract(Record record) { taskVariable.setType(TaskVariableTypeEnum.valOf(record.get(TABLE.TYPE).intValue())); taskVariable.setDefaultValue(record.get(TABLE.DEFAULT_VALUE)); taskVariable.setDescription(record.get(TABLE.DESCRIPTION)); - taskVariable.setChangeable(Bool.isTrue(record.get(TABLE.IS_CHANGEABLE).intValue())); - taskVariable.setRequired(Bool.isTrue(record.get(TABLE.IS_REQUIRED).intValue())); + taskVariable.setChangeable(Bool.isTrue(record.get(TABLE.IS_CHANGEABLE).byteValue())); + taskVariable.setRequired(Bool.isTrue(record.get(TABLE.IS_REQUIRED).byteValue())); return taskVariable; } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateStepDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateStepDAOImpl.java index 8efeaeb7d2..56418fdfc8 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateStepDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateStepDAOImpl.java @@ -31,7 +31,11 @@ import com.tencent.bk.job.manage.model.dto.task.TaskStepDTO; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; -import org.jooq.*; +import org.jooq.Condition; +import org.jooq.DSLContext; +import org.jooq.Record11; +import org.jooq.Result; +import org.jooq.UpdateSetMoreStep; import org.jooq.generated.tables.TaskTemplateStep; import org.jooq.generated.tables.records.TaskTemplateStepRecord; import org.jooq.types.UByte; @@ -112,7 +116,7 @@ public long insertStep(TaskStepDTO taskStep) { .columns(TABLE.TEMPLATE_ID, TABLE.NAME, TABLE.TYPE, TABLE.PREVIOUS_STEP_ID, TABLE.NEXT_STEP_ID, TABLE.IS_DELETED) .values(ULong.valueOf(taskStep.getTemplateId()), taskStep.getName(), - UByte.valueOf(taskStep.getType().getType()), ULong.valueOf(taskStep.getPreviousStepId()), + UByte.valueOf(taskStep.getType().getValue()), ULong.valueOf(taskStep.getPreviousStepId()), ULong.valueOf(taskStep.getNextStepId()), UByte.valueOf(0)) .returning(TABLE.ID).fetchOne(); return record.getId().longValue(); @@ -152,7 +156,7 @@ public boolean updateStepById(TaskStepDTO taskStep) { List conditions = new ArrayList<>(); conditions.add(TABLE.ID.eq(ULong.valueOf(taskStep.getId()))); conditions.add(TABLE.TEMPLATE_ID.eq(ULong.valueOf(taskStep.getTemplateId()))); - conditions.add(TABLE.TYPE.eq(UByte.valueOf(taskStep.getType().getType()))); + conditions.add(TABLE.TYPE.eq(UByte.valueOf(taskStep.getType().getValue()))); return 1 == updateStep.where(conditions).limit(1).execute(); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/host/HostCache.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/host/HostCache.java index db60c03d78..1314ef064b 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/host/HostCache.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/host/HostCache.java @@ -26,7 +26,6 @@ import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; -import com.tencent.bk.job.common.model.dto.IpDTO; import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.manage.model.db.CacheHostDO; import lombok.extern.slf4j.Slf4j; @@ -49,7 +48,6 @@ public class HostCache { private final RedisTemplate redisTemplate; private final AppScopeMappingService appScopeMappingService; - private final String HOST_KEY_PREFIX = "job:manage:host:"; @Autowired public HostCache(@Qualifier("jsonRedisTemplate") RedisTemplate redisTemplate, @@ -59,13 +57,28 @@ public HostCache(@Qualifier("jsonRedisTemplate") RedisTemplate r } /** - * 批量查询缓存的主机 + * 批量根据ip查询缓存的主机 * - * @param hosts 主机列表 + * @param cloudIps 主机ip(云区域+IP)列表 + * @return 缓存的主机列表, 按照传入的hosts排序。如果该主机不存在，List中对应索引的值为null + */ + public List batchGetHostsByIps(List cloudIps) { + List hostKeys = cloudIps.stream().map(this::buildHostIpKey).collect(Collectors.toList()); + return getHostsByKeys(hostKeys); + } + + /** + * 批量根据hostId查询缓存的主机 + * + * @param hostIds 主机ID列表 * @return 缓存的主机列表, 按照传入的hosts排序。如果该主机不存在，List中对应索引的值为null */ - public List batchGetHosts(List hosts) { - List hostKeys = hosts.stream().map(this::buildHostKey).collect(Collectors.toList()); + public List batchGetHostsByHostIds(List hostIds) { + List hostKeys = hostIds.stream().map(this::buildHostIdKey).collect(Collectors.toList()); + return getHostsByKeys(hostKeys); + } + + private List getHostsByKeys(List hostKeys) { try { return (List) redisTemplate.opsForValue().multiGet(hostKeys); } catch (Exception e) { @@ -80,8 +93,10 @@ public List batchGetHosts(List hosts) { * @param applicationHostDTO 主机 */ public void deleteHost(ApplicationHostDTO applicationHostDTO) { - String hostKey = buildHostKey(applicationHostDTO); - redisTemplate.delete(hostKey); + String hostIpKey = buildHostIpKey(applicationHostDTO); + String hostIdKey = buildHostIdKey(applicationHostDTO); + redisTemplate.delete(hostIpKey); + redisTemplate.delete(hostIdKey); } /** @@ -90,7 +105,6 @@ public void deleteHost(ApplicationHostDTO applicationHostDTO) { * @param applicationHostDTO 主机 */ public void addOrUpdateHost(ApplicationHostDTO applicationHostDTO) { - String hostKey = buildHostKey(applicationHostDTO); CacheHostDO cacheHost = new CacheHostDO(); cacheHost.setBizId(applicationHostDTO.getBizId()); if (applicationHostDTO.getAppId() == null) { @@ -102,7 +116,10 @@ public void addOrUpdateHost(ApplicationHostDTO applicationHostDTO) { cacheHost.setCloudAreaId(applicationHostDTO.getCloudAreaId()); cacheHost.setIp(applicationHostDTO.getIp()); cacheHost.setHostId(applicationHostDTO.getHostId()); - redisTemplate.opsForValue().set(hostKey, cacheHost, 1, TimeUnit.DAYS); + String hostIpKey = buildHostIpKey(applicationHostDTO); + String hostIdKey = buildHostIdKey(applicationHostDTO); + redisTemplate.opsForValue().set(hostIpKey, cacheHost, 1, TimeUnit.DAYS); + redisTemplate.opsForValue().set(hostIdKey, cacheHost, 1, TimeUnit.DAYS); } /** @@ -114,18 +131,25 @@ public void addOrUpdateHosts(List hosts) { hosts.forEach(this::addOrUpdateHost); } - private String buildHostKey(ApplicationHostDTO applicationHostDTO) { - return buildHostKey(applicationHostDTO.getCloudAreaId(), + + private String buildHostIpKey(ApplicationHostDTO applicationHostDTO) { + return buildHostIpKey(applicationHostDTO.getCloudAreaId(), applicationHostDTO.getIp()); } - private String buildHostKey(IpDTO host) { - String cloudIp = host.getCloudAreaId() + ":" + host.getIp(); - return HOST_KEY_PREFIX + cloudIp; + private String buildHostIdKey(ApplicationHostDTO applicationHostDTO) { + return buildHostIdKey(applicationHostDTO.getHostId()); + } + + private String buildHostIpKey(String cloudIp) { + return "job:manage:host:" + cloudIp; + } + + private String buildHostIdKey(long hostId) { + return "job:manage:host:hostId:" + hostId; } - private String buildHostKey(Long cloudAreaId, String ip) { - String cloudIp = cloudAreaId + ":" + ip; - return HOST_KEY_PREFIX + cloudIp; + private String buildHostIpKey(Long bkCloudId, String ip) { + return buildHostIpKey(bkCloudId + ":" + ip); } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/db/CacheAppDO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/db/CacheAppDO.java index 25041c95e6..f4d242bd79 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/db/CacheAppDO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/db/CacheAppDO.java @@ -24,23 +24,21 @@ package com.tencent.bk.job.manage.model.db; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; -import com.tencent.bk.job.common.annotation.CompatibleImplementation; -import com.tencent.bk.job.common.constant.AppTypeEnum; import com.tencent.bk.job.common.model.dto.ApplicationAttrsDO; import com.tencent.bk.job.common.model.dto.ApplicationDTO; import com.tencent.bk.job.common.model.dto.ResourceScope; import lombok.Getter; import lombok.Setter; -import java.util.List; - /** * Redis 缓存业务DO */ @Getter @Setter @JsonInclude(JsonInclude.Include.NON_EMPTY) +@JsonIgnoreProperties(ignoreUnknown = true) public class CacheAppDO { /** * 业务ID @@ -62,30 +60,6 @@ public class CacheAppDO { */ private String name; - /** - * 业务类型 - */ - @CompatibleImplementation(explain = "兼容字段，等业务集全部迁移到cmdb之后可以删除", version = "3.6.x") - private Integer appType; - - /** - * 子业务 - */ - @CompatibleImplementation(explain = "兼容字段，等业务集全部迁移到cmdb之后可以删除", version = "3.6.x") - private List subBizIds; - - /** - * 业务运维 - */ - @CompatibleImplementation(explain = "兼容字段，等业务集全部迁移到cmdb之后可以删除", version = "3.6.x") - private String maintainers; - - /** - * 临时字段-业务初始部门ID - */ - @CompatibleImplementation(explain = "兼容字段，等业务集全部迁移到cmdb之后可以删除", version = "3.6.x") - private Long operateDeptId; - /** * 业务属性 */ @@ -100,10 +74,6 @@ public static CacheAppDO fromApplicationDTO(ApplicationDTO application) { cacheAppDO.setScopeType(application.getScope().getType().getValue()); cacheAppDO.setScopeId(application.getScope().getId()); cacheAppDO.setName(application.getName()); - cacheAppDO.setAppType(application.getAppType().getValue()); - cacheAppDO.setMaintainers(application.getMaintainers()); - cacheAppDO.setSubBizIds(application.getSubBizIds()); - cacheAppDO.setOperateDeptId(application.getOperateDeptId()); cacheAppDO.setAttrs(application.getAttrs()); return cacheAppDO; } @@ -116,10 +86,6 @@ public static ApplicationDTO toApplicationDTO(CacheAppDO cacheAppDO) { application.setId(cacheAppDO.getId()); application.setScope(new ResourceScope(cacheAppDO.getScopeType(), cacheAppDO.getScopeId())); application.setName(cacheAppDO.getName()); - application.setAppType(AppTypeEnum.valueOf(cacheAppDO.getAppType())); - application.setMaintainers(cacheAppDO.getMaintainers()); - application.setSubBizIds(cacheAppDO.getSubBizIds()); - application.setOperateDeptId(cacheAppDO.getOperateDeptId()); application.setAttrs(cacheAppDO.getAttrs()); return application; } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/db/CacheHostDO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/db/CacheHostDO.java index 7570a03e38..124307ddbf 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/db/CacheHostDO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/db/CacheHostDO.java @@ -50,6 +50,11 @@ public class CacheHostDO { */ private Long hostId; + /** + * 主机Agent ID + */ + private String agentId; + /** * 业务ID */ diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskStepConverter.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskStepConverter.java index 4d63785dcd..91332ee493 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskStepConverter.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskStepConverter.java @@ -38,7 +38,7 @@ public static ServiceTaskStepDTO convertToServiceTaskStepDTO(TaskStepDTO taskSte ServiceTaskStepDTO serviceTaskStepDTO = new ServiceTaskStepDTO(); serviceTaskStepDTO.setId(taskStepDTO.getId()); serviceTaskStepDTO.setName(taskStepDTO.getName()); - serviceTaskStepDTO.setType(taskStepDTO.getType().getType()); + serviceTaskStepDTO.setType(taskStepDTO.getType().getValue()); serviceTaskStepDTO.setEnable(taskStepDTO.getEnable()); serviceTaskStepDTO.setApprovalStepInfo( TaskApprovalStepConverter.convertToServiceTaskApprovalStepDTO( diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskStepDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskStepDTO.java index be1f8f92e7..9062a72ed1 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskStepDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskStepDTO.java @@ -94,7 +94,7 @@ public static TaskStepVO toVO(TaskStepDTO taskStep) { TaskStepVO stepVO = new TaskStepVO(); stepVO.setId(taskStep.getId()); stepVO.setTemplateStepId(taskStep.getTemplateStepId()); - stepVO.setType(taskStep.getType().getType()); + stepVO.setType(taskStep.getType().getValue()); stepVO.setName(taskStep.getName()); stepVO.setScriptStepInfo(TaskScriptStepDTO.toVO(taskStep.getScriptStepInfo())); stepVO.setFileStepInfo(TaskFileStepDTO.toVO(taskStep.getFileStepInfo())); @@ -150,7 +150,7 @@ public static EsbStepV3DTO toEsbStepV3(TaskStepDTO taskStep) { EsbStepV3DTO esbStep = new EsbStepV3DTO(); esbStep.setId(taskStep.getId()); esbStep.setName(taskStep.getName()); - esbStep.setType(taskStep.getType().getType()); + esbStep.setType(taskStep.getType().getValue()); switch (taskStep.getType()) { case SCRIPT: esbStep.setScriptInfo(TaskScriptStepDTO.toEsbScriptInfoV3(taskStep.getScriptStepInfo())); @@ -174,7 +174,7 @@ public static ServiceTaskStepDTO toServiceDTO(TaskStepDTO taskStep) { ServiceTaskStepDTO serviceTaskStep = new ServiceTaskStepDTO(); serviceTaskStep.setId(taskStep.getId()); serviceTaskStep.setName(taskStep.getName()); - serviceTaskStep.setType(taskStep.getType().getType()); + serviceTaskStep.setType(taskStep.getType().getValue()); serviceTaskStep.setEnable(taskStep.getEnable()); switch (taskStep.getType()) { case SCRIPT: diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/whiteip/CloudIPDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/whiteip/CloudIPDTO.java index 28a2fb61f7..e076757372 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/whiteip/CloudIPDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/whiteip/CloudIPDTO.java @@ -24,7 +24,7 @@ package com.tencent.bk.job.manage.model.dto.whiteip; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import lombok.AllArgsConstructor; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -53,8 +53,8 @@ public String getCloudIP() { return cloudAreaId + ":" + ip; } - public IpDTO toIpDTO() { - return new IpDTO(cloudAreaId, ip); + public HostDTO toHostDTO() { + return new HostDTO(cloudAreaId, ip); } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/ApplicationService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/ApplicationService.java index 0dc0614391..a44989035a 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/ApplicationService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/ApplicationService.java @@ -24,7 +24,6 @@ package com.tencent.bk.job.manage.service; -import com.tencent.bk.job.common.annotation.CompatibleImplementation; import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; import com.tencent.bk.job.common.exception.NotFoundException; import com.tencent.bk.job.common.model.dto.ApplicationDTO; @@ -145,18 +144,6 @@ public interface ApplicationService { */ List listAllApps(); - /** - * 判断用户是否有业务权限 - *

- * 接入权限中心后不应使用本接口 - * - * @param appId 业务 ID - * @param username 用户名 - * @return 是否有业务权限 - */ - @CompatibleImplementation(explain = "兼容方法，等业务集全部迁移到cmdb之后可以删除", version = "3.6.x") - boolean checkAppPermission(long appId, String username); - /** * 创建业务 * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/HostService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/HostService.java index 35743a413f..dc8d80da12 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/HostService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/HostService.java @@ -30,9 +30,10 @@ import com.tencent.bk.job.common.model.dto.AppResourceScope; import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; import com.tencent.bk.job.common.model.dto.DynamicGroupInfoDTO; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.model.vo.HostInfoVO; import com.tencent.bk.job.manage.common.consts.whiteip.ActionScopeEnum; +import com.tencent.bk.job.manage.model.inner.ServiceListAppHostResultDTO; import com.tencent.bk.job.manage.model.web.request.AgentStatisticsReq; import com.tencent.bk.job.manage.model.web.request.ipchooser.AppTopologyTreeNode; import com.tencent.bk.job.manage.model.web.request.ipchooser.ListHostByBizTopologyNodesReq; @@ -182,22 +183,19 @@ AgentStatistics getAgentStatistics(String username, void fillAgentStatus(List hosts); - Boolean existsHost(Long bizId, String ip); - /** - * 检查主机是否在业务下 + * 获取业务下的主机 * - * @param appId Job业务ID - * @param hostIps 被检查的主机 - * @return 非法的主机 + * @param appId Job业务ID + * @param hosts 主机列表 */ - List checkAppHosts(Long appId, List hostIps); + ServiceListAppHostResultDTO listAppHosts(Long appId, List hosts); /** - * 根据主机IP批量获取主机。如果在同步的主机中不存在，那么从cmdb查询 + * 根据主机批量获取主机。如果在同步的主机中不存在，那么从cmdb查询 * - * @param hostIps 主机IP + * @param hosts 主机 * @return 主机 */ - List listHosts(Collection hostIps); + List listHosts(Collection hosts); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/NotifyService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/NotifyService.java index a7dd1898a3..eb2fe3e108 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/NotifyService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/NotifyService.java @@ -95,12 +95,12 @@ Set findUserByResourceRoles(Long appId, String triggerUser, Integer reso PageTemplateVO getPageTemplate(String username); - Integer sendNotificationsToUsers(ServiceUserNotificationDTO serviceUserNotificationDTO); + Integer asyncSendNotificationsToUsers(ServiceUserNotificationDTO serviceUserNotificationDTO); - Integer sendNotificationsToUsersByChannel(ServiceUserNotificationDTO serviceUserNotificationDTO, - List channelTypeList); + Integer asyncSendNotificationsByChannel(ServiceUserNotificationDTO serviceUserNotificationDTO, + List channelTypeList); - Integer sendNotificationsToAdministrators(ServiceNotificationMessage serviceNotificationMessage); + Integer asyncSendNotificationsToAdministrators(ServiceNotificationMessage serviceNotificationMessage); Integer sendTemplateNotification(ServiceTemplateNotificationDTO templateNotificationDTO); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/PaaSService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/PaaSService.java index 765c18b559..bf3f6f3cae 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/PaaSService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/PaaSService.java @@ -40,11 +40,11 @@ public interface PaaSService { List getAllChannelList(String bkToken, String uin) throws IOException; - boolean sendMsg( + void sendMsg( String msgType, String sender, Set receivers, String title, String content - ) throws Exception; + ); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/AppRoleServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/AppRoleServiceImpl.java index 17be0646bc..740b15acbe 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/AppRoleServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/AppRoleServiceImpl.java @@ -25,6 +25,7 @@ package com.tencent.bk.job.manage.service.impl; import com.tencent.bk.job.common.cc.model.AppRoleDTO; +import com.tencent.bk.job.common.cc.sdk.BizSetCmdbClient; import com.tencent.bk.job.common.cc.sdk.CmdbClientFactory; import com.tencent.bk.job.common.cc.sdk.IBizCmdbClient; import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; @@ -34,14 +35,11 @@ import com.tencent.bk.job.manage.service.AppRoleService; import com.tencent.bk.job.manage.service.ApplicationService; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; -import java.util.Arrays; import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Set; @@ -50,33 +48,31 @@ public class AppRoleServiceImpl implements AppRoleService { private final ApplicationService applicationService; + private final BizSetCmdbClient bizSetCmdbClient; @Autowired - public AppRoleServiceImpl(@Lazy ApplicationService applicationService) { + public AppRoleServiceImpl(@Lazy ApplicationService applicationService, + BizSetCmdbClient bizSetCmdbClient) { this.applicationService = applicationService; + this.bizSetCmdbClient = bizSetCmdbClient; } @Override public List listAppRoles(String lang) { IBizCmdbClient bizCmdbClient = CmdbClientFactory.getCmdbClient(lang); - return bizCmdbClient.getAppRoleList(); + return bizCmdbClient.listRoles(); } @Override public Set listAppUsersByRole(Long appId, String role) { - ApplicationDTO applicationDTO = applicationService.getAppByAppId(appId); - ResourceScope scope = applicationDTO.getScope(); + ApplicationDTO application = applicationService.getAppByAppId(appId); + ResourceScope scope = application.getScope(); if (scope.getType() == ResourceScopeTypeEnum.BIZ) { IBizCmdbClient bizCmdbClient = CmdbClientFactory.getCmdbClient(JobContextUtil.getUserLang()); - return bizCmdbClient.getAppUsersByRole(appId, role); + return bizCmdbClient.listUsersByRole(Long.valueOf(scope.getId()), role); } else if (scope.getType() == ResourceScopeTypeEnum.BIZ_SET) { // 业务集当前只支持运维人员 - if ("bk_biz_maintainer".equals(role)) { - String maintainerStr = applicationDTO.getMaintainers(); - if (StringUtils.isNotBlank(maintainerStr)) { - return new HashSet<>(Arrays.asList(maintainerStr.split("[,;]"))); - } - } + return bizSetCmdbClient.listUsersByRole(Long.valueOf(scope.getId()), role); } else { log.warn("Not supported resourceScope:{}", scope); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ApplicationServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ApplicationServiceImpl.java index f1a2f9aae6..9b3abe25ee 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ApplicationServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ApplicationServiceImpl.java @@ -35,7 +35,6 @@ import com.tencent.bk.job.manage.service.AccountService; import com.tencent.bk.job.manage.service.ApplicationService; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.jooq.DSLContext; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -228,40 +227,6 @@ public List listAllApps() { return applicationDAO.listAllApps(); } - @Override - public boolean checkAppPermission(long appId, String username) { - if (StringUtils.isBlank(username)) { - return false; - } - username = username.trim(); - ApplicationDTO appInfo = getAppByAppId(appId); - if (log.isDebugEnabled()) { - log.debug("Get app info for {}|{}", appId, appInfo); - } - if (appInfo == null) { - log.warn("Check app permission, app[{}] is not exist!", appId); - return false; - } - String maintainers = appInfo.getMaintainers(); - if (log.isDebugEnabled()) { - log.debug("Checking app {} user {}|{}", appId, username, maintainers); - } - if (StringUtils.isNotBlank(maintainers)) { - for (String maintainer : maintainers.split("[,;]")) { - if (StringUtils.isBlank(maintainer)) { - continue; - } - if (log.isDebugEnabled()) { - log.debug("Checking...|{}|{}|{}", appId, username, maintainer); - } - if (username.equals(maintainer.trim())) { - return true; - } - } - } - return false; - } - @Override public void updateApp(ApplicationDTO application) { log.info("Update app: {}", JsonUtils.toJson(application)); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/GlobalSettingsServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/GlobalSettingsServiceImpl.java index c42faa1828..15f1b0fda4 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/GlobalSettingsServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/GlobalSettingsServiceImpl.java @@ -55,8 +55,6 @@ import com.tencent.bk.job.manage.model.dto.notify.AvailableEsbChannelDTO; import com.tencent.bk.job.manage.model.dto.notify.NotifyEsbChannelDTO; import com.tencent.bk.job.manage.model.dto.notify.NotifyTemplateDTO; -import com.tencent.bk.job.manage.model.inner.ServiceNotificationMessage; -import com.tencent.bk.job.manage.model.inner.ServiceUserNotificationDTO; import com.tencent.bk.job.manage.model.web.request.globalsetting.AccountNameRule; import com.tencent.bk.job.manage.model.web.request.globalsetting.AccountNameRulesReq; import com.tencent.bk.job.manage.model.web.request.globalsetting.FileUploadSettingReq; @@ -80,6 +78,7 @@ import com.tencent.bk.job.manage.model.web.vo.notify.UserVO; import com.tencent.bk.job.manage.service.GlobalSettingsService; import com.tencent.bk.job.manage.service.NotifyService; +import com.tencent.bk.job.manage.service.impl.notify.NotifySendService; import com.tencent.bk.job.manage.service.impl.notify.NotifyUserService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; @@ -94,7 +93,6 @@ import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -121,6 +119,7 @@ public class GlobalSettingsServiceImpl implements GlobalSettingsService { private final NotifyEsbChannelDAO notifyEsbChannelDAO; private final AvailableEsbChannelDAO availableEsbChannelDAO; private final NotifyService notifyService; + private final NotifySendService notifySendService; private final NotifyUserService notifyUserService; private final GlobalSettingDAO globalSettingDAO; private final NotifyTemplateDAO notifyTemplateDAO; @@ -140,7 +139,8 @@ public GlobalSettingsServiceImpl( , NotifyEsbChannelDAO notifyEsbChannelDAO , AvailableEsbChannelDAO availableEsbChannelDAO , NotifyService notifyService - , NotifyUserService notifyUserService, + , NotifySendService notifySendService, + NotifyUserService notifyUserService, GlobalSettingDAO globalSettingDAO , NotifyTemplateDAO notifyTemplateDAO, MessageI18nService i18nService, @@ -152,6 +152,7 @@ public GlobalSettingsServiceImpl( this.notifyEsbChannelDAO = notifyEsbChannelDAO; this.availableEsbChannelDAO = availableEsbChannelDAO; this.notifyService = notifyService; + this.notifySendService = notifySendService; this.notifyUserService = notifyUserService; this.globalSettingDAO = globalSettingDAO; this.notifyTemplateDAO = notifyTemplateDAO; @@ -632,12 +633,15 @@ public Integer saveChannelTemplate(String username, ChannelTemplateReq req) { public Integer sendChannelTemplate(String username, ChannelTemplatePreviewReq req) { List receiverList = StringUtil.strToList(req.getReceiverStr(), String.class, NotifyConsts.SEPERATOR_COMMA); - return notifyService.sendNotificationsToUsersByChannel( - new ServiceUserNotificationDTO( - new HashSet<>(receiverList), - new ServiceNotificationMessage( - req.getTitle(), req.getContent())), - Collections.singletonList(req.getChannelCode())); + Set receiverSet = new HashSet<>(receiverList); + notifySendService.sendUserChannelNotify( + null, + receiverSet, + req.getChannelCode(), + req.getTitle(), + req.getContent() + ); + return receiverSet.size(); } @Override diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/HostServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/HostServiceImpl.java index c8fd6b052f..f4d2b3d673 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/HostServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/HostServiceImpl.java @@ -46,7 +46,7 @@ import com.tencent.bk.job.common.model.dto.ApplicationDTO; import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; import com.tencent.bk.job.common.model.dto.DynamicGroupInfoDTO; -import com.tencent.bk.job.common.model.dto.IpDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.model.dto.ResourceScope; import com.tencent.bk.job.common.model.vo.CloudAreaInfoVO; import com.tencent.bk.job.common.model.vo.HostInfoVO; @@ -63,6 +63,7 @@ import com.tencent.bk.job.manage.model.db.CacheHostDO; import com.tencent.bk.job.manage.model.dto.HostTopoDTO; import com.tencent.bk.job.manage.model.dto.whiteip.CloudIPDTO; +import com.tencent.bk.job.manage.model.inner.ServiceListAppHostResultDTO; import com.tencent.bk.job.manage.model.web.request.AgentStatisticsReq; import com.tencent.bk.job.manage.model.web.request.ipchooser.AppTopologyTreeNode; import com.tencent.bk.job.manage.model.web.request.ipchooser.ListHostByBizTopologyNodesReq; @@ -504,10 +505,6 @@ public List getAppDynamicGroupList(String username, @Override public List getBizDynamicGroupHostList(String username, Long bizId, List dynamicGroupIdList) { - ApplicationDTO applicationInfo = applicationService.getAppByScope( - new ResourceScope(ResourceScopeTypeEnum.BIZ, bizId.toString()) - ); - String maintainer = applicationInfo.getMaintainers().split("[,;]")[0]; Map ccGroupInfoMap = new HashMap<>(); Map> bizId2GroupIdMap = new HashMap<>(); @@ -715,11 +712,6 @@ public void fillAgentStatus(List hosts) { } } - @Override - public Boolean existsHost(Long bizId, String ip) { - return applicationHostDAO.existsHost(dslContext, bizId, ip); - } - @Override public CcTopologyNodeVO listAppTopologyHostCountTree(String username, AppResourceScope appResourceScope) { @@ -942,10 +934,11 @@ private void separateNotInAppIP(Long appId, }); // 对于本地不在目标业务下的主机再到CMDB查询 IBizCmdbClient bizCmdbClient = CmdbClientFactory.getCmdbClient(); - List ipDTOList = notInAppIPListByLocal.parallelStream() - .map(CloudIPDTO::toIpDTO) + List ipDTOList = notInAppIPListByLocal.parallelStream() + .map(CloudIPDTO::toHostDTO) .collect(Collectors.toList()); - List cmdbExistHosts = bizCmdbClient.listHostsByIps(ipDTOList); + List cmdbExistHosts = bizCmdbClient.listHostsByIps( + ipDTOList.stream().map(HostDTO::toCloudIp).collect(Collectors.toList())); Map cmdbHostsMap = new HashMap<>(); Set cmdbExistCloudIPSet = new HashSet<>(); cmdbExistHosts.forEach(host -> { @@ -1056,7 +1049,7 @@ public List getHostsByIp( IBizCmdbClient cmdbClient = CmdbClientFactory.getCmdbClient(); List cmdbHosts = cmdbClient.listHostsByIps( validIPList.parallelStream() - .map(CloudIPDTO::toIpDTO) + .map(CloudIPDTO::getCloudIP) .collect(Collectors.toList()) ); hostDTOList.addAll(cmdbHosts); @@ -1289,51 +1282,6 @@ public AgentStatistics getAgentStatistics(String username, Long appId, AgentStat return new AgentStatistics(normalCount, abnormalCount); } - @Override - public List checkAppHosts(Long appId, - List hostIps) { - ApplicationDTO application = applicationService.getAppByAppId(appId); - - if (application.isAllBizSet()) { - // cmdb全业务包含了所有业务下的主机，不需要再判断 - return Collections.emptyList(); - } - - List includeBizIds = buildIncludeBizIdList(application); - if (CollectionUtils.isEmpty(includeBizIds)) { - log.warn("App do not contains any biz, appId:{}", appId); - return hostIps; - } - - List hostsInOtherApp = new ArrayList<>(); - List notExistHosts = new ArrayList<>(); - - // 从缓存中查询主机信息，并判断主机是否在业务下 - checkCachedHosts(hostIps, includeBizIds, hostsInOtherApp, notExistHosts); - - // 如果主机不在缓存中，那么从已同步到Job的主机中查询 - if (CollectionUtils.isNotEmpty(notExistHosts)) { - log.info("Hosts not in cached, check hosts by synced hosts. notCacheHosts: {}", notExistHosts); - checkSyncHosts(includeBizIds, hostsInOtherApp, notExistHosts); - } - - // 如果主机不在已同步到Job的主机中, 那么从cmdb直接查询 - if (CollectionUtils.isNotEmpty(notExistHosts)) { - log.info("Hosts not synced, check hosts by cmdb. notSyncedHosts: {}", notExistHosts); - checkHostsFromCmdb(includeBizIds, hostsInOtherApp, notExistHosts); - } - - List invalidHosts = new ArrayList<>(); - if (CollectionUtils.isNotEmpty(notExistHosts) || CollectionUtils.isNotEmpty(hostsInOtherApp)) { - invalidHosts.addAll(notExistHosts); - invalidHosts.addAll(hostsInOtherApp.stream() - .map(host -> new IpDTO(host.getCloudAreaId(), host.getIp())).collect(Collectors.toList())); - log.info("Contains invalid hosts, appId: {}, notExistHosts: {}, hostsInOtherApp: {}", - appId, notExistHosts, hostsInOtherApp); - } - return invalidHosts; - } - private List buildIncludeBizIdList(ApplicationDTO application) { List bizIdList = new ArrayList<>(); if (application.isBiz()) { @@ -1346,29 +1294,30 @@ private List buildIncludeBizIdList(ApplicationDTO application) { return bizIdList; } - private void checkCachedHosts(List hostIps, + private void checkCachedHosts(List hosts, List includeBizIds, List hostsInOtherApp, - List notExistHosts) { - List cacheHosts = hostCache.batchGetHosts(hostIps); - for (int i = 0; i < hostIps.size(); i++) { - IpDTO hostIp = hostIps.get(i); + List notExistHosts) { + List cacheHosts = + hostCache.batchGetHostsByIps(hosts.stream().map(HostDTO::toCloudIp).collect(Collectors.toList())); + for (int i = 0; i < hosts.size(); i++) { + HostDTO host = hosts.get(i); CacheHostDO cacheHost = cacheHosts.get(i); if (cacheHost != null) { if (!includeBizIds.contains(cacheHost.getBizId())) { - hostsInOtherApp.add(new BasicAppHost(cacheHost.getBizId(), - cacheHost.getCloudAreaId(), cacheHost.getIp())); + hostsInOtherApp.add(new BasicAppHost(cacheHost.getBizId(), cacheHost.getHostId(), + cacheHost.getCloudAreaId(), cacheHost.getIp(), cacheHost.getAgentId())); } } else { - notExistHosts.add(hostIp); + notExistHosts.add(host.toCloudIp()); } } } private void checkSyncHosts(List includeBizIds, List hostsInOtherApp, - List notExistHosts) { - List appHosts = applicationHostDAO.listHosts(notExistHosts); + List notExistHosts) { + List appHosts = applicationHostDAO.listHostsByIps(notExistHosts); if (CollectionUtils.isNotEmpty(appHosts)) { for (ApplicationHostDTO appHost : appHosts) { if (appHost.getBizId() == null || appHost.getBizId() <= 0) { @@ -1376,15 +1325,12 @@ private void checkSyncHosts(List includeBizIds, // DB中缓存的主机可能没有业务信息(依赖的主机事件还没有处理),那么暂时跳过该主机 continue; } - IpDTO hostIp = new IpDTO(appHost.getCloudAreaId(), appHost.getIp()); - if (appHost.getBizId() == JobConstants.PUBLIC_APP_ID) { - continue; - } - notExistHosts.remove(hostIp); + + notExistHosts.remove(appHost.getCloudIp()); hostCache.addOrUpdateHost(appHost); if (!includeBizIds.contains(appHost.getBizId())) { - hostsInOtherApp.add(new BasicAppHost(appHost.getBizId(), - appHost.getCloudAreaId(), appHost.getIp())); + hostsInOtherApp.add(new BasicAppHost(appHost.getBizId(), appHost.getHostId(), + appHost.getCloudAreaId(), appHost.getIp(), appHost.getAgentId())); } } } @@ -1392,14 +1338,14 @@ private void checkSyncHosts(List includeBizIds, private void checkHostsFromCmdb(List includeBizIds, List hostsInOtherApp, - List notExistHosts) { + List notExistHosts) { IBizCmdbClient bizCmdbClient = CmdbClientFactory.getCmdbClient(); try { List cmdbExistHosts = bizCmdbClient.listHostsByIps(notExistHosts); if (CollectionUtils.isNotEmpty(cmdbExistHosts)) { - List cmdbExistHostIps = cmdbExistHosts.stream() - .map(host -> new IpDTO(host.getCloudAreaId(), host.getIp())) + List cmdbExistHostIps = cmdbExistHosts.stream() + .map(ApplicationHostDTO::getCloudIp) .collect(Collectors.toList()); notExistHosts.removeAll(cmdbExistHostIps); log.info("sync new hosts from cmdb, hosts:{}", cmdbExistHosts); @@ -1408,8 +1354,8 @@ private void checkHostsFromCmdb(List includeBizIds, cmdbExistHosts.forEach(syncHost -> { if (!includeBizIds.contains(syncHost.getBizId())) { - hostsInOtherApp.add(new BasicAppHost(syncHost.getBizId(), - syncHost.getCloudAreaId(), syncHost.getIp())); + hostsInOtherApp.add(new BasicAppHost(syncHost.getBizId(), syncHost.getHostId(), + syncHost.getCloudAreaId(), syncHost.getIp(), syncHost.getAgentId())); } }); } @@ -1418,46 +1364,313 @@ private void checkHostsFromCmdb(List includeBizIds, } } - public List listHosts(Collection hostIps) { - // 从Job已同步的主机中查询 - List syncedHosts = applicationHostDAO.listHosts(hostIps); - Map hostsMap = new HashMap<>(); - if (CollectionUtils.isNotEmpty(syncedHosts)) { - syncedHosts.forEach(syncedHost -> hostsMap.put(syncedHost.getCloudIp(), syncedHost)); + public List listHosts(Collection hosts) { + Pair, List> hostResult = listHostsFromCacheOrCmdb(hosts); + if (CollectionUtils.isEmpty(hostResult.getRight())) { + return Collections.emptyList(); + } + return hostResult.getRight().stream().map(basicAppHost -> { + ApplicationHostDTO appHost = new ApplicationHostDTO(); + appHost.setHostId(basicAppHost.getHostId()); + appHost.setCloudAreaId(basicAppHost.getBkCloudId()); + appHost.setIp(basicAppHost.getIp()); + appHost.setAgentId(basicAppHost.getAgentId()); + appHost.setBizId(basicAppHost.getBizId()); + return appHost; + }).collect(Collectors.toList()); + } + + @Data + private static class BasicAppHost { + private Long bizId; + private Long hostId; + private Long bkCloudId; + private String ip; + private String agentId; + + private BasicAppHost(Long bizId, Long hostId, Long bkCloudId, String ip, String agentId) { + this.bizId = bizId; + this.hostId = hostId; + this.bkCloudId = bkCloudId; + this.ip = ip; + this.agentId = agentId; + } + + public HostDTO toHostDTO() { + HostDTO host = new HostDTO(); + host.setHostId(hostId); + host.setBkCloudId(bkCloudId); + host.setIp(ip); + host.setAgentId(agentId); + return host; } - List resultHosts = new ArrayList<>(); - List notSyncedHostIps = new ArrayList<>(); - hostIps.forEach(hostIp -> { - ApplicationHostDTO host = hostsMap.get(hostIp.convertToStrIp()); - if (host == null || host.getBizId() == null || host.getBizId() <= 0) { - notSyncedHostIps.add(hostIp); + } + + @Override + public ServiceListAppHostResultDTO listAppHosts(Long appId, + List hosts) { + ServiceListAppHostResultDTO result = new ServiceListAppHostResultDTO(); + ApplicationDTO application = applicationService.getAppByAppId(appId); + + Pair, List> hostResult = listHostsFromCacheOrCmdb(hosts); + List notExistHosts = hostResult.getLeft(); + List existHosts = hostResult.getRight(); + List validHosts = new ArrayList<>(); + List notInAppHosts = new ArrayList<>(); + + if (application.isAllBizSet()) { + // 如果是全业务，所以主机都是合法的 + result.setNotExistHosts(notExistHosts); + result.setValidHosts(existHosts.stream().map(BasicAppHost::toHostDTO).collect(Collectors.toList())); + result.setNotInAppHosts(Collections.emptyList()); + return result; + } + + // 普通业务集和普通业务需要判断主机是否归属于业务 + List includeBizIds = buildIncludeBizIdList(application); + if (CollectionUtils.isEmpty(includeBizIds)) { + log.warn("App do not contains any biz, appId:{}", appId); + result.setValidHosts(Collections.emptyList()); + result.setNotExistHosts(Collections.emptyList()); + result.setNotInAppHosts(hosts); + return result; + } + + existHosts.forEach(existHost -> { + if (includeBizIds.contains(existHost.getBizId())) { + validHosts.add(existHost.toHostDTO()); } else { - resultHosts.add(host); + notInAppHosts.add(existHost.toHostDTO()); } }); - // 如果从已同步的主机中没有查询到的主机或者主机信息不完整，那么从cmdb实时查询 - if (CollectionUtils.isNotEmpty(notSyncedHostIps)) { - IBizCmdbClient bizCmdbClient = CmdbClientFactory.getCmdbClient(); - List hostsFromCmdb = bizCmdbClient.listHostsByIps(notSyncedHostIps); - if (CollectionUtils.isNotEmpty(hostsFromCmdb)) { - resultHosts.addAll(hostsFromCmdb); + + if (CollectionUtils.isNotEmpty(notExistHosts) || CollectionUtils.isNotEmpty(notInAppHosts)) { + log.info("Contains invalid hosts, appId: {}, notExistHosts: {}, hostsInOtherApp: {}", + appId, notExistHosts, notInAppHosts); + } + + result.setNotExistHosts(notExistHosts); + result.setValidHosts(validHosts); + result.setNotInAppHosts(notInAppHosts); + + return result; + } + + private Pair, List> listHostsFromCacheOrCmdb(Collection hosts) { + List appHosts = new ArrayList<>(); + List notExistHosts = new ArrayList<>(); + List hostIds = new ArrayList<>(); + List cloudIps = new ArrayList<>(); + hosts.forEach(host -> { + if (host.getHostId() != null) { + hostIds.add(host.getHostId()); + } else { + cloudIps.add(host.toCloudIp()); + } + }); + if (CollectionUtils.isNotEmpty(hostIds)) { + Pair, List> result = listHostsByStrategy(hostIds, new ListHostByHostIdsStrategy()); + appHosts.addAll(result.getRight()); + if (CollectionUtils.isNotEmpty(result.getLeft())) { + result.getLeft().forEach(notExistHostId -> { + notExistHosts.add(HostDTO.fromHostId(notExistHostId)); + }); + } + } + if (CollectionUtils.isNotEmpty(cloudIps)) { + Pair, List> result = listHostsByStrategy(cloudIps, new ListHostByIpsStrategy()); + appHosts.addAll(result.getRight()); + if (CollectionUtils.isNotEmpty(result.getLeft())) { + result.getLeft().forEach(notExistCloudIp -> { + notExistHosts.add(HostDTO.fromCloudIp(notExistCloudIp)); + }); + } + } + return Pair.of(notExistHosts, appHosts); + } + + private Pair, List> listHostsByStrategy(List keys, + ListHostStrategy listHostStrategy) { + List appHosts = new ArrayList<>(); + List notExistKeys = null; + + if (CollectionUtils.isNotEmpty(keys)) { + Pair, List> result = listHostStrategy.listHostsFromCache(keys); + notExistKeys = result.getLeft(); + if (CollectionUtils.isNotEmpty(result.getRight())) { + appHosts.addAll(result.getRight()); } } - return resultHosts; + if (CollectionUtils.isNotEmpty(notExistKeys)) { + Pair, List> result = listHostStrategy.listHostsFromDb(notExistKeys); + notExistKeys = result.getLeft(); + if (CollectionUtils.isNotEmpty(result.getRight())) { + appHosts.addAll(result.getRight()); + } + } + + if (CollectionUtils.isNotEmpty(notExistKeys)) { + Pair, List> result = listHostStrategy.listHostsFromCmdb(notExistKeys); + notExistKeys = result.getLeft(); + if (CollectionUtils.isNotEmpty(result.getRight())) { + appHosts.addAll(result.getRight()); + } + } + + return Pair.of(notExistKeys, appHosts); } - @Data - private static class BasicAppHost { - private Long bizId; - private Long cloudAreaId; - private String ip; + /** + * 主机查询策略 + * + * @param 查询使用的主机KEY + */ + private interface ListHostStrategy { + Pair, List> listHostsFromCache(List keys); - private BasicAppHost(Long bizId, Long cloudAreaId, String ip) { - this.bizId = bizId; - this.cloudAreaId = cloudAreaId; - this.ip = ip; + Pair, List> listHostsFromDb(List keys); + + Pair, List> listHostsFromCmdb(List keys); + } + + /** + * 根据ip查询主机 + */ + private class ListHostByIpsStrategy implements ListHostStrategy { + @Override + public Pair, List> listHostsFromCache(List cloudIps) { + List appHosts = new ArrayList<>(); + List notExistCloudIps = new ArrayList<>(); + List cacheHosts = hostCache.batchGetHostsByIps(cloudIps); + for (int i = 0; i < cloudIps.size(); i++) { + String cloudIp = cloudIps.get(i); + CacheHostDO cacheHost = cacheHosts.get(i); + if (cacheHost != null) { + BasicAppHost appHost = new BasicAppHost(cacheHost.getBizId(), cacheHost.getHostId(), + cacheHost.getCloudAreaId(), cacheHost.getIp(), cacheHost.getAgentId()); + appHosts.add(appHost); + } else { + notExistCloudIps.add(cloudIp); + } + } + return Pair.of(notExistCloudIps, appHosts); + } + + @Override + public Pair, List> listHostsFromDb(List cloudIps) { + List appHosts = new ArrayList<>(); + List notExistCloudIps = new ArrayList<>(cloudIps); + List hostsInDb = applicationHostDAO.listHostsByIps(cloudIps); + if (CollectionUtils.isNotEmpty(hostsInDb)) { + for (ApplicationHostDTO appHost : hostsInDb) { + if (appHost.getBizId() == null || appHost.getBizId() <= 0) { + log.info("Host: {}|{} missing bizId, skip!", appHost.getHostId(), appHost.getCloudIp()); + // DB中缓存的主机可能没有业务信息(依赖的主机事件还没有处理),那么暂时跳过该主机 + continue; + } + hostCache.addOrUpdateHost(appHost); + notExistCloudIps.remove(appHost.getCloudIp()); + BasicAppHost host = new BasicAppHost(appHost.getBizId(), appHost.getHostId(), + appHost.getCloudAreaId(), appHost.getIp(), appHost.getAgentId()); + appHosts.add(host); + } + } + return Pair.of(notExistCloudIps, appHosts); + } + + @Override + public Pair, List> listHostsFromCmdb(List cloudIps) { + List appHosts = new ArrayList<>(); + List notExistCloudIps = new ArrayList<>(cloudIps); + + IBizCmdbClient bizCmdbClient = CmdbClientFactory.getCmdbClient(); + List cmdbExistHosts = bizCmdbClient.listHostsByIps(cloudIps); + if (CollectionUtils.isNotEmpty(cmdbExistHosts)) { + List cmdbExistHostIds = cmdbExistHosts.stream() + .map(ApplicationHostDTO::getCloudIp) + .collect(Collectors.toList()); + notExistCloudIps.removeAll(cmdbExistHostIds); + log.info("sync new hosts from cmdb, hosts:{}", cmdbExistHosts); + + hostCache.addOrUpdateHosts(cmdbExistHosts); + + cmdbExistHosts.forEach(syncHost -> { + BasicAppHost host = new BasicAppHost(syncHost.getBizId(), syncHost.getHostId(), + syncHost.getCloudAreaId(), syncHost.getIp(), syncHost.getAgentId()); + appHosts.add(host); + }); + } + return Pair.of(notExistCloudIps, appHosts); + } + } + + private class ListHostByHostIdsStrategy implements ListHostStrategy { + @Override + public Pair, List> listHostsFromCache(List hostIds) { + List appHosts = new ArrayList<>(); + List notExistHostIds = new ArrayList<>(); + List cacheHosts = hostCache.batchGetHostsByHostIds(hostIds); + for (int i = 0; i < hostIds.size(); i++) { + long hostId = hostIds.get(i); + CacheHostDO cacheHost = cacheHosts.get(i); + if (cacheHost != null) { + BasicAppHost appHost = new BasicAppHost(cacheHost.getBizId(), cacheHost.getHostId(), + cacheHost.getCloudAreaId(), cacheHost.getIp(), cacheHost.getAgentId()); + appHosts.add(appHost); + } else { + notExistHostIds.add(hostId); + } + } + return Pair.of(notExistHostIds, appHosts); + } + + @Override + public Pair, List> listHostsFromDb(List hostIds) { + List appHosts = new ArrayList<>(); + List notExistHostIds = new ArrayList<>(hostIds); + List hostsInDb = applicationHostDAO.listHostInfoByHostIds(hostIds); + if (CollectionUtils.isNotEmpty(hostsInDb)) { + for (ApplicationHostDTO appHost : hostsInDb) { + if (appHost.getBizId() == null || appHost.getBizId() <= 0) { + log.info("Host: {}|{} missing bizId, skip!", appHost.getHostId(), appHost.getCloudIp()); + // DB中缓存的主机可能没有业务信息(依赖的主机事件还没有处理),那么暂时跳过该主机 + continue; + } + hostCache.addOrUpdateHost(appHost); + notExistHostIds.remove(appHost.getHostId()); + BasicAppHost host = new BasicAppHost(appHost.getBizId(), appHost.getHostId(), + appHost.getCloudAreaId(), appHost.getIp(), appHost.getAgentId()); + appHosts.add(host); + } + } + return Pair.of(notExistHostIds, appHosts); + } + + @Override + public Pair, List> listHostsFromCmdb(List hostIds) { + List appHosts = new ArrayList<>(); + List notExistHostIds = new ArrayList<>(hostIds); + + IBizCmdbClient bizCmdbClient = CmdbClientFactory.getCmdbClient(); + List cmdbExistHosts = bizCmdbClient.listHostsByHostIds(hostIds); + if (CollectionUtils.isNotEmpty(cmdbExistHosts)) { + List cmdbExistHostIds = cmdbExistHosts.stream() + .map(ApplicationHostDTO::getHostId) + .collect(Collectors.toList()); + notExistHostIds.removeAll(cmdbExistHostIds); + log.info("sync new hosts from cmdb, hosts:{}", cmdbExistHosts); + + hostCache.addOrUpdateHosts(cmdbExistHosts); + + cmdbExistHosts.forEach(syncHost -> { + BasicAppHost host = new BasicAppHost(syncHost.getBizId(), syncHost.getHostId(), + syncHost.getCloudAreaId(), syncHost.getIp(), syncHost.getAgentId()); + appHosts.add(host); + }); + } + return Pair.of(notExistHostIds, appHosts); } } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/PaaSServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/PaaSServiceImpl.java index 0c6d54a18b..f36fabc004 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/PaaSServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/PaaSServiceImpl.java @@ -86,14 +86,14 @@ public List getAllChannelList(String bkToken, String uin) { @Override @EsbApiTimed - public boolean sendMsg( + public void sendMsg( String msgType, String sender, Set receivers, String title, String content - ) throws Exception { + ) { IPaasClient paasClient = getClientWithCurrentLang(); - return paasClient.sendMsg(msgType, sender, receivers, title, content); + paasClient.sendMsg(msgType, sender, receivers, title, content); } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/WatchableSendMsgService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/WatchableSendMsgService.java index 08b7c1a176..0bc97942a2 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/WatchableSendMsgService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/WatchableSendMsgService.java @@ -54,7 +54,7 @@ public WatchableSendMsgService(PaaSService paaSService, } @EsbApiTimed - public boolean sendMsg( + public void sendMsg( Long appId, long createTimeMillis, String msgType, @@ -62,14 +62,11 @@ public boolean sendMsg( Set receivers, String title, String content - ) throws Exception { + ) { String sendStatus = MetricsConstants.TAG_VALUE_SEND_STATUS_FAILED; try { - boolean result = paaSService.sendMsg(msgType, sender, receivers, title, content); - if (result) { - sendStatus = MetricsConstants.TAG_VALUE_SEND_STATUS_SUCCESS; - } - return result; + paaSService.sendMsg(msgType, sender, receivers, title, content); + sendStatus = MetricsConstants.TAG_VALUE_SEND_STATUS_SUCCESS; } finally { long delayMillis = System.currentTimeMillis() - createTimeMillis; Tags tags = Tags.of(MetricsConstants.TAG_KEY_MSG_TYPE, msgType); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/NotifySendService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/NotifySendService.java index 2ef3bd28d7..93819840db 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/NotifySendService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/NotifySendService.java @@ -24,7 +24,6 @@ package com.tencent.bk.job.manage.service.impl.notify; -import brave.Tracing; import com.tencent.bk.job.common.trace.executors.TraceableExecutorService; import com.tencent.bk.job.manage.dao.notify.EsbUserInfoDAO; import com.tencent.bk.job.manage.metrics.MetricsConstants; @@ -34,6 +33,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cloud.sleuth.Tracer; import org.springframework.stereotype.Service; import java.util.Collections; @@ -57,7 +57,7 @@ public class NotifySendService { @Autowired public NotifySendService(WatchableSendMsgService watchableSendMsgService, EsbUserInfoDAO esbUserInfoDAO, - Tracing tracing, + Tracer tracer, MeterRegistry meterRegistry) { this.watchableSendMsgService = watchableSendMsgService; this.esbUserInfoDAO = esbUserInfoDAO; @@ -69,7 +69,7 @@ public NotifySendService(WatchableSendMsgService watchableSendMsgService, TimeUnit.SECONDS, new LinkedBlockingQueue<>(10) ), - tracing + tracer ); measureNotifySendExecutor(meterRegistry); } @@ -112,7 +112,8 @@ private SendNotifyTask buildSendTask(Long appId, return task; } - public void sendUserChannelNotify(Long appId, Set receivers, String channel, String title, String content) { + public void asyncSendUserChannelNotify(Long appId, Set receivers, String channel, String title, + String content) { if (CollectionUtils.isEmpty(receivers)) { log.warn("receivers is empty of channel {}, do not send notification", channel); return; @@ -127,7 +128,30 @@ public void sendUserChannelNotify(Long appId, Set receivers, String chan notifySendExecutor.submit(buildSendTask(appId, receivers, channel, title, content)); } - public void sendNotifyMessages(Long appId, Map> channelUsersMap, String title, String content) { - channelUsersMap.forEach((channel, userSet) -> sendUserChannelNotify(appId, userSet, channel, title, content)); + public void sendUserChannelNotify(Long appId, + Set receivers, + String channel, + String title, + String content) { + if (CollectionUtils.isEmpty(receivers)) { + log.warn("receivers is empty of channel {}, do not send notification", channel); + return; + } + watchableSendMsgService.sendMsg( + appId, + System.currentTimeMillis(), + channel, + null, + receivers, + title, + content + ); + } + + public void asyncSendNotifyMessages(Long appId, Map> channelUsersMap, String title, + String content) { + channelUsersMap.forEach((channel, userSet) -> + asyncSendUserChannelNotify(appId, userSet, channel, title, content) + ); } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/NotifyServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/NotifyServiceImpl.java index 6cc3e60a1b..4e69223252 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/NotifyServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/NotifyServiceImpl.java @@ -418,7 +418,7 @@ public Integer sendSimpleNotification(ServiceNotificationDTO notification) { } ServiceNotificationMessage notificationMessage = notifyMessageMap.get(new ArrayList<>(notifyMessageMap.keySet()).get(0)); - notifySendService.sendNotifyMessages( + notifySendService.asyncSendNotifyMessages( notification.getTriggerDTO().getAppId(), channelUsersMap, notificationMessage.getTitle(), @@ -552,7 +552,7 @@ private List getTriggerPolicys(Long appId, // 3.默认业务 // 业务未配置消息通知策略才使用默认策略 if (!appNotifyPolicyConfigured(appId) && CollectionUtils.isEmpty(triggerPolicyList)) { - triggerPolicyList = systemDefaultNotifyPolices(resourceType, triggerType, resourceExecuteStatus); + triggerPolicyList = systemDefaultNotifyPolices(triggerType, resourceType, resourceExecuteStatus); } return triggerPolicyList; } @@ -635,22 +635,22 @@ public PageTemplateVO getPageTemplate(String username) { } @Override - public Integer sendNotificationsToUsers(ServiceUserNotificationDTO serviceUserNotificationDTO) { + public Integer asyncSendNotificationsToUsers(ServiceUserNotificationDTO serviceUserNotificationDTO) { // 获取所有可用渠道 List availableChannelTypeList = getAvailableChannelTypeList(); - return sendNotificationsToUsersByChannel(serviceUserNotificationDTO, availableChannelTypeList); + return asyncSendNotificationsByChannel(serviceUserNotificationDTO, availableChannelTypeList); } @Override - public Integer sendNotificationsToUsersByChannel(ServiceUserNotificationDTO serviceUserNotificationDTO, - List channelTypeList) { + public Integer asyncSendNotificationsByChannel(ServiceUserNotificationDTO serviceUserNotificationDTO, + List channelTypeList) { // 组装通知map Map> channelUsersMap = new HashMap<>(); for (String channelType : channelTypeList) { channelUsersMap.put(channelType, serviceUserNotificationDTO.getReceivers()); } ServiceNotificationMessage notificationMessage = serviceUserNotificationDTO.getNotificationMessage(); - notifySendService.sendNotifyMessages( + notifySendService.asyncSendNotifyMessages( null, channelUsersMap, notificationMessage.getTitle(), @@ -660,9 +660,9 @@ public Integer sendNotificationsToUsersByChannel(ServiceUserNotificationDTO serv } @Override - public Integer sendNotificationsToAdministrators(ServiceNotificationMessage serviceNotificationMessage) { + public Integer asyncSendNotificationsToAdministrators(ServiceNotificationMessage serviceNotificationMessage) { List administrators = localPermissionService.getAdministrators(); - return sendNotificationsToUsers(new ServiceUserNotificationDTO(new HashSet<>(administrators), + return asyncSendNotificationsToUsers(new ServiceUserNotificationDTO(new HashSet<>(administrators), serviceNotificationMessage)); } @@ -714,7 +714,7 @@ public Integer sendTemplateNotification(ServiceTemplateNotificationDTO templateN ); //发送消息通知 if (notifyMsg != null) { - notifySendService.sendUserChannelNotify( + notifySendService.asyncSendUserChannelNotify( appId, userSet, channel, @@ -771,7 +771,7 @@ public Integer triggerTemplateNotification(ServiceTriggerTemplateNotificationDTO log.warn(PrefConsts.TAG_PREF_SLOW + watch.prettyPrint()); } if (notificationMessage != null) { - notifySendService.sendUserChannelNotify( + notifySendService.asyncSendUserChannelNotify( appId, userSet, channel, diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/SendNotifyTask.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/SendNotifyTask.java index 1d9d3fd549..1409cb46da 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/SendNotifyTask.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/SendNotifyTask.java @@ -67,8 +67,8 @@ public void run() { return; } try { - boolean sendResult = sendMsgWithRetry(); - if (sendResult) { + boolean result = sendMsgWithRetry(); + if (result) { logSendSuccess(); } else { handleSendFail(null); @@ -82,22 +82,27 @@ private void logInvalidReceivers() { log.warn("receivers is null or empty, skip, msgType={},title={}", msgType, title); } - private boolean sendMsgWithRetry() throws Exception { + private boolean sendMsgWithRetry() { int count = 0; - boolean sendResult = false; - while (!sendResult && count < NOTIFY_MAX_RETRY_COUNT) { + boolean result = false; + while (!result && count < NOTIFY_MAX_RETRY_COUNT) { count += 1; - sendResult = watchableSendMsgService.sendMsg( - appId, - createTimeMillis, - msgType, - sender, - receivers, - title, - content - ); + try { + watchableSendMsgService.sendMsg( + appId, + createTimeMillis, + msgType, + sender, + receivers, + title, + content + ); + result = true; + } catch (Exception e) { + log.error("Fail to sendMsg", e); + } } - return sendResult; + return result; } private void logSendSuccess() { @@ -111,9 +116,9 @@ private void handleSendFail(Exception e) { logIgnoreToSend(); return; } - FormattingTuple msg = MessageFormatter.format( + FormattingTuple msg = MessageFormatter.arrayFormat( "Fail to send notify:({},{},{},{})", - new String[]{ + new Object[]{ String.join(",", receivers), msgType, title, diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/AbstractCmdbResourceEventWatcher.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/AbstractCmdbResourceEventWatcher.java index 714183ab1a..4880cbbab6 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/AbstractCmdbResourceEventWatcher.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/AbstractCmdbResourceEventWatcher.java @@ -1,7 +1,5 @@ package com.tencent.bk.job.manage.service.impl.sync; -import brave.Span; -import brave.Tracing; import com.tencent.bk.job.common.cc.model.result.ResourceEvent; import com.tencent.bk.job.common.cc.model.result.ResourceWatchResult; import com.tencent.bk.job.common.redis.util.LockUtils; @@ -12,6 +10,8 @@ import com.tencent.bk.job.common.util.json.JsonUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; +import org.springframework.cloud.sleuth.Span; +import org.springframework.cloud.sleuth.Tracer; import org.springframework.data.redis.core.RedisTemplate; import java.util.List; @@ -28,7 +28,7 @@ public abstract class AbstractCmdbResourceEventWatcher extends Thread { */ private final String machineIp; private final RedisTemplate redisTemplate; - private final Tracing tracing; + private final Tracer tracer; /** * 事件监听任务分布式锁KEY */ @@ -41,10 +41,10 @@ public abstract class AbstractCmdbResourceEventWatcher extends Thread { public AbstractCmdbResourceEventWatcher(String watcherResourceName, RedisTemplate redisTemplate, - Tracing tracing) { + Tracer tracer) { this.machineIp = IpUtils.getFirstMachineIP(); this.redisTemplate = redisTemplate; - this.tracing = tracing; + this.tracer = tracer; this.watcherResourceName = watcherResourceName; this.setName(watcherResourceName); this.redisLockKey = "watch-cmdb-" + this.watcherResourceName + "-lock"; @@ -127,7 +127,7 @@ private void watchEvent() { Span span = null; try { ResourceWatchResult watchResult; - span = this.tracing.tracer().newTrace(); + span = this.tracer.nextSpan(); if (cursor == null) { // 从10分钟前开始watch long startTime = System.currentTimeMillis() / 1000 - 10 * 60; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSetEventWatcher.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSetEventWatcher.java index 2c137ee60a..7fb06b3827 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSetEventWatcher.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSetEventWatcher.java @@ -1,6 +1,5 @@ package com.tencent.bk.job.manage.service.impl.sync; -import brave.Tracing; import com.tencent.bk.job.common.cc.model.req.ResourceWatchReq; import com.tencent.bk.job.common.cc.model.result.BizSetEventDetail; import com.tencent.bk.job.common.cc.model.result.ResourceEvent; @@ -12,6 +11,7 @@ import lombok.extern.slf4j.Slf4j; import org.jooq.exception.DataAccessException; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cloud.sleuth.Tracer; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Component; @@ -27,11 +27,11 @@ public class BizSetEventWatcher extends AbstractCmdbResourceEventWatcher redisTemplate, - Tracing tracing, + Tracer tracer, ApplicationService applicationService, BizSetService bizSetService, BizSetCmdbClient bizSetCmdbClient) { - super("bizSet", redisTemplate, tracing); + super("bizSet", redisTemplate, tracer); this.applicationService = applicationService; this.bizSetService = bizSetService; this.bizSetCmdbClient = bizSetCmdbClient; @@ -98,7 +98,6 @@ private void updateBizSetProps(ApplicationDTO originApp, ApplicationDTO updateAp originApp.setName(updateApp.getName()); originApp.setBkSupplierAccount(updateApp.getBkSupplierAccount()); originApp.setLanguage(updateApp.getLanguage()); - originApp.setMaintainers(updateApp.getMaintainers()); originApp.setTimeZone(updateApp.getTimeZone()); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSetRelationEventWatcher.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSetRelationEventWatcher.java index d1a42df2e7..b8e4b963b0 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSetRelationEventWatcher.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSetRelationEventWatcher.java @@ -1,6 +1,5 @@ package com.tencent.bk.job.manage.service.impl.sync; -import brave.Tracing; import com.tencent.bk.job.common.cc.model.req.ResourceWatchReq; import com.tencent.bk.job.common.cc.model.result.BizSetRelationEventDetail; import com.tencent.bk.job.common.cc.model.result.ResourceEvent; @@ -14,6 +13,7 @@ import com.tencent.bk.job.manage.service.impl.BizSetService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cloud.sleuth.Tracer; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Component; @@ -31,11 +31,11 @@ public class BizSetRelationEventWatcher extends AbstractCmdbResourceEventWatcher @Autowired public BizSetRelationEventWatcher(RedisTemplate redisTemplate, - Tracing tracing, + Tracer tracer, ApplicationService applicationService, BizSetService bizSetService, BizSetCmdbClient bizSetCmdbClient) { - super("bizSetRelation", redisTemplate, tracing); + super("bizSetRelation", redisTemplate, tracer); this.applicationService = applicationService; this.bizSetService = bizSetService; this.bizSetCmdbClient = bizSetCmdbClient; @@ -68,7 +68,6 @@ protected void handleEvent(ResourceEvent event) { if (cacheApplication == null || cacheApplication.isDeleted()) { return; } - cacheApplication.setSubBizIds(latestSubBizIds); ApplicationAttrsDO attrs = cacheApplication.getAttrs(); if (attrs != null) { attrs.setSubBizIds(latestSubBizIds); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSetSyncService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSetSyncService.java index 4020e5fa3f..9017f5591f 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSetSyncService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSetSyncService.java @@ -28,7 +28,6 @@ import com.tencent.bk.job.common.cc.model.bizset.BizSetInfo; import com.tencent.bk.job.common.cc.model.bizset.BizSetScope; import com.tencent.bk.job.common.cc.sdk.IBizSetCmdbClient; -import com.tencent.bk.job.common.constant.AppTypeEnum; import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; import com.tencent.bk.job.common.model.dto.ApplicationAttrsDO; import com.tencent.bk.job.common.model.dto.ApplicationDTO; @@ -81,7 +80,7 @@ public void syncBizSetFromCMDB() { return; } log.info("[{}] Begin to sync bizSet from cmdb", Thread.currentThread().getName()); - List ccBizSets = bizSetCmdbClient.getAllBizSetApps(); + List ccBizSets = bizSetCmdbClient.listAllBizSets(); if (log.isInfoEnabled()) { log.info("Sync cmdb bizSet result: {}", JsonUtils.toJson(ccBizSets)); } @@ -180,18 +179,14 @@ private ApplicationDTO convertBizSetToApplication(BizSetInfo bizSetInfo) { ApplicationDTO appInfoDTO = new ApplicationDTO(); appInfoDTO.setBkSupplierAccount(bizSetInfo.getSupplierAccount()); appInfoDTO.setName(bizSetInfo.getName()); - appInfoDTO.setMaintainers(bizSetInfo.getMaintainer()); - appInfoDTO.setOperateDeptId(bizSetInfo.getOperateDeptId()); appInfoDTO.setTimeZone(bizSetInfo.getTimezone()); BizSetScope scope = bizSetInfo.getScope(); ApplicationAttrsDO attrs = new ApplicationAttrsDO(); if (scope != null && scope.isMatchAll()) { // 全业务 - appInfoDTO.setAppType(AppTypeEnum.ALL_APP); attrs.setMatchAllBiz(true); } else { // 普通业务集 - appInfoDTO.setAppType(AppTypeEnum.APP_SET); attrs.setMatchAllBiz(false); if (CollectionUtils.isNotEmpty(bizSetInfo.getBizList())) { attrs.setSubBizIds(bizSetInfo.getBizList().stream().map(BizInfo::getId).collect(Collectors.toList())); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizWatchThread.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizWatchThread.java index 1434482b1a..0aec6549b2 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizWatchThread.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizWatchThread.java @@ -237,7 +237,6 @@ private void updateBizProps(ApplicationDTO originApp, ApplicationDTO updateApp) originApp.setName(updateApp.getName()); originApp.setBkSupplierAccount(updateApp.getBkSupplierAccount()); originApp.setLanguage(updateApp.getLanguage()); - originApp.setMaintainers(updateApp.getMaintainers()); originApp.setTimeZone(updateApp.getTimeZone()); } } diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/EsbCmdbClient.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/EsbCmdbClient.java deleted file mode 100644 index 2abfed73c3..0000000000 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/EsbCmdbClient.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.upgrader.client; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.tencent.bk.job.common.cc.model.bizset.BatchUpdateBizSetReq; -import com.tencent.bk.job.common.cc.model.bizset.BizSetFilter; -import com.tencent.bk.job.common.cc.model.bizset.BizSetInfo; -import com.tencent.bk.job.common.cc.model.bizset.CreateBizSetReq; -import com.tencent.bk.job.common.cc.model.bizset.Page; -import com.tencent.bk.job.common.cc.model.bizset.Rule; -import com.tencent.bk.job.common.cc.model.bizset.SearchBizSetReq; -import com.tencent.bk.job.common.cc.model.bizset.SearchBizSetResp; -import com.tencent.bk.job.common.esb.model.EsbReq; -import com.tencent.bk.job.common.esb.model.EsbResp; -import com.tencent.bk.job.common.esb.sdk.AbstractEsbSdkClient; -import lombok.extern.slf4j.Slf4j; -import org.apache.http.client.methods.HttpPost; - -import java.util.ArrayList; -import java.util.List; - -@Slf4j -public class EsbCmdbClient extends AbstractEsbSdkClient { - - private final String cmdbSupplierAccount; - - private static final String FIELD_KEY_BIZ_SET_ID = "bk_biz_set_id"; - - private static final String SEARCH_BUSINESS_SET = "/api/c/compapi/v2/cc/list_business_set/"; - private static final String CREATE_BUSINESS_SET = "/api/c/compapi/v2/cc/create_business_set/"; - private static final String BATCH_UPDATE_BUSINESS_SET = "/api/c/compapi/v2/cc/batch_update_business_set/"; - - public EsbCmdbClient(String esbHostUrl, - String appCode, - String appSecret, - String lang, - String cmdbSupplierAccount) { - super(esbHostUrl, appCode, appSecret, lang, false); - this.cmdbSupplierAccount = cmdbSupplierAccount; - } - - public T makeCmdbBaseReq(Class reqClass) { - return makeBaseReqByWeb(reqClass, null, "admin", cmdbSupplierAccount); - } - - /** - * 根据Id查询业务集信息 - * - * @param id 业务集id - * @return 业务集信息 - */ - public List searchBizSetById(Long id) { - SearchBizSetReq req = makeCmdbBaseReq(SearchBizSetReq.class); - req.setPage(new Page()); - BizSetFilter filter = new BizSetFilter(); - filter.setCondition(BizSetFilter.CONDITION_AND); - List rules = new ArrayList<>(); - rules.add(new Rule(FIELD_KEY_BIZ_SET_ID, Rule.OPERATOR_EQUAL, id)); - filter.setRules(rules); - req.setFilter(filter); - EsbResp resp = getEsbRespByReq( - HttpPost.METHOD_NAME, - SEARCH_BUSINESS_SET, - req, - new TypeReference>() { - }); - if (!resp.getResult() || resp.getData() == null) { - return null; - } - return resp.getData().getBizSetList(); - } - - /** - * 创建业务集 - * - * @param createBizSetReq 业务集信息 - * @return 业务集id - */ - public Long createBizSet(CreateBizSetReq createBizSetReq) { - EsbResp resp = getEsbRespByReq( - HttpPost.METHOD_NAME, - CREATE_BUSINESS_SET, - createBizSetReq, - new TypeReference>() { - }); - return resp.getData(); - } - - /** - * 批量更新业务集 - * - * @param batchUpdateBizSetReq 业务集更新请求信息 - * @return 业务集id - */ - public Boolean batchUpdateBizSet(BatchUpdateBizSetReq batchUpdateBizSetReq) { - EsbResp resp = getEsbRespByReq( - HttpPost.METHOD_NAME, - BATCH_UPDATE_BUSINESS_SET, - batchUpdateBizSetReq, - new TypeReference>() { - }); - return resp.getResult(); - } - -} diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/AppInfo.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/AppInfo.java index b59460d2cb..90d9a98f19 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/AppInfo.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/AppInfo.java @@ -24,17 +24,11 @@ package com.tencent.bk.job.upgrader.model; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.tencent.bk.job.common.annotation.CompatibleImplementation; -import com.tencent.bk.job.common.constant.AppTypeEnum; -import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; import com.tencent.bk.job.common.model.dto.ApplicationAttrsDO; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import java.util.List; - @Data @NoArgsConstructor @AllArgsConstructor @@ -57,28 +51,13 @@ public class AppInfo { */ private String name; - /** - * 业务类型 - */ - private Integer appType; - /** * 运维 */ private String maintainers; - /** - * 子业务 - */ - private List subAppIds; - private String owner; - /** - * 初始运维部门Id - */ - private Long operateDeptId; - /** * 时区 */ @@ -94,18 +73,4 @@ public class AppInfo { */ private ApplicationAttrsDO attrs; - /** - * 是否是全业务 - */ - @CompatibleImplementation(explain = "兼容方法，等业务集全部迁移到cmdb之后需要移除对appType的判断", version = "3.6.x") - @JsonIgnore - public boolean isAllBizSet() { - return appType == AppTypeEnum.ALL_APP.getValue() - || (ResourceScopeTypeEnum.BIZ_SET.getValue().equals(scopeType) - && attrs != null && attrs.getMatchAllBiz() != null && attrs.getMatchAllBiz()); - } - - public String getIdAndName() { - return "(" + getId() + "," + getName() + ")"; - } } diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/BizSetAuthMigrationTask.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/BizSetAuthMigrationTask.java deleted file mode 100644 index 1d439be52c..0000000000 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/BizSetAuthMigrationTask.java +++ /dev/null @@ -1,349 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.upgrader.task; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.google.common.base.Charsets; -import com.tencent.bk.job.common.iam.client.EsbIamClient; -import com.tencent.bk.job.common.iam.dto.BatchAuthByPathReq; -import com.tencent.bk.job.common.iam.dto.EsbIamAction; -import com.tencent.bk.job.common.iam.dto.EsbIamBatchAuthedPolicy; -import com.tencent.bk.job.common.iam.dto.EsbIamBatchPathResource; -import com.tencent.bk.job.common.iam.dto.EsbIamPathItem; -import com.tencent.bk.job.common.iam.dto.EsbIamSubject; -import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.upgrader.anotation.ExecuteTimeEnum; -import com.tencent.bk.job.upgrader.anotation.RequireTaskParam; -import com.tencent.bk.job.upgrader.anotation.UpgradeTask; -import com.tencent.bk.job.upgrader.anotation.UpgradeTaskInputParam; -import com.tencent.bk.job.upgrader.client.JobClient; -import com.tencent.bk.job.upgrader.model.AppInfo; -import com.tencent.bk.job.upgrader.task.param.JobManageServerAddress; -import com.tencent.bk.job.upgrader.task.param.ParamNameConsts; -import io.micrometer.core.instrument.util.IOUtils; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import java.io.InputStream; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.stream.Collectors; - -/** - * 账号使用权限迁移任务 - */ -@Slf4j -@RequireTaskParam(value = { - @UpgradeTaskInputParam(value = JobManageServerAddress.class) -}) -@UpgradeTask( - dataStartVersion = "3.0.0.0", - targetVersion = "3.5.0", - targetExecuteTime = ExecuteTimeEnum.MAKE_UP) -public class BizSetAuthMigrationTask extends BaseUpgradeTask { - - private static final String PLACEHOLDER_BIZ_SET_ID = "{{biz_set_id}}"; - private static final String PLACEHOLDER_BIZ_SET_NAME = "{{biz_set_name}}"; - - private JobClient jobManageClient; - private EsbIamClient esbIamClient; - private List bizSetAppInfoList; - private Map bizSetAppInfoMap; - - private String getJobHostUrlByAddress(String address) { - if (!address.startsWith("http://") && !address.startsWith("https://")) { - address = "http://" + address; - } - return address; - } - - public BizSetAuthMigrationTask(Properties properties) { - super(properties); - } - - @Override - public void init() { - String jobAuthToken = getJobAuthToken(); - jobManageClient = new JobClient( - getJobHostUrlByAddress((String) getProperties().get(ParamNameConsts.INPUT_PARAM_JOB_MANAGE_SERVER_ADDRESS)), - jobAuthToken - ); - this.bizSetAppInfoList = getAllBizSetAppInfoFromManage(); - bizSetAppInfoMap = new HashMap<>(); - bizSetAppInfoList.forEach(appInfo -> { - bizSetAppInfoMap.put(appInfo.getId(), appInfo.getName()); - }); - } - - private EsbIamClient getEsbIamClient() { - Properties properties = getProperties(); - if (esbIamClient == null) { - esbIamClient = new EsbIamClient( - (String) properties.get(ParamNameConsts.CONFIG_PROPERTY_ESB_SERVICE_URL), - (String) properties.get(ParamNameConsts.CONFIG_PROPERTY_APP_CODE), - (String) properties.get(ParamNameConsts.CONFIG_PROPERTY_APP_SECRET), - false - ); - } - return esbIamClient; - } - - private List getAllBizSetAppInfoFromManage() { - try { - return jobManageClient.listBizSetApps(); - } catch (Exception e) { - log.error("Fail to get normal apps from job-manage, please confirm job-manage version>=3.3.5.0"); - throw e; - } - } - - /** - * 调用权限中心接口进行批量授权 - * - * @param esbIamSubject 授权对象 - * @param actions 授权操作列表 - * @param resourceList 批量依赖资源路径 - * @return 是否授权成功 - */ - private boolean batchAuth(EsbIamSubject esbIamSubject, - List actions, - List resourceList) { - try { - List batchAuthedPolicy = getEsbIamClient().batchAuthByPath(actions, - esbIamSubject, resourceList, null); - log.info("batchAuthedPolicy={}", JsonUtils.toJson(batchAuthedPolicy)); - return true; - } catch (Exception e) { - log.error("Fail to auth subject {} to {}", JsonUtils.toJson(esbIamSubject), JsonUtils.toJson(actions), e); - return false; - } - } - - public void printSeparateLine() { - log.info("=================================================="); - } - - private void resetResourceListTpl(List reqList) { - fillResourceListTpl(reqList, PLACEHOLDER_BIZ_SET_ID, PLACEHOLDER_BIZ_SET_NAME); - } - - private void fillResourceListTpl(List reqlist, - String bizSetId, - String bizSetName) { - for (BatchAuthByPathReq batchAuthByPathReq : reqlist) { - List resourceList = batchAuthByPathReq.getResources(); - for (EsbIamBatchPathResource esbIamBatchPathResource : resourceList) { - List> paths = esbIamBatchPathResource.getPaths(); - if (CollectionUtils.isEmpty(paths)) { - continue; - } - for (List path : paths) { - if (CollectionUtils.isEmpty(path)) { - continue; - } - for (EsbIamPathItem esbIamPathItem : path) { - esbIamPathItem.setId(bizSetId); - esbIamPathItem.setName(bizSetName); - } - } - } - } - } - - private void recordAuthActionResult(boolean result, - List successActionList, - List failedActionList, - String maintainer, - String bizSetAppStr, - BatchAuthByPathReq req) { - if (result) { - log.info( - "[Action] Success to auth maintainer:{}, app:{}, actions:{}", - maintainer, - bizSetAppStr, - JsonUtils.toJson(req.getActions()) - ); - successActionList.addAll( - req.getActions().parallelStream() - .map(EsbIamAction::getId) - .collect(Collectors.toList()) - ); - } else { - log.warn( - "[Action] Fail to auth maintainer:{}, app:{}, actions:{}, resources:{}", - maintainer, - bizSetAppStr, - JsonUtils.toJson(req.getActions()), - JsonUtils.toJson(req.getResources()) - ); - failedActionList.addAll( - req.getActions().parallelStream() - .map(EsbIamAction::getId) - .collect(Collectors.toList()) - ); - } - } - - private void recordAuthMaintainerResult(List failedActionList, - List successMaintainerList, - List failedMaintainerList, - String maintainer, - int successActionCount, - int actionCount, - String bizSetAppStr) { - // 所有Action全部授权成功 - if (failedActionList.isEmpty()) { - successMaintainerList.add(maintainer); - log.info( - "[Maintainer] Success to auth maintainer:{} to {} actions of app {}", - maintainer, - successActionCount, - bizSetAppStr - ); - } else { - failedMaintainerList.add(maintainer); - log.warn( - "[Maintainer] Fail to auth maintainer:{} to {}/{} actions of app {}, failed actions:{}", - maintainer, - failedActionList, - actionCount, - bizSetAppStr, - JsonUtils.toJson(failedActionList) - ); - } - } - - private void recordAuthBizSetResult(List failedMaintainerList, - List successBizSetList, - List failedBizSetList, - int successMaintainerCount, - int maintainerCount, - String bizSetAppStr) { - if (failedMaintainerList.isEmpty()) { - successBizSetList.add(bizSetAppStr); - log.info( - "[BizSet] Success to auth {} maintainers of app {}", - successMaintainerCount, - bizSetAppStr - ); - } else { - failedBizSetList.add(bizSetAppStr); - log.warn( - "[BizSet] Fail to auth {}/{} maintainers of app {}, failed maintainers:{}", - failedMaintainerList.size(), - maintainerCount, - bizSetAppStr, - JsonUtils.toJson(failedMaintainerList) - ); - } - } - - private void showGlobalAuthResult(List failedBizSetList, - List successBizSetList) { - if (failedBizSetList.isEmpty()) { - log.info( - "[Global] Success to auth {} bizSetApps", - successBizSetList.size() - ); - } else { - log.warn( - "[Global] Fail to auth {}/{} bizSetApps, failed bizSetApps:{}", - failedBizSetList.size(), - bizSetAppInfoList.size(), - JsonUtils.toJson(failedBizSetList) - ); - } - } - - @Override - public int execute(String[] args) { - log.info(getName() + " for version " + getTargetVersion() + " begin to run..."); - // 1.权限模板数据读取 - String bizSetTplFileName = "biz_set_auth_template.json"; - InputStream ins = getClass().getClassLoader().getResourceAsStream(bizSetTplFileName); - String authTemplateStr = IOUtils.toString(ins, Charsets.UTF_8); - List reqList = JsonUtils.fromJson( - authTemplateStr, new TypeReference>() { - }); - log.info("{} auth request template(s) loaded", reqList.size()); - printSeparateLine(); - log.info("Begin to auth according to job app info:"); - // 2.业务集运维人员授权 - List successBizSetList = new ArrayList<>(); - List failedBizSetList = new ArrayList<>(); - for (AppInfo bizSetAppInfo : bizSetAppInfoList) { - resetResourceListTpl(reqList); - fillResourceListTpl(reqList, bizSetAppInfo.getId().toString(), bizSetAppInfo.getName()); - String maintainerStr = bizSetAppInfo.getMaintainers(); - if (StringUtils.isBlank(maintainerStr)) { - log.warn( - "maintainer is blank, app:{},{}, ignore", - bizSetAppInfo.getId(), - bizSetAppInfo.getName() - ); - continue; - } - String[] maintainers = maintainerStr.split("[,;]"); - List maintainerList = new ArrayList<>(); - for (String maintainer : maintainers) { - if (!maintainerList.contains(maintainer)) { - maintainerList.add(maintainer); - } - } - List successMaintainerList = new ArrayList<>(); - List failedMaintainerList = new ArrayList<>(); - for (String maintainer : maintainerList) { - EsbIamSubject esbIamSubject = new EsbIamSubject("user", maintainer); - List successActionList = new ArrayList<>(); - List failedActionList = new ArrayList<>(); - int actionCount = 0; - for (BatchAuthByPathReq req : reqList) { - actionCount += req.getActions().size(); - boolean result = batchAuth(esbIamSubject, req.getActions(), req.getResources()); - recordAuthActionResult( - result, successActionList, failedActionList, - maintainer, bizSetAppInfo.getIdAndName(), req - ); - } - recordAuthMaintainerResult( - failedActionList, successMaintainerList, failedMaintainerList, - maintainer, successActionList.size(), actionCount, bizSetAppInfo.getIdAndName() - ); - } - recordAuthBizSetResult( - failedMaintainerList, successBizSetList, failedBizSetList, - successMaintainerList.size(), maintainerList.size(), bizSetAppInfo.getIdAndName() - ); - } - showGlobalAuthResult(failedBizSetList, successBizSetList); - log.info("auth bizSetApps finished!"); - printSeparateLine(); - return 0; - } -} diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/BizSetCreateMigrationTask.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/BizSetCreateMigrationTask.java deleted file mode 100644 index 073576d774..0000000000 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/BizSetCreateMigrationTask.java +++ /dev/null @@ -1,311 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.upgrader.task; - -import com.tencent.bk.job.common.cc.model.bizset.BasicBizSet; -import com.tencent.bk.job.common.cc.model.bizset.BatchUpdateBizSetData; -import com.tencent.bk.job.common.cc.model.bizset.BatchUpdateBizSetReq; -import com.tencent.bk.job.common.cc.model.bizset.BizSetAttr; -import com.tencent.bk.job.common.cc.model.bizset.BizSetFilter; -import com.tencent.bk.job.common.cc.model.bizset.BizSetInfo; -import com.tencent.bk.job.common.cc.model.bizset.BizSetScope; -import com.tencent.bk.job.common.cc.model.bizset.CreateBizSetReq; -import com.tencent.bk.job.common.cc.model.bizset.Rule; -import com.tencent.bk.job.common.constant.AppTypeEnum; -import com.tencent.bk.job.common.constant.JobConstants; -import com.tencent.bk.job.common.util.FileUtil; -import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.upgrader.anotation.ExecuteTimeEnum; -import com.tencent.bk.job.upgrader.anotation.RequireTaskParam; -import com.tencent.bk.job.upgrader.anotation.UpgradeTask; -import com.tencent.bk.job.upgrader.anotation.UpgradeTaskInputParam; -import com.tencent.bk.job.upgrader.client.EsbCmdbClient; -import com.tencent.bk.job.upgrader.client.JobClient; -import com.tencent.bk.job.upgrader.model.AppInfo; -import com.tencent.bk.job.upgrader.task.param.JobManageServerAddress; -import com.tencent.bk.job.upgrader.task.param.ParamNameConsts; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.helpers.FormattingTuple; -import org.slf4j.helpers.MessageFormatter; -import org.springframework.util.CollectionUtils; - -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Properties; - -/** - * 业务集、全业务向CMDB迁移的资源创建任务 - */ -@Slf4j -@RequireTaskParam(value = { - @UpgradeTaskInputParam(value = JobManageServerAddress.class) -}) -@UpgradeTask( - dataStartVersion = "3.0.0.0", - targetVersion = "3.5.0", - targetExecuteTime = ExecuteTimeEnum.AFTER_UPDATE_JOB) -public class BizSetCreateMigrationTask extends BaseUpgradeTask { - - private JobClient jobManageClient; - - private EsbCmdbClient esbCmdbClient; - private List bizSetAppInfoList; - - private String getJobHostUrlByAddress(String address) { - if (!address.startsWith("http://") && !address.startsWith("https://")) { - address = "http://" + address; - } - return address; - } - - public BizSetCreateMigrationTask(Properties properties) { - super(properties); - } - - @Override - public void init() { - String appCode = (String) properties.get(ParamNameConsts.CONFIG_PROPERTY_APP_CODE); - String appSecret = (String) properties.get(ParamNameConsts.CONFIG_PROPERTY_APP_SECRET); - String esbBaseUrl = (String) properties.get(ParamNameConsts.CONFIG_PROPERTY_ESB_SERVICE_URL); - String cmdbSupplierAccount = - (String) properties.get(ParamNameConsts.CONFIG_PROPERTY_CMDB_DEFAULT_SUPPLIER_ACCOUNT); - String jobAuthToken = getJobAuthToken(); - jobManageClient = new JobClient( - getJobHostUrlByAddress((String) properties.get(ParamNameConsts.INPUT_PARAM_JOB_MANAGE_SERVER_ADDRESS)), - jobAuthToken - ); - esbCmdbClient = new EsbCmdbClient(esbBaseUrl, appCode, appSecret, "zh-cn", cmdbSupplierAccount); - // 从job-manage拉取业务集/全业务信息 - this.bizSetAppInfoList = getAllBizSetAppInfoFromManage(); - } - - private List getAllBizSetAppInfoFromManage() { - try { - return jobManageClient.listBizSetApps(); - } catch (Exception e) { - log.error("Fail to get special apps from job-manage, please confirm job-manage version>=3.5.0.0"); - throw e; - } - } - - /** - * 为业务集构造选择业务的过滤条件 - * - * @param appInfo 业务集信息 - * @return 过滤条件 - */ - private BizSetFilter buildAppSetFilter(AppInfo appInfo) { - BizSetFilter filter = new BizSetFilter(); - filter.setCondition(BizSetFilter.CONDITION_AND); - - List rules = new ArrayList<>(); - // 指定所有子业务ID - List subAppIds = appInfo.getSubAppIds(); - if (subAppIds == null) { - subAppIds = new ArrayList<>(); - } - // 指定业务所属部门ID - Long operateDeptId = appInfo.getOperateDeptId(); - // 空业务集 - if (CollectionUtils.isEmpty(subAppIds) && operateDeptId == null) { - Rule subAppIdsRule = new Rule(); - subAppIdsRule.setField("bk_biz_id"); - subAppIdsRule.setOperator(Rule.OPERATOR_IN); - subAppIdsRule.setValue(subAppIds); - rules.add(subAppIdsRule); - filter.setRules(rules); - return filter; - } - if (!CollectionUtils.isEmpty(subAppIds)) { - Rule subAppIdsRule = new Rule(); - subAppIdsRule.setField("bk_biz_id"); - subAppIdsRule.setOperator(Rule.OPERATOR_IN); - subAppIdsRule.setValue(subAppIds); - rules.add(subAppIdsRule); - } - if (operateDeptId != null) { - Rule operateDeptIdRule = new Rule(); - operateDeptIdRule.setField("bk_operate_dept_id"); - operateDeptIdRule.setOperator(Rule.OPERATOR_IN); - operateDeptIdRule.setValue(Collections.singletonList(operateDeptId)); - rules.add(operateDeptIdRule); - } - - filter.setRules(rules); - return filter; - } - - /** - * 构造符合CMDB规则的运维人员字段 - * - * @param rawStr Job原始运维人员字段 - * @return 符合CMDB规则的运维人员字段 - */ - private String buildMaintainerStr(String rawStr) { - String[] maintainers = rawStr.split("[,;]"); - return String.join(",", maintainers); - } - - /** - * 获取业务集最终在CMDB中的ID，用于判定存在性决定是否创建 - * - * @param appInfo 业务信息 - * @return 业务集ID - */ - private long getFinalBizSetId(AppInfo appInfo) { - if (StringUtils.isNotBlank(appInfo.getScopeId())) { - return Long.parseLong(appInfo.getScopeId()); - } - return appInfo.getId(); - } - - /** - * 调用CMDB接口更新业务集属性信息 - * - * @param attr 业务集属性 - * @param scope 业务选择范围 - * @return 是否成功更新 - */ - private Boolean updateBizSet(BizSetAttr attr, BizSetScope scope) { - BatchUpdateBizSetReq batchUpdateBizSetReq = esbCmdbClient.makeCmdbBaseReq(BatchUpdateBizSetReq.class); - batchUpdateBizSetReq.setBizSetIds(Collections.singletonList(attr.getId())); - BatchUpdateBizSetData data = new BatchUpdateBizSetData(); - data.setAttr(attr); - data.setScope(scope); - batchUpdateBizSetReq.setData(data); - return esbCmdbClient.batchUpdateBizSet(batchUpdateBizSetReq); - } - - /** - * 根据Job中现存业务集/全业务信息向CMDB创建业务集/全业务 - * - * @param appInfo 业务集/全业务信息 - * @return 业务集是否已存在于CMDB中 - */ - private boolean createOrUpdateCMDBResourceForApp(AppInfo appInfo) { - CreateBizSetReq createBizSetReq = esbCmdbClient.makeCmdbBaseReq(CreateBizSetReq.class); - String desc = "Auto created by bk-job migration"; - String supplierAccount = (String) properties.get(ParamNameConsts.CONFIG_PROPERTY_CMDB_DEFAULT_SUPPLIER_ACCOUNT); - BizSetAttr attr = BizSetAttr.builder() - .id(getFinalBizSetId(appInfo)) - .name(appInfo.getName()) - .desc(desc) - .maintainer(buildMaintainerStr(appInfo.getMaintainers())) - .timeZone(appInfo.getTimeZone()) - .language(appInfo.getLanguage()) - .supplierAccount(supplierAccount) - .build(); - createBizSetReq.setAttr(attr); - BizSetScope scope = new BizSetScope(); - if (appInfo.isAllBizSet()) { - // 全业务 - scope.setMatchAll(true); - scope.setFilter(null); - } else if (appInfo.getAppType() == AppTypeEnum.APP_SET.getValue()) { - // 普通业务集 - scope.setMatchAll(false); - scope.setFilter(buildAppSetFilter(appInfo)); - } else { - log.warn("Not support app type:{}", appInfo.getAppType()); - return false; - } - createBizSetReq.setScope(scope); - try { - List bizSetList = esbCmdbClient.searchBizSetById(attr.getId()); - if (CollectionUtils.isEmpty(bizSetList)) { - Long bizSetId = esbCmdbClient.createBizSet(createBizSetReq); - if (bizSetId != null && bizSetId > 0) { - log.info("bizSet {} created", bizSetId); - } else { - log.warn("fail to create bizSet {}", attr.getId()); - return false; - } - } else { - if (attr.getId() == JobConstants.DEFAULT_ALL_BIZ_SET_ID) { - // CMDB内置的全业务，不更新 - log.info("bizSet {} is all-business bizSet created by cmdb, ignore", attr.getId()); - } else { - // 更新业务集 - log.info("bizSet {} already exists, update:{}", attr.getId(), updateBizSet(attr, scope)); - } - } - return true; - } catch (Exception e) { - FormattingTuple msg = MessageFormatter.format("Fail to create bizSet {}", createBizSetReq); - log.error(msg.getMessage(), e); - return false; - } - } - - @Override - public int execute(String[] args) { - log.info(getName() + " for version " + getTargetVersion() + " begin to run..."); - List successfulBizSetList = new ArrayList<>(); - List failedBizSetList = new ArrayList<>(); - int successCount = 0; - for (AppInfo appInfo : bizSetAppInfoList) { - BasicBizSet bizSet = new BasicBizSet(getFinalBizSetId(appInfo), appInfo.getName()); - // 1.调用CMDB接口创建业务集/全业务 - if (createOrUpdateCMDBResourceForApp(appInfo)) { - successCount += 1; - successfulBizSetList.add(bizSet); - } else { - failedBizSetList.add(bizSet); - } - } - if (successCount == bizSetAppInfoList.size()) { - log.info("all {} bizSets migrated to CMDB", successCount); - } else { - log.warn( - "{}/{} bizSets migrated to CMDB, please check log to confirm failed bizSets", - successCount, - bizSetAppInfoList.size() - ); - log.warn("Failed bizSets:{}", JsonUtils.toJson(failedBizSetList)); - } - // 2.生成更新CMDB数据库的业务集信息Json文件 - String content = JsonUtils.toJson(successfulBizSetList); - InputStream ins = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - try { - File targetFile = new File("biz_set_list.json"); - FileUtil.writeInsToFile(ins, targetFile.getAbsolutePath()); - log.info("biz_set_list.json generated, please save it to continue the next step! path:{}", - targetFile.getAbsolutePath()); - } catch (InterruptedException e) { - log.error("Fail to gen biz_set_list.json", e); - return 1; - } - if (!failedBizSetList.isEmpty()) { - return 1; - } - return 0; - } -} diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/BizSetMigrationStatusUpdateTask.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/BizSetMigrationStatusUpdateTask.java deleted file mode 100644 index 9e3cc47309..0000000000 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/BizSetMigrationStatusUpdateTask.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.upgrader.task; - -import com.tencent.bk.job.upgrader.anotation.ExecuteTimeEnum; -import com.tencent.bk.job.upgrader.anotation.RequireTaskParam; -import com.tencent.bk.job.upgrader.anotation.UpgradeTask; -import com.tencent.bk.job.upgrader.anotation.UpgradeTaskInputParam; -import com.tencent.bk.job.upgrader.client.JobClient; -import com.tencent.bk.job.upgrader.task.param.JobManageServerAddress; -import com.tencent.bk.job.upgrader.task.param.ParamNameConsts; -import lombok.extern.slf4j.Slf4j; - -import java.util.Properties; - -/** - * 业务集、全业务向CMDB迁移完成后的迁移状态更新任务 - */ -@Slf4j -@RequireTaskParam(value = { - @UpgradeTaskInputParam(value = JobManageServerAddress.class) -}) -@UpgradeTask( - dataStartVersion = "3.0.0.0", - targetVersion = "3.5.0", - targetExecuteTime = ExecuteTimeEnum.MAKE_UP) -public class BizSetMigrationStatusUpdateTask extends BaseUpgradeTask { - - private JobClient jobManageClient; - - private String getJobHostUrlByAddress(String address) { - if (!address.startsWith("http://") && !address.startsWith("https://")) { - address = "http://" + address; - } - return address; - } - - public BizSetMigrationStatusUpdateTask(Properties properties) { - super(properties); - } - - @Override - public void init() { - String jobAuthToken = getJobAuthToken(); - jobManageClient = new JobClient( - getJobHostUrlByAddress((String) properties.get(ParamNameConsts.INPUT_PARAM_JOB_MANAGE_SERVER_ADDRESS)), - jobAuthToken - ); - } - - @Override - public int execute(String[] args) { - log.info(getName() + " for version " + getTargetVersion() + " begin to run..."); - boolean status = true; - try { - if (args.length >= 4) { - status = Boolean.parseBoolean(args[3].trim().toLowerCase()); - log.info("status from param:{}", status); - } - } catch (Exception e) { - log.warn("Fail to parse status from param:{}", args[3]); - return 1; - } - try { - log.info("try to set bizSet migration status:{}", status); - jobManageClient.setBizSetMigrationStatus(status); - } catch (Exception e) { - log.error("Fail to update migration status", e); - return 1; - } - return 0; - } -} diff --git a/src/frontend/.eslintrc.js b/src/frontend/.eslintrc.js index 4152f6ea1a..6505bd6970 100644 --- a/src/frontend/.eslintrc.js +++ b/src/frontend/.eslintrc.js @@ -50,9 +50,12 @@ module.exports = { }, // add your custom rules hered rules: { + // 要求或禁止末尾逗号 'comma-dangle': ['error', 'always-multiline'], + // 强制在代码块中使用一致的大括号风格 // https://eslint.org/docs/rules/brace-style 'brace-style': ['error', '1tbs', { allowSingleLine: false }], + // 强制在 parseInt() 使用基数参数 radix: 'error', 'no-restricted-properties': [2], 'no-case-declarations': 'error', diff --git a/src/frontend/lib/icon-cool/fonts/iconcool.eot b/src/frontend/lib/bk-icon/fonts/iconcool.eot similarity index 100% rename from src/frontend/lib/icon-cool/fonts/iconcool.eot rename to src/frontend/lib/bk-icon/fonts/iconcool.eot diff --git a/src/frontend/lib/icon-cool/fonts/iconcool.svg b/src/frontend/lib/bk-icon/fonts/iconcool.svg similarity index 100% rename from src/frontend/lib/icon-cool/fonts/iconcool.svg rename to src/frontend/lib/bk-icon/fonts/iconcool.svg diff --git a/src/frontend/lib/icon-cool/fonts/iconcool.ttf b/src/frontend/lib/bk-icon/fonts/iconcool.ttf similarity index 100% rename from src/frontend/lib/icon-cool/fonts/iconcool.ttf rename to src/frontend/lib/bk-icon/fonts/iconcool.ttf diff --git a/src/frontend/lib/icon-cool/fonts/iconcool.woff b/src/frontend/lib/bk-icon/fonts/iconcool.woff similarity index 100% rename from src/frontend/lib/icon-cool/fonts/iconcool.woff rename to src/frontend/lib/bk-icon/fonts/iconcool.woff diff --git a/src/frontend/lib/icon-cool/iconcool.js b/src/frontend/lib/bk-icon/iconcool.js similarity index 100% rename from src/frontend/lib/icon-cool/iconcool.js rename to src/frontend/lib/bk-icon/iconcool.js diff --git a/src/frontend/lib/icon-cool/iconcool.json b/src/frontend/lib/bk-icon/iconcool.json similarity index 100% rename from src/frontend/lib/icon-cool/iconcool.json rename to src/frontend/lib/bk-icon/iconcool.json diff --git a/src/frontend/lib/icon-cool/style.css b/src/frontend/lib/bk-icon/style.css similarity index 100% rename from src/frontend/lib/icon-cool/style.css rename to src/frontend/lib/bk-icon/style.css diff --git a/src/frontend/package.json b/src/frontend/package.json index ac32e4f991..8b7f85d74e 100644 --- a/src/frontend/package.json +++ b/src/frontend/package.json @@ -1,6 +1,6 @@ { "name": "job", - "version": "1.0.0", + "version": "3.5.0", "description": "job", "author": "blueking", "scripts": { @@ -9,24 +9,23 @@ "dev-external": "webpack serve --https --hot --env development --env mode=external --config ./webpack.config.js", "build": "webpack --env --config ./webpack.config.js", "lint:js": "eslint --fix --ext .js,.vue ./src", - "lint:style": "stylelint -v --fix ./**/*.{css,vue}" + "lint:style": "stylelint --fix **/**/*.{css,vue}" }, "keywords": [], "license": "ISC", "dependencies": { "@blueking/paas-login": "0.0.11", "@blueking/user-selector": "^1.0.7", - "@icon-cool/bk-icon-job": "0.0.11", - "@vue/composition-api": "^1.0.0-beta.1", + "@vue/composition-api": "^1.5.0", "ace-builds": "^1.4.12", - "axios": "^0.26.0", - "bk-magic-vue": "^2.4.14", - "core-js": "^3.18.1", + "axios": "^0.27.2", + "bk-magic-vue": "^2.4.9-beta.1", + "core-js": "^3.22.4", "cron-parser-custom": "^2.13.0", "diff": "^5.0.0", "diff2html": "^3.4.4", - "highlight.js": "^11.2.0", - "html2canvas": "^1.0.0-rc.7", + "highlight.js": "^11.4.0", + "html2canvas": "^1.4.0", "js-base64": "^3.6.1", "js-cookie": "^3.0.1", "lodash": "^4.17.21", @@ -40,10 +39,10 @@ "vuex": "3.6.2" }, "devDependencies": { - "@babel/core": "^7.17.8", + "@babel/core": "^7.17.10", "@babel/eslint-parser": "^7.16.3", - "@babel/plugin-proposal-class-properties": "^7.13.0", - "@babel/plugin-proposal-export-namespace-from": "^7.12.13", + "@babel/plugin-proposal-class-properties": "^7.16.7", + "@babel/plugin-proposal-export-namespace-from": "^7.16.7", "@babel/plugin-syntax-dynamic-import": "^7.8.3", "@babel/plugin-syntax-jsx": "^7.12.13", "@babel/plugin-transform-modules-commonjs": "^7.13.8", @@ -53,29 +52,30 @@ "@vue/babel-plugin-transform-vue-jsx": "1.2.1", "@vue/babel-sugar-inject-h": "1.2.2", "@vue/eslint-config-standard": "^6.0.0", - "babel-loader": "^8.2.4", + "babel-loader": "^8.2.5", "babel-plugin-import-bk-magic-vue": "^2.1.10", "babel-plugin-lodash": "^3.3.4", "copy-webpack-plugin": "^10.2.4", "css-loader": "^6.5.1", "css-minimizer-webpack-plugin": "^3.1.3", "dotenv": "^16.0.0", - "eslint": "^8.12.0", - "eslint-config-standard": "^16.0.2", + "eslint": "^8.14.0", + "eslint-config-standard": "^17.0.0", "eslint-plugin-html": "^6.1.2", - "eslint-plugin-import": "^2.25.3", + "eslint-plugin-import": "^2.25.4", + "eslint-plugin-n": "^15.2.0", "eslint-plugin-node": "^11.1.0", "eslint-plugin-promise": "^6.0.0", "eslint-plugin-vue": "8.0.3", "eslint-webpack-plugin": "^3.1.0", "figlet": "^1.5.0", - "html-loader": "^3.0.1", + "html-loader": "^3.1.0", "html-webpack-plugin": "^5.5.0", "lodash-webpack-plugin": "^0.11.6", "markdown-loader": "^8.0.0", "mini-css-extract-plugin": "^2.6.0", "node-xlsx": "^0.21.0", - "postcss": "^8.4.12", + "postcss": "^8.4.13", "postcss-advanced-variables": "~3.0.0", "postcss-atroot": "^0.2.3", "postcss-extend-rule": "^4.0.0", @@ -85,13 +85,13 @@ "postcss-loader": "^6.2.0", "postcss-mixins": "^9.0.2", "postcss-nested": "^5.0.6", - "postcss-preset-env": "^7.4.3", + "postcss-preset-env": "^7.5.0", "postcss-property-lookup": "^3.0.0", "postcss-syntax": "^0.36.2", "postcss-url": "^10.1.3", "speed-measure-webpack-plugin": "^1.5.0", "style-loader": "^3.3.1", - "stylelint": "^14.6.1", + "stylelint": "^14.8.2", "stylelint-config-standard": "^25.0.0", "stylelint-order": "^5.0.0", "stylelint-scss": "^4.2.0", @@ -103,7 +103,7 @@ "vue-template-compiler": "2.6.14", "webpack": "^5.63.0", "webpack-cli": "^4.6.0", - "webpack-dev-server": "^4.4.0", + "webpack-dev-server": "^4.9.0", "webpack-merge": "^5.2.0" }, "engines": { diff --git a/src/frontend/src/App.vue b/src/frontend/src/App.vue index 6395e901dc..ed478eb385 100644 --- a/src/frontend/src/App.vue +++ b/src/frontend/src/App.vue @@ -34,7 +34,8 @@ + :arrow="false" + :tippy-options="{ hideOnClick: false }">
- +
@@ -68,7 +73,10 @@
{{ $t('问题反馈') }}
 - +
{{ currentUser.username }} diff --git a/src/frontend/src/components/app-select/index.vue b/src/frontend/src/components/app-select/index.vue index 12b4bc5ed5..0782e203ca 100644 --- a/src/frontend/src/components/app-select/index.vue +++ b/src/frontend/src/components/app-select/index.vue @@ -33,7 +33,7 @@ @@ -73,7 +73,7 @@ @click="handleAppChange(app)">
{{ app.name }} - (#{{ app.scopeId }}) + ({{ app.scopeId }})
-