From 21579f2bb0d94beb5091ec38ce57745c7b49316d Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Mon, 25 Dec 2023 20:34:39 +0800 Subject: [PATCH] =?UTF-8?q?feature:=20Job=20=E6=94=AF=E6=8C=81=E5=AE=B9?= =?UTF-8?q?=E5=99=A8=E6=89=A7=E8=A1=8C=20-=20=E8=84=9A=E6=9C=AC=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=20#2631?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../exception/NotImplementedException.java | 2 +- .../tencent/bk/job/common/gse/IGseClient.java | 15 + .../bk/job/common/gse/v1/GseV1ApiClient.java | 19 +- .../bk/job/common/gse/v2/GseV2ApiClient.java | 8 +- .../bk/job/common/gse/v2/model/Agent.java | 6 + .../v2/model/AtomicFileTaskResultContent.java | 64 ++- .../gse/v2/model}/ExecuteObjectGseKey.java | 34 +- .../model/GetExecuteScriptResultRequest.java | 11 +- .../model/GetTransferFileResultRequest.java | 30 +- ...ava => ScriptExecuteObjectTaskResult.java} | 33 +- .../common/gse/v2/model/ScriptTaskResult.java | 2 +- .../gse/v2/model/TerminateGseTaskRequest.java | 10 +- .../model/web/vo/AgentTaskExecutionVO.java | 6 +- .../StepInstanceDAOImplIntegrationTest.java | 10 +- .../impl/EsbFastExecuteSQLResourceImpl.java | 2 +- .../EsbFastExecuteScriptResourceImpl.java | 2 +- .../v2/impl/EsbFastPushFileResourceImpl.java | 2 +- .../EsbGetJobInstanceLogResourceImpl.java | 12 +- .../EsbGetStepInstanceStatusResourceImpl.java | 2 +- .../impl/EsbPushConfigFileResourceImpl.java | 2 +- ...atchGetJobInstanceIpLogV3ResourceImpl.java | 6 +- .../v3/EsbFastExecuteSQLV3ResourceImpl.java | 2 +- .../EsbFastExecuteScriptV3ResourceImpl.java | 2 +- .../v3/EsbFastTransferFileV3ResourceImpl.java | 2 +- .../EsbGetJobInstanceIpLogV3ResourceImpl.java | 12 +- ...sbGetStepInstanceDetailV3ResourceImpl.java | 4 +- .../v3/EsbPushConfigFileResourceV3Impl.java | 2 +- .../web/impl/WebExecuteTaskResourceImpl.java | 4 +- .../WebTaskExecutionResultResourceImpl.java | 18 +- .../web/impl/WebTaskInstanceResourceImpl.java | 4 +- .../bk/job/execute/dao/FileAgentTaskDAO.java | 7 +- .../execute/dao/FileExecuteObjectTaskDAO.java | 6 +- .../job/execute/dao/ScriptAgentTaskDAO.java | 7 +- .../dao/ScriptExecuteObjectTaskDAO.java | 6 +- .../dao/impl/FileAgentTaskDAOImpl.java | 7 +- .../impl/FileExecuteObjectTaskDAOImpl.java | 6 +- .../dao/impl/ScriptAgentTaskDAOImpl.java | 7 +- .../impl/ScriptExecuteObjectTaskDAOImpl.java | 6 +- .../execute/dao/impl/StepInstanceDAOImpl.java | 4 +- .../executor/AbstractGseTaskStartCommand.java | 59 +-- .../executor/FileGseTaskStartCommand.java | 159 +++--- .../executor/FileGseTaskStopCommand.java | 8 +- .../SQLScriptGseTaskStartCommand.java | 2 +- .../executor/ScriptGseTaskStartCommand.java | 70 +-- .../executor/ScriptGseTaskStopCommand.java | 8 +- .../engine/listener/GseStepEventHandler.java | 2 +- .../execute/engine/model/ExecuteObject.java | 85 +++- .../bk/job/execute/engine/model/JobFile.java | 21 +- .../execute/engine/model/LogPullProgress.java | 5 +- .../result/AbstractResultHandleTask.java | 106 ++-- .../engine/result/FileResultHandleTask.java | 422 ++++++++------- .../engine/result/ScriptResultHandleTask.java | 309 +++++------ .../JobLastHostsVariableResolver.java | 4 +- .../model/ExecuteObjectCompositeKey.java | 127 +++-- .../job/execute/model/ExecuteObjectTask.java | 21 +- .../job/execute/model/ExecuteObjectsDTO.java | 103 ++++ ....java => FileExecuteObjectLogContent.java} | 16 +- ...ava => ScriptExecuteObjectLogContent.java} | 41 +- .../execute/model/StepInstanceBaseDTO.java | 39 +- .../bk/job/execute/model/StepInstanceDTO.java | 31 +- .../service/ExecuteObjectTaskService.java | 6 +- .../bk/job/execute/service/LogService.java | 171 ++++--- .../execute/service/StepInstanceService.java | 10 + .../TaskInstanceExecuteObjectService.java | 86 ++-- .../AbstractExecuteObjectTaskServiceImpl.java | 50 +- .../FileExecuteObjectTaskServiceImpl.java | 151 ++---- .../service/impl/LogExportServiceImpl.java | 17 +- .../execute/service/impl/LogServiceImpl.java | 480 ++++++++++-------- .../service/impl/NotifyServiceImpl.java | 2 +- .../impl/RollingConfigServiceImpl.java | 6 +- .../ScriptExecuteObjectTaskServiceImpl.java | 128 ++--- .../service/impl/StepInstanceServiceImpl.java | 45 +- .../service/impl/TaskExecuteServiceImpl.java | 42 +- .../TaskInstanceExecuteObjectServiceImpl.java | 45 +- .../service/impl/TaskResultServiceImpl.java | 6 +- .../service/ServiceExecuteObjectLogDTO.java | 2 +- ... => ServiceExecuteObjectScriptLogDTO.java} | 43 +- .../service/impl/ServiceLogResourceImpl.java | 8 +- .../common/consts/task/TaskFileTypeEnum.java | 3 - 79 files changed, 1940 insertions(+), 1383 deletions(-) rename src/backend/{job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result => commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model}/ExecuteObjectGseKey.java (63%) rename src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/{ScriptAgentTaskResult.java => ScriptExecuteObjectTaskResult.java} (67%) rename src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/{FileIpLogContent.java => FileExecuteObjectLogContent.java} (83%) rename src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/{ScriptHostLogContent.java => ScriptExecuteObjectLogContent.java} (68%) rename src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/{ServiceScriptLogDTO.java => ServiceExecuteObjectScriptLogDTO.java} (77%) diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/NotImplementedException.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/NotImplementedException.java index feb0dfc65c..f5b3decfe9 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/NotImplementedException.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/NotImplementedException.java @@ -64,7 +64,7 @@ public NotImplementedException(String message, Throwable cause, Integer errorCod } public NotImplementedException(String message, Throwable cause, Integer errorCode, - Object[] errorParams) { + Object[] errorParams) { super(message, cause, ErrorType.UNIMPLEMENTED, errorCode, errorParams); } } diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/IGseClient.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/IGseClient.java index ced873d0dc..3a7ec74429 100644 --- a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/IGseClient.java +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/IGseClient.java @@ -58,6 +58,21 @@ default List buildAgents(Collection agentIds, String user, String .collect(Collectors.toList()); } + /** + * 设置 Agent 认证信息 + * + * @param agents agent列表 + * @param user 用户 + * @param password 密码 + * @return Agent + */ + default List fillAgentAuthInfo(Collection agents, String user, String password) { + return agents.stream().peek(agent -> { + agent.setUser(user); + agent.setPwd(password); + }).collect(Collectors.toList()); + } + /** * 构建目标Agent * diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/GseV1ApiClient.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/GseV1ApiClient.java index 8ff8ff9a3f..77151f4fff 100644 --- a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/GseV1ApiClient.java +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/GseV1ApiClient.java @@ -42,7 +42,7 @@ import com.tencent.bk.job.common.gse.v2.model.GetExecuteScriptResultRequest; import com.tencent.bk.job.common.gse.v2.model.GetTransferFileResultRequest; import com.tencent.bk.job.common.gse.v2.model.GseTaskResponse; -import com.tencent.bk.job.common.gse.v2.model.ScriptAgentTaskResult; +import com.tencent.bk.job.common.gse.v2.model.ScriptExecuteObjectTaskResult; import com.tencent.bk.job.common.gse.v2.model.ScriptTaskResult; import com.tencent.bk.job.common.gse.v2.model.SourceFile; import com.tencent.bk.job.common.gse.v2.model.TargetFile; @@ -283,10 +283,10 @@ private static api_host convertToApiHost(String cloudIp) { private ScriptTaskResult toScriptTaskResult(api_task_detail_result resultV1) { ScriptTaskResult result = new ScriptTaskResult(); if (CollectionUtils.isNotEmpty(resultV1.getResult())) { - List agentTaskResults = + List agentTaskResults = resultV1.getResult().stream() .map(agentTaskResultV1 -> { - ScriptAgentTaskResult agentTaskResult = new ScriptAgentTaskResult(); + ScriptExecuteObjectTaskResult agentTaskResult = new ScriptExecuteObjectTaskResult(); agentTaskResult.setAgentId(agentTaskResultV1.getGse_composite_id() + ":" + agentTaskResultV1.getIp()); agentTaskResult.setAtomicTaskId(agentTaskResultV1.getAtomic_task_id()); @@ -439,8 +439,9 @@ private List toV1CopyFileInfoRequest(TransferFileRequest re @Override public FileTaskResult getTransferFileResult(GetTransferFileResultRequest request) { api_map_rsp rsp; - if (CollectionUtils.isNotEmpty(request.getAgentIds())) { - rsp = pullCopyFileResult(request.getTaskId(), request.getAgentIds()); + if (CollectionUtils.isNotEmpty(request.getAgents())) { + rsp = pullCopyFileResult(request.getTaskId(), + request.getAgents().stream().map(Agent::getAgentId).collect(Collectors.toList())); } else if (StringUtils.isNotBlank(request.getTaskId())) { rsp = pullCopyFileResult(request.getTaskId()); } else { @@ -622,11 +623,11 @@ private api_stop_task_request toV1StopTaskRequest(TerminateGseTaskRequest reques GseTaskTypeEnum taskType) { api_stop_task_request stopRequest = new api_stop_task_request(); stopRequest.setStop_task_id(request.getTaskId()); - if (CollectionUtils.isNotEmpty(request.getAgentIds())) { - stopRequest.setAgents(request.getAgentIds().stream() - .map(agentId -> + if (CollectionUtils.isNotEmpty(request.getAgents())) { + stopRequest.setAgents(request.getAgents().stream() + .map(agent -> // 终止任务并不不需要账号密码,此处传入为了绕过thrift协议的校验 - buildAgent(agentId, buildEmptyApiAuth())) + buildAgent(agent.getAgentId(), buildEmptyApiAuth())) .collect(Collectors.toList())); } stopRequest.setType(taskType.getValue()); diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/GseV2ApiClient.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/GseV2ApiClient.java index c3f9bc67a7..56c17dda9e 100644 --- a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/GseV2ApiClient.java +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/GseV2ApiClient.java @@ -36,11 +36,11 @@ @Slf4j public class GseV2ApiClient extends AbstractBkApiClient implements IGseClient { - private static final String URI_ASYNC_EXECUTE_SCRIPT = "/api/v2/task/async_execute_script"; - private static final String URI_GET_EXECUTE_SCRIPT_RESULT = "/api/v2/task/get_execute_script_result"; private static final String URI_LIST_AGENT_STATE = "/api/v2/cluster/list_agent_state"; - private static final String URI_ASYNC_TRANSFER_FILE = "/api/v2/task/async_transfer_file"; - private static final String URI_GET_TRANSFER_FILE_RESULT = "/api/v2/task/async/get_transfer_file_result"; + private static final String URI_ASYNC_EXECUTE_SCRIPT = "/api/v2/task/extensions/async_execute_script"; + private static final String URI_GET_EXECUTE_SCRIPT_RESULT = "/api/v2/task/extensions/get_execute_script_result"; + private static final String URI_ASYNC_TRANSFER_FILE = "/api/v2/task/extensions/async_push_file"; + private static final String URI_GET_TRANSFER_FILE_RESULT = "/api/v2/task/extensions/get_transfer_file_result"; private final BkApiAuthorization gseBkApiAuthorization; public GseV2ApiClient(MeterRegistry meterRegistry, diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/Agent.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/Agent.java index 7f555a6639..c4aef5ab0d 100644 --- a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/Agent.java +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/Agent.java @@ -14,6 +14,12 @@ public class Agent { @JsonProperty("bk_agent_id") private String agentId; + /** + * 目标容器 ID, 空则为主机 + */ + @JsonProperty("bk_container_id") + private String containerId; + /** * 执行账号名 */ diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/AtomicFileTaskResultContent.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/AtomicFileTaskResultContent.java index 5e6902e907..b24dc977e0 100644 --- a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/AtomicFileTaskResultContent.java +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/AtomicFileTaskResultContent.java @@ -1,5 +1,6 @@ package com.tencent.bk.job.common.gse.v2.model; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.tencent.bk.job.common.gse.constants.FileDistModeEnum; import com.tencent.bk.job.common.gse.constants.FileTaskTypeEnum; @@ -30,6 +31,12 @@ public class AtomicFileTaskResultContent { @JsonProperty("source_agent_id") private String sourceAgentId; + /** + * 文件源 container id + */ + @JsonProperty("source_container_id") + private String sourceContainerId; + /** * 源文件目录，下发任务时指定的源文件路径 */ @@ -43,11 +50,17 @@ public class AtomicFileTaskResultContent { private String sourceFileName; /** - * 分发目标主机 agent id + * 分发目标 agent id */ @JsonProperty("dest_agent_id") private String destAgentId; + /** + * 分发目标容器 container id + */ + @JsonProperty("dest_container_id") + private String destContainerId; + /** * 目标目录 */ @@ -130,6 +143,10 @@ public class AtomicFileTaskResultContent { */ private FileTaskTypeEnum taskType; + @JsonIgnore + private ExecuteObjectGseKey sourceExecuteObjectGseKey; + @JsonIgnore + private ExecuteObjectGseKey destExecuteObjectGseKey; public boolean isDownloadMode() { return FileDistModeEnum.DOWNLOAD.getValue().equals(this.mode); @@ -162,20 +179,25 @@ public String getStandardDestFilePath() { public String getTaskId() { if (taskId == null) { - this.taskId = buildTaskId(mode, sourceAgentId, getStandardSourceFilePath(), destAgentId, - getStandardDestFilePath()); + this.taskId = buildTaskId(mode, getSourceExecuteObjectGseKey(), getStandardSourceFilePath(), + getDestExecuteObjectGseKey(), getStandardDestFilePath()); } return taskId; } - public static String buildTaskId(Integer mode, String sourceAgentId, String sourceFilePath, String destAgentId, + public static String buildTaskId(Integer mode, + ExecuteObjectGseKey sourceExecuteObjectGseKey, + String sourceFilePath, + ExecuteObjectGseKey destExecuteObjectGseKey, String destFilePath) { String taskId; if (FileDistModeEnum.getFileDistMode(mode) == FileDistModeEnum.DOWNLOAD) { - taskId = concat(mode.toString(), sourceAgentId, GseFilePathUtils.standardizedGSEFilePath(sourceFilePath), - destAgentId, destFilePath); + taskId = concat(mode.toString(), sourceExecuteObjectGseKey.getKey(), + GseFilePathUtils.standardizedGSEFilePath(sourceFilePath), + destExecuteObjectGseKey.getKey(), destFilePath); } else { - taskId = concat(mode.toString(), sourceAgentId, GseFilePathUtils.standardizedGSEFilePath(sourceFilePath)); + taskId = concat(mode.toString(), sourceExecuteObjectGseKey.getKey(), + GseFilePathUtils.standardizedGSEFilePath(sourceFilePath)); } return taskId; } @@ -199,4 +221,32 @@ private static String concat(String... strArgs) { public boolean isApiProtocolBeforeV2() { return this.protocolVersion == null || this.protocolVersion < 2; } + + @JsonIgnore + public ExecuteObjectGseKey getDestExecuteObjectGseKey() { + if (destExecuteObjectGseKey != null) { + return destExecuteObjectGseKey; + } + if (StringUtils.isNotEmpty(destContainerId)) { + // bk_container_id 不为空，说明是容器执行对象 + destExecuteObjectGseKey = ExecuteObjectGseKey.ofContainer(destAgentId, destContainerId); + } else { + destExecuteObjectGseKey = ExecuteObjectGseKey.ofHost(destAgentId); + } + return destExecuteObjectGseKey; + } + + @JsonIgnore + public ExecuteObjectGseKey getSourceExecuteObjectGseKey() { + if (sourceExecuteObjectGseKey != null) { + return sourceExecuteObjectGseKey; + } + if (StringUtils.isNotEmpty(sourceContainerId)) { + // bk_container_id 不为空，说明是容器执行对象 + sourceExecuteObjectGseKey = ExecuteObjectGseKey.ofContainer(sourceAgentId, sourceContainerId); + } else { + sourceExecuteObjectGseKey = ExecuteObjectGseKey.ofHost(destAgentId); + } + return sourceExecuteObjectGseKey; + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ExecuteObjectGseKey.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/ExecuteObjectGseKey.java similarity index 63% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ExecuteObjectGseKey.java rename to src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/ExecuteObjectGseKey.java index 6e06ceb4ef..9609083682 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ExecuteObjectGseKey.java +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/ExecuteObjectGseKey.java @@ -22,28 +22,39 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.execute.engine.result; +package com.tencent.bk.job.common.gse.v2.model; import com.tencent.bk.job.common.constant.ExecuteObjectTypeEnum; +import lombok.Getter; import java.util.Objects; +import java.util.StringJoiner; /** * 执行对象KEY, 用于跟 GSE 交互 */ +@Getter public class ExecuteObjectGseKey { - private final String key; + private String agentId; + private String containerId; + private String key; - private ExecuteObjectGseKey(String key) { - this.key = key; + private ExecuteObjectGseKey() { } - public static ExecuteObjectGseKey ofHostKey(String agentId) { - return new ExecuteObjectGseKey(ExecuteObjectTypeEnum.HOST.getValue() + ":" + agentId); + public static ExecuteObjectGseKey ofHost(String agentId) { + ExecuteObjectGseKey executeObjectGseKey = new ExecuteObjectGseKey(); + executeObjectGseKey.agentId = agentId; + executeObjectGseKey.key = ExecuteObjectTypeEnum.HOST.getValue() + ":" + agentId; + return executeObjectGseKey; } - public static ExecuteObjectGseKey ofContainerKey(String containerId) { - return new ExecuteObjectGseKey(ExecuteObjectTypeEnum.CONTAINER.getValue() + ":" + containerId); + public static ExecuteObjectGseKey ofContainer(String agentId, String containerId) { + ExecuteObjectGseKey executeObjectGseKey = new ExecuteObjectGseKey(); + executeObjectGseKey.agentId = agentId; + executeObjectGseKey.containerId = containerId; + executeObjectGseKey.key = ExecuteObjectTypeEnum.HOST.getValue() + ":" + agentId; + return executeObjectGseKey; } @Override @@ -58,4 +69,11 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(key); } + + @Override + public String toString() { + return new StringJoiner(", ", ExecuteObjectGseKey.class.getSimpleName() + "[", "]") + .add("key='" + key + "'") + .toString(); + } } diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/GetExecuteScriptResultRequest.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/GetExecuteScriptResultRequest.java index 273cf5fea8..5719d0a22f 100644 --- a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/GetExecuteScriptResultRequest.java +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/GetExecuteScriptResultRequest.java @@ -34,6 +34,12 @@ public static class AgentTask { @JsonProperty("bk_agent_id") private String agentId; + /** + * 目标容器 ID, 空则为主机 + */ + @JsonProperty("bk_container_id") + private String containerId; + /** * 脚本任务 */ @@ -55,9 +61,10 @@ public static class AtomicTask { private int offset; } - public void addAgentTaskQuery(String agentId, Integer atomicTaskId, int offset) { + public void addAgentTaskQuery(ExecuteObjectGseKey executeObjectGseKey, Integer atomicTaskId, int offset) { AgentTask agentTask = new AgentTask(); - agentTask.setAgentId(agentId); + agentTask.setAgentId(executeObjectGseKey.getAgentId()); + agentTask.setContainerId(executeObjectGseKey.getContainerId()); AtomicTask atomicTask = new AtomicTask(); atomicTask.setAtomicTaskId(atomicTaskId); atomicTask.setOffset(offset); diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/GetTransferFileResultRequest.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/GetTransferFileResultRequest.java index 10d13951a1..b9d671528b 100644 --- a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/GetTransferFileResultRequest.java +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/GetTransferFileResultRequest.java @@ -5,6 +5,8 @@ import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; +import java.util.ArrayList; +import java.util.Collection; import java.util.List; /** @@ -21,8 +23,30 @@ public class GetTransferFileResultRequest extends GseReq { private String taskId; /** - * 过滤结果的agentId + * 过滤结果的agent */ - @JsonProperty("agent_id_list") - private List agentIds; + @JsonProperty("agents") + private List agents; + + public void addAgentQuery(ExecuteObjectGseKey executeObjectGseKey) { + if (agents == null) { + agents = new ArrayList<>(); + } + Agent agent = new Agent(); + agent.setAgentId(executeObjectGseKey.getAgentId()); + agent.setContainerId(executeObjectGseKey.getContainerId()); + agents.add(agent); + } + + public void batchAddAgentQuery(Collection executeObjectGseKeys) { + if (agents == null) { + agents = new ArrayList<>(); + } + executeObjectGseKeys.forEach(executeObjectGseKey -> { + Agent agent = new Agent(); + agent.setAgentId(executeObjectGseKey.getAgentId()); + agent.setContainerId(executeObjectGseKey.getContainerId()); + agents.add(agent); + }); + } } diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/ScriptAgentTaskResult.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/ScriptExecuteObjectTaskResult.java similarity index 67% rename from src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/ScriptAgentTaskResult.java rename to src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/ScriptExecuteObjectTaskResult.java index be61cf3f98..78cd446b60 100644 --- a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/ScriptAgentTaskResult.java +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/ScriptExecuteObjectTaskResult.java @@ -1,5 +1,6 @@ package com.tencent.bk.job.common.gse.v2.model; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; import lombok.NoArgsConstructor; @@ -8,17 +9,23 @@ import java.util.StringJoiner; /** - * GSE - Agent脚本任务执行结果 + * GSE - 脚本任务执行结果 */ @Data @NoArgsConstructor -public class ScriptAgentTaskResult { +public class ScriptExecuteObjectTaskResult { /** * agent id */ @JsonProperty("bk_agent_id") private String agentId; + /** + * container id + */ + @JsonProperty("bk_container_id") + private String containerId; + /** * 脚本原子任务ID */ @@ -78,15 +85,35 @@ public class ScriptAgentTaskResult { */ private long contentLength; + /** + * 执行目标对应的 GSE KEY。非协议内容 + */ + @JsonIgnore + private ExecuteObjectGseKey executeObjectGseKey; + public void setScreen(String screen) { this.screen = screen; this.contentLength = StringUtils.isEmpty(screen) ? 0 : screen.length(); } + @JsonIgnore + public ExecuteObjectGseKey getExecuteObjectGseKey() { + if (executeObjectGseKey != null) { + return executeObjectGseKey; + } + if (StringUtils.isNotEmpty(containerId)) { + // bk_container_id 不为空，说明是容器执行对象 + executeObjectGseKey = ExecuteObjectGseKey.ofContainer(agentId, containerId); + } else { + executeObjectGseKey = ExecuteObjectGseKey.ofHost(agentId); + } + return executeObjectGseKey; + } + @Override public String toString() { - return new StringJoiner(", ", ScriptAgentTaskResult.class.getSimpleName() + "[", "]") + return new StringJoiner(", ", ScriptExecuteObjectTaskResult.class.getSimpleName() + "[", "]") .add("agentId=" + agentId) .add("atomicTaskId=" + atomicTaskId) .add("status=" + status) diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/ScriptTaskResult.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/ScriptTaskResult.java index c020d693d7..7c8c554733 100644 --- a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/ScriptTaskResult.java +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/ScriptTaskResult.java @@ -14,5 +14,5 @@ public class ScriptTaskResult { /** * 执行结果 */ - private List result; + private List result; } diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/TerminateGseTaskRequest.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/TerminateGseTaskRequest.java index 1a522e01fd..1c1966a3cf 100644 --- a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/TerminateGseTaskRequest.java +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/TerminateGseTaskRequest.java @@ -23,10 +23,10 @@ public class TerminateGseTaskRequest extends GseReq { private String taskId; /** - * 脚本命令定义 + * 目标 Agent 列表 */ - @JsonProperty("agent_id_list") - private List agentIds = new ArrayList<>(); + @JsonProperty("agents") + private List agents = new ArrayList<>(); /** * 是否是GSE V2 Task; 根据gseV2Task判断请求GSE V1/v2 @@ -35,10 +35,10 @@ public class TerminateGseTaskRequest extends GseReq { private boolean gseV2Task; public TerminateGseTaskRequest(String taskId, - List agentIds, + List agents, boolean gseV2Task) { this.taskId = taskId; - this.agentIds = agentIds; + this.agents = agents; this.gseV2Task = gseV2Task; } } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/AgentTaskExecutionVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/AgentTaskExecutionVO.java index 57537a128c..674ce16612 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/AgentTaskExecutionVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/AgentTaskExecutionVO.java @@ -25,15 +25,19 @@ package com.tencent.bk.job.execute.model.web.vo; import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import com.tencent.bk.job.common.util.json.DecimalFormatJsonSerializer; import com.tencent.bk.job.common.util.json.LongTimestampSerializer; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; -@ApiModel("执行对象任务执行信息") +@ApiModel("Agent任务执行信息") @Data @Deprecated +@CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.DEPLOY, + explain = "使用 ExecuteObjectTaskVO 替换。发布完成后可以删除") public class AgentTaskExecutionVO { @ApiModelProperty("执行次数") private Integer retryCount; diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImplIntegrationTest.java index 0e2a8716ca..bed6928d04 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImplIntegrationTest.java +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImplIntegrationTest.java @@ -85,10 +85,10 @@ public void testGetStepInstanceBase() { assertThat(stepInstance.getEndTime()).isEqualTo(1572868801000L); assertThat(stepInstance.getTotalTime()).isEqualTo(1111L); assertThat(stepInstance.getCreateTime()).isEqualTo(1572868800000L); - assertThat(stepInstance.getTargetServers()).isNotNull(); + assertThat(stepInstance.getTargetExecuteObjects()).isNotNull(); List expectedServer = new ArrayList<>(); expectedServer.add(new HostDTO(0L, "127.0.0.1")); - assertThat(stepInstance.getTargetServers().getIpList()).containsAll(expectedServer); + assertThat(stepInstance.getTargetExecuteObjects().getIpList()).containsAll(expectedServer); assertThat(stepInstance.getBatch()).isEqualTo(0); } @@ -104,7 +104,7 @@ public void testAddStepInstanceBase() { List ipList = new ArrayList<>(); ipList.add(new HostDTO(0L, "127.0.0.1")); servers.setIpList(ipList); - stepInstanceDTO.setTargetServers(servers); + stepInstanceDTO.setTargetExecuteObjects(servers); stepInstanceDTO.setOperator("admin"); stepInstanceDTO.setStatus(RunStatusEnum.SUCCESS); stepInstanceDTO.setExecuteCount(0); @@ -125,10 +125,10 @@ public void testAddStepInstanceBase() { assertThat(returnStepInstance.getStepId()).isEqualTo(1L); assertThat(returnStepInstance.getName()).isEqualTo("task1-step1"); assertThat(returnStepInstance.getExecuteType()).isEqualTo(StepExecuteTypeEnum.EXECUTE_SCRIPT.getValue()); - assertThat(returnStepInstance.getTargetServers().getIpList()).hasSize(1); + assertThat(returnStepInstance.getTargetExecuteObjects().getIpList()).hasSize(1); List expectedServer = new ArrayList<>(); expectedServer.add(new HostDTO(0L, "127.0.0.1")); - assertThat(returnStepInstance.getTargetServers().getIpList()).containsAll(expectedServer); + assertThat(returnStepInstance.getTargetExecuteObjects().getIpList()).containsAll(expectedServer); assertThat(returnStepInstance.getOperator()).isEqualTo("admin"); assertThat(returnStepInstance.getStatus()).isEqualTo(RunStatusEnum.SUCCESS); assertThat(returnStepInstance.getExecuteCount()).isEqualTo(0); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastExecuteSQLResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastExecuteSQLResourceImpl.java index 345b08b089..0ba8c69691 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastExecuteSQLResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastExecuteSQLResourceImpl.java @@ -196,7 +196,7 @@ private StepInstanceDTO buildFastSQLStepInstance(String username, EsbFastExecute JobConstants.DEFAULT_JOB_TIMEOUT_SECONDS : request.getTimeout()); stepInstance.setExecuteType(StepExecuteTypeEnum.EXECUTE_SQL.getValue()); stepInstance.setStatus(RunStatusEnum.BLANK); - stepInstance.setTargetServers(convertToStandardServers(request.getTargetServer(), request.getIpList(), + stepInstance.setTargetExecuteObjects(convertToStandardServers(request.getTargetServer(), request.getIpList(), request.getDynamicGroupIdList())); AccountDTO account = accountService.getAccountById(request.getDbAccountId()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastExecuteScriptResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastExecuteScriptResourceImpl.java index 5d0c028eb5..78e6b2a4ba 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastExecuteScriptResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastExecuteScriptResourceImpl.java @@ -211,7 +211,7 @@ private StepInstanceDTO buildFastScriptStepInstance(String username, EsbFastExec stepInstance.setExecuteType(StepExecuteTypeEnum.EXECUTE_SCRIPT.getValue()); stepInstance.setStatus(RunStatusEnum.BLANK); - stepInstance.setTargetServers(convertToStandardServers(request.getTargetServer(), request.getIpList(), + stepInstance.setTargetExecuteObjects(convertToStandardServers(request.getTargetServer(), request.getIpList(), request.getDynamicGroupIdList())); AccountDTO account = accountService.getSystemAccountByAlias(request.getAccount(), request.getAppId()); if (account == null) { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastPushFileResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastPushFileResourceImpl.java index 8fd9a1a53c..ec56439c88 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastPushFileResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastPushFileResourceImpl.java @@ -218,7 +218,7 @@ private StepInstanceDTO buildFastFileStepInstance(String username, EsbFastPushFi stepInstance.setFileTargetPath(request.getTargetPath()); stepInstance.setFileSourceList(convertFileSource(request.getAppId(), request.getFileSources())); stepInstance.setAppId(request.getAppId()); - stepInstance.setTargetServers(convertToStandardServers(request.getTargetServer(), request.getIpList(), + stepInstance.setTargetExecuteObjects(convertToStandardServers(request.getTargetServer(), request.getIpList(), request.getDynamicGroupIdList())); stepInstance.setOperator(username); stepInstance.setStatus(RunStatusEnum.BLANK); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetJobInstanceLogResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetJobInstanceLogResourceImpl.java index 4b2d5f49aa..72bfddc15f 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetJobInstanceLogResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetJobInstanceLogResourceImpl.java @@ -40,9 +40,9 @@ import com.tencent.bk.job.execute.api.esb.v2.EsbGetJobInstanceLogResource; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.model.ExecuteObjectTaskDetail; -import com.tencent.bk.job.execute.model.FileIpLogContent; +import com.tencent.bk.job.execute.model.FileExecuteObjectLogContent; import com.tencent.bk.job.execute.model.ResultGroupDTO; -import com.tencent.bk.job.execute.model.ScriptHostLogContent; +import com.tencent.bk.job.execute.model.ScriptExecuteObjectLogContent; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.esb.v2.EsbStepInstanceResultAndLog; import com.tencent.bk.job.execute.model.esb.v2.request.EsbGetJobInstanceLogRequest; @@ -171,14 +171,14 @@ private void addLogContent(StepInstanceBaseDTO stepInstance, List hostLogContentList = logService.batchGetScriptHostLogContent(jobCreateDate, + List hostLogContentList = logService.batchGetScriptExecuteObjectLogContent(jobCreateDate, stepInstance.getId(), stepInstance.getExecuteCount(), null, queryHosts); if (CollectionUtils.isEmpty(hostLogContentList)) { @@ -184,7 +184,7 @@ private void buildFileLogs(EsbIpLogsV3DTO esbIpLogs, StepInstanceBaseDTO stepIns List queryHosts) { esbIpLogs.setLogType(LogTypeEnum.FILE.getValue()); - ServiceExecuteObjectLogsDTO ipLogs = logService.batchGetFileIpLogContent( + ServiceExecuteObjectLogsDTO ipLogs = logService.batchGetFileExecuteObjectLogContent( stepInstance.getId(), stepInstance.getExecuteCount(), null, queryHosts); if (ipLogs == null || CollectionUtils.isEmpty(ipLogs.getIpLogs())) { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastExecuteSQLV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastExecuteSQLV3ResourceImpl.java index 7f8c17f971..22e3764d03 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastExecuteSQLV3ResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastExecuteSQLV3ResourceImpl.java @@ -185,7 +185,7 @@ private StepInstanceDTO buildFastSQLStepInstance(String username, EsbFastExecute JobConstants.DEFAULT_JOB_TIMEOUT_SECONDS : request.getTimeout()); stepInstance.setExecuteType(StepExecuteTypeEnum.EXECUTE_SQL.getValue()); stepInstance.setStatus(RunStatusEnum.BLANK); - stepInstance.setTargetServers(convertToServersDTO(request.getTargetServer())); + stepInstance.setTargetExecuteObjects(convertToServersDTO(request.getTargetServer())); stepInstance.setDbAccountId(request.getDbAccountId()); stepInstance.setOperator(username); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastExecuteScriptV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastExecuteScriptV3ResourceImpl.java index e7156b6b8f..70a97af242 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastExecuteScriptV3ResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastExecuteScriptV3ResourceImpl.java @@ -218,7 +218,7 @@ private StepInstanceDTO buildFastScriptStepInstance(String username, EsbFastExec stepInstance.setExecuteType(StepExecuteTypeEnum.EXECUTE_SCRIPT.getValue()); stepInstance.setStatus(RunStatusEnum.BLANK); - stepInstance.setTargetServers(convertToServersDTO(request.getTargetServer())); + stepInstance.setTargetExecuteObjects(convertToServersDTO(request.getTargetServer())); stepInstance.setAccountId(request.getAccountId()); stepInstance.setAccountAlias(request.getAccountAlias()); stepInstance.setOperator(username); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastTransferFileV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastTransferFileV3ResourceImpl.java index 58752a3f37..a3ba93b10c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastTransferFileV3ResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastTransferFileV3ResourceImpl.java @@ -253,7 +253,7 @@ private StepInstanceDTO buildFastFileStepInstance(String username, stepInstance.setFileTargetName(request.getTargetName()); stepInstance.setFileSourceList(convertFileSource(request.getFileSources())); stepInstance.setAppId(request.getAppId()); - stepInstance.setTargetServers(convertToServersDTO(request.getTargetServer())); + stepInstance.setTargetExecuteObjects(convertToServersDTO(request.getTargetServer())); stepInstance.setOperator(username); stepInstance.setStatus(RunStatusEnum.BLANK); stepInstance.setCreateTime(DateUtils.currentTimeMillis()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceIpLogV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceIpLogV3ResourceImpl.java index ac8bbe5efc..b7ef9f3412 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceIpLogV3ResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceIpLogV3ResourceImpl.java @@ -38,8 +38,8 @@ import com.tencent.bk.job.common.model.ValidateResult; import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.service.AppScopeMappingService; -import com.tencent.bk.job.execute.model.FileIpLogContent; -import com.tencent.bk.job.execute.model.ScriptHostLogContent; +import com.tencent.bk.job.execute.model.FileExecuteObjectLogContent; +import com.tencent.bk.job.execute.model.ScriptExecuteObjectLogContent; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.esb.v3.EsbFileLogV3DTO; import com.tencent.bk.job.execute.model.esb.v3.EsbIpLogV3DTO; @@ -129,7 +129,7 @@ private ValidateResult checkRequest(EsbGetJobInstanceIpLogV3Request request) { private void buildScriptLog(EsbIpLogV3DTO ipLog, Long stepInstanceId, Integer executeCount, Long hostId, Long cloudAreaId, String ip) { ipLog.setLogType(LogTypeEnum.SCRIPT.getValue()); - ScriptHostLogContent logContent = logService.getScriptHostLogContent(stepInstanceId, + ScriptExecuteObjectLogContent logContent = logService.getScriptExecuteObjectLogContent(stepInstanceId, executeCount, null, HostDTO.fromHostIdOrCloudIp(hostId, cloudAreaId, ip)); if (logContent != null && StringUtils.isNotBlank(logContent.getContent())) { ipLog.setScriptLogContent(logContent.getContent()); @@ -142,9 +142,9 @@ private void buildScriptLog(EsbIpLogV3DTO ipLog, Long stepInstanceId, Integer ex private void buildFileLog(EsbIpLogV3DTO ipLog, Long stepInstanceId, Integer executeCount, Long hostId, Long cloudAreaId, String ip) { ipLog.setLogType(LogTypeEnum.FILE.getValue()); - FileIpLogContent downloadIpLog = logService.getFileIpLogContent(stepInstanceId, executeCount, null, + FileExecuteObjectLogContent downloadIpLog = logService.getFileExecuteObjectLogContent(stepInstanceId, executeCount, null, HostDTO.fromHostIdOrCloudIp(hostId, cloudAreaId, ip), FileDistModeEnum.DOWNLOAD.getValue()); - List uploadTaskLogs = logService.batchGetFileSourceIpLogContent( + List uploadTaskLogs = logService.batchGetFileSourceExecuteObjectLogContent( stepInstanceId, executeCount, null); List fileLogs = new ArrayList<>(); @@ -172,7 +172,7 @@ private void buildFileLog(EsbIpLogV3DTO ipLog, Long stepInstanceId, Integer exec ipLog.setFileLogs(fileLogs); } - private List buildDownloadFileLogs(FileIpLogContent downloadIpLog) { + private List buildDownloadFileLogs(FileExecuteObjectLogContent downloadIpLog) { List downloadFileLogs = new ArrayList<>(); if (downloadIpLog != null && CollectionUtils.isNotEmpty(downloadIpLog.getFileTaskLogs())) { downloadFileLogs = downloadIpLog.getFileTaskLogs().stream().map(this::toEsbFileLogV3DTO) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceDetailV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceDetailV3ResourceImpl.java index a80cc4a559..bc73985f44 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceDetailV3ResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceDetailV3ResourceImpl.java @@ -129,7 +129,7 @@ private EsbStepV3DTO convertToEsbStepV3DTO(StepInstanceDTO stepInstance) { } scriptStepInfo.setScriptTimeout(getTimeout(stepInstance)); scriptStepInfo.setAccount(getAccount(stepInstance)); - scriptStepInfo.setServer(stepInstance.getTargetServers().toEsbServerV3DTO()); + scriptStepInfo.setServer(stepInstance.getTargetExecuteObjects().toEsbServerV3DTO()); scriptStepInfo.setIgnoreError(stepInstance.isIgnoreError() ? 1 : 0); esbStepV3DTO.setScriptInfo(scriptStepInfo); break; @@ -221,7 +221,7 @@ private EsbFileDestinationV3DTO getFileDestination(StepInstanceDTO stepInstance) EsbFileDestinationV3DTO esbFileDestinationV3DTO = new EsbFileDestinationV3DTO(); esbFileDestinationV3DTO.setPath(stepInstance.getFileTargetPath()); esbFileDestinationV3DTO.setAccount(getAccount(stepInstance)); - esbFileDestinationV3DTO.setServer(stepInstance.getTargetServers().toEsbServerV3DTO()); + esbFileDestinationV3DTO.setServer(stepInstance.getTargetExecuteObjects().toEsbServerV3DTO()); return esbFileDestinationV3DTO; } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbPushConfigFileResourceV3Impl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbPushConfigFileResourceV3Impl.java index c822b49230..b517af4b23 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbPushConfigFileResourceV3Impl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbPushConfigFileResourceV3Impl.java @@ -153,7 +153,7 @@ private StepInstanceDTO buildFastFileStepInstance( stepInstance.setFileTargetPath(request.getTargetPath()); stepInstance.setFileSourceList(convertConfigFileSource(username, configFileList)); stepInstance.setAppId(request.getAppId()); - stepInstance.setTargetServers(convertToServersDTO(request.getTargetServer())); + stepInstance.setTargetExecuteObjects(convertToServersDTO(request.getTargetServer())); stepInstance.setOperator(username); stepInstance.setStatus(RunStatusEnum.BLANK); stepInstance.setCreateTime(DateUtils.currentTimeMillis()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebExecuteTaskResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebExecuteTaskResourceImpl.java index 580bf6ff41..0b4602a93a 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebExecuteTaskResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebExecuteTaskResourceImpl.java @@ -321,7 +321,7 @@ private StepInstanceDTO buildFastScriptStepInstance(String userName, Long appId, stepInstance.setName(request.getName()); stepInstance.setStepId(-1L); stepInstance.setAppId(appId); - stepInstance.setTargetServers(convertToServersDTO(request.getTargetServers())); + stepInstance.setTargetExecuteObjects(convertToServersDTO(request.getTargetServers())); if (request.getScriptLanguage().equals(ScriptTypeEnum.SQL.getValue())) { stepInstance.setDbAccountId(request.getAccount()); stepInstance.setExecuteType(StepExecuteTypeEnum.EXECUTE_SQL.getValue()); @@ -477,7 +477,7 @@ private StepInstanceDTO buildFastFileStepInstance(String userName, Long appId, W stepInstance.setName(request.getName()); ExecuteFileDestinationInfoVO fileDestination = request.getFileDestination(); stepInstance.setAccountId(fileDestination.getAccountId()); - stepInstance.setTargetServers(convertToServersDTO(fileDestination.getServer())); + stepInstance.setTargetExecuteObjects(convertToServersDTO(fileDestination.getServer())); stepInstance.setFileTargetPath(fileDestination.getPath()); stepInstance.setStepId(-1L); stepInstance.setExecuteType(StepExecuteTypeEnum.SEND_FILE.getValue()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java index dbe63d2282..547dbabddd 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java @@ -59,9 +59,9 @@ import com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum; import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; import com.tencent.bk.job.execute.model.ExecuteObjectTaskDetail; -import com.tencent.bk.job.execute.model.FileIpLogContent; +import com.tencent.bk.job.execute.model.FileExecuteObjectLogContent; import com.tencent.bk.job.execute.model.ResultGroupDTO; -import com.tencent.bk.job.execute.model.ScriptHostLogContent; +import com.tencent.bk.job.execute.model.ScriptExecuteObjectLogContent; import com.tencent.bk.job.execute.model.StepExecutionDTO; import com.tencent.bk.job.execute.model.StepExecutionDetailDTO; import com.tencent.bk.job.execute.model.StepExecutionRecordDTO; @@ -632,13 +632,13 @@ public Response getScriptLogContentByHost(String username, Integer batch) { auditAndAuthViewStepInstance(username, appResourceScope, stepInstanceId); - ScriptHostLogContent scriptHostLogContent = logService.getScriptHostLogContent(stepInstanceId, executeCount, + ScriptExecuteObjectLogContent scriptExecuteObjectLogContent = logService.getScriptExecuteObjectLogContent(stepInstanceId, executeCount, batch, HostDTO.fromHostId(hostId)); IpScriptLogContentVO ipScriptLogContentVO = new IpScriptLogContentVO(); - if (scriptHostLogContent != null) { - ipScriptLogContentVO.setDisplayIp(scriptHostLogContent.getCloudIp()); - ipScriptLogContentVO.setLogContent(scriptHostLogContent.getContent()); - ipScriptLogContentVO.setFinished(scriptHostLogContent.isFinished()); + if (scriptExecuteObjectLogContent != null) { + ipScriptLogContentVO.setDisplayIp(scriptExecuteObjectLogContent.getCloudIp()); + ipScriptLogContentVO.setLogContent(scriptExecuteObjectLogContent.getContent()); + ipScriptLogContentVO.setFinished(scriptExecuteObjectLogContent.isFinished()); } return Response.buildSuccessResp(ipScriptLogContentVO); } @@ -826,7 +826,7 @@ public Response getFileLogContentByHost(String username, result.setFileDistributionDetails(fileDistDetailVOS); if ("download".equals(mode)) { - FileIpLogContent downloadLog = logService.getFileIpLogContent(stepInstanceId, executeCount, batch, + FileExecuteObjectLogContent downloadLog = logService.getFileExecuteObjectLogContent(stepInstanceId, executeCount, batch, HostDTO.fromHostIdOrCloudIp(hostId, ip), FileDistModeEnum.DOWNLOAD.getValue()); // downloadLog为null说明步骤还未下发至GSE就被终止 if (downloadLog != null && CollectionUtils.isNotEmpty(downloadLog.getFileTaskLogs())) { @@ -840,7 +840,7 @@ public Response getFileLogContentByHost(String username, } Collections.sort(fileDistDetailVOS); } else { - List fileTaskLogs = logService.batchGetFileSourceIpLogContent(stepInstanceId, + List fileTaskLogs = logService.batchGetFileSourceExecuteObjectLogContent(stepInstanceId, executeCount, batch); if (CollectionUtils.isNotEmpty(fileTaskLogs)) { fileTaskLogs.forEach(fileTaskLog -> { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskInstanceResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskInstanceResourceImpl.java index 55c7441353..7beeb0ba30 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskInstanceResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskInstanceResourceImpl.java @@ -223,7 +223,7 @@ private ExecuteStepVO convertToStepVO(StepInstanceDTO stepInstance) { scriptStepVO.setScriptSource(stepInstance.getScriptSource()); scriptStepVO.setScriptId(stepInstance.getScriptId()); scriptStepVO.setScriptVersionId(stepInstance.getScriptVersionId()); - scriptStepVO.setExecuteTarget(stepInstance.getTargetServers().convertToTaskTargetVO()); + scriptStepVO.setExecuteTarget(stepInstance.getTargetExecuteObjects().convertToTaskTargetVO()); scriptStepVO.setIgnoreError(stepInstance.isIgnoreError() ? 1 : 0); stepVO.setScriptStepInfo(scriptStepVO); } else if (stepType == StepExecuteTypeEnum.SEND_FILE) { @@ -238,7 +238,7 @@ private ExecuteStepVO convertToStepVO(StepInstanceDTO stepInstance) { } else { fileDestinationInfoVO.setPath(stepInstance.getFileTargetPath()); } - fileDestinationInfoVO.setServer(stepInstance.getTargetServers().convertToTaskTargetVO()); + fileDestinationInfoVO.setServer(stepInstance.getTargetExecuteObjects().convertToTaskTargetVO()); fileStepVO.setFileDestination(fileDestinationInfoVO); fileStepVO.setIgnoreError(stepInstance.isIgnoreError() ? 1 : 0); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileAgentTaskDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileAgentTaskDAO.java index b77b91e29e..76dd2557a2 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileAgentTaskDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileAgentTaskDAO.java @@ -25,6 +25,7 @@ package com.tencent.bk.job.execute.dao; import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import com.tencent.bk.job.common.constant.Order; import com.tencent.bk.job.execute.model.ExecuteObjectTask; import com.tencent.bk.job.execute.model.ResultGroupBaseDTO; @@ -37,21 +38,21 @@ * FileAgentTaskDAO */ @Deprecated -@CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x") +@CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA) public interface FileAgentTaskDAO { /** * 批量新增Agent任务 * * @param agentTasks Agent任务列表 */ - void batchSaveAgentTasks(Collection agentTasks); + void batchSaveAgentTasks(Collection agentTasks); /** * 批量更新Agent任务 * * @param agentTasks Agent任务 */ - void batchUpdateAgentTasks(Collection agentTasks); + void batchUpdateAgentTasks(Collection agentTasks); /** * 获取步骤成功执行的Agent任务数量 diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileExecuteObjectTaskDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileExecuteObjectTaskDAO.java index 88c4576206..ddf15f76bf 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileExecuteObjectTaskDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileExecuteObjectTaskDAO.java @@ -41,14 +41,14 @@ public interface FileExecuteObjectTaskDAO { * * @param tasks 任务列表 */ - void batchSaveTasks(Collection tasks); + void batchSaveTasks(Collection tasks); /** * 批量更新任务 * * @param tasks 任务 */ - void batchUpdateTasks(Collection tasks); + void batchUpdateTasks(Collection tasks); /** * 获取步骤成功执行的任务数量 @@ -139,7 +139,7 @@ ExecuteObjectTask getTaskByExecuteObjectId(Long stepInstanceId, Integer executeCount, Integer batch, FileTaskModeEnum mode, - long executeObjectId); + String executeObjectId); /** diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptAgentTaskDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptAgentTaskDAO.java index bad90ed970..04b6bbd989 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptAgentTaskDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptAgentTaskDAO.java @@ -25,6 +25,7 @@ package com.tencent.bk.job.execute.dao; import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import com.tencent.bk.job.common.constant.Order; import com.tencent.bk.job.execute.model.ExecuteObjectTask; import com.tencent.bk.job.execute.model.ResultGroupBaseDTO; @@ -36,21 +37,21 @@ * ScriptAgentTaskDAO */ @Deprecated -@CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x") +@CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA) public interface ScriptAgentTaskDAO { /** * 批量保存Agent任务 * * @param agentTasks Agent任务 */ - void batchSaveAgentTasks(Collection agentTasks); + void batchSaveAgentTasks(Collection agentTasks); /** * 批量更新Agent任务 * * @param agentTasks Agent任务 */ - void batchUpdateAgentTasks(Collection agentTasks); + void batchUpdateAgentTasks(Collection agentTasks); /** * 获取步骤成功执行的Agent任务数量 diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptExecuteObjectTaskDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptExecuteObjectTaskDAO.java index 1c687324df..04eae8f9cb 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptExecuteObjectTaskDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptExecuteObjectTaskDAO.java @@ -40,14 +40,14 @@ public interface ScriptExecuteObjectTaskDAO { * * @param tasks 任务 */ - void batchSaveTasks(Collection tasks); + void batchSaveTasks(Collection tasks); /** * 批量更新任务 * * @param tasks 任务 */ - void batchUpdateTasks(Collection tasks); + void batchUpdateTasks(Collection tasks); /** * 获取步骤成功执行的任务数量 @@ -138,7 +138,7 @@ List listTasks(Long stepInstanceId, ExecuteObjectTask getTaskByExecuteObjectId(Long stepInstanceId, Integer executeCount, Integer batch, - long executeObjectId); + String executeObjectId); /** * 判断步骤实例的执行对象任务记录是否存在 diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java index 54ba2e6e43..a847fc3fa4 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java @@ -25,6 +25,7 @@ package com.tencent.bk.job.execute.dao.impl; import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import com.tencent.bk.job.common.constant.Order; import com.tencent.bk.job.execute.dao.FileAgentTaskDAO; import com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum; @@ -61,7 +62,7 @@ @Repository @Deprecated -@CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x") +@CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA) public class FileAgentTaskDAOImpl implements FileAgentTaskDAO { private static final GseFileAgentTask T_GSE_FILE_AGENT_TASK = GseFileAgentTask.GSE_FILE_AGENT_TASK; @@ -89,7 +90,7 @@ public FileAgentTaskDAOImpl(@Qualifier("job-execute-dsl-context") DSLContext CTX } @Override - public void batchSaveAgentTasks(Collection agentTasks) { + public void batchSaveAgentTasks(Collection agentTasks) { String sql = "insert into gse_file_agent_task (step_instance_id, execute_count, actual_execute_count, batch," + "mode, host_id, agent_id ,gse_task_id,status, start_time, end_time, total_time, error_code)" + " values (?,?,?,?,?,?,?,?,?,?,?,?,?)"; @@ -116,7 +117,7 @@ public void batchSaveAgentTasks(Collection agentTasks) { } @Override - public void batchUpdateAgentTasks(Collection agentTasks) { + public void batchUpdateAgentTasks(Collection agentTasks) { if (CollectionUtils.isEmpty(agentTasks)) { return; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileExecuteObjectTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileExecuteObjectTaskDAOImpl.java index 2cb51bfa45..74ff863908 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileExecuteObjectTaskDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileExecuteObjectTaskDAOImpl.java @@ -38,12 +38,12 @@ public class FileExecuteObjectTaskDAOImpl implements FileExecuteObjectTaskDAO { @Override - public void batchSaveTasks(Collection tasks) { + public void batchSaveTasks(Collection tasks) { } @Override - public void batchUpdateTasks(Collection tasks) { + public void batchUpdateTasks(Collection tasks) { } @@ -96,7 +96,7 @@ public ExecuteObjectTask getTaskByExecuteObjectId(Long stepInstanceId, Integer executeCount, Integer batch, FileTaskModeEnum mode, - long executeObjectId) { + String executeObjectId) { return null; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java index d16285019d..78c3675cbd 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java @@ -25,6 +25,7 @@ package com.tencent.bk.job.execute.dao.impl; import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import com.tencent.bk.job.common.constant.JobConstants; import com.tencent.bk.job.common.constant.Order; import com.tencent.bk.job.execute.dao.ScriptAgentTaskDAO; @@ -61,7 +62,7 @@ @Repository @Deprecated -@CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x") +@CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA) public class ScriptAgentTaskDAOImpl implements ScriptAgentTaskDAO { private static final GseScriptAgentTask T_GSE_SCRIPT_AGENT_TASK = GseScriptAgentTask.GSE_SCRIPT_AGENT_TASK; @@ -91,7 +92,7 @@ public ScriptAgentTaskDAOImpl(@Qualifier("job-execute-dsl-context") DSLContext C } @Override - public void batchSaveAgentTasks(Collection agentTasks) { + public void batchSaveAgentTasks(Collection agentTasks) { String sql = "insert into gse_script_agent_task (step_instance_id, execute_count, actual_execute_count, batch," + " host_id, agent_id, gse_task_id, status, start_time, end_time, total_time, error_code, exit_code, tag," + " log_offset)" @@ -121,7 +122,7 @@ public void batchSaveAgentTasks(Collection agentTasks) { } @Override - public void batchUpdateAgentTasks(Collection agentTasks) { + public void batchUpdateAgentTasks(Collection agentTasks) { if (CollectionUtils.isEmpty(agentTasks)) { return; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptExecuteObjectTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptExecuteObjectTaskDAOImpl.java index f7650a5e7e..a7b3708afd 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptExecuteObjectTaskDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptExecuteObjectTaskDAOImpl.java @@ -37,12 +37,12 @@ public class ScriptExecuteObjectTaskDAOImpl implements ScriptExecuteObjectTaskDAO { @Override - public void batchSaveTasks(Collection tasks) { + public void batchSaveTasks(Collection tasks) { } @Override - public void batchUpdateTasks(Collection tasks) { + public void batchUpdateTasks(Collection tasks) { } @@ -95,7 +95,7 @@ public List listTasksByGseTaskId(Long gseTaskId) { public ExecuteObjectTask getTaskByExecuteObjectId(Long stepInstanceId, Integer executeCount, Integer batch, - long executeObjectId) { + String executeObjectId) { return null; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImpl.java index 9c84112e95..502904de11 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImpl.java @@ -146,7 +146,7 @@ public Long addStepInstanceBase(StepInstanceBaseDTO stepInstance) { stepInstance.getStartTime(), stepInstance.getEndTime(), stepInstance.getTotalTime(), - stepInstance.getTargetServers() == null ? null : JsonUtils.toJson(stepInstance.getTargetServers()), + stepInstance.getTargetExecuteObjects() == null ? null : JsonUtils.toJson(stepInstance.getTargetExecuteObjects()), stepInstance.getCreateTime(), stepInstance.isIgnoreError() ? Byte.valueOf("1") : Byte.valueOf("0"), stepInstance.getStepNum(), @@ -444,7 +444,7 @@ private StepInstanceBaseDTO extractBaseInfo(Record record) { stepInstance.setTotalTime(record.get(t.TOTAL_TIME)); if (StringUtils.isNotBlank(record.get(t.TARGET_SERVERS))) { ExecuteObjectsDTO targetServers = JsonUtils.fromJson(record.get(t.TARGET_SERVERS), ExecuteObjectsDTO.class); - stepInstance.setTargetServers(targetServers); + stepInstance.setTargetExecuteObjects(targetServers); } stepInstance.setCreateTime(record.get(t.CREATE_TIME)); stepInstance.setIgnoreError(JooqDataTypeUtil.toInteger(record.get(t.IGNORE_ERROR)) != null diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/AbstractGseTaskStartCommand.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/AbstractGseTaskStartCommand.java index 86eb268c04..1e9743262c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/AbstractGseTaskStartCommand.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/AbstractGseTaskStartCommand.java @@ -26,8 +26,8 @@ import com.tencent.bk.job.common.constant.TaskVariableTypeEnum; import com.tencent.bk.job.common.gse.GseClient; +import com.tencent.bk.job.common.gse.v2.model.ExecuteObjectGseKey; import com.tencent.bk.job.common.gse.v2.model.GseTaskResponse; -import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.config.JobExecuteConfig; @@ -41,6 +41,7 @@ import com.tencent.bk.job.execute.engine.result.ResultHandleManager; import com.tencent.bk.job.execute.engine.result.ha.ResultHandleTaskKeepaliveManager; import com.tencent.bk.job.execute.model.ExecuteObjectTask; +import com.tencent.bk.job.execute.model.ExecuteObjectTaskDetail; import com.tencent.bk.job.execute.model.GseTaskDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.StepInstanceVariableValuesDTO; @@ -89,9 +90,9 @@ public abstract class AbstractGseTaskStartCommand extends AbstractGseTaskCommand */ protected String requestId; /** - * GSE 目标 Agent 任务, Map + * GSE任务与JOB执行对象任务的映射关系 */ - protected Map targetAgentTaskMap = new HashMap<>(); + protected Map targetExecuteObjectTaskMap = new HashMap<>(); /** * 全局参数分析结果 */ @@ -105,17 +106,9 @@ public abstract class AbstractGseTaskStartCommand extends AbstractGseTaskCommand */ protected Map globalVariables = new HashMap<>(); /** - * Agent ID 与 host 映射关系 + * 执行对象任务列表 */ - protected Map agentIdHostMap; - /** - * hostId 与 host 映射关系 - */ - protected Map hostIdHostMap; - /** - * Agent tasks - */ - protected List agentTasks; + protected List executeObjectTasks; AbstractGseTaskStartCommand(ResultHandleManager resultHandleManager, @@ -162,8 +155,6 @@ public abstract class AbstractGseTaskStartCommand extends AbstractGseTaskCommand this.gseTasksExceptionCounter = gseTasksExceptionCounter; this.requestId = requestId; this.stepInstanceService = stepInstanceService; - this.agentIdHostMap = stepInstanceService.computeStepHosts(stepInstance, HostDTO::getAgentId); - this.hostIdHostMap = stepInstanceService.computeStepHosts(stepInstance, HostDTO::getHostId); } @@ -243,29 +234,31 @@ private boolean startGseTaskIfNotAvailable(StopWatch watch) { } } - private void initAgentTasks() { - this.agentTasks = executeObjectTaskService.listTasksByGseTaskId(gseTask.getId()); - updateUninstalledAgentTasks(this.agentTasks); + private void initExecuteObjectTasks() { + this.executeObjectTasks = executeObjectTaskService.listTasksByGseTaskId(stepInstance, gseTask.getId()); + updateUninstalledExecuteObjectTasks(this.executeObjectTasks); - agentTasks.stream() + executeObjectTasks.stream() .filter(ExecuteObjectTask::isTarget) - .filter(agentTask -> !agentTask.isAgentIdEmpty()) - .forEach(agentTask -> this.targetAgentTaskMap.put(agentTask.getAgentId(), agentTask)); + .filter(executeObjectTask -> !executeObjectTask.getExecuteObject().isAgentIdEmpty()) + .forEach(executeObjectTask -> + this.targetExecuteObjectTaskMap.put( + executeObjectTask.getExecuteObject().toExecuteObjectGseKey(), executeObjectTask)); } - private void updateUninstalledAgentTasks(Collection agentTasks) { - List invalidAgentTasks = agentTasks.stream() - .filter(ExecuteObjectTask::isAgentIdEmpty) + private void updateUninstalledExecuteObjectTasks(Collection executeObjectTasks) { + List invalidExecuteObjectTasks = executeObjectTasks.stream() + .filter(executeObjectTask -> executeObjectTask.getExecuteObject().isAgentIdEmpty()) .collect(Collectors.toList()); - if (CollectionUtils.isNotEmpty(invalidAgentTasks)) { - log.warn("{} contains invalid agent tasks: {}", gseTaskInfo, invalidAgentTasks); - invalidAgentTasks.forEach(agentTask -> { - agentTask.setStatus(ExecuteObjectTaskStatusEnum.AGENT_NOT_INSTALLED); - agentTask.setStartTime(System.currentTimeMillis()); - agentTask.setEndTime(System.currentTimeMillis()); - agentTask.calculateTotalTime(); + if (CollectionUtils.isNotEmpty(invalidExecuteObjectTasks)) { + log.warn("{} contains invalid execute object tasks: {}", gseTaskInfo, invalidExecuteObjectTasks); + invalidExecuteObjectTasks.forEach(executeObjectTask -> { + executeObjectTask.setStatus(ExecuteObjectTaskStatusEnum.AGENT_NOT_INSTALLED); + executeObjectTask.setStartTime(System.currentTimeMillis()); + executeObjectTask.setEndTime(System.currentTimeMillis()); + executeObjectTask.calculateTotalTime(); }); - executeObjectTaskService.batchUpdateTasks(agentTasks); + executeObjectTaskService.batchUpdateTasks(executeObjectTasks); } } @@ -345,7 +338,7 @@ private void finishGseTask(RunStatusEnum gseTaskStatus) { * 初始化执行上下文，在GSE任务下发前调用 */ protected void initExecutionContext() { - initAgentTasks(); + initExecuteObjectTasks(); initVariables(); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/FileGseTaskStartCommand.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/FileGseTaskStartCommand.java index 17c06b664a..295ff7c215 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/FileGseTaskStartCommand.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/FileGseTaskStartCommand.java @@ -28,12 +28,12 @@ import com.tencent.bk.job.common.constant.NotExistPathHandlerEnum; import com.tencent.bk.job.common.gse.GseClient; import com.tencent.bk.job.common.gse.v2.model.Agent; +import com.tencent.bk.job.common.gse.v2.model.ExecuteObjectGseKey; import com.tencent.bk.job.common.gse.v2.model.FileTransferTask; import com.tencent.bk.job.common.gse.v2.model.GseTaskResponse; import com.tencent.bk.job.common.gse.v2.model.SourceFile; import com.tencent.bk.job.common.gse.v2.model.TargetFile; import com.tencent.bk.job.common.gse.v2.model.TransferFileRequest; -import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.util.DataSizeConverter; import com.tencent.bk.job.common.util.FilePathUtils; import com.tencent.bk.job.common.util.date.DateUtils; @@ -44,6 +44,7 @@ import com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum; import com.tencent.bk.job.execute.engine.evict.TaskEvictPolicyExecutor; import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; import com.tencent.bk.job.execute.engine.model.FileDest; import com.tencent.bk.job.execute.engine.model.JobFile; import com.tencent.bk.job.execute.engine.result.FileResultHandleTask; @@ -53,6 +54,7 @@ import com.tencent.bk.job.execute.engine.util.MacroUtil; import com.tencent.bk.job.execute.model.AccountDTO; import com.tencent.bk.job.execute.model.ExecuteObjectTask; +import com.tencent.bk.job.execute.model.ExecuteObjectTaskDetail; import com.tencent.bk.job.execute.model.FileDetailDTO; import com.tencent.bk.job.execute.model.FileSourceDTO; import com.tencent.bk.job.execute.model.GseTaskDTO; @@ -87,7 +89,7 @@ @Slf4j public class FileGseTaskStartCommand extends AbstractGseTaskStartCommand { - private final FileExecuteObjectTaskService fileAgentTaskService; + private final FileExecuteObjectTaskService fileExecuteObjectTaskService; /** * 待分发文件，文件传输的源文件 */ @@ -100,10 +102,11 @@ public class FileGseTaskStartCommand extends AbstractGseTaskStartCommand { * 本地文件的存储根目录 */ private final String fileStorageRootPath; + /** - * GSE 源 Agent 任务, Map + * 文件源-GSE任务与JOB执行对象任务的映射关系 */ - protected Map sourceAgentTaskMap = new HashMap<>(); + protected Map sourceExecuteObjectTaskMap = new HashMap<>(); /** * 源文件与目标文件路径映射关系 */ @@ -117,7 +120,7 @@ public class FileGseTaskStartCommand extends AbstractGseTaskStartCommand { public FileGseTaskStartCommand(ResultHandleManager resultHandleManager, TaskInstanceService taskInstanceService, GseTaskService gseTaskService, - FileExecuteObjectTaskService fileAgentTaskService, + FileExecuteObjectTaskService fileExecuteObjectTaskService, AccountService accountService, TaskInstanceVariableService taskInstanceVariableService, StepInstanceVariableValueService stepInstanceVariableValueService, @@ -140,7 +143,7 @@ public FileGseTaskStartCommand(ResultHandleManager resultHandleManager, super(resultHandleManager, taskInstanceService, gseTaskService, - fileAgentTaskService, + fileExecuteObjectTaskService, accountService, taskInstanceVariableService, stepInstanceVariableValueService, @@ -159,7 +162,7 @@ public FileGseTaskStartCommand(ResultHandleManager resultHandleManager, stepInstance, gseTask, stepInstanceService); - this.fileAgentTaskService = fileAgentTaskService; + this.fileExecuteObjectTaskService = fileExecuteObjectTaskService; this.fileStorageRootPath = fileStorageRootPath; } @@ -175,7 +178,7 @@ protected void initExecutionContext() { // 解析源<->目标文件映射 parseSrcDestFileMap(); // 初始化执行对象任务 - initFileSourceGseAgentTasks(); + initFileSourceExecuteObjectTasks(); // 保存文件子任务的初始状态 saveInitialFileTaskLogs(); } @@ -187,7 +190,7 @@ private void parseSrcDestFileMap() { allSrcDestFileMap = JobSrcFileUtils.buildSourceDestPathMapping(allSrcFiles, targetDir, stepInstance.getFileTargetName()); allSrcDestFileMap.forEach((sreFile, destFile) -> { - if (isAgentInstalled(sreFile.getHost().getAgentId())) { + if (isAgentInstalled(sreFile.getExecuteObject())) { srcDestFileMap.put(sreFile, destFile); } }); @@ -213,7 +216,7 @@ private void resolveFileSource() { private void parseSrcFiles() { allSrcFiles = JobSrcFileUtils.parseSrcFiles(stepInstance, fileStorageRootPath); srcFiles = allSrcFiles.stream() - .filter(file -> isAgentInstalled(file.getHost().getAgentId())) + .filter(file -> isAgentInstalled(file.getExecuteObject())) .collect(Collectors.toSet()); // 设置源文件所在主机账号信息 setAccountInfoForSourceFiles(srcFiles); @@ -268,33 +271,40 @@ private void resolvedTargetPathWithVariable() { } /** - * 初始化源文件服务器上传任务状态 + * 初始化源文件上传任务状态 */ - private void initFileSourceGseAgentTasks() { - Set sourceHosts = new HashSet<>(); + private void initFileSourceExecuteObjectTasks() { + Set sourceExecuteObjects = new HashSet<>(); if (allSrcFiles != null) { for (JobFile sendFile : allSrcFiles) { - if (sendFile.getHost() != null) { - sourceHosts.add(sendFile.getHost()); + if (sendFile.getExecuteObject() != null) { + sourceExecuteObjects.add(sendFile.getExecuteObject()); } } } - List fileSourceGseAgentTasks = new ArrayList<>(); - for (HostDTO sourceHost : sourceHosts) { - ExecuteObjectTask agentTask = new ExecuteObjectTask(stepInstanceId, executeCount, batch, sourceHost.getHostId(), - sourceHost.getAgentId()); - agentTask.setActualExecuteCount(executeCount); - agentTask.setFileTaskMode(FileTaskModeEnum.UPLOAD); - agentTask.setGseTaskId(gseTask.getId()); + List executeObjectTasks = new ArrayList<>(); + boolean isSupportExecuteObject = stepInstance.isSupportExecuteObject(); + for (ExecuteObject sourceExecuteObject : sourceExecuteObjects) { + ExecuteObjectTaskDetail executeObjectTask; + if (isSupportExecuteObject) { + + } else { + + } + + new ExecuteObjectTaskDetail(stepInstanceId, executeCount, batch, Exe); + executeObjectTask.setActualExecuteCount(executeCount); + executeObjectTask.setFileTaskMode(FileTaskModeEnum.UPLOAD); + executeObjectTask.setGseTaskId(gseTask.getId()); if (StringUtils.isNotEmpty(sourceHost.getAgentId())) { - agentTask.setStatus(ExecuteObjectTaskStatusEnum.WAITING); - sourceAgentTaskMap.put(sourceHost.getAgentId(), agentTask); + executeObjectTask.setStatus(ExecuteObjectTaskStatusEnum.WAITING); + sourceExecuteObjectTaskMap.put(sourceHost.getAgentId(), executeObjectTask); } else { - agentTask.setStatus(ExecuteObjectTaskStatusEnum.FAILED); + executeObjectTask.setStatus(ExecuteObjectTaskStatusEnum.FAILED); } - fileSourceGseAgentTasks.add(agentTask); + executeObjectTasks.add(executeObjectTask); } - fileAgentTaskService.batchSaveTasks(fileSourceGseAgentTasks); + fileExecuteObjectTaskService.batchSaveTasks(executeObjectTasks); } @Override @@ -310,11 +320,15 @@ protected GseTaskResponse startGseTask() { return GseTaskResponse.fail(GseTaskResponse.ERROR_CODE_FAIL, "account is empty"); } - List targetAgents = gseClient.buildAgents(targetAgentTaskMap.keySet(), accountInfo.getAccount(), + List targetAgents = gseClient.fillAgentAuthInfo( + targetExecuteObjectTaskMap.values().stream() + .map(executeObjectTask -> executeObjectTask.getExecuteObject().toGseAgent()) + .collect(Collectors.toList()), + accountInfo.getAccount(), accountInfo.getPassword()); // 构造GSE文件分发请求 for (JobFile file : srcFiles) { - Agent srcAgent = gseClient.buildAgent(file.getHost().getAgentId(), file.getAccount(), file.getPassword()); + Agent srcAgent = buildAgent(file.getExecuteObject(), file.getAccount(), file.getPassword()); SourceFile sourceFile = new SourceFile(file.getFileName(), file.getDir(), srcAgent); FileDest fileDest = srcDestFileMap.get(file); @@ -339,6 +353,13 @@ protected GseTaskResponse startGseTask() { return gseClient.asyncTransferFile(request); } + private Agent buildAgent(ExecuteObject executeObject, String user, String password) { + Agent agent = executeObject.toGseAgent(); + agent.setUser(user); + agent.setPwd(password); + return agent; + } + private void setSpeedLimit(TransferFileRequest request, StepInstanceDTO stepInstance) { if (stepInstance.getFileDownloadSpeedLimit() != null && stepInstance.getFileDownloadSpeedLimit() > 0) { // KB -> MB @@ -359,7 +380,7 @@ private void setSpeedLimit(TransferFileRequest request, StepInstanceDTO stepInst */ private void saveInitialFileTaskLogs() { try { - Map logs = new HashMap<>(); + Map logs = new HashMap<>(); addInitialFileUploadTaskLogs(logs); addInitialFileDownloadTaskLogs(logs); // 调用logService写入MongoDB @@ -369,43 +390,42 @@ private void saveInitialFileTaskLogs() { } } - private void addInitialFileUploadTaskLogs(Map logs) { + private void addInitialFileUploadTaskLogs(Map logs) { // 每个要分发的源文件一条上传日志 for (JobFile file : allSrcFiles) { - Long sourceHostId = file.getHost().getHostId(); - ServiceExecuteObjectLogDTO hostTaskLog = initServiceLogDTOIfAbsent(logs, stepInstanceId, executeCount, - sourceHostId, file.getHost().toCloudIp()); - boolean isAgentInstalled = isAgentInstalled(file.getHost().getAgentId()); + ServiceExecuteObjectLogDTO hostTaskLog = initExecuteObjectLogIfAbsent(stepInstance, logs, + file.getExecuteObject()); + boolean isAgentInstalled = isAgentInstalled(file.getExecuteObject()); FileDistStatusEnum status = isAgentInstalled ? FileDistStatusEnum.WAITING : FileDistStatusEnum.FAILED; hostTaskLog.addFileTaskLog( logService.buildUploadServiceFileTaskLogDTO( - file, status, "--", "--", "--", + stepInstance, file, status, "--", "--", "--", isAgentInstalled ? null : "Agent is not installed")); } } - private boolean isAgentInstalled(String agentId) { - return StringUtils.isNotEmpty(agentId); + private boolean isAgentInstalled(ExecuteObject executeObject) { + return !executeObject.isAgentIdEmpty(); } - private void addInitialFileDownloadTaskLogs(Map logs) { + private void addInitialFileDownloadTaskLogs(Map logs) { // 每个目标IP从每个要分发的源文件下载的一条下载日志 - agentTasks.stream() + executeObjectTasks.stream() .filter(ExecuteObjectTask::isTarget) - .forEach(targetAgentTask -> { - HostDTO targetHost = hostIdHostMap.get(targetAgentTask.getHostId()); - boolean isTargetAgentInstalled = isAgentInstalled(targetHost.getAgentId()); - ServiceExecuteObjectLogDTO ipTaskLog = initServiceLogDTOIfAbsent(logs, stepInstanceId, executeCount, - targetHost.getHostId(), targetHost.toCloudIp()); + .forEach(targetExecuteObjectTask -> { + boolean isTargetAgentInstalled = isAgentInstalled(targetExecuteObjectTask.getExecuteObject()); + ServiceExecuteObjectLogDTO executeObjectLogs = initExecuteObjectLogIfAbsent(stepInstance, logs, + targetExecuteObjectTask.getExecuteObject()); for (JobFile file : allSrcFiles) { - boolean isSourceAgentInstalled = isAgentInstalled(file.getHost().getAgentId()); + boolean isSourceAgentInstalled = isAgentInstalled(file.getExecuteObject()); FileDistStatusEnum status = isTargetAgentInstalled && isSourceAgentInstalled ? FileDistStatusEnum.WAITING : FileDistStatusEnum.FAILED; - ipTaskLog.addFileTaskLog( + executeObjectLogs.addFileTaskLog( logService.buildDownloadServiceFileTaskLogDTO( + stepInstance, file, - targetHost, + targetExecuteObjectTask.getExecuteObject(), getDestPath(file), status, "--", @@ -419,21 +439,28 @@ private void addInitialFileDownloadTaskLogs(Map logs, long stepInstanceId, - int executeCount, Long hostId, String cloudIp) { - ServiceExecuteObjectLogDTO hostTaskLog = logs.get(hostId); - if (hostTaskLog == null) { - hostTaskLog = new ServiceExecuteObjectLogDTO(); - hostTaskLog.setStepInstanceId(stepInstanceId); - hostTaskLog.setExecuteCount(executeCount); - hostTaskLog.setHostId(hostId); - hostTaskLog.setCloudIp(cloudIp); - logs.put(hostId, hostTaskLog); + private ServiceExecuteObjectLogDTO initExecuteObjectLogIfAbsent( + StepInstanceDTO stepInstance, + Map logs, + ExecuteObject executeObject + ) { + ServiceExecuteObjectLogDTO executeObjectLogDTO = logs.get(executeObject.toExecuteObjectGseKey()); + if (executeObjectLogDTO == null) { + executeObjectLogDTO = new ServiceExecuteObjectLogDTO(); + executeObjectLogDTO.setStepInstanceId(stepInstanceId); + executeObjectLogDTO.setExecuteCount(executeCount); + } + if (stepInstance.isSupportExecuteObject()) { + executeObjectLogDTO.setExecuteObjectId(executeObject.getId()); + } else { + executeObjectLogDTO.setHostId(executeObject.getHost().getHostId()); + executeObjectLogDTO.setCloudIp(executeObject.getHost().toCloudIp()); } - return hostTaskLog; + logs.put(executeObject.toExecuteObjectGseKey(), executeObjectLogDTO); + return executeObjectLogDTO; } - private void writeLogs(Map executionLogs) { + private void writeLogs(Map executionLogs) { if (log.isDebugEnabled()) { log.debug("Write file task initial logs, executionLogs: {}", executionLogs); } @@ -461,28 +488,28 @@ protected void addResultHandleTask() { taskExecuteMQEventDispatcher, resultHandleTaskKeepaliveManager, taskEvictPolicyExecutor, - fileAgentTaskService, + fileExecuteObjectTaskService, stepInstanceService, gseClient, taskInstance, stepInstance, taskVariablesAnalyzeResult, - targetAgentTaskMap, - sourceAgentTaskMap, + targetExecuteObjectTaskMap, + sourceExecuteObjectTaskMap, gseTask, srcDestFileMap, requestId, - agentTasks); + executeObjectTasks); resultHandleManager.handleDeliveredTask(fileResultHandleTask); } @Override protected boolean checkGseTaskExecutable() { - if (this.targetAgentTaskMap.isEmpty()) { + if (this.targetExecuteObjectTaskMap.isEmpty()) { log.warn("File gse task target agent is empty, can not execute! Set gse task status fail"); return false; } - if (this.sourceAgentTaskMap.isEmpty()) { + if (this.sourceExecuteObjectTaskMap.isEmpty()) { log.warn("File gse task source agent is empty, can not execute! Set gse task status fail"); return false; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/FileGseTaskStopCommand.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/FileGseTaskStopCommand.java index adb318ddba..af4112a89a 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/FileGseTaskStopCommand.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/FileGseTaskStopCommand.java @@ -29,6 +29,7 @@ import com.tencent.bk.job.common.gse.v2.model.TerminateGseTaskRequest; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.model.ExecuteObjectTask; +import com.tencent.bk.job.execute.model.ExecuteObjectTaskDetail; import com.tencent.bk.job.execute.model.GseTaskDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; @@ -57,7 +58,7 @@ public FileGseTaskStopCommand(AgentService agentService, super(agentService, accountService, gseTaskService, - executeObjectTaskService, + executeObjectTaskService, tracer, gseClient, taskInstance, @@ -68,8 +69,9 @@ public FileGseTaskStopCommand(AgentService agentService, @Override public void execute() { log.info("Stop gse file task, gseTask:" + gseTaskInfo); - List agentTasks = executeObjectTaskService.listTasksByGseTaskId(gseTask.getId()); - List agentIds = agentTasks.stream() + List executeObjectTasks = + executeObjectTaskService.listTasksByGseTaskId(stepInstance, gseTask.getId()); + List agentIds = executeObjectTasks.stream() .map(ExecuteObjectTask::getAgentId) .distinct() .collect(Collectors.toList()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/SQLScriptGseTaskStartCommand.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/SQLScriptGseTaskStartCommand.java index e8feebdf47..d0a76a430b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/SQLScriptGseTaskStartCommand.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/SQLScriptGseTaskStartCommand.java @@ -178,7 +178,7 @@ protected ExecuteScriptRequest buildScriptRequest() { AccountDTO accountInfo = getAccountBean(stepInstance.getAccountId(), stepInstance.getAccount(), stepInstance.getAppId()); - List agentList = gseClient.buildAgents(targetAgentTaskMap.keySet(), + List agentList = gseClient.buildAgents(targetExecuteObjectTaskMap.keySet(), accountInfo.getAccount(), accountInfo.getPassword()); builder.addScriptTask(agentList, scriptFilePath, publicScriptName, buildRunSqlShellParams(sqlScriptFileName), diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/ScriptGseTaskStartCommand.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/ScriptGseTaskStartCommand.java index 6962e0a40f..246fdceae4 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/ScriptGseTaskStartCommand.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/ScriptGseTaskStartCommand.java @@ -30,7 +30,6 @@ import com.tencent.bk.job.common.gse.v2.model.Agent; import com.tencent.bk.job.common.gse.v2.model.ExecuteScriptRequest; import com.tencent.bk.job.common.gse.v2.model.GseTaskResponse; -import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.service.VariableResolver; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; @@ -40,6 +39,7 @@ import com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum; import com.tencent.bk.job.execute.engine.evict.TaskEvictPolicyExecutor; import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; import com.tencent.bk.job.execute.engine.model.TaskVariablesAnalyzeResult; import com.tencent.bk.job.execute.engine.result.ResultHandleManager; @@ -52,7 +52,7 @@ import com.tencent.bk.job.execute.engine.variable.VariableResolveResult; import com.tencent.bk.job.execute.engine.variable.VariableResolveUtils; import com.tencent.bk.job.execute.model.AccountDTO; -import com.tencent.bk.job.execute.model.ExecuteObjectTask; +import com.tencent.bk.job.execute.model.ExecuteObjectTaskDetail; import com.tencent.bk.job.execute.model.GseTaskDTO; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; @@ -69,7 +69,7 @@ import com.tencent.bk.job.execute.service.StepInstanceVariableValueService; import com.tencent.bk.job.execute.service.TaskInstanceService; import com.tencent.bk.job.execute.service.TaskInstanceVariableService; -import com.tencent.bk.job.logsvr.model.service.ServiceScriptLogDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceExecuteObjectScriptLogDTO; import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; @@ -81,6 +81,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @Slf4j public class ScriptGseTaskStartCommand extends AbstractGseTaskStartCommand { @@ -268,15 +269,22 @@ private ExecuteScriptRequest buildRequestWithoutAnyParam(StepInstanceDTO stepIns ScriptRequestBuilder builder = new ScriptRequestBuilder(); builder.addScriptFile(scriptFilePath, scriptFileName, scriptContent); - AccountDTO accountInfo = getAccountBean(stepInstance.getAccountId(), stepInstance.getAccount(), - stepInstance.getAppId()); - List agents = gseClient.buildAgents(targetAgentTaskMap.keySet(), - accountInfo.getAccount(), accountInfo.getPassword()); + List agents = buildTargetAgents(); builder.addScriptTask(agents, scriptFilePath, scriptFileName, resolvedScriptParam, timeout); return builder.build(); } + private List buildTargetAgents() { + AccountDTO account = getAccountBean(stepInstance.getAccountId(), stepInstance.getAccount(), + stepInstance.getAppId()); + return gseClient.fillAgentAuthInfo( + targetExecuteObjectTaskMap.values().stream().map(executeObjectTask -> + executeObjectTask.getExecuteObject().toGseAgent()).collect(Collectors.toList()), + account.getAccount(), + account.getPassword()); + } + /** * 创建下发请求-仅包含常量 */ @@ -303,11 +311,9 @@ private ExecuteScriptRequest buildRequestWithConstParamOnly(StepInstanceDTO step String resolvedScriptParam = resolveScriptParamVariables(scriptParam); int timeout = TimeoutUtils.adjustTaskTimeout(stepInstance.getTimeout()); - AccountDTO accountInfo = getAccountBean(stepInstance.getAccountId(), stepInstance.getAccount(), - stepInstance.getAppId()); - List agentList = gseClient.buildAgents(targetAgentTaskMap.keySet(), - accountInfo.getAccount(), accountInfo.getPassword()); - builder.addScriptTask(agentList, scriptFilePath, wrapperScriptFileName, resolvedScriptParam, timeout); + List agents = buildTargetAgents(); + + builder.addScriptTask(agents, scriptFilePath, wrapperScriptFileName, resolvedScriptParam, timeout); return builder.build(); } @@ -424,10 +430,7 @@ private ExecuteScriptRequest buildRequestWithChangeableParam(StepInstanceDTO ste String resolvedScriptParam = resolveScriptParamVariables(scriptParam); int timeout = TimeoutUtils.adjustTaskTimeout(stepInstance.getTimeout()); - AccountDTO accountInfo = getAccountBean(stepInstance.getAccountId(), stepInstance.getAccount(), - stepInstance.getAppId()); - List agentList = gseClient.buildAgents(targetAgentTaskMap.keySet(), - accountInfo.getAccount(), accountInfo.getPassword()); + List agents = buildTargetAgents(); ScriptRequestBuilder builder = new ScriptRequestBuilder(); @@ -459,8 +462,8 @@ private ExecuteScriptRequest buildRequestWithChangeableParam(StepInstanceDTO ste builder.addScriptFile(scriptFilePath, wrapperScriptFileName, wrapperScriptContent); builder.addScriptFile(scriptFilePath, getJobParamScriptFileName, getJobParamScriptContent); - builder.addScriptTask(agentList, scriptFilePath, wrapperScriptFileName, resolvedScriptParam, timeout); - builder.addScriptTask(agentList, scriptFilePath, getJobParamScriptFileName, null, timeout); + builder.addScriptTask(agents, scriptFilePath, wrapperScriptFileName, resolvedScriptParam, timeout); + builder.addScriptTask(agents, scriptFilePath, getJobParamScriptFileName, null, timeout); return builder.build(); } @@ -657,10 +660,10 @@ protected final void addResultHandleTask() { taskInstance, stepInstance, taskVariablesAnalyzeResult, - targetAgentTaskMap, + targetExecuteObjectTaskMap, gseTask, requestId, - agentTasks); + executeObjectTasks); resultHandleManager.handleDeliveredTask(scriptResultHandleTask); } @@ -673,31 +676,32 @@ protected final void handleStartGseTaskError(GseTaskResponse gseTaskResponse) { String errorMsg = "GSE Job failed:" + gseTaskResponse.getErrorMessage(); int errorMsgLength = errorMsg.length(); - List scriptLogs = new ArrayList<>(targetAgentTaskMap.size()); - for (ExecuteObjectTask agentTask : targetAgentTaskMap.values()) { - HostDTO host = agentIdHostMap.get(agentTask.getAgentId()); + List scriptLogs = new ArrayList<>(targetExecuteObjectTaskMap.size()); + for (ExecuteObjectTaskDetail executeObjectTask : targetExecuteObjectTaskMap.values()) { + ExecuteObject executeObject = executeObjectTask.getExecuteObject(); // 日志输出 - ServiceScriptLogDTO scriptLog = logService.buildSystemScriptLog( - host, + ServiceExecuteObjectScriptLogDTO scriptLog = logService.buildSystemScriptLog( + stepInstance, + executeObject, errorMsg, - agentTask.getScriptLogOffset() + errorMsgLength, + executeObjectTask.getScriptLogOffset() + errorMsgLength, now); scriptLogs.add(scriptLog); // AgentTask 结果更新 - agentTask.setGseTaskId(gseTask.getId()); - agentTask.setStartTime(gseTask.getStartTime()); - agentTask.setEndTime(now); - agentTask.setTotalTime(TaskCostCalculator.calculate(gseTask.getStartTime(), now, null)); - agentTask.setStatus(ExecuteObjectTaskStatusEnum.SUBMIT_FAILED); + executeObjectTask.setGseTaskId(gseTask.getId()); + executeObjectTask.setStartTime(gseTask.getStartTime()); + executeObjectTask.setEndTime(now); + executeObjectTask.setTotalTime(TaskCostCalculator.calculate(gseTask.getStartTime(), now, null)); + executeObjectTask.setStatus(ExecuteObjectTaskStatusEnum.SUBMIT_FAILED); } logService.batchWriteScriptLog(taskInstance.getCreateTime(), stepInstanceId, executeCount, batch, scriptLogs); - scriptAgentTaskService.batchUpdateTasks(targetAgentTaskMap.values()); + scriptAgentTaskService.batchUpdateTasks(targetExecuteObjectTaskMap.values()); } @Override protected boolean checkGseTaskExecutable() { - if (this.targetAgentTaskMap.isEmpty()) { + if (this.targetExecuteObjectTaskMap.isEmpty()) { log.warn("Script gse task target agent is empty, can not execute! Set gse task status fail"); return false; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/ScriptGseTaskStopCommand.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/ScriptGseTaskStopCommand.java index c2d23cf6fc..a043be11fb 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/ScriptGseTaskStopCommand.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/ScriptGseTaskStopCommand.java @@ -29,6 +29,7 @@ import com.tencent.bk.job.common.gse.v2.model.TerminateGseTaskRequest; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.model.ExecuteObjectTask; +import com.tencent.bk.job.execute.model.ExecuteObjectTaskDetail; import com.tencent.bk.job.execute.model.GseTaskDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; @@ -57,7 +58,7 @@ public ScriptGseTaskStopCommand(AgentService agentService, super(agentService, accountService, gseTaskService, - executeObjectTaskService, + executeObjectTaskService, tracer, gseClient, taskInstance, @@ -68,8 +69,9 @@ public ScriptGseTaskStopCommand(AgentService agentService, @Override public void execute() { log.info("Stop gse script task, gseTask:" + gseTaskInfo); - List agentTasks = executeObjectTaskService.listTasksByGseTaskId(gseTask.getId()); - List terminateAgentIds = agentTasks.stream() + List executeObjectTasks = + executeObjectTaskService.listTasksByGseTaskId(stepInstance, gseTask.getId()); + List terminateAgentIds = executeObjectTasks.stream() .map(ExecuteObjectTask::getAgentId) .distinct() .collect(Collectors.toList()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java index 13eadda497..62946f1ad0 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java @@ -267,7 +267,7 @@ private void saveGseAgentTasksForStartStep(Long gseTaskId, // 普通步骤，启动的时候需要初始化所有AgentTask List agentTasks = new ArrayList<>( buildInitialGseAgentTasks(stepInstanceId, executeCount, executeCount, batch, - gseTaskId, stepInstance.getTargetServers().getIpList())); + gseTaskId, stepInstance.getTargetExecuteObjects().getIpList())); saveAgentTasks(stepInstance, agentTasks); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/ExecuteObject.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/ExecuteObject.java index 9d9a1e867f..2e1cacb288 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/ExecuteObject.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/ExecuteObject.java @@ -25,16 +25,23 @@ package com.tencent.bk.job.execute.engine.model; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; import com.tencent.bk.job.common.annotation.PersistenceObject; import com.tencent.bk.job.common.constant.ExecuteObjectTypeEnum; +import com.tencent.bk.job.common.gse.v2.model.Agent; +import com.tencent.bk.job.common.gse.v2.model.ExecuteObjectGseKey; import com.tencent.bk.job.common.model.dto.Container; import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.model.ExecuteObjectCompositeKey; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; import lombok.ToString; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +import java.util.Objects; /** * 作业执行对象通用模型 @@ -69,14 +76,23 @@ public class ExecuteObject implements Cloneable { */ private HostDTO host; - public ExecuteObject(String id, Container container) { - this.id = id; + /** + * 执行对象对应的 。不会被持久化 + */ + @JsonIgnore + private ExecuteObjectGseKey executeObjectGseKey; + + /** + * 执行对象对应的组合KEY + */ + private ExecuteObjectCompositeKey executeObjectCompositeKey; + + public ExecuteObject(Container container) { this.type = ExecuteObjectTypeEnum.CONTAINER; this.container = container; } - public ExecuteObject(String id, HostDTO host) { - this.id = id; + public ExecuteObject(HostDTO host) { this.type = ExecuteObjectTypeEnum.HOST; this.host = host; } @@ -99,4 +115,65 @@ public ExecuteObject clone() { } return clone; } + + @JsonIgnore + public boolean isExecuteObjectFeatureEnabled() { + // 如果执行对象的特性生效，那么这里的 ID 不为空 + return StringUtils.isNotEmpty(id); + } + + @JsonIgnore + public boolean isHost() { + return type == ExecuteObjectTypeEnum.HOST; + } + + @JsonIgnore + public boolean isContainer() { + return type == ExecuteObjectTypeEnum.CONTAINER; + } + + public ExecuteObjectGseKey toExecuteObjectGseKey() { + if (executeObjectGseKey != null) { + return executeObjectGseKey; + } + if (isHost()) { + executeObjectGseKey = ExecuteObjectGseKey.ofHost(host.getAgentId()); + } else { + executeObjectGseKey = ExecuteObjectGseKey.ofContainer(container.getAgentId(), container.getContainerId()); + } + return executeObjectGseKey; + } + + @JsonIgnore + public boolean isAgentIdEmpty() { + if (isHost()) { + return StringUtils.isNotEmpty(getHost().getAgentId()); + } else { + return StringUtils.isNotEmpty(getContainer().getAgentId()); + } + } + + public Agent toGseAgent() { + Agent agent = new Agent(); + if (isHost()) { + agent.setAgentId(host.getAgentId()); + } else { + agent.setAgentId(container.getAgentId()); + agent.setContainerId(container.getContainerId()); + } + return agent; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ExecuteObject that = (ExecuteObject) o; + return getExecuteObjectGseKey().equals(that.getExecuteObjectGseKey()); + } + + @Override + public int hashCode() { + return Objects.hash(getExecuteObjectGseKey()); + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/JobFile.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/JobFile.java index 5de4477ef8..f46300f9d0 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/JobFile.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/JobFile.java @@ -24,7 +24,6 @@ package com.tencent.bk.job.execute.engine.model; -import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.util.FilePathUtils; import com.tencent.bk.job.manage.common.consts.task.TaskFileTypeEnum; import lombok.Data; @@ -42,9 +41,9 @@ public class JobFile { */ private TaskFileTypeEnum fileType; /** - * 源文件主机 + * 源文件执行对象 */ - private HostDTO host; + private ExecuteObject executeObject; /** * 文件路径(用户输入) */ @@ -94,7 +93,7 @@ public class JobFile { /** * @param fileType 文件类型 - * @param host 源文件主机 + * @param executeObject 源文件执行对象 * @param filePath 文件路径 * @param dir 目录名称 * @param fileName 文件名 @@ -103,7 +102,7 @@ public class JobFile { * @param displayFilePath 要展示的文件路径 */ public JobFile(TaskFileTypeEnum fileType, - HostDTO host, + ExecuteObject executeObject, String filePath, String dir, String fileName, @@ -111,7 +110,7 @@ public JobFile(TaskFileTypeEnum fileType, String password, String displayFilePath) { this.fileType = fileType; - this.host = host; + this.executeObject = executeObject; this.filePath = filePath; this.dir = dir; this.fileName = fileName; @@ -122,7 +121,7 @@ public JobFile(TaskFileTypeEnum fileType, /** * @param fileType 文件类型 - * @param host 源文件主机 + * @param executeObject 源文件执行对象 * @param filePath 文件路径 * @param displayFilePath 要展示的文件路径 * @param dir 目录名称 @@ -132,7 +131,7 @@ public JobFile(TaskFileTypeEnum fileType, * @param accountAlias 账号别名 */ public JobFile(TaskFileTypeEnum fileType, - HostDTO host, + ExecuteObject executeObject, String filePath, String displayFilePath, String dir, @@ -141,7 +140,7 @@ public JobFile(TaskFileTypeEnum fileType, Long accountId, String accountAlias) { this.fileType = fileType; - this.host = host; + this.executeObject = executeObject; this.filePath = filePath; this.displayFilePath = displayFilePath; this.dir = dir; @@ -184,8 +183,8 @@ public String getUniqueKey() { StringBuilder sb = new StringBuilder(); sb.append(fileType.name()).append(":"); if (fileType == TaskFileTypeEnum.SERVER) { - // 远程文件分发，需要源主机信息才能唯一确定一个源文件 - sb.append(host.getUniqueKey()).append(":"); + // 远程文件分发，需要源执行对象信息才能唯一确定一个源文件 + sb.append(executeObject.toExecuteObjectGseKey()).append(":"); } sb.append(getStandardFilePath()); this.uniqueKey = sb.toString(); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/LogPullProgress.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/LogPullProgress.java index a496896532..b8a6687405 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/LogPullProgress.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/LogPullProgress.java @@ -24,6 +24,7 @@ package com.tencent.bk.job.execute.engine.model; +import com.tencent.bk.job.common.gse.v2.model.ExecuteObjectGseKey; import lombok.Data; /** @@ -32,9 +33,9 @@ @Data public class LogPullProgress { /** - * bk_agent_id + * 执行对象GSE KEY */ - private String agentId; + private ExecuteObjectGseKey executeObjectGseKey; /** * 任务编号 */ diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/AbstractResultHandleTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/AbstractResultHandleTask.java index 78dbc48d7a..0e8aef3719 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/AbstractResultHandleTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/AbstractResultHandleTask.java @@ -26,7 +26,7 @@ import com.tencent.bk.job.common.constant.JobConstants; import com.tencent.bk.job.common.gse.GseClient; -import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.common.gse.v2.model.ExecuteObjectGseKey; import com.tencent.bk.job.common.redis.util.LockUtils; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.common.util.json.JsonUtils; @@ -45,6 +45,7 @@ import com.tencent.bk.job.execute.engine.model.TaskVariablesAnalyzeResult; import com.tencent.bk.job.execute.engine.result.ha.ResultHandleTaskKeepaliveManager; import com.tencent.bk.job.execute.model.ExecuteObjectTask; +import com.tencent.bk.job.execute.model.ExecuteObjectTaskDetail; import com.tencent.bk.job.execute.model.GseTaskDTO; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; @@ -58,7 +59,6 @@ import com.tencent.bk.job.execute.service.TaskInstanceVariableService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; import org.springframework.util.StopWatch; import java.util.Collection; @@ -132,13 +132,13 @@ public abstract class AbstractResultHandleTask implements ContinuousScheduled */ protected GseTaskDTO gseTask; /** - * Agent 任务 + * 执行对象任务列表 */ - protected List agentTasks; + protected List executeObjectTasks; /** - * GSE 目标主机任务执行结果，Map + * GSE任务与JOB执行对象任务的映射关系 */ - protected Map targetAgentTasks; + protected Map targetExecuteObjectTasks; /** * 全局参数分析结果 */ @@ -150,23 +150,19 @@ public abstract class AbstractResultHandleTask implements ContinuousScheduled /** * 任务包含的所有目标服务器 */ - protected Set targetAgentIds = new HashSet<>(); + protected Set targetExecuteObjectGseKeys = new HashSet<>(); /** * 未结束的目标服务器 */ - protected Set notFinishedTargetAgentIds = new HashSet<>(); + protected Set notFinishedTargetExecuteObjectGseKeys = new HashSet<>(); /** * 已经分析结果完成的目标服务器 */ - protected Set analyseFinishedTargetAgentIds = new HashSet<>(); + protected Set analyseFinishedTargetExecuteObjectGseKeys = new HashSet<>(); /** * 执行成功的目标服务器 */ - protected Set successTargetAgentIds = new HashSet<>(); - /** - * Agent ID 与 host 映射关系 - */ - protected Map agentIdHostMap; + protected Set successTargetExecuteObjectGseKeys = new HashSet<>(); /** * 任务成功被终止 */ @@ -230,10 +226,10 @@ protected AbstractResultHandleTask(TaskInstanceService taskInstanceService, TaskInstanceDTO taskInstance, StepInstanceDTO stepInstance, TaskVariablesAnalyzeResult taskVariablesAnalyzeResult, - Map targetAgentTasks, + Map targetExecuteObjectTasks, GseTaskDTO gseTask, String requestId, - List agentTasks) { + List executeObjectTasks) { this.taskInstanceService = taskInstanceService; this.gseTaskService = gseTaskService; this.logService = logService; @@ -253,15 +249,14 @@ protected AbstractResultHandleTask(TaskInstanceService taskInstanceService, this.appId = stepInstance.getAppId(); this.stepInstanceId = stepInstance.getId(); this.taskVariablesAnalyzeResult = taskVariablesAnalyzeResult; - this.targetAgentTasks = targetAgentTasks; + this.targetExecuteObjectTasks = targetExecuteObjectTasks; this.gseTask = gseTask; - this.agentTasks = agentTasks; + this.executeObjectTasks = executeObjectTasks; this.gseTaskInfo = buildGseTaskInfo(stepInstance.getTaskInstanceId(), gseTask); - targetAgentTasks.values().forEach(agentTask -> this.targetAgentIds.add(agentTask.getAgentId())); - this.notFinishedTargetAgentIds.addAll(targetAgentIds); - - this.agentIdHostMap = stepInstanceService.computeStepHosts(stepInstance, HostDTO::getAgentId); + targetExecuteObjectTasks.values().forEach(executeObjectTask -> + this.targetExecuteObjectGseKeys.add(executeObjectTask.getExecuteObject().toExecuteObjectGseKey())); + this.notFinishedTargetExecuteObjectGseKeys.addAll(targetExecuteObjectGseKeys); // 如果是执行方案，需要初始化全局变量 if (taskInstance.isPlanInstance()) { @@ -271,7 +266,9 @@ protected AbstractResultHandleTask(TaskInstanceService taskInstanceService, } } - this.hasInvalidHost = agentTasks.stream().anyMatch(agentTask -> StringUtils.isEmpty(agentTask.getAgentId())); + this.hasInvalidHost = + executeObjectTasks.stream().anyMatch( + executeObjectTask -> executeObjectTask.getExecuteObject().isAgentIdEmpty()); } private String buildGseTaskInfo(Long jobInstanceId, GseTaskDTO gseTask) { @@ -443,7 +440,7 @@ private boolean shouldSkipStep() { private void saveStatusWhenSkip() { List notFinishedGseAgentTasks = - targetAgentTasks.values().stream().filter(not(ExecuteObjectTask::isFinished)).collect(Collectors.toList()); + targetExecuteObjectTasks.values().stream().filter(not(ExecuteObjectTask::isFinished)).collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(notFinishedGseAgentTasks)) { notFinishedGseAgentTasks.forEach(agentTask -> { agentTask.setStatus(ExecuteObjectTaskStatusEnum.UNKNOWN); @@ -493,7 +490,8 @@ private boolean checkEmptyGseResult(GseTaskResult gseTaskResult) { if (currentTimeMillis - latestPullGseLogSuccessTimeMillis >= GSE_TASK_EMPTY_RESULT_MAX_TOLERATION_MILLS) { log.warn("[{}]: Execution result log always empty!", gseTaskInfo); this.executeResult = GseTaskExecuteResult.FAILED; - saveFailInfoForUnfinishedAgentTask(ExecuteObjectTaskStatusEnum.LOG_ERROR, "Execution result log always empty."); + saveFailInfoForUnfinishedAgentTask(ExecuteObjectTaskStatusEnum.LOG_ERROR, "Execution result log " + + "always empty."); finishGseTask(GseTaskExecuteResult.FAILED, true); isAbnormal = true; } @@ -508,7 +506,8 @@ private boolean checkPullResult(GseLogBatchPullResult gseLogBatchPullResult) log.error("[{}] Pull gse task result error, errorMsg: {}", gseTaskInfo, gseLogBatchPullResult.getErrorMsg()); this.executeResult = GseTaskExecuteResult.FAILED; - saveFailInfoForUnfinishedAgentTask(ExecuteObjectTaskStatusEnum.LOG_ERROR, gseLogBatchPullResult.getErrorMsg()); + saveFailInfoForUnfinishedAgentTask(ExecuteObjectTaskStatusEnum.LOG_ERROR, + gseLogBatchPullResult.getErrorMsg()); finishGseTask(GseTaskExecuteResult.FAILED, true); return false; } @@ -519,25 +518,30 @@ private boolean checkPullResult(GseLogBatchPullResult gseLogBatchPullResult) /** * 设置目标gent任务结束状态 * - * @param agentId agentId - * @param startTime 起始时间 - * @param endTime 终止时间 - * @param agentTask 日志 - */ - protected void dealTargetAgentFinish(String agentId, Long startTime, Long endTime, ExecuteObjectTask agentTask) { - log.info("[{}]: Deal target agent finished| agentId={}| startTime:{}, endTime:{}, agentTask:{}", - gseTaskInfo, agentId, startTime, endTime, JsonUtils.toJsonWithoutSkippedFields(agentTask)); - - notFinishedTargetAgentIds.remove(agentId); - analyseFinishedTargetAgentIds.add(agentId); + * @param executeObjectGseKey executeObjectGseKey + * @param startTime 起始时间 + * @param endTime 终止时间 + * @param executeObjectTask 执行对象任务 + */ + protected void dealTargetExecuteObjectFinish(ExecuteObjectGseKey executeObjectGseKey, + Long startTime, + Long endTime, + ExecuteObjectTask executeObjectTask) { + log.info("[{}]: Deal target agent finished| executeObjectGseKey={}| startTime:{}, endTime:{}, " + + "executeObjectTask:{}", + gseTaskInfo, executeObjectGseKey, startTime, endTime, + JsonUtils.toJsonWithoutSkippedFields(executeObjectTask)); + + notFinishedTargetExecuteObjectGseKeys.remove(executeObjectGseKey); + analyseFinishedTargetExecuteObjectGseKeys.add(executeObjectGseKey); if (endTime - startTime <= 0) { - agentTask.setTotalTime(100L); + executeObjectTask.setTotalTime(100L); } else { - agentTask.setTotalTime(endTime - startTime); + executeObjectTask.setTotalTime(endTime - startTime); } - agentTask.setStartTime(startTime); - agentTask.setEndTime(endTime); + executeObjectTask.setStartTime(startTime); + executeObjectTask.setEndTime(endTime); } /** @@ -597,12 +601,12 @@ private RunStatusEnum analyseGseTaskStatus(GseTaskExecuteResult result) { /** * 批量更新AgentTask并重置changed标志 * - * @param agentTasks 执行对象任务列表 + * @param executeObjectTasks 执行对象任务列表 */ - protected void batchSaveChangedGseAgentTasks(Collection agentTasks) { - if (CollectionUtils.isNotEmpty(agentTasks)) { + protected void batchSaveChangedGseAgentTasks(Collection executeObjectTasks) { + if (CollectionUtils.isNotEmpty(executeObjectTasks)) { List changedGseAgentTasks = - agentTasks.stream().filter(ExecuteObjectTask::isChanged).collect(Collectors.toList()); + executeObjectTasks.stream().filter(ExecuteObjectTask::isChanged).collect(Collectors.toList()); executeObjectTaskService.batchUpdateTasks(changedGseAgentTasks); changedGseAgentTasks.forEach(agentTask -> agentTask.setChanged(false)); } @@ -610,16 +614,16 @@ protected void batchSaveChangedGseAgentTasks(Collection agent protected void saveFailInfoForUnfinishedAgentTask(ExecuteObjectTaskStatusEnum status, String errorMsg) { log.info("[{}]: Deal unfinished agent result| notFinishedTargetAgentIds : {}", - gseTaskInfo, notFinishedTargetAgentIds); + gseTaskInfo, notFinishedTargetExecuteObjectGseKeys); long startTime = (gseTask != null && gseTask.getStartTime() != null) ? gseTask.getStartTime() : System.currentTimeMillis(); - for (String agentId : notFinishedTargetAgentIds) { - ExecuteObjectTask agentTask = targetAgentTasks.get(agentId); + for (ExecuteObjectGseKey executeObjectGseKey : notFinishedTargetExecuteObjectGseKeys) { + ExecuteObjectTask agentTask = targetExecuteObjectTasks.get(executeObjectGseKey); agentTask.setStartTime(startTime); agentTask.setEndTime(System.currentTimeMillis()); agentTask.setStatus(status); } - batchSaveChangedGseAgentTasks(targetAgentTasks.values()); + batchSaveChangedGseAgentTasks(targetExecuteObjectTasks.values()); } @Override @@ -675,14 +679,14 @@ public String getTaskType() { * 是否所有目标Agent上的任务都完成 */ protected boolean isAllTargetAgentTasksDone() { - return this.analyseFinishedTargetAgentIds.size() == this.targetAgentIds.size(); + return this.analyseFinishedTargetExecuteObjectGseKeys.size() == this.targetExecuteObjectGseKeys.size(); } /** * 是否所有目标Agent上的任务都执行成功 */ protected boolean isAllTargetAgentTasksSuccess() { - return this.targetAgentIds.size() == this.successTargetAgentIds.size(); + return this.targetExecuteObjectGseKeys.size() == this.successTargetExecuteObjectGseKeys.size(); } /** diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/FileResultHandleTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/FileResultHandleTask.java index 79afd564b4..f99bbd635d 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/FileResultHandleTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/FileResultHandleTask.java @@ -32,9 +32,9 @@ import com.tencent.bk.job.common.gse.constants.GSECode; import com.tencent.bk.job.common.gse.v2.model.AtomicFileTaskResult; import com.tencent.bk.job.common.gse.v2.model.AtomicFileTaskResultContent; +import com.tencent.bk.job.common.gse.v2.model.ExecuteObjectGseKey; import com.tencent.bk.job.common.gse.v2.model.FileTaskResult; import com.tencent.bk.job.common.gse.v2.model.GetTransferFileResultRequest; -import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.util.feature.FeatureExecutionContext; import com.tencent.bk.job.common.util.feature.FeatureIdConstants; import com.tencent.bk.job.common.util.feature.FeatureToggle; @@ -45,6 +45,7 @@ import com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum; import com.tencent.bk.job.execute.engine.evict.TaskEvictPolicyExecutor; import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; import com.tencent.bk.job.execute.engine.model.FileDest; import com.tencent.bk.job.execute.engine.model.FileGseTaskResult; import com.tencent.bk.job.execute.engine.model.GseLogBatchPullResult; @@ -55,6 +56,7 @@ import com.tencent.bk.job.execute.engine.result.ha.ResultHandleTaskKeepaliveManager; import com.tencent.bk.job.execute.engine.util.GseUtils; import com.tencent.bk.job.execute.model.ExecuteObjectTask; +import com.tencent.bk.job.execute.model.ExecuteObjectTaskDetail; import com.tencent.bk.job.execute.model.GseTaskDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; @@ -84,7 +86,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; /** * 文件任务执行结果处理 @@ -92,41 +93,41 @@ @Slf4j public class FileResultHandleTask extends AbstractResultHandleTask { /** - * GSE 源 Agent 任务, Map + * 文件源-GSE任务与JOB执行对象任务的映射关系 */ - private final Map sourceAgentTasks; + private final Map sourceExecuteObjectTaskMap; /** * 任务包含的源服务器 */ - private final Set sourceAgentIds = new HashSet<>(); + private final Set sourceExecuteObjectGseKeys = new HashSet<>(); /** * 已经分析结果完成的源服务器 */ - protected Set analyseFinishedSourceAgentIds = new HashSet<>(); + protected Set analyseFinishedSourceExecuteObjectGseKeys = new HashSet<>(); /** - * 成功的文件下载任务，key: agentId, value: 成功的文件任务名称 + * 成功的文件下载任务，key: ExecuteObjectGseKey, value: 成功的文件任务名称 */ - private final Map> successDownloadFileMap = new HashMap<>(); + private final Map> successDownloadFileMap = new HashMap<>(); /** - * 已经结束的文件下载任务，key: agentId, value: 已完成文件任务名称 + * 已经结束的文件下载任务，key: ExecuteObjectGseKey, value: 已完成文件任务名称 */ - private final Map> finishedDownloadFileMap = new HashMap<>(); + private final Map> finishedDownloadFileMap = new HashMap<>(); /** - * 下载文件任务数，key: agentId, value: 主机对应的文件任务数 + * 下载文件任务数，key: ExecuteObjectGseKey, value: 主机对应的文件任务数 */ - private final Map fileDownloadTaskNumMap = new HashMap<>(); + private final Map fileDownloadTaskNumMap = new HashMap<>(); /** - * 成功的文件上传任务，key: agentId, value: 成功的文件任务名称 + * 成功的文件上传任务，key: ExecuteObjectGseKey, value: 成功的文件任务名称 */ - private final Map> successUploadFileMap = new HashMap<>(); + private final Map> successUploadFileMap = new HashMap<>(); /** - * 已经结束的文件上传任务，key: agentId, value: 已完成文件任务名称 + * 已经结束的文件上传任务，key: ExecuteObjectGseKey, value: 已完成文件任务名称 */ - private final Map> finishedUploadFileMap = new HashMap<>(); + private final Map> finishedUploadFileMap = new HashMap<>(); /** - * 上传文件任务数，key: agentId, value: 主机对应的文件任务数 + * 上传文件任务数，key: ExecuteObjectGseKey, value: 主机对应的文件任务数 */ - private final Map fileUploadTaskNumMap = new HashMap<>(); + private final Map fileUploadTaskNumMap = new HashMap<>(); /** * 文件任务进度表 */ @@ -138,7 +139,7 @@ public class FileResultHandleTask extends AbstractResultHandleTask> compatibleDestSrcMap = null; /** - * 源文件 + * 源文件. key: file-key; value: 文件 */ private final Map srcFilesMap = new HashMap<>(); @@ -149,7 +150,7 @@ public class FileResultHandleTask extends AbstractResultHandleTask notFinishedSourceAgentIds = new HashSet<>(); + protected Set notFinishedSourceExecuteObjectGseKeys = new HashSet<>(); /** * 文件任务执行结果处理调度策略 */ @@ -159,9 +160,9 @@ public class FileResultHandleTask extends AbstractResultHandleTask targetAgentTasks, - Map sourceAgentTasks, + Map targetExecuteObjectTaskMap, + Map sourceExecuteObjectTaskMap, GseTaskDTO gseTask, Map srcDestFileMap, String requestId, - List agentTasks) { + List executeObjectTasks) { super(taskInstanceService, gseTaskService, logService, @@ -198,11 +199,11 @@ public FileResultHandleTask(TaskInstanceService taskInstanceService, taskInstance, stepInstance, taskVariablesAnalyzeResult, - targetAgentTasks, + targetExecuteObjectTaskMap, gseTask, requestId, - agentTasks); - this.sourceAgentTasks = sourceAgentTasks; + executeObjectTasks); + this.sourceExecuteObjectTaskMap = sourceExecuteObjectTaskMap; this.srcDestFileMap = srcDestFileMap; initSrcFilesMap(srcDestFileMap.keySet()); @@ -211,41 +212,43 @@ public FileResultHandleTask(TaskInstanceService taskInstanceService, log.info("InitFileResultHandleTask|stepInstanceId: {}|fileSourceAgentIds: {}|targetAgentIds: {}|" + "fileUploadTaskNumMap: {}|fileDownloadTaskNumMap: {}", - stepInstance.getId(), sourceAgentIds, targetAgentIds, fileUploadTaskNumMap, + stepInstance.getId(), sourceExecuteObjectGseKeys, targetExecuteObjectGseKeys, fileUploadTaskNumMap, fileDownloadTaskNumMap); } private void initSrcFilesMap(Collection srcFiles) { srcFiles.forEach(srcFile -> srcFilesMap.put( - buildSrcFileKey(srcFile.getHost().getAgentId(), srcFile.getStandardFilePath()), + buildSrcFileKey(srcFile.getExecuteObject().getExecuteObjectGseKey(), srcFile.getStandardFilePath()), srcFile) ); } - private String buildSrcFileKey(String srcAgentId, String standardSrcFilePath) { - return srcAgentId + ":" + standardSrcFilePath; + private String buildSrcFileKey(ExecuteObjectGseKey executeObjectGseKey, String standardSrcFilePath) { + return executeObjectGseKey.getKey() + ":" + standardSrcFilePath; } /** * 初始化文件任务计数器 */ private void initFileTaskNumMap() { - for (String agentId : this.targetAgentIds) { - this.fileDownloadTaskNumMap.put(agentId, this.srcDestFileMap.size()); + for (ExecuteObjectGseKey executeObjectGseKey : this.targetExecuteObjectGseKeys) { + this.fileDownloadTaskNumMap.put(executeObjectGseKey, this.srcDestFileMap.size()); } for (JobFile sendFile : this.srcDestFileMap.keySet()) { - String agentId = sendFile.getHost().getAgentId(); - this.fileUploadTaskNumMap.put(agentId, this.fileUploadTaskNumMap.get(agentId) == null ? 1 : - (this.fileUploadTaskNumMap.get(agentId) + 1)); + ExecuteObjectGseKey executeObjectGseKey = sendFile.getExecuteObject().toExecuteObjectGseKey(); + this.fileUploadTaskNumMap.put(executeObjectGseKey, + this.fileUploadTaskNumMap.get(executeObjectGseKey) == null ? + 1 : (this.fileUploadTaskNumMap.get(executeObjectGseKey) + 1)); } } private void initSourceAgentIds() { - sourceAgentTasks.values().forEach(agentTask -> { - this.notFinishedSourceAgentIds.add(agentTask.getAgentId()); - this.sourceAgentIds.add(agentTask.getAgentId()); + sourceExecuteObjectTaskMap.values().forEach(executeObjectTask -> { + this.notFinishedSourceExecuteObjectGseKeys.add( + executeObjectTask.getExecuteObject().toExecuteObjectGseKey()); + this.sourceExecuteObjectGseKeys.add(executeObjectTask.getExecuteObject().toExecuteObjectGseKey()); }); } @@ -255,12 +258,12 @@ GseLogBatchPullResult pullGseTaskResultInBatches() { request.setGseV2Task(gseV2Task); request.setTaskId(gseTask.getGseTaskId()); - if (CollectionUtils.isNotEmpty(this.analyseFinishedSourceAgentIds) - || CollectionUtils.isNotEmpty(this.analyseFinishedTargetAgentIds)) { - List notFinishedAgentIds = new ArrayList<>(); - notFinishedAgentIds.addAll(notFinishedSourceAgentIds); - notFinishedAgentIds.addAll(notFinishedTargetAgentIds); - request.setAgentIds(notFinishedAgentIds.stream().distinct().collect(Collectors.toList())); + if (CollectionUtils.isNotEmpty(this.analyseFinishedSourceExecuteObjectGseKeys) + || CollectionUtils.isNotEmpty(this.analyseFinishedTargetExecuteObjectGseKeys)) { + List notFinishedExecuteObjectGseKeys = new ArrayList<>(); + notFinishedExecuteObjectGseKeys.addAll(notFinishedSourceExecuteObjectGseKeys); + notFinishedExecuteObjectGseKeys.addAll(notFinishedTargetExecuteObjectGseKeys); + request.batchAddAgentQuery(notFinishedExecuteObjectGseKeys); } FileTaskResult result = gseClient.getTransferFileResult(request); GseLogBatchPullResult pullResult; @@ -296,8 +299,9 @@ GseTaskExecuteResult analyseGseTaskResult(GseTaskResult taskDeta AtomicFileTaskResultContent content = result.getContent(); boolean isDownloadResult = content.isDownloadMode(); - String agentId = isDownloadResult ? content.getDestAgentId() : content.getSourceAgentId(); - analyseFileResult(agentId, jobAtomicFileTaskResult, executionLogs, isDownloadResult); + ExecuteObjectGseKey executeObjectGseKey = isDownloadResult ? + content.getDestExecuteObjectGseKey() : content.getSourceExecuteObjectGseKey(); + analyseFileResult(executeObjectGseKey, jobAtomicFileTaskResult, executionLogs, isDownloadResult); } watch.stop(); @@ -308,8 +312,8 @@ GseTaskExecuteResult analyseGseTaskResult(GseTaskResult taskDeta // 保存任务执行结果 watch.start("saveAgentTasks"); - batchSaveChangedGseAgentTasks(targetAgentTasks.values()); - batchSaveChangedGseAgentTasks(sourceAgentTasks.values()); + batchSaveChangedGseAgentTasks(targetExecuteObjectTasks.values()); + batchSaveChangedGseAgentTasks(sourceExecuteObjectTaskMap.values()); watch.stop(); log.info("[{}] Analyse gse task result -> " + @@ -317,10 +321,10 @@ GseTaskExecuteResult analyseGseTaskResult(GseTaskResult taskDeta "analyseFinishedTargetAgentIds={}, analyseFinishedSourceAgentIds={}, finishedDownloadFileMap={}, " + "successDownloadFileMap={}, finishedUploadFileMap={}, successUploadFileMap={}", this.gseTaskInfo, - this.notFinishedTargetAgentIds, - this.notFinishedSourceAgentIds, - this.analyseFinishedTargetAgentIds, - this.analyseFinishedSourceAgentIds, + this.notFinishedTargetExecuteObjectGseKeys, + this.notFinishedSourceExecuteObjectGseKeys, + this.analyseFinishedTargetExecuteObjectGseKeys, + this.analyseFinishedSourceExecuteObjectGseKeys, this.finishedDownloadFileMap, this.successDownloadFileMap, this.finishedUploadFileMap, @@ -349,8 +353,8 @@ private void compatibleProtocolBeforeV2(AtomicFileTaskResultContent content) { if (map == null) { map = new HashMap<>(); } - if (StringUtils.isNotBlank(jobFile.getHost().getIp())) { - map.put(jobFile.getHost().getIp(), jobFile); + if (StringUtils.isNotBlank(jobFile.getExecuteObject().getHost().getIp())) { + map.put(jobFile.getExecuteObject().getHost().getIp(), jobFile); } return map; }); @@ -372,7 +376,7 @@ private void compatibleProtocolBeforeV2(AtomicFileTaskResultContent content) { destPath, srcIp); return; } - content.setSourceAgentId(srcFile.getHost().toCloudIp()); + content.setSourceAgentId(srcFile.getExecuteObject().getHost().toCloudIp()); content.setSourceFileDir(srcFile.getDir()); content.setSourceFileName(srcFile.getFileName()); // 重置 @@ -390,11 +394,11 @@ private boolean isSupportProtocolBeforeV2() { ); } - private void analyseFileResult(String agentId, + private void analyseFileResult(ExecuteObjectGseKey executeObjectGseKey, JobAtomicFileTaskResult result, Map executionLogs, boolean isDownloadResult) { - ExecuteObjectTask agentTask = result.getAgentTask(); + ExecuteObjectTask agentTask = result.getExecuteObjectTask(); if (agentTask.getStartTime() == null) { agentTask.setStartTime(System.currentTimeMillis()); } @@ -405,10 +409,10 @@ private void analyseFileResult(String agentId, analyseRunningFileResult(result, executionLogs, agentTask); break; case FINISHED: - analyseSuccessFileResult(agentId, result, executionLogs, isDownloadResult); + analyseSuccessFileResult(executeObjectGseKey, result, executionLogs, isDownloadResult); break; case TERMINATE: - analyseTerminatedFileResult(agentId, result, executionLogs, isDownloadResult); + analyseTerminatedFileResult(executeObjectGseKey, result, executionLogs, isDownloadResult); break; default: analyseFailedFileResult(result, executionLogs, isDownloadResult); @@ -424,7 +428,7 @@ private void analyseRunningFileResult(JobAtomicFileTaskResult result, agentTask.setStartTime(result.getResult().getContent().getStartTime()); } - private void analyseSuccessFileResult(String agentId, + private void analyseSuccessFileResult(ExecuteObjectGseKey executeObjectGseKey, JobAtomicFileTaskResult result, Map executionLogs, boolean isDownloadResult) { @@ -432,17 +436,17 @@ private void analyseSuccessFileResult(String agentId, parseExecutionLog(result, executionLogs); if (isDownloadResult) { addFinishedFile(true, true, - content.getDestAgentId(), content.getTaskId()); + content.getDestExecuteObjectGseKey(), content.getTaskId()); } else { addFinishedFile(true, false, - content.getSourceAgentId(), content.getTaskId()); + content.getSourceExecuteObjectGseKey(), content.getTaskId()); } // 分析日志，更新successAgentIds、notStartedAgentIds等状态集合 - analyseAgentTaskResult(GSECode.AtomicErrorCode.FINISHED.getValue(), agentId, content.getStartTime(), - content.getEndTime(), isDownloadResult); + analyseAgentTaskResult(GSECode.AtomicErrorCode.FINISHED.getValue(), executeObjectGseKey, + content.getStartTime(), content.getEndTime(), isDownloadResult); } - private void analyseTerminatedFileResult(String agentId, + private void analyseTerminatedFileResult(ExecuteObjectGseKey executeObjectGseKey, JobAtomicFileTaskResult result, Map executionLogs, boolean isDownloadResult) { @@ -450,21 +454,22 @@ private void analyseTerminatedFileResult(String agentId, parseExecutionLog(result, executionLogs); if (isDownloadResult) { addFinishedFile(false, true, - content.getDestAgentId(), content.getTaskId()); + content.getDestExecuteObjectGseKey(), content.getTaskId()); } else { addFinishedFile(false, false, - content.getSourceAgentId(), content.getTaskId()); + content.getSourceExecuteObjectGseKey(), content.getTaskId()); } - analyseAgentTaskResult(GSECode.AtomicErrorCode.TERMINATE.getValue(), agentId, content.getStartTime(), - content.getEndTime(), isDownloadResult); + analyseAgentTaskResult(GSECode.AtomicErrorCode.TERMINATE.getValue(), executeObjectGseKey, + content.getStartTime(), content.getEndTime(), isDownloadResult); this.isTerminatedSuccess = true; } private JobAtomicFileTaskResult buildJobAtomicFileTaskResult(AtomicFileTaskResult atomicFileTaskResult) { AtomicFileTaskResultContent content = atomicFileTaskResult.getContent(); boolean isDownloadResult = content.isDownloadMode(); - String agentId = isDownloadResult ? content.getDestAgentId() : content.getSourceAgentId(); - JobFile srcFile = srcFilesMap.get(buildSrcFileKey(content.getSourceAgentId(), + ExecuteObjectGseKey executeObjectGseKey = isDownloadResult ? + content.getDestExecuteObjectGseKey() : content.getSourceExecuteObjectGseKey(); + JobFile srcFile = srcFilesMap.get(buildSrcFileKey(content.getSourceExecuteObjectGseKey(), content.getStandardSourceFilePath())); if (srcFile == null) { log.error("Src file not found, sourceAgentId: {}, filePath:{}", content.getSourceAgentId(), @@ -472,31 +477,46 @@ private JobAtomicFileTaskResult buildJobAtomicFileTaskResult(AtomicFileTaskResul throw new InternalException("Parse src file fail", ErrorCode.INTERNAL_ERROR); } - ExecuteObjectTask agentTask = getAgentTask(isDownloadResult, agentId); + ExecuteObjectTask executeObjectTask = getExecuteObjectTask(isDownloadResult, executeObjectGseKey); - HostDTO sourceHost = agentIdHostMap.get(content.getSourceAgentId()); - if (sourceHost == null) { - log.error("Src host not found, sourceAgentId: {}", content.getSourceAgentId()); - throw new InternalException("Parse src host fail", ErrorCode.INTERNAL_ERROR); + ExecuteObject sourceExecuteObject = findExecuteObject(sourceExecuteObjectTaskMap, + content.getSourceExecuteObjectGseKey()); + if (sourceExecuteObject == null) { + log.error("Src execute object not found, sourceExecuteObjectGseKey: {}", + content.getSourceExecuteObjectGseKey()); + throw new InternalException("Parse src execute object fail", ErrorCode.INTERNAL_ERROR); } - HostDTO targetHost = null; + ExecuteObject targetExecuteObject = null; if (isDownloadResult) { - targetHost = agentIdHostMap.get(content.getDestAgentId()); - if (targetHost == null) { - log.error("Src host not found, sourceAgentId: {}", content.getSourceAgentId()); + targetExecuteObject = findExecuteObject(targetExecuteObjectTasks, + content.getDestExecuteObjectGseKey()); + if (targetExecuteObject == null) { + log.error("Target execute object not found, destExecuteObjectGseKey: {}", + content.getDestExecuteObjectGseKey()); throw new InternalException("Parse target host fail", ErrorCode.INTERNAL_ERROR); } } - return new JobAtomicFileTaskResult(atomicFileTaskResult, sourceHost, targetHost, srcFile, agentTask); + return new JobAtomicFileTaskResult(atomicFileTaskResult, sourceExecuteObject, targetExecuteObject, + srcFile, executeObjectTask); } - private ExecuteObjectTask getAgentTask(boolean isDownloadResult, String agentId) { + private ExecuteObject findExecuteObject(Map executeObjectTaskMap, + ExecuteObjectGseKey executeObjectGseKey) { + ExecuteObjectTaskDetail executeObjectTask = executeObjectTaskMap.get(executeObjectGseKey); + if (executeObjectGseKey == null) { + return null; + } + return executeObjectTask.getExecuteObject(); + } + + private ExecuteObjectTaskDetail getExecuteObjectTask(boolean isDownloadResult, + ExecuteObjectGseKey executeObjectGseKey) { if (isDownloadResult) { - return targetAgentTasks.get(agentId); + return targetExecuteObjectTasks.get(executeObjectGseKey); } else { - return sourceAgentTasks.get(agentId); + return sourceExecuteObjectTaskMap.get(executeObjectGseKey); } } @@ -541,7 +561,7 @@ private GseTaskExecuteResult analyseExecuteResult() { } private boolean isAllSourceAgentTasksDone() { - return this.notFinishedSourceAgentIds.isEmpty() && this.notFinishedTargetAgentIds.isEmpty(); + return this.notFinishedSourceExecuteObjectGseKeys.isEmpty() && this.notFinishedTargetExecuteObjectGseKeys.isEmpty(); } private boolean shouldAnalyse(AtomicFileTaskResult result) { @@ -551,30 +571,30 @@ private boolean shouldAnalyse(AtomicFileTaskResult result) { AtomicFileTaskResultContent content = result.getContent(); boolean isDownloadMode = content.isDownloadMode(); - String agentId = isDownloadMode ? - content.getDestAgentId() : content.getSourceAgentId(); + ExecuteObjectGseKey executeObjectGseKey = isDownloadMode ? + content.getDestExecuteObjectGseKey() : content.getSourceExecuteObjectGseKey(); if (isDownloadMode) { - if (this.analyseFinishedTargetAgentIds.contains(agentId) // 该Agent已经分析结束，不需要再分析 + if (this.analyseFinishedTargetExecuteObjectGseKeys.contains(executeObjectGseKey) // 该Agent已经分析结束，不需要再分析 // 该文件下载任务已结束 - || (this.finishedDownloadFileMap.get(agentId) != null - && this.finishedDownloadFileMap.get(agentId).contains(content.getTaskId()))) { + || (this.finishedDownloadFileMap.get(executeObjectGseKey) != null + && this.finishedDownloadFileMap.get(executeObjectGseKey).contains(content.getTaskId()))) { return false; } // 不属于当前任务的目标Agent - if (!this.fileDownloadTaskNumMap.containsKey(agentId)) { - log.warn("[{}] Unexpected target agentId {}. result: {}", gseTaskInfo, agentId, + if (!this.fileDownloadTaskNumMap.containsKey(executeObjectGseKey)) { + log.warn("[{}] Unexpected target executeObjectGseKey {}. result: {}", gseTaskInfo, executeObjectGseKey, JsonUtils.toJson(result)); return false; } } else { - if (this.analyseFinishedSourceAgentIds.contains(agentId) // 该Agent已经分析结束，不需要再分析 - || (this.finishedUploadFileMap.get(agentId) != null - && this.finishedUploadFileMap.get(agentId).contains(content.getTaskId()))) { + if (this.analyseFinishedSourceExecuteObjectGseKeys.contains(executeObjectGseKey) // 该Agent已经分析结束，不需要再分析 + || (this.finishedUploadFileMap.get(executeObjectGseKey) != null + && this.finishedUploadFileMap.get(executeObjectGseKey).contains(content.getTaskId()))) { return false; } // 不属于当前任务的源Agent - if (!this.fileUploadTaskNumMap.containsKey(agentId)) { - log.warn("[{}] Unexpected source agentId {}. result: {}", gseTaskInfo, agentId, + if (!this.fileUploadTaskNumMap.containsKey(executeObjectGseKey)) { + log.warn("[{}] Unexpected source executeObjectGseKey {}. result: {}", gseTaskInfo, executeObjectGseKey, JsonUtils.toJson(result)); return false; } @@ -595,40 +615,41 @@ private void analyseFailedFileResult(JobAtomicFileTaskResult result, private void dealDownloadTaskFail(AtomicFileTaskResult result, Map executionLogs) { AtomicFileTaskResultContent content = result.getContent(); - dealDownloadTaskFail(executionLogs, content.getSourceAgentId(), content.getStandardSourceFilePath(), - content.getDestAgentId(), content.getStandardDestFilePath(), result.getErrorCode(), + dealDownloadTaskFail(executionLogs, content.getSourceExecuteObjectGseKey(), content.getStandardSourceFilePath(), + content.getDestExecuteObjectGseKey(), content.getStandardDestFilePath(), result.getErrorCode(), buildErrorLogContent(result), content.getStartTime(), content.getEndTime()); } private void dealDownloadTaskFail(Map executionLogs, - String sourceAgentId, + ExecuteObjectGseKey sourceExecuteObjectGseKey, String sourceFilePath, - String targetAgentId, + ExecuteObjectGseKey destExecuteObjectGseKey, String destFilePath, Integer errorCode, String errorMsg, Long startTime, Long endTime) { - JobFile srcFile = srcFilesMap.get(buildSrcFileKey(sourceAgentId, sourceFilePath)); - HostDTO targetHost = agentIdHostMap.get(targetAgentId); + JobFile srcFile = srcFilesMap.get(buildSrcFileKey(sourceExecuteObjectGseKey, sourceFilePath)); + ExecuteObject targetExecuteObject = findExecuteObject(targetExecuteObjectTasks, destExecuteObjectGseKey); FileDest fileDest = srcDestFileMap.get(srcFile); // 记录目标IP单个文件下载任务的结束状态 addFinishedFile(false, true, - targetAgentId, + destExecuteObjectGseKey, AtomicFileTaskResultContent.buildTaskId( FileDistModeEnum.DOWNLOAD.getValue(), - sourceAgentId, + sourceExecuteObjectGseKey, sourceFilePath, - targetAgentId, + destExecuteObjectGseKey, destFilePath) ); // 每个目标IP增加一条下载失败的日志到日志总Map中 addFileTaskLog(executionLogs, logService.buildDownloadServiceFileTaskLogDTO( + stepInstance, srcFile, - targetHost, + targetExecuteObject, fileDest.getDestPath(), FileDistStatusEnum.FAILED, null, @@ -637,7 +658,7 @@ private void dealDownloadTaskFail(Map executio errorMsg ) ); - analyseAgentTaskResult(errorCode, targetAgentId, getTimeOrDefault(startTime), + analyseAgentTaskResult(errorCode, destExecuteObjectGseKey, getTimeOrDefault(startTime), getTimeOrDefault(endTime), true); } @@ -655,16 +676,17 @@ private long getTimeOrDefault(Long time) { private void dealUploadTaskFail(JobAtomicFileTaskResult result, Map executionLogs) { AtomicFileTaskResultContent content = result.getResult().getContent(); - String sourceAgentId = content.getSourceAgentId(); + ExecuteObjectGseKey sourceExecuteObjectGseKey = content.getSourceExecuteObjectGseKey(); JobFile srcFile = result.getSrcFile(); Long startTime = getTimeOrDefault(content.getStartTime()); Long endTime = getTimeOrDefault(content.getEndTime()); // 记录源IP单个文件上传任务的结束状态 - addFinishedFile(false, false, sourceAgentId, content.getTaskId()); + addFinishedFile(false, false, sourceExecuteObjectGseKey, content.getTaskId()); // 增加一条上传源失败的上传日志 addFileTaskLog(executionLogs, logService.buildUploadServiceFileTaskLogDTO( + stepInstance, srcFile, FileDistStatusEnum.FAILED, null, @@ -672,15 +694,16 @@ private void dealUploadTaskFail(JobAtomicFileTaskResult result, null, buildErrorLogContent(result.getResult())) ); - analyseAgentTaskResult(result.getResult().getErrorCode(), sourceAgentId, startTime, endTime, false); + analyseAgentTaskResult(result.getResult().getErrorCode(), sourceExecuteObjectGseKey, startTime, endTime, false); // 源失败了，会影响所有目标IP对应的agent上的download任务 - for (String targetAgentId : this.targetAgentIds) { + for (ExecuteObjectGseKey targetExecuteObjectGseKey : this.targetExecuteObjectGseKeys) { FileDest fileDest = srcDestFileMap.get(srcFile); - dealDownloadTaskFail(executionLogs, - sourceAgentId, + dealDownloadTaskFail( + executionLogs, + sourceExecuteObjectGseKey, content.getStandardSourceFilePath(), - targetAgentId, + targetExecuteObjectGseKey, fileDest.getDestPath(), result.getResult().getErrorCode(), buildErrorLogContent(result.getResult()), @@ -706,28 +729,28 @@ private String buildErrorLogContent(AtomicFileTaskResult result) { /** * 根据errorCode、fileNum、successNum更新successAgentIds状态集合与agentTask状态 * - * @param errorCode GSE错误码 - * @param agentId agentId - * @param fileNum 文件总数 - * @param successNum 成功分发的文件总数 - * @param isDownload 是否为下载结果 - * @param agentTask 执行对象任务 + * @param errorCode GSE错误码 + * @param executeObjectGseKey executeObjectGseKey + * @param fileNum 文件总数 + * @param successNum 成功分发的文件总数 + * @param isDownload 是否为下载结果 + * @param agentTask 执行对象任务 */ private void analyseAgentStatus(int errorCode, - String agentId, + ExecuteObjectGseKey executeObjectGseKey, int fileNum, int successNum, boolean isDownload, ExecuteObjectTask agentTask) { // 文件任务成功数=任务总数 if (successNum >= fileNum) { - if (hasInvalidSourceHost) { + if (hasInvalidSourceExecuteObject) { // 如果包含了非法的源文件主机，即使GSE任务（已过滤非法主机)执行成功，那么对于这个主机来说，整体上任务状态是失败 agentTask.setStatus(ExecuteObjectTaskStatusEnum.FAILED); } else { agentTask.setStatus(ExecuteObjectTaskStatusEnum.SUCCESS); if (isDownload) { - this.successTargetAgentIds.add(agentId); + this.successTargetExecuteObjectGseKeys.add(executeObjectGseKey); } } } else { @@ -742,43 +765,47 @@ private void analyseAgentStatus(int errorCode, /** * 分析日志，更新successAgentIds、notStartedAgentIds等状态集合，用于判定最终整体任务状态 * - * @param errorCode GSE错误码 - * @param agentId agentId - * @param startTime 任务起始时间 - * @param endTime 任务终止时间 - * @param isDownloadResult 是否为下载结果 + * @param errorCode GSE错误码 + * @param executeObjectGseKey executeObjectGseKey + * @param startTime 任务起始时间 + * @param endTime 任务终止时间 + * @param isDownloadResult 是否为下载结果 */ - private void analyseAgentTaskResult(int errorCode, String agentId, long startTime, long endTime, + private void analyseAgentTaskResult(int errorCode, + ExecuteObjectGseKey executeObjectGseKey, + long startTime, + long endTime, boolean isDownloadResult) { int finishedNum; int fileNum; int successNum; if (isDownloadResult) { - finishedNum = this.finishedDownloadFileMap.get(agentId) == null ? 0 : - this.finishedDownloadFileMap.get(agentId).size(); - fileNum = this.fileDownloadTaskNumMap.get(agentId) == null ? 0 : - this.fileDownloadTaskNumMap.get(agentId); - successNum = this.successDownloadFileMap.get(agentId) == null ? 0 : - this.successDownloadFileMap.get(agentId).size(); + finishedNum = this.finishedDownloadFileMap.get(executeObjectGseKey) == null ? 0 : + this.finishedDownloadFileMap.get(executeObjectGseKey).size(); + fileNum = this.fileDownloadTaskNumMap.get(executeObjectGseKey) == null ? 0 : + this.fileDownloadTaskNumMap.get(executeObjectGseKey); + successNum = this.successDownloadFileMap.get(executeObjectGseKey) == null ? 0 : + this.successDownloadFileMap.get(executeObjectGseKey).size(); } else { - finishedNum = this.finishedUploadFileMap.get(agentId) == null ? 0 : - this.finishedUploadFileMap.get(agentId).size(); - successNum = this.successUploadFileMap.get(agentId) == null ? 0 : - this.successUploadFileMap.get(agentId).size(); - fileNum = this.fileUploadTaskNumMap.get(agentId) == null ? 0 : this.fileUploadTaskNumMap.get(agentId); + finishedNum = this.finishedUploadFileMap.get(executeObjectGseKey) == null ? 0 : + this.finishedUploadFileMap.get(executeObjectGseKey).size(); + successNum = this.successUploadFileMap.get(executeObjectGseKey) == null ? 0 : + this.successUploadFileMap.get(executeObjectGseKey).size(); + fileNum = this.fileUploadTaskNumMap.get(executeObjectGseKey) == null ? + 0 : this.fileUploadTaskNumMap.get(executeObjectGseKey); } - ExecuteObjectTask agentTask = getAgentTask(isDownloadResult, agentId); + ExecuteObjectTask agentTask = getExecuteObjectTask(isDownloadResult, executeObjectGseKey); if (finishedNum >= fileNum) { - log.info("[{}] Analyse Agent task finished! agentId: {}, finishedTaskNum: {}, expectedTaskNum: {}", - gseTaskInfo, agentId, finishedNum, fileNum); + log.info("[{}] Analyse Agent task finished! executeObjectGseKey: {}, finishedTaskNum: {}, " + + "expectedTaskNum: {}", gseTaskInfo, executeObjectGseKey, finishedNum, fileNum); // 更新AgentTask结果 if (isDownloadResult) { - dealTargetAgentFinish(agentId, startTime, endTime, agentTask); + dealTargetExecuteObjectFinish(executeObjectGseKey, startTime, endTime, agentTask); } else { - dealUploadAgentFinished(agentId, startTime, endTime, agentTask); + dealUploadAgentFinished(executeObjectGseKey, startTime, endTime, agentTask); } - analyseAgentStatus(errorCode, agentId, fileNum, successNum, isDownloadResult, agentTask); + analyseAgentStatus(errorCode, executeObjectGseKey, fileNum, successNum, isDownloadResult, agentTask); } else { agentTask.setStatus(ExecuteObjectTaskStatusEnum.RUNNING); } @@ -787,30 +814,36 @@ private void analyseAgentTaskResult(int errorCode, String agentId, long startTim /** * 设置源执行对象任务结束状态 * - * @param agentId agentId - * @param startTime 起始时间 - * @param endTime 终止时间 - * @param agentTask 日志 + * @param executeObjectGseKey executeObjectGseKey + * @param startTime 起始时间 + * @param endTime 终止时间 + * @param executeObjectTask 执行对象任务 */ - private void dealUploadAgentFinished(String agentId, Long startTime, Long endTime, ExecuteObjectTask agentTask) { - log.info("[{}]: Deal source agent finished| agentId={}| startTime:{}, endTime:{}, agentTask:{}", - gseTaskInfo, agentId, startTime, endTime, JsonUtils.toJsonWithoutSkippedFields(agentTask)); - - this.notFinishedSourceAgentIds.remove(agentId); - this.analyseFinishedSourceAgentIds.add(agentId); + private void dealUploadAgentFinished(ExecuteObjectGseKey executeObjectGseKey, + Long startTime, + Long endTime, + ExecuteObjectTask executeObjectTask) { + log.info("[{}]: Deal source agent finished| executeObjectGseKey={}| startTime:{}, endTime:{}, " + + "executeObjectTask:{}", + gseTaskInfo, executeObjectGseKey, startTime, endTime, + JsonUtils.toJsonWithoutSkippedFields(executeObjectTask)); + + this.notFinishedSourceExecuteObjectGseKeys.remove(executeObjectGseKey); + this.analyseFinishedSourceExecuteObjectGseKeys.add(executeObjectGseKey); if (endTime - startTime <= 0) { - agentTask.setTotalTime(100L); + executeObjectTask.setTotalTime(100L); } else { - agentTask.setTotalTime(endTime - startTime); + executeObjectTask.setTotalTime(endTime - startTime); } - agentTask.setStartTime(startTime); - agentTask.setEndTime(endTime); + executeObjectTask.setStartTime(startTime); + executeObjectTask.setEndTime(endTime); } - /* + /** * 从执行结果生成执行日志 */ - private void parseExecutionLog(JobAtomicFileTaskResult result, Map executionLogs) { + private void parseExecutionLog(JobAtomicFileTaskResult result, + Map executionLogs) { AtomicFileTaskResultContent content = result.getResult().getContent(); Integer mode = content.getMode(); JobFile srcFile = result.getSrcFile(); @@ -847,12 +880,12 @@ private void parseExecutionLog(JobAtomicFileTaskResult result, Map execu /** * 向某个IP上传/下载文件的已结束状态Map、已成功状态Map中添加记录 * - * @param isSuccess 单个文件上传/下载任务是否成功 - * @param isDownloadMode 是否为下载 - * @param agentId bk_agent_id - * @param taskId 单个文件任务唯一Key + * @param isSuccess 单个文件上传/下载任务是否成功 + * @param isDownloadMode 是否为下载 + * @param executeObjectGseKey executeObjectGseKey + * @param taskId 单个文件任务唯一Key */ - private void addFinishedFile(boolean isSuccess, boolean isDownloadMode, String agentId, String taskId) { + private void addFinishedFile(boolean isSuccess, + boolean isDownloadMode, + ExecuteObjectGseKey executeObjectGseKey, + String taskId) { if (isDownloadMode) { - addFinishedFile(isSuccess, agentId, taskId, finishedDownloadFileMap, successDownloadFileMap); + addFinishedFile(isSuccess, executeObjectGseKey, taskId, finishedDownloadFileMap, successDownloadFileMap); } else { - addFinishedFile(isSuccess, agentId, taskId, finishedUploadFileMap, successUploadFileMap); + addFinishedFile(isSuccess, executeObjectGseKey, taskId, finishedUploadFileMap, successUploadFileMap); } } - private void addFinishedFile(boolean isSuccess, String agentId, String taskId, - Map> finishedFileMap, - Map> successFileMap) { - Set finishedFileSet = finishedFileMap.computeIfAbsent(agentId, k -> Sets.newHashSet()); + private void addFinishedFile(boolean isSuccess, + ExecuteObjectGseKey executeObjectGseKey, + String taskId, + Map> finishedFileMap, + Map> successFileMap) { + Set finishedFileSet = finishedFileMap.computeIfAbsent( + executeObjectGseKey, k -> Sets.newHashSet()); finishedFileSet.add(taskId); if (isSuccess) { - Set successFileSet = successFileMap.computeIfAbsent(agentId, k -> Sets.newHashSet()); + Set successFileSet = + successFileMap.computeIfAbsent(executeObjectGseKey, k -> Sets.newHashSet()); successFileSet.add(taskId); } } @@ -1000,13 +1040,13 @@ private static class JobAtomicFileTaskResult { */ private AtomicFileTaskResult result; /** - * 源主机 + * 源执行对象 */ - private HostDTO sourceHost; + private ExecuteObject sourceExecuteObject; /** - * 目标主机 + * 目标执行对象 */ - private HostDTO targetHost; + private ExecuteObject targetExecuteObject; /** * 源文件 */ @@ -1014,18 +1054,18 @@ private static class JobAtomicFileTaskResult { /** * Agent 任务 */ - private ExecuteObjectTask agentTask; + private ExecuteObjectTask executeObjectTask; public JobAtomicFileTaskResult(AtomicFileTaskResult result, - HostDTO sourceHost, - HostDTO targetHost, + ExecuteObject sourceExecuteObject, + ExecuteObject targetExecuteObject, JobFile srcFile, - ExecuteObjectTask agentTask) { + ExecuteObjectTask executeObjectTask) { this.result = result; - this.sourceHost = sourceHost; - this.targetHost = targetHost; + this.sourceExecuteObject = sourceExecuteObject; + this.targetExecuteObject = targetExecuteObject; this.srcFile = srcFile; - this.agentTask = agentTask; + this.executeObjectTask = executeObjectTask; } } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScriptResultHandleTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScriptResultHandleTask.java index 618a837dad..83fdf9e74c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScriptResultHandleTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScriptResultHandleTask.java @@ -27,8 +27,9 @@ import com.tencent.bk.job.common.gse.GseClient; import com.tencent.bk.job.common.gse.constants.GSECode; import com.tencent.bk.job.common.gse.v1.GseReadTimeoutException; +import com.tencent.bk.job.common.gse.v2.model.ExecuteObjectGseKey; import com.tencent.bk.job.common.gse.v2.model.GetExecuteScriptResultRequest; -import com.tencent.bk.job.common.gse.v2.model.ScriptAgentTaskResult; +import com.tencent.bk.job.common.gse.v2.model.ScriptExecuteObjectTaskResult; import com.tencent.bk.job.common.gse.v2.model.ScriptTaskResult; import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.util.CollectionUtil; @@ -38,6 +39,7 @@ import com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum; import com.tencent.bk.job.execute.engine.evict.TaskEvictPolicyExecutor; import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; import com.tencent.bk.job.execute.engine.model.GseLogBatchPullResult; import com.tencent.bk.job.execute.engine.model.GseTaskExecuteResult; import com.tencent.bk.job.execute.engine.model.GseTaskResult; @@ -48,6 +50,7 @@ import com.tencent.bk.job.execute.engine.result.ha.ResultHandleTaskKeepaliveManager; import com.tencent.bk.job.execute.engine.util.GseUtils; import com.tencent.bk.job.execute.model.ExecuteObjectTask; +import com.tencent.bk.job.execute.model.ExecuteObjectTaskDetail; import com.tencent.bk.job.execute.model.GseTaskDTO; import com.tencent.bk.job.execute.model.HostVariableValuesDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; @@ -61,7 +64,7 @@ import com.tencent.bk.job.execute.service.StepInstanceVariableValueService; import com.tencent.bk.job.execute.service.TaskInstanceService; import com.tencent.bk.job.execute.service.TaskInstanceVariableService; -import com.tencent.bk.job.logsvr.model.service.ServiceScriptLogDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceExecuteObjectScriptLogDTO; import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -95,17 +98,17 @@ public class ScriptResultHandleTask extends AbstractResultHandleTask + * 脚本任务执行日志进度 */ - private final Map logPullProgressMap = new HashMap<>(); + private final Map logPullProgressMap = new HashMap<>(); /** - * 保存命名空间参数，Map> + * 保存命名空间参数，Map> */ - private final Map> namespaceParamValues = new HashMap<>(); + private final Map> namespaceParamValues = new HashMap<>(); /** - * 保存可变全局参数，Map> + * 保存可变全局参数，Map> */ - private final Map> changeableGlobalParamValues = new HashMap<>(); + private final Map> changeableGlobalParamValues = new HashMap<>(); /** * 脚本任务结果处理调度策略 */ @@ -113,7 +116,7 @@ public class ScriptResultHandleTask extends AbstractResultHandleTask> pullAgentIdBatches = new LinkedList<>(); + private List> pullExecuteObjectGseKeyBatches = new LinkedList<>(); /** * 当前使用的批次大小 */ @@ -129,16 +132,6 @@ public class ScriptResultHandleTask extends AbstractResultHandleTask agentTaskMap, + Map executeObjectTaskMap, GseTaskDTO gseTask, String requestId, - List agentTasks) { + List executeObjectTasks) { super(taskInstanceService, gseTaskService, logService, @@ -171,37 +164,42 @@ public ScriptResultHandleTask(TaskInstanceService taskInstanceService, taskInstance, stepInstance, taskVariablesAnalyzeResult, - agentTaskMap, + executeObjectTaskMap, gseTask, requestId, - agentTasks); - initLogPullProcess(agentTaskMap.values()); + executeObjectTasks); + initLogPullProcess(executeObjectTaskMap.values()); } - private void initLogPullProcess(Collection agentTasks) { - agentTasks.forEach(agentTask -> { + private void initLogPullProcess(Collection executeObjectTasks) { + executeObjectTasks.forEach(executeObjectTaskDetail -> { + ExecuteObjectGseKey executeObjectGseKey = + executeObjectTaskDetail.getExecuteObject().toExecuteObjectGseKey(); LogPullProgress process = new LogPullProgress(); - process.setAgentId(agentTask.getAgentId()); - process.setByteOffset(agentTask.getScriptLogOffset()); + process.setExecuteObjectGseKey(executeObjectGseKey); + process.setByteOffset(executeObjectTaskDetail.getScriptLogOffset()); process.setAtomicTaskId(0); - logPullProgressMap.put(agentTask.getAgentId(), process); + logPullProgressMap.put(executeObjectGseKey, process); }); } @Override GseLogBatchPullResult pullGseTaskResultInBatches() { - if (pullAgentIdBatches.isEmpty()) { - List queryAgentIdList = new ArrayList<>(notFinishedTargetAgentIds); - pullAgentIdBatches = CollectionUtil.partitionList(queryAgentIdList, currentBatchSize); + if (pullExecuteObjectGseKeyBatches.isEmpty()) { + List queryExecuteObjectGseKeyList + = new ArrayList<>(notFinishedTargetExecuteObjectGseKeys); + pullExecuteObjectGseKeyBatches = CollectionUtil.partitionList( + queryExecuteObjectGseKeyList, currentBatchSize); } return tryPullGseResultWithRetry(); } private GseLogBatchPullResult tryPullGseResultWithRetry() { - List pullLogAgentIds = pullAgentIdBatches.get(pullResultBatchesIndex.get() - 1); + List pullLogExecuteObjectGseKeys + = pullExecuteObjectGseKeyBatches.get(pullResultBatchesIndex.get() - 1); try { - ScriptTaskResult result = pullGseTaskResult(pullLogAgentIds); - boolean isLastBatch = pullResultBatchesIndex.get() == pullAgentIdBatches.size(); + ScriptTaskResult result = pullGseTaskResult(pullLogExecuteObjectGseKeys); + boolean isLastBatch = pullResultBatchesIndex.get() == pullExecuteObjectGseKeyBatches.size(); GseLogBatchPullResult batchPullResult = new GseLogBatchPullResult<>(true, isLastBatch, new ScriptGseTaskResult(result), null); if (isLastBatch) { @@ -215,7 +213,7 @@ private GseLogBatchPullResult tryPullGseResultWithRetry() { if (isSuccess) { log.info("Reduce batch size and rebuild batch list successfully, currentBatchSize: {}, batches: {}. " + "Retry pull!", - this.currentBatchSize, this.pullAgentIdBatches); + this.currentBatchSize, this.pullExecuteObjectGseKeyBatches); return tryPullGseResultWithRetry(); } else { log.warn("Try pull gse log with min batch size, but fail!"); @@ -226,19 +224,19 @@ private GseLogBatchPullResult tryPullGseResultWithRetry() { private void resetBatch() { pullResultBatchesIndex = new AtomicInteger(1); - pullAgentIdBatches.clear(); + pullExecuteObjectGseKeyBatches.clear(); } - private ScriptTaskResult pullGseTaskResult(List agentIds) { + private ScriptTaskResult pullGseTaskResult(List executeObjectGseKeys) { GetExecuteScriptResultRequest request = new GetExecuteScriptResultRequest(); request.setGseV2Task(gseV2Task); request.setTaskId(gseTask.getGseTaskId()); - agentIds.forEach(agentId -> { - LogPullProgress progress = logPullProgressMap.get(agentId); + executeObjectGseKeys.forEach(executeObjectGseKey -> { + LogPullProgress progress = logPullProgressMap.get(executeObjectGseKey); if (progress == null) { - request.addAgentTaskQuery(agentId, 0, 0); + request.addAgentTaskQuery(executeObjectGseKey, 0, 0); } else { - request.addAgentTaskQuery(agentId, progress.getAtomicTaskId(), progress.getByteOffset()); + request.addAgentTaskQuery(executeObjectGseKey, progress.getAtomicTaskId(), progress.getByteOffset()); } }); @@ -252,16 +250,17 @@ private boolean tryReduceBatchSizeAndRebuildBatchList() { return false; } currentBatchSize = getBatchSizeLessThanCurrentSize(); - List> newBatchList = new ArrayList<>(); - List leftAgentIds = new ArrayList<>(); + List> newBatchList = new ArrayList<>(); + List leftExecuteObjectGseKeys = new ArrayList<>(); if (pullResultBatchesIndex.get() > 1) { - List> pullFinishedBatchList = pullAgentIdBatches.subList(0, pullResultBatchesIndex.get() - 1); + List> pullFinishedBatchList = + pullExecuteObjectGseKeyBatches.subList(0, pullResultBatchesIndex.get() - 1); newBatchList.addAll(pullFinishedBatchList); } - pullAgentIdBatches.subList(pullResultBatchesIndex.get() - 1, pullAgentIdBatches.size()) - .forEach(leftAgentIds::addAll); - newBatchList.addAll(CollectionUtil.partitionList(leftAgentIds, currentBatchSize)); - pullAgentIdBatches = newBatchList; + pullExecuteObjectGseKeyBatches.subList(pullResultBatchesIndex.get() - 1, pullExecuteObjectGseKeyBatches.size()) + .forEach(leftExecuteObjectGseKeys::addAll); + newBatchList.addAll(CollectionUtil.partitionList(leftExecuteObjectGseKeys, currentBatchSize)); + pullExecuteObjectGseKeyBatches = newBatchList; return true; } @@ -281,33 +280,35 @@ GseTaskExecuteResult analyseGseTaskResult(GseTaskResult taskDe return analyseExecuteResult(); } long currentTime = DateUtils.currentTimeMillis(); // 当前时间 - List scriptLogs = new ArrayList<>(); + List scriptLogs = new ArrayList<>(); StopWatch watch = new StopWatch("analyse-gse-script-task"); watch.start("analyse"); - for (ScriptAgentTaskResult agentTaskResult : taskDetail.getResult().getResult()) { - String agentId = agentTaskResult.getAgentId(); - if (!shouldAnalyse(agentTaskResult)) { + for (ScriptExecuteObjectTaskResult executeObjectTaskResult : taskDetail.getResult().getResult()) { + ExecuteObjectGseKey executeObjectGseKey = executeObjectTaskResult.getExecuteObjectGseKey(); + if (!shouldAnalyse(executeObjectTaskResult)) { continue; } - ExecuteObjectTask agentTask = targetAgentTasks.get(agentId); - if (agentTask == null) { - log.warn("[{}] No agent task found for agentId {}. result: {}", gseTaskInfo, agentId, - JsonUtils.toJson(agentTaskResult)); + ExecuteObjectTask executeObjectTask = targetExecuteObjectTasks.get(executeObjectGseKey); + if (executeObjectTask == null) { + log.warn("[{}] No execute object task found for executeObjectGseKey {}. result: {}", + gseTaskInfo, executeObjectGseKey, JsonUtils.toJson(executeObjectTaskResult)); continue; } - log.info("[{}]: Analyse agent task result, result: {}", gseTaskInfo, agentTaskResult); + log.info("[{}]: Analyse execute object task result, result: {}", gseTaskInfo, executeObjectTaskResult); /*为了解决shell上下文传参的问题，在下发用户脚本的时候，实际上下发两个脚本。第一个脚本是用户脚本，第二个脚本 *是获取上下文参数的脚本。所以m_id=0的是用户脚本的执行日志，需要分析记录；m_id=1的，则是获取上下文参数 *输出的日志内容，不需要记录，仅需要从日志分析提取上下文参数*/ - boolean isUserScriptResult = agentTaskResult.getAtomicTaskId() == 0; + boolean isUserScriptResult = executeObjectTaskResult.getAtomicTaskId() == 0; if (isUserScriptResult) { - addScriptLogsAndRefreshPullProgress(scriptLogs, agentTaskResult, agentId, agentTask, currentTime); + addScriptLogsAndRefreshPullProgress(scriptLogs, executeObjectTaskResult, + executeObjectGseKey, executeObjectTask, currentTime); } - analyseAgentResult(agentTaskResult, agentTask, agentId, isUserScriptResult, currentTime); + analyseAgentResult(executeObjectTaskResult, executeObjectTask, executeObjectGseKey, isUserScriptResult, + currentTime); } watch.stop(); @@ -316,11 +317,12 @@ GseTaskExecuteResult analyseGseTaskResult(GseTaskResult taskDe watch.stop(); watch.start("saveGseAgentTasks"); - batchSaveChangedGseAgentTasks(targetAgentTasks.values()); + batchSaveChangedGseAgentTasks(targetExecuteObjectTasks.values()); watch.stop(); log.info("[{}] Analyse gse task result -> notFinishedTargetAgentIds={}, analyseFinishedTargetAgentIds={}", - this.gseTaskInfo, this.notFinishedTargetAgentIds, this.analyseFinishedTargetAgentIds); + this.gseTaskInfo, this.notFinishedTargetExecuteObjectGseKeys, + this.analyseFinishedTargetExecuteObjectGseKeys); GseTaskExecuteResult rst = analyseExecuteResult(); if (!rst.getResultCode().equals(GseTaskExecuteResult.RESULT_CODE_RUNNING)) { @@ -335,76 +337,87 @@ GseTaskExecuteResult analyseGseTaskResult(GseTaskResult taskDe return rst; } - private boolean shouldAnalyse(ScriptAgentTaskResult agentTaskResult) { - String agentId = agentTaskResult.getAgentId(); + private boolean shouldAnalyse(ScriptExecuteObjectTaskResult agentTaskResult) { + ExecuteObjectGseKey executeObjectGseKey = agentTaskResult.getExecuteObjectGseKey(); // 该Agent已经日志分析结束，不要再分析 - if (this.analyseFinishedTargetAgentIds.contains(agentId)) { + if (this.analyseFinishedTargetExecuteObjectGseKeys.contains(executeObjectGseKey)) { return false; } - if (!this.targetAgentIds.contains(agentId)) { - log.warn("[{}] Unexpected target agentId {}. result: {}", gseTaskInfo, agentId, + if (!this.targetExecuteObjectGseKeys.contains(executeObjectGseKey)) { + log.warn("[{}] Unexpected target executeObjectGseKey {}. result: {}", gseTaskInfo, executeObjectGseKey, JsonUtils.toJson(agentTaskResult)); return false; } return true; } - private void addScriptLogsAndRefreshPullProgress(List logs, - ScriptAgentTaskResult agentTaskResult, - String agentId, - ExecuteObjectTask agentTask, + private void addScriptLogsAndRefreshPullProgress(List logs, + ScriptExecuteObjectTaskResult executeObjectTaskResult, + ExecuteObjectGseKey executeObjectGseKey, + ExecuteObjectTask executeObjectTask, long currentTime) { - HostDTO host = agentIdHostMap.get(agentTask.getAgentId()); - if (GSECode.AtomicErrorCode.getErrorCode(agentTaskResult.getErrorCode()) == GSECode.AtomicErrorCode.ERROR) { - logs.add(logService.buildSystemScriptLog(host, - agentTaskResult.getErrorMsg(), agentTask.getScriptLogOffset(), currentTime)); + ExecuteObject executeObject = findExecuteObject(executeObjectGseKey); + if (GSECode.AtomicErrorCode.getErrorCode(executeObjectTaskResult.getErrorCode()) + == GSECode.AtomicErrorCode.ERROR) { + logs.add(logService.buildSystemScriptLog(stepInstance, executeObject, + executeObjectTaskResult.getErrorMsg(), executeObjectTask.getScriptLogOffset(), currentTime)); } else { - String content = agentTaskResult.getScreen(); + String content = executeObjectTaskResult.getScreen(); if (StringUtils.isEmpty(content)) { return; } - int offset = agentTask.getScriptLogOffset(); + int offset = executeObjectTask.getScriptLogOffset(); if (StringUtils.isNotEmpty(content)) { int bytes = content.getBytes(StandardCharsets.UTF_8).length; offset += bytes; - agentTask.setScriptLogOffset(offset); + executeObjectTask.setScriptLogOffset(offset); } - logs.add(new ServiceScriptLogDTO(host, offset, agentTaskResult.getScreen())); + logs.add(logService.buildScriptLog(stepInstance, executeObject, + executeObjectTaskResult.getScreen(), offset)); } // 刷新日志拉取偏移量 - refreshPullLogProgress(agentTaskResult.getScreen(), agentId, agentTaskResult.getAtomicTaskId()); + refreshPullLogProgress(executeObjectTaskResult.getScreen(), executeObjectGseKey, + executeObjectTaskResult.getAtomicTaskId()); } - private void saveScriptLogContent(List logs) { + private ExecuteObject findExecuteObject(ExecuteObjectGseKey executeObjectGseKey) { + return targetExecuteObjectTasks.get(executeObjectGseKey).getExecuteObject(); + } + + private void saveScriptLogContent(List logs) { logService.batchWriteScriptLog(taskInstance.getCreateTime(), stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), logs); } - private void analyseAgentResult(ScriptAgentTaskResult agentResult, ExecuteObjectTask agentTask, String agentId, - boolean isUserScriptResult, long currentTime) { + private void analyseAgentResult(ScriptExecuteObjectTaskResult executeObjectTaskResult, + ExecuteObjectTask executeObjectTask, + ExecuteObjectGseKey executeObjectGseKey, + boolean isUserScriptResult, + long currentTime) { boolean isShellScript = (stepInstance.getScriptType().equals(ScriptTypeEnum.SHELL.getValue())); - if (agentTask.getStartTime() == null) { - agentTask.setStartTime(currentTime); + if (executeObjectTask.getStartTime() == null) { + executeObjectTask.setStartTime(currentTime); } - agentTask.setErrorCode(agentResult.getErrorCode()); - if (GSECode.AtomicErrorCode.getErrorCode(agentResult.getErrorCode()) == GSECode.AtomicErrorCode.ERROR) { + executeObjectTask.setErrorCode(executeObjectTaskResult.getErrorCode()); + if (GSECode.AtomicErrorCode.getErrorCode(executeObjectTaskResult.getErrorCode()) + == GSECode.AtomicErrorCode.ERROR) { // 脚本执行失败 - dealAgentFinish(agentId, agentResult, agentTask); - agentTask.setStatus(GseUtils.getStatusByGseErrorCode(agentResult.getErrorCode())); - } else if (GSECode.AtomicErrorCode.getErrorCode(agentResult.getErrorCode()) + dealExecuteObjectFinish(executeObjectGseKey, executeObjectTaskResult, executeObjectTask); + executeObjectTask.setStatus(GseUtils.getStatusByGseErrorCode(executeObjectTaskResult.getErrorCode())); + } else if (GSECode.AtomicErrorCode.getErrorCode(executeObjectTaskResult.getErrorCode()) == GSECode.AtomicErrorCode.TERMINATE) { - dealAgentFinish(agentId, agentResult, agentTask); - agentTask.setStatus(ExecuteObjectTaskStatusEnum.GSE_TASK_TERMINATE_SUCCESS); + dealExecuteObjectFinish(executeObjectGseKey, executeObjectTaskResult, executeObjectTask); + executeObjectTask.setStatus(ExecuteObjectTaskStatusEnum.GSE_TASK_TERMINATE_SUCCESS); this.isTerminatedSuccess = true; } else { // 分析GSE的返回状态 - GSECode.Status status = GSECode.Status.getStatus(agentResult.getStatus()); + GSECode.Status status = GSECode.Status.getStatus(executeObjectTaskResult.getStatus()); switch (status) { case UNSTART: // 0：原子任务已派发； case RUNNING: // 1：原子任务执行中； - agentTask.setStatus(ExecuteObjectTaskStatusEnum.RUNNING); + executeObjectTask.setStatus(ExecuteObjectTaskStatusEnum.RUNNING); break; case SUCCESS: if (isShellScript && isUserScriptResult) { @@ -412,58 +425,59 @@ private void analyseAgentResult(ScriptAgentTaskResult agentResult, ExecuteObject && (taskVariablesAnalyzeResult.isExistChangeableGlobalVar() || taskVariablesAnalyzeResult.isExistNamespaceVar())) { //对于包含云参或者上下文参数的任务，下发任务的时候包含了2个任务；第一个是执行用户脚本；第二个获取参数的值 - agentTask.setStatus(ExecuteObjectTaskStatusEnum.RUNNING); - refreshPullLogProgress("", agentId, 1); + executeObjectTask.setStatus(ExecuteObjectTaskStatusEnum.RUNNING); + refreshPullLogProgress("", executeObjectGseKey, 1); } else { //普通任务，拉取日志，设置为成功 - dealAgentFinish(agentId, agentResult, agentTask); - agentTask.setStatus(ExecuteObjectTaskStatusEnum.SUCCESS); - if (this.targetAgentIds.contains(agentId)) { - successTargetAgentIds.add(agentId); + dealExecuteObjectFinish(executeObjectGseKey, executeObjectTaskResult, executeObjectTask); + executeObjectTask.setStatus(ExecuteObjectTaskStatusEnum.SUCCESS); + if (this.targetExecuteObjectGseKeys.contains(executeObjectGseKey)) { + successTargetExecuteObjectGseKeys.add(executeObjectGseKey); } } } else { //获取输出参数的任务执行完成，需要分析日志 - dealAgentFinish(agentId, agentResult, agentTask); - agentTask.setStatus(ExecuteObjectTaskStatusEnum.SUCCESS); - if (this.targetAgentIds.contains(agentId)) { - successTargetAgentIds.add(agentId); + dealExecuteObjectFinish(executeObjectGseKey, executeObjectTaskResult, executeObjectTask); + executeObjectTask.setStatus(ExecuteObjectTaskStatusEnum.SUCCESS); + if (this.targetExecuteObjectGseKeys.contains(executeObjectGseKey)) { + successTargetExecuteObjectGseKeys.add(executeObjectGseKey); } - parseVariableValueFromResult(agentResult, agentId); + parseVariableValueFromResult(executeObjectTaskResult, executeObjectGseKey); } if (isUserScriptResult) { - agentTask.setTag(agentResult.getTag()); + executeObjectTask.setTag(executeObjectTaskResult.getTag()); } break; case TIMEOUT: - dealAgentFinish(agentId, agentResult, agentTask); - agentTask.setStatus(ExecuteObjectTaskStatusEnum.SCRIPT_TIMEOUT); + dealExecuteObjectFinish(executeObjectGseKey, executeObjectTaskResult, executeObjectTask); + executeObjectTask.setStatus(ExecuteObjectTaskStatusEnum.SCRIPT_TIMEOUT); break; case DISCARD: - dealAgentFinish(agentId, agentResult, agentTask); - agentTask.setStatus(ExecuteObjectTaskStatusEnum.SCRIPT_TERMINATE); + dealExecuteObjectFinish(executeObjectGseKey, executeObjectTaskResult, executeObjectTask); + executeObjectTask.setStatus(ExecuteObjectTaskStatusEnum.SCRIPT_TERMINATE); break; default: - dealAgentFinish(agentId, agentResult, agentTask); - int errCode = agentResult.getErrorCode(); - int exitCode = agentResult.getExitCode(); + dealExecuteObjectFinish(executeObjectGseKey, executeObjectTaskResult, executeObjectTask); + int errCode = executeObjectTaskResult.getErrorCode(); + int exitCode = executeObjectTaskResult.getExitCode(); if (errCode == 0) { if (exitCode != 0) { - agentTask.setStatus(ExecuteObjectTaskStatusEnum.SCRIPT_NOT_ZERO_EXIT_CODE); + executeObjectTask.setStatus(ExecuteObjectTaskStatusEnum.SCRIPT_NOT_ZERO_EXIT_CODE); } else { - agentTask.setStatus(ExecuteObjectTaskStatusEnum.SCRIPT_FAILED); + executeObjectTask.setStatus(ExecuteObjectTaskStatusEnum.SCRIPT_FAILED); } - agentTask.setTag(agentResult.getTag()); + executeObjectTask.setTag(executeObjectTaskResult.getTag()); } break; } } } - private void parseVariableValueFromResult(ScriptAgentTaskResult agentTaskResult, String agentId) { - if (agentTaskResult.getAtomicTaskId() == 1 - && agentTaskResult.getStatus() == GSECode.Status.SUCCESS.getValue()) { - String paramsContent = agentTaskResult.getScreen(); + private void parseVariableValueFromResult(ScriptExecuteObjectTaskResult executeObjectTaskResult, + ExecuteObjectGseKey executeObjectGseKey) { + if (executeObjectTaskResult.getAtomicTaskId() == 1 + && executeObjectTaskResult.getStatus() == GSECode.Status.SUCCESS.getValue()) { + String paramsContent = executeObjectTaskResult.getScreen(); if (!StringUtils.isEmpty(paramsContent)) { String[] varKeyAndValueStrArray = paramsContent.split("\n"); for (String varKeyAndValueStr : varKeyAndValueStrArray) { @@ -482,12 +496,12 @@ private void parseVariableValueFromResult(ScriptAgentTaskResult agentTaskResult, continue; } if (taskVariablesAnalyzeResult.isNamespaceVar(paramName)) { - Map param2Values = namespaceParamValues.computeIfAbsent(agentId, - k -> new HashMap<>()); + Map param2Values = namespaceParamValues.computeIfAbsent( + executeObjectGseKey, k -> new HashMap<>()); param2Values.put(paramName, parseShellEscapeValue(paramValue)); } else if (taskVariablesAnalyzeResult.isChangeableGlobalVar(paramName)) { - Map param2Values = changeableGlobalParamValues.computeIfAbsent(agentId, - k -> new HashMap<>()); + Map param2Values = changeableGlobalParamValues.computeIfAbsent( + executeObjectGseKey, k -> new HashMap<>()); param2Values.put(paramName, parseShellEscapeValue(paramValue)); } } @@ -539,7 +553,7 @@ private List buildChangeableGlobalVarValues() { return null; } List varValues = null; - for (Map.Entry> entry : changeableGlobalParamValues.entrySet()) { + for (Map.Entry> entry : changeableGlobalParamValues.entrySet()) { // 如果存在多个主机，那么随机选择其中一个主机上面的参数值 varValues = toVariableValuesList(entry.getValue()); } @@ -551,10 +565,14 @@ private List buildNamespaceVariableValues() { return null; } List hostVariableValuesList = new ArrayList<>(); - for (Map.Entry> entry : namespaceParamValues.entrySet()) { + for (Map.Entry> entry : namespaceParamValues.entrySet()) { HostVariableValuesDTO hostVariableValues = new HostVariableValuesDTO(); List paramValues = toVariableValuesList(entry.getValue()); - HostDTO host = agentIdHostMap.get(entry.getKey()); + ExecuteObject executeObject = findExecuteObject((entry.getKey())); + if (!executeObject.isHost()) { + continue; + } + HostDTO host = executeObject.getHost(); hostVariableValues.setHostId(host.getHostId()); hostVariableValues.setCloudIpv4(host.toCloudIp()); hostVariableValues.setCloudIpv6(host.toCloudIpv6()); @@ -576,9 +594,12 @@ private List toVariableValuesList(Map varNameV return variableValues; } - private void dealAgentFinish(String agentId, ScriptAgentTaskResult agentTaskResult, ExecuteObjectTask agentTask) { - dealTargetAgentFinish(agentId, agentTaskResult.getStartTime(), agentTaskResult.getEndTime(), agentTask); - agentTask.setExitCode(getExitCode(agentTaskResult.getExitCode())); + private void dealExecuteObjectFinish(ExecuteObjectGseKey executeObjectGseKey, + ScriptExecuteObjectTaskResult executeObjectTaskResult, + ExecuteObjectTask executeObjectTask) { + dealTargetExecuteObjectFinish(executeObjectGseKey, executeObjectTaskResult.getStartTime(), + executeObjectTaskResult.getEndTime(), executeObjectTask); + executeObjectTask.setExitCode(getExitCode(executeObjectTaskResult.getExitCode())); } private int getExitCode(int exitCode) { @@ -592,20 +613,20 @@ private int getExitCode(int exitCode) { /** * 刷新拉取日志进度 * - * @param context 日志内容 - * @param agentId bk_agent_id - * @param mid gse任务的m_id + * @param context 日志内容 + * @param executeObjectGseKey executeObjectGseKey + * @param mid gse任务的m_id */ - private void refreshPullLogProgress(String context, String agentId, int mid) { + private void refreshPullLogProgress(String context, ExecuteObjectGseKey executeObjectGseKey, int mid) { int increase = 0; if (null != context) { increase = context.getBytes(StandardCharsets.UTF_8).length; } - LogPullProgress progress = logPullProgressMap.get(agentId); + LogPullProgress progress = logPullProgressMap.get(executeObjectGseKey); if (null == progress) { progress = new LogPullProgress(); - logPullProgressMap.put(agentId, progress); + logPullProgressMap.put(executeObjectGseKey, progress); } int prevMid = progress.getAtomicTaskId(); if (prevMid != mid) { @@ -614,7 +635,7 @@ private void refreshPullLogProgress(String context, String agentId, int mid) { } else { progress.setByteOffset(progress.getByteOffset() + increase); } - progress.setAgentId(agentId); + progress.setExecuteObjectGseKey(executeObjectGseKey); progress.setAtomicTaskId(mid); } @@ -638,11 +659,13 @@ protected void saveFailInfoForUnfinishedAgentTask(ExecuteObjectTaskStatusEnum st super.saveFailInfoForUnfinishedAgentTask(status, errorMsg); long endTime = System.currentTimeMillis(); if (StringUtils.isNotEmpty(errorMsg)) { - List scriptLogs = notFinishedTargetAgentIds.stream().map(agentId -> { - ExecuteObjectTask agentTask = targetAgentTasks.get(agentId); - HostDTO host = agentIdHostMap.get(agentId); - return logService.buildSystemScriptLog(host, errorMsg, agentTask.getScriptLogOffset(), endTime); - }).collect(Collectors.toList()); + List scriptLogs = + notFinishedTargetExecuteObjectGseKeys.stream().map(executeObjectGseKey -> { + ExecuteObjectTask executeObjectTask = targetExecuteObjectTasks.get(executeObjectGseKey); + ExecuteObject executeObject = findExecuteObject(executeObjectGseKey); + return logService.buildSystemScriptLog(stepInstance, executeObject, errorMsg, + executeObjectTask.getScriptLogOffset(), endTime); + }).collect(Collectors.toList()); logService.batchWriteScriptLog(taskInstance.getCreateTime(), stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), scriptLogs); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/JobLastHostsVariableResolver.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/JobLastHostsVariableResolver.java index 1b8a270440..a100dac89e 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/JobLastHostsVariableResolver.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/JobLastHostsVariableResolver.java @@ -141,8 +141,8 @@ private List listAgentTasks(StepInstanceDTO stepInstanc private Set extractAllHosts(StepInstanceDTO stepInstance) { Set hosts = new HashSet<>(); - if (CollectionUtils.isNotEmpty(stepInstance.getTargetServers().getIpList())) { - hosts.addAll(stepInstance.getTargetServers().getIpList()); + if (CollectionUtils.isNotEmpty(stepInstance.getTargetExecuteObjects().getIpList())) { + hosts.addAll(stepInstance.getTargetExecuteObjects().getIpList()); } if (CollectionUtils.isNotEmpty(stepInstance.getFileSourceList())) { stepInstance.getFileSourceList().forEach(fileSource -> { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ExecuteObjectCompositeKey.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ExecuteObjectCompositeKey.java index c4acbfd307..dc01d5bb94 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ExecuteObjectCompositeKey.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ExecuteObjectCompositeKey.java @@ -27,70 +27,135 @@ import com.tencent.bk.job.common.constant.ExecuteObjectTypeEnum; import lombok.Getter; import lombok.Setter; -import lombok.ToString; + +import java.util.Objects; /** * 执行对象复合 KEY */ @Getter @Setter -@ToString public class ExecuteObjectCompositeKey { /** - * 执行对象 ID + * 执行对象复合 KEY 的类型 */ - private String executeObjectId; + private CompositeKeyType compositeKeyType; /** * 执行对象类型 */ private ExecuteObjectTypeEnum executeObjectType; + + /** + * 执行对象 ID + */ + private String executeObjectId; + + + /** + * 执行对象对应资源的 ID + */ + private Long resourceId; /** * 主机 ID */ private Long hostId; + /** + * 容器 ID + */ + private Long containerId; + /** * 管控区域+ipv4 方式标识主机 */ private String cloudIp; - /** - * 容器 ID - */ - private Long containerId; + private ExecuteObjectCompositeKey() { + } + + + public static ExecuteObjectCompositeKey of(ExecuteObjectTypeEnum executeObjectType, String executeObjectId) { + ExecuteObjectCompositeKey key = new ExecuteObjectCompositeKey(); + key.setCompositeKeyType(CompositeKeyType.EXECUTE_OBJECT_ID); + key.setExecuteObjectType(executeObjectType); + key.setExecuteObjectId(executeObjectId); + return key; + } - public static ExecuteObjectCompositeKey of(String executeObjectId) { - ExecuteObjectCompositeKey query = new ExecuteObjectCompositeKey(); - query.setExecuteObjectId(executeObjectId); - return query; + public static ExecuteObjectCompositeKey ofHost(Long hostId) { + ExecuteObjectCompositeKey key = new ExecuteObjectCompositeKey(); + key.setCompositeKeyType(CompositeKeyType.RESOURCE_ID); + key.setResourceId(hostId); + key.setExecuteObjectType(ExecuteObjectTypeEnum.HOST); + key.setHostId(hostId); + return key; } - public static ExecuteObjectCompositeKey ofHost(Long hostId, String cloudIp) { - ExecuteObjectCompositeKey query = new ExecuteObjectCompositeKey(); - query.setExecuteObjectType(ExecuteObjectTypeEnum.HOST); - query.setHostId(hostId); - return query; + public static ExecuteObjectCompositeKey ofHost(String cloudIp) { + ExecuteObjectCompositeKey key = new ExecuteObjectCompositeKey(); + key.setCompositeKeyType(CompositeKeyType.HOST_CLOUD_IP); + key.setExecuteObjectType(ExecuteObjectTypeEnum.HOST); + key.setCloudIp(cloudIp); + return key; } public static ExecuteObjectCompositeKey ofContainer(Long containerId) { - ExecuteObjectCompositeKey query = new ExecuteObjectCompositeKey(); - query.setExecuteObjectType(ExecuteObjectTypeEnum.CONTAINER); - query.setContainerId(containerId); - return query; + ExecuteObjectCompositeKey key = new ExecuteObjectCompositeKey(); + key.setCompositeKeyType(CompositeKeyType.RESOURCE_ID); + key.setResourceId(containerId); + key.setExecuteObjectType(ExecuteObjectTypeEnum.CONTAINER); + key.setContainerId(containerId); + return key; } - /** - * 获取执行对象资源 ID - */ - public String getResourceId() { - switch (executeObjectType) { - case HOST: - return hostId != null ? String.valueOf(hostId) : null; - case CONTAINER: - return containerId != null ? String.valueOf(containerId) : null; + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ExecuteObjectCompositeKey that = (ExecuteObjectCompositeKey) o; + if (this.getCompositeKeyType() != that.getCompositeKeyType()) { + return false; + } + switch (this.getCompositeKeyType()) { + case EXECUTE_OBJECT_ID: + return this.getExecuteObjectId().equals(that.getExecuteObjectId()); + case RESOURCE_ID: + return this.getExecuteObjectType() == that.getExecuteObjectType() + && this.getResourceId().equals(that.getResourceId()); + case HOST_CLOUD_IP: + return this.getCloudIp().equals(that.getCloudIp()); + } + return false; + } + + @Override + public int hashCode() { + switch (this.getCompositeKeyType()) { + case EXECUTE_OBJECT_ID: + return Objects.hash(compositeKeyType, executeObjectId); + case RESOURCE_ID: + return Objects.hash(compositeKeyType, resourceId); + case HOST_CLOUD_IP: + return Objects.hash(compositeKeyType, cloudIp); default: - return null; + return 0; + } + } + + private enum CompositeKeyType { + EXECUTE_OBJECT_ID(1), + RESOURCE_ID(2), + HOST_CLOUD_IP(3); + + private final int value; + + CompositeKeyType(int value) { + this.value = value; + } + + public int getValue() { + return value; } } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ExecuteObjectTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ExecuteObjectTask.java index 6929ed75ff..7d7478505e 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ExecuteObjectTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ExecuteObjectTask.java @@ -25,6 +25,7 @@ package com.tencent.bk.job.execute.model; import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum; import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; import lombok.Getter; @@ -63,18 +64,20 @@ public class ExecuteObjectTask { /** * 执行对象 ID */ - private Long executeObjId; + private String executeObjectId; /** * 主机 ID */ @Deprecated - @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x") + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA, + explain = "兼容老数据，数据失效后可删除") private Long hostId; /** * 主机 agentId */ @Deprecated - @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x") + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA, + explain = "兼容老数据，数据失效后可删除") private String agentId; /** * 任务状态 @@ -121,23 +124,23 @@ public class ExecuteObjectTask { */ private volatile boolean changed; - public ExecuteObjectTask(long stepInstanceId, int executeCount, int batch, Long executeObjId) { + public ExecuteObjectTask(long stepInstanceId, int executeCount, int batch, String executeObjectId) { this.stepInstanceId = stepInstanceId; this.executeCount = executeCount; this.batch = batch; - this.executeObjId = executeObjId; + this.executeObjectId = executeObjectId; } public ExecuteObjectTask(long stepInstanceId, int executeCount, int batch, FileTaskModeEnum fileTaskMode, - Long executeObjId) { + String executeObjectId) { this.stepInstanceId = stepInstanceId; this.executeCount = executeCount; this.batch = batch; this.fileTaskMode = fileTaskMode; - this.executeObjId = executeObjId; + this.executeObjectId = executeObjectId; } public ExecuteObjectTask(ExecuteObjectTask executeObjectTask) { @@ -146,7 +149,9 @@ public ExecuteObjectTask(ExecuteObjectTask executeObjectTask) { this.actualExecuteCount = executeObjectTask.getActualExecuteCount(); this.batch = executeObjectTask.getBatch(); this.fileTaskMode = executeObjectTask.getFileTaskMode(); - this.executeObjId = executeObjectTask.getExecuteObjId(); + this.executeObjectId = executeObjectTask.getExecuteObjectId(); + this.hostId = executeObjectTask.getHostId(); + this.agentId = executeObjectTask.getAgentId(); this.status = executeObjectTask.getStatus(); this.startTime = executeObjectTask.getStartTime(); this.endTime = executeObjectTask.getEndTime(); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ExecuteObjectsDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ExecuteObjectsDTO.java index 6a02af3d8a..f055443f9d 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ExecuteObjectsDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ExecuteObjectsDTO.java @@ -44,6 +44,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; /** @@ -255,4 +256,106 @@ public EsbServerV3DTO toEsbServerV3DTO() { } return esbServerV3DTO; } + + public ExecuteObject findExecuteObjectByCompositeKey(ExecuteObjectCompositeKey executeObjectCompositeKey) { + if (executeObjectCompositeKey.getExecuteObjectId() != null) { + return executeObjects.stream() + .filter(executeObject -> executeObjectCompositeKey.getExecuteObjectId().equals(executeObject.getId())) + .findFirst() + .orElse(null); + } else if (executeObjectCompositeKey.getHostId() != null) { + // 兼容使用 hostId 的方式 + if (CollectionUtils.isNotEmpty(executeObjects)) { + return executeObjects.stream() + .filter(executeObject -> executeObject.isHost() + && executeObjectCompositeKey.getHostId().equals(executeObject.getHost().getHostId())) + .findFirst() + .orElse(null); + } else { + HostDTO matchHost = ipList.stream() + .filter(host -> executeObjectCompositeKey.getHostId().equals(host.getHostId())) + .findFirst() + .orElse(null); + return matchHost == null ? null : new ExecuteObject(matchHost); + } + } else if (executeObjectCompositeKey.getCloudIp() != null) { + // 兼容使用 云区域+ip 的方式 + if (CollectionUtils.isNotEmpty(executeObjects)) { + return executeObjects.stream() + .filter(executeObject -> executeObject.isHost() + && executeObjectCompositeKey.getCloudIp().equals(executeObject.getHost().toCloudIp())) + .findFirst() + .orElse(null); + } else { + HostDTO matchHost = ipList.stream() + .filter(host -> executeObjectCompositeKey.getCloudIp().equals(host.toCloudIp())) + .findFirst() + .orElse(null); + return matchHost == null ? null : new ExecuteObject(matchHost); + } + } else { + throw new IllegalArgumentException("InvalidExecuteObjectCompositeKey"); + } + } + + public List findExecuteObjectByCompositeKeys( + Collection executeObjectCompositeKeys) { + if (CollectionUtils.isEmpty(executeObjectCompositeKeys)) { + return Collections.emptyList(); + } + ExecuteObjectCompositeKey anyKey = executeObjectCompositeKeys.stream().findFirst().orElse(null); + Objects.requireNonNull(anyKey); + + if (anyKey.getExecuteObjectId() != null) { + List executeObjectIds = + executeObjectCompositeKeys.stream() + .map(ExecuteObjectCompositeKey::getExecuteObjectId) + .collect(Collectors.toList()); + return executeObjects.stream() + .filter(executeObject -> executeObjectIds.contains(executeObject.getId())) + .collect(Collectors.toList()); + } else if (anyKey.getHostId() != null) { + // 兼容使用 hostId 的方式 + List hostIds = + executeObjectCompositeKeys.stream() + .map(ExecuteObjectCompositeKey::getHostId) + .collect(Collectors.toList()); + if (CollectionUtils.isNotEmpty(executeObjects)) { + return executeObjects.stream() + .filter(executeObject -> executeObject.isHost() + && hostIds.contains(executeObject.getHost().getHostId())) + .collect(Collectors.toList()); + } else { + List matchHosts = ipList.stream() + .filter(host -> hostIds.contains(host.getHostId())) + .collect(Collectors.toList()); + if (CollectionUtils.isEmpty(matchHosts)) { + return Collections.emptyList(); + } + return matchHosts.stream().map(ExecuteObject::new).collect(Collectors.toList()); + } + } else if (anyKey.getCloudIp() != null) { + // 兼容使用 云区域+ip 的方式 + List cloudIps = + executeObjectCompositeKeys.stream() + .map(ExecuteObjectCompositeKey::getCloudIp) + .collect(Collectors.toList()); + if (CollectionUtils.isNotEmpty(executeObjects)) { + return executeObjects.stream() + .filter(executeObject -> executeObject.isHost() + && cloudIps.contains(executeObject.getHost().toCloudIp())) + .collect(Collectors.toList()); + } else { + List matchHosts = ipList.stream() + .filter(host -> cloudIps.contains(host.toCloudIp())) + .collect(Collectors.toList()); + if (CollectionUtils.isEmpty(matchHosts)) { + return Collections.emptyList(); + } + return matchHosts.stream().map(ExecuteObject::new).collect(Collectors.toList()); + } + } else { + throw new IllegalArgumentException("InvalidExecuteObjectCompositeKey"); + } + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FileIpLogContent.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FileExecuteObjectLogContent.java similarity index 83% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FileIpLogContent.java rename to src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FileExecuteObjectLogContent.java index a836caeff8..dcf682084d 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FileIpLogContent.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FileExecuteObjectLogContent.java @@ -24,6 +24,7 @@ package com.tencent.bk.job.execute.model; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; import com.tencent.bk.job.logsvr.model.service.ServiceFileTaskLogDTO; import lombok.Data; import lombok.NoArgsConstructor; @@ -36,7 +37,7 @@ */ @Data @NoArgsConstructor -public class FileIpLogContent { +public class FileExecuteObjectLogContent { /** * 步骤实例ID */ @@ -46,9 +47,9 @@ public class FileIpLogContent { */ private int executeCount; /** - * 目标IP + * 目标执行对象 */ - private String ip; + private ExecuteObject executeObject; /** * 日志内容 */ @@ -62,11 +63,14 @@ public class FileIpLogContent { */ private boolean finished; - public FileIpLogContent(long stepInstanceId, int executeCount, String ip, - List fileTaskLogs, boolean finished) { + public FileExecuteObjectLogContent(long stepInstanceId, + int executeCount, + ExecuteObject executeObject, + List fileTaskLogs, + boolean finished) { this.stepInstanceId = stepInstanceId; this.executeCount = executeCount; - this.ip = ip; + this.executeObject = executeObject; this.fileTaskLogs = fileTaskLogs; this.finished = finished; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ScriptHostLogContent.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ScriptExecuteObjectLogContent.java similarity index 68% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ScriptHostLogContent.java rename to src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ScriptExecuteObjectLogContent.java index 314767b5c4..03aab6c440 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ScriptHostLogContent.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ScriptExecuteObjectLogContent.java @@ -24,16 +24,16 @@ package com.tencent.bk.job.execute.model; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; import lombok.Data; import lombok.NoArgsConstructor; -import org.apache.commons.lang3.StringUtils; /** * 脚本日志内容 */ @Data @NoArgsConstructor -public class ScriptHostLogContent { +public class ScriptExecuteObjectLogContent { /** * 步骤实例ID */ @@ -43,17 +43,9 @@ public class ScriptHostLogContent { */ private int executeCount; /** - * 主机ID + * 执行对象 */ - private Long hostId; - /** - * 目标云区域ID:ipv4 - */ - private String cloudIp; - /** - * 目标云区域ID:ipv6 - */ - private String cloudIpv6; + private ExecuteObject executeObject; /** * 日志内容 */ @@ -63,28 +55,15 @@ public class ScriptHostLogContent { */ private boolean finished; - public ScriptHostLogContent(long stepInstanceId, - int executeCount, - Long hostId, - String cloudIp, - String cloudIpv6, - String content, - boolean finished) { + public ScriptExecuteObjectLogContent(long stepInstanceId, + int executeCount, + ExecuteObject executeObject, + String content, + boolean finished) { this.stepInstanceId = stepInstanceId; this.executeCount = executeCount; - this.hostId = hostId; - this.cloudIp = cloudIp; - this.cloudIpv6 = cloudIpv6; + this.executeObject = executeObject; this.content = content; this.finished = finished; } - - /** - * 获取主机的ip，优先返回ipv4 - * - * @return 主机ipv4/ipv6, ipv4 优先 - */ - public String getPrimaryIp() { - return StringUtils.isNotEmpty(cloudIp) ? cloudIp : cloudIpv6; - } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceBaseDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceBaseDTO.java index e84555f19e..7930ddbfc1 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceBaseDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceBaseDTO.java @@ -27,11 +27,15 @@ import com.tencent.bk.job.common.gse.util.AgentUtils; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; import lombok.Getter; import lombok.Setter; import lombok.ToString; +import org.apache.commons.collections4.CollectionUtils; +import java.util.Collection; +import java.util.List; import java.util.Set; /** @@ -105,9 +109,9 @@ public class StepInstanceBaseDTO { */ protected boolean ignoreError; /** - * 目标服务器 + * 执行目标 */ - protected ExecuteObjectsDTO targetServers; + protected ExecuteObjectsDTO targetExecuteObjects; /** * 不合法的服务器 */ @@ -129,6 +133,11 @@ public class StepInstanceBaseDTO { */ protected Long rollingConfigId; + /** + * 是否支持执行对象特性 + */ + private Boolean supportExecuteObject; + /** * 获取步骤类型 * @@ -173,8 +182,8 @@ public boolean isScriptStep() { } public int getTargetServerTotalCount() { - if (this.targetServers != null && this.targetServers.getIpList() != null) { - return this.targetServers.getIpList().size(); + if (this.targetExecuteObjects != null && this.targetExecuteObjects.getIpList() != null) { + return this.targetExecuteObjects.getIpList().size(); } else { return 0; } @@ -206,8 +215,28 @@ public String getUniqueKey() { */ public boolean isTargetGseV2Agent() { // 只需要判断任意一个即可，因为前置校验已经保证所有的主机的agentId全部都是V1或者V2 - boolean isTargetGseV1Agent = this.targetServers.getIpList().stream() + boolean isTargetGseV1Agent = this.targetExecuteObjects.getIpList().stream() .anyMatch(host -> AgentUtils.isGseV1AgentId(host.getAgentId())); return !isTargetGseV1Agent; } + + /** + * 通过执行目标判断是否支持"执行对象" + */ + public boolean isSupportExecuteObject() { + if (supportExecuteObject == null) { + supportExecuteObject = CollectionUtils.isNotEmpty(targetExecuteObjects.getExecuteObjects()); + } + return supportExecuteObject; + + } + + public ExecuteObject findExecuteObjectByCompositeKey(ExecuteObjectCompositeKey executeObjectCompositeKey) { + return targetExecuteObjects.findExecuteObjectByCompositeKey(executeObjectCompositeKey); + } + + public List findExecuteObjectByCompositeKeys( + Collection executeObjectCompositeKeys) { + return targetExecuteObjects.findExecuteObjectByCompositeKeys(executeObjectCompositeKeys); + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceDTO.java index 667ed6790e..bf999e9204 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceDTO.java @@ -25,6 +25,10 @@ package com.tencent.bk.job.execute.model; import com.tencent.bk.job.common.constant.DuplicateHandlerEnum; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; +import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; import lombok.Getter; import lombok.Setter; import lombok.ToString; @@ -169,7 +173,7 @@ public StepInstanceDTO(StepInstanceBaseDTO stepInstanceBase) { this.totalTime = stepInstanceBase.totalTime; this.createTime = stepInstanceBase.createTime; this.ignoreError = stepInstanceBase.ignoreError; - this.targetServers = stepInstanceBase.targetServers; + this.targetExecuteObjects = stepInstanceBase.targetExecuteObjects; this.rollingConfigId = stepInstanceBase.rollingConfigId; this.batch = stepInstanceBase.getBatch(); } @@ -224,4 +228,29 @@ public void fillConfirmStepInfo(ConfirmStepInstanceDTO confirmStepInstance) { this.confirmRoles = confirmStepInstance.getConfirmRoles(); this.notifyChannels = confirmStepInstance.getNotifyChannels(); } + + @Override + public ExecuteObject findExecuteObjectByCompositeKey(ExecuteObjectCompositeKey executeObjectCompositeKey) { + if (stepType == TaskStepTypeEnum.SCRIPT) { + return targetExecuteObjects.findExecuteObjectByCompositeKey(executeObjectCompositeKey); + } else if (stepType == TaskStepTypeEnum.FILE) { + ExecuteObject executeObject = + targetExecuteObjects.findExecuteObjectByCompositeKey(executeObjectCompositeKey); + if (executeObject != null) { + return executeObject; + } + for (FileSourceDTO fileSource : fileSourceList) { + if (fileSource.getServers() == null) { + continue; + } + executeObject = fileSource.getServers().findExecuteObjectByCompositeKey(executeObjectCompositeKey); + if (executeObject != null) { + return executeObject; + } + } + return null; + } else { + throw new InternalException("Not support method invoke for step", ErrorCode.INTERNAL_ERROR); + } + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ExecuteObjectTaskService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ExecuteObjectTaskService.java index c35dede883..b25429c4a9 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ExecuteObjectTaskService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ExecuteObjectTaskService.java @@ -44,14 +44,14 @@ public interface ExecuteObjectTaskService { * * @param tasks 任务列表 */ - void batchSaveTasks(Collection tasks); + void batchSaveTasks(Collection tasks); /** * 批量更新执行对象任务 * * @param tasks 执行对象任务 */ - void batchUpdateTasks(Collection tasks); + void batchUpdateTasks(Collection tasks); /** * 获取执行成功的执行对象任务数量 @@ -70,7 +70,7 @@ public interface ExecuteObjectTaskService { * @param gseTaskId GSE任务ID * @return 执行对象任务 */ - List listTasksByGseTaskId(StepInstanceBaseDTO stepInstance, Long gseTaskId); + List listTasksByGseTaskId(StepInstanceBaseDTO stepInstance, Long gseTaskId); /** * 获取执行对象任务 diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/LogService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/LogService.java index 587cca1472..07cdd88da3 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/LogService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/LogService.java @@ -24,16 +24,18 @@ package com.tencent.bk.job.execute.service; -import com.tencent.bk.job.common.exception.ServiceException; import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.common.constants.FileDistStatusEnum; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; import com.tencent.bk.job.execute.engine.model.JobFile; -import com.tencent.bk.job.execute.model.FileIpLogContent; -import com.tencent.bk.job.execute.model.ScriptHostLogContent; +import com.tencent.bk.job.execute.model.ExecuteObjectCompositeKey; +import com.tencent.bk.job.execute.model.FileExecuteObjectLogContent; +import com.tencent.bk.job.execute.model.ScriptExecuteObjectLogContent; +import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; +import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.logsvr.model.service.ServiceExecuteObjectLogDTO; -import com.tencent.bk.job.logsvr.model.service.ServiceExecuteObjectLogsDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceExecuteObjectScriptLogDTO; import com.tencent.bk.job.logsvr.model.service.ServiceFileTaskLogDTO; -import com.tencent.bk.job.logsvr.model.service.ServiceScriptLogDTO; import java.util.List; @@ -45,16 +47,32 @@ public interface LogService { /** * 构造job系统日志 * - * @param host 主机 + * @param stepInstance 步骤实例 + * @param executeObject 执行对象 * @param content 日志原始内容 * @param offset 日志偏移 - 字节 * @param logTimeInMillSeconds 日志时间 * @return 系统日志 */ - ServiceScriptLogDTO buildSystemScriptLog(HostDTO host, - String content, - int offset, - Long logTimeInMillSeconds); + ServiceExecuteObjectScriptLogDTO buildSystemScriptLog(StepInstanceBaseDTO stepInstance, + ExecuteObject executeObject, + String content, + int offset, + Long logTimeInMillSeconds); + + /** + * 构造脚本日志 + * + * @param stepInstance 步骤实例 + * @param executeObject 执行对象 + * @param content 日志原始内容 + * @param offset 日志偏移 - 字节 + * @return 系统日志 + */ + ServiceExecuteObjectScriptLogDTO buildScriptLog(StepInstanceBaseDTO stepInstance, + ExecuteObject executeObject, + String content, + int offset); /** * 写脚本执行日志 @@ -65,47 +83,62 @@ ServiceScriptLogDTO buildSystemScriptLog(HostDTO host, * @param batch 滚动执行批次;非滚动步骤传入null * @param scriptLogs 脚本日志 */ - void batchWriteScriptLog(long jobCreateTime, long stepInstanceId, int executeCount, Integer batch, - List scriptLogs); + void batchWriteScriptLog(long jobCreateTime, + long stepInstanceId, + int executeCount, + Integer batch, + List scriptLogs); /** * 获取脚本执行日志 * - * @param stepInstanceId 步骤实例 ID - * @param executeCount 执行次数 - * @param batch 滚动执行批次;非滚动步骤传入null - * @param host 主机 + * @param stepInstance 步骤实例 + * @param executeCount 执行次数 + * @param batch 滚动执行批次;非滚动步骤传入null + * @param key 执行对象复合 KEY * @return 日志内容 */ - ScriptHostLogContent getScriptHostLogContent(long stepInstanceId, int executeCount, Integer batch, HostDTO host); + ScriptExecuteObjectLogContent getScriptExecuteObjectLogContent(StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch, + ExecuteObjectCompositeKey key); /** * 批量获取脚本执行日志 * - * @param jobCreateDateStr 作业创建时间 - * @param stepInstanceId 步骤实例 ID - * @param executeCount 执行次数 - * @param batch 滚动执行批次;非滚动步骤传入null - * @param hosts 主机列表,最大支持1000个 + * @param jobCreateDateStr 作业创建时间 + * @param stepInstance 步骤实例 + * @param executeCount 执行次数 + * @param batch 滚动执行批次;非滚动步骤传入null + * @param executeObjectCompositeKeys 执行对象复合 KEY 列表,最大支持1000个 * @return 日志内容 */ - List batchGetScriptHostLogContent(String jobCreateDateStr, long stepInstanceId, - int executeCount, Integer batch, List hosts); + List batchGetScriptExecuteObjectLogContent( + String jobCreateDateStr, + StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch, + List executeObjectCompositeKeys + ); /** * 获取脚本执行日志 * - * @param stepInstanceId 步骤实例 ID - * @param executeCount 执行次数 - * @param batch 滚动执行批次;非滚动步骤传入null - * @param host 主机 - * @param mode 文件传输模式 + * @param stepInstance 步骤实例 + * @param executeCount 执行次数 + * @param batch 滚动执行批次;非滚动步骤传入null + * @param executeObjectCompositeKey 执行对象复合 KEY + * @param mode 文件传输模式 * @return 日志内容 - * @throws ServiceException 异常 */ - FileIpLogContent getFileIpLogContent(long stepInstanceId, int executeCount, Integer batch, HostDTO host, - Integer mode); + FileExecuteObjectLogContent getFileExecuteObjectLogContent( + StepInstanceDTO stepInstance, + int executeCount, + Integer batch, + ExecuteObjectCompositeKey executeObjectCompositeKey, + Integer mode + ); /** * 获取脚本执行日志 @@ -116,7 +149,9 @@ FileIpLogContent getFileIpLogContent(long stepInstanceId, int executeCount, Inte * @param taskIds 文件任务ID列表 * @return 日志内容 */ - List getFileLogContentByTaskIds(long stepInstanceId, int executeCount, Integer batch, + List getFileLogContentByTaskIds(long stepInstanceId, + int executeCount, + Integer batch, List taskIds); /** @@ -127,23 +162,25 @@ List getFileLogContentByTaskIds(long stepInstanceId, int * @param batch 滚动执行批次;非滚动步骤传入null * @return 日志内容 */ - List batchGetFileSourceIpLogContent(long stepInstanceId, - int executeCount, - Integer batch); + List batchGetFileSourceExecuteObjectLogContent(long stepInstanceId, + int executeCount, + Integer batch); /** * 获取文件任务文件源日志 * - * @param stepInstanceId 步骤实例 ID - * @param executeCount 执行次数 - * @param batch 滚动执行批次;非滚动步骤传入null - * @param hosts 主机列表 + * @param stepInstance 步骤实例 + * @param executeCount 执行次数 + * @param batch 滚动执行批次;非滚动步骤传入null + * @param executeObjectCompositeKeys 执行对象复合 KEY 列表 * @return 日志内容 */ - ServiceExecuteObjectLogsDTO batchGetFileIpLogContent(long stepInstanceId, - int executeCount, - Integer batch, - List hosts); + List batchGetFileExecuteObjectLogContent( + StepInstanceDTO stepInstance, + int executeCount, + Integer batch, + List executeObjectCompositeKeys + ); /** * 根据关键字获取对应的ip @@ -154,7 +191,7 @@ ServiceExecuteObjectLogsDTO batchGetFileIpLogContent(long stepInstanceId, * @param keyword 关键字 * @return ips */ - List getIpsByContentKeyword(long stepInstanceId, int executeCount, Integer batch, String keyword); + List getHostsByContentKeyword(long stepInstanceId, int executeCount, Integer batch, String keyword); /** * 写文件日志日志 - 指定时间 @@ -171,22 +208,24 @@ void writeFileLogsWithTimestamp(long jobCreateTime, /** * 写文件日志日志 * - * @param jobCreateTime 任务创建时间 - * @param hostFileLogs 文件任务执行日志 + * @param jobCreateTime 任务创建时间 + * @param executeObjectLogs 文件任务执行日志 */ - void writeFileLogs(long jobCreateTime, List hostFileLogs); + void writeFileLogs(long jobCreateTime, List executeObjectLogs); /** * 构造上传文件任务日志 * - * @param srcFile 源文件 - * @param status 任务状态 - * @param size 源文件大小 - * @param speed 上传速度 - * @param process 进度 - * @param content 日志内容 + * @param stepInstance 步骤实例 + * @param srcFile 源文件 + * @param status 任务状态 + * @param size 源文件大小 + * @param speed 上传速度 + * @param process 进度 + * @param content 日志内容 */ - ServiceFileTaskLogDTO buildUploadServiceFileTaskLogDTO(JobFile srcFile, + ServiceFileTaskLogDTO buildUploadServiceFileTaskLogDTO(StepInstanceDTO stepInstance, + JobFile srcFile, FileDistStatusEnum status, String size, String speed, @@ -196,17 +235,19 @@ ServiceFileTaskLogDTO buildUploadServiceFileTaskLogDTO(JobFile srcFile, /** * 构造下载文件任务日志 * - * @param srcFile 源文件 - * @param targetHost 目标主机 - * @param targetPath 目标路径 - * @param status 任务状态 - * @param size 源文件大小 - * @param speed 下载速度 - * @param process 进度 - * @param content 日志内容 + * @param stepInstance 步骤实例 + * @param srcFile 源文件 + * @param targetExecuteObject 目标执行对象 + * @param targetPath 目标路径 + * @param status 任务状态 + * @param size 源文件大小 + * @param speed 下载速度 + * @param process 进度 + * @param content 日志内容 */ - ServiceFileTaskLogDTO buildDownloadServiceFileTaskLogDTO(JobFile srcFile, - HostDTO targetHost, + ServiceFileTaskLogDTO buildDownloadServiceFileTaskLogDTO(StepInstanceDTO stepInstance, + JobFile srcFile, + ExecuteObject targetExecuteObject, String targetPath, FileDistStatusEnum status, String size, diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceService.java index 60450804fa..609a49fb58 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceService.java @@ -25,6 +25,7 @@ package com.tencent.bk.job.execute.service; import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import java.util.Map; @@ -85,6 +86,15 @@ Map computeStepHosts(StepInstanceBaseDTO stepInstance, Map computeStepHosts(long stepInstanceId, Function keyMapper); + /** + * 获取步骤包含的执行对象(源+目标) + * + * @param stepInstance 步骤实例 + * @return 步骤实例包含的主机列表 + */ + Map computeStepExecuteObjects(StepInstanceBaseDTO stepInstance, + Function keyMapper); + /** * 获取步骤实例 * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskInstanceExecuteObjectService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskInstanceExecuteObjectService.java index 7847b82f96..0de3aef5f6 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskInstanceExecuteObjectService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskInstanceExecuteObjectService.java @@ -24,51 +24,45 @@ package com.tencent.bk.job.execute.service; -import com.tencent.bk.job.common.constant.ExecuteObjectTypeEnum; -import com.tencent.bk.job.execute.engine.model.ExecuteObject; - -import java.util.Collection; -import java.util.List; - public interface TaskInstanceExecuteObjectService { - /** - * 批量保存执行对象 - * - * @param executeObjects 执行对象 - */ - void batchSaveExecuteObjects(Collection executeObjects); - - /** - * 根据作业实例ID获取执行对象 - * - * @param jobInstanceId 作业实例ID - * @return 执行对象 - */ - List listExecuteObjectsByJobInstanceId(long jobInstanceId); - - /** - * 根据执行对象ID批量查询执行对象 - * - * @param executeObjectIds 执行对象ID集合 - * @return 执行对象 - */ - List listExecuteObjectsByIds(Collection executeObjectIds); - - - /** - * 根据执行对象ID查询执行对象 - * - * @param executeObjectId 执行对象ID - * @return 执行对象 - */ - ExecuteObject getExecuteObjectById(long executeObjectId); - - /** - * 根据执行对象ID查询执行对象 - * - * @param executeObjectType 执行对象类型 - * @param resourceId 执行对象资源ID - * @return 执行对象 - */ - ExecuteObject getExecuteObject(ExecuteObjectTypeEnum executeObjectType, String resourceId); +// /** +// * 批量保存执行对象 +// * +// * @param executeObjects 执行对象 +// */ +// void batchSaveExecuteObjects(Collection executeObjects); +// +// /** +// * 根据作业实例ID获取执行对象 +// * +// * @param jobInstanceId 作业实例ID +// * @return 执行对象 +// */ +// List listExecuteObjectsByJobInstanceId(long jobInstanceId); +// +// /** +// * 根据执行对象ID批量查询执行对象 +// * +// * @param executeObjectIds 执行对象ID集合 +// * @return 执行对象 +// */ +// List listExecuteObjectsByIds(Collection executeObjectIds); +// +// +// /** +// * 根据执行对象ID查询执行对象 +// * +// * @param executeObjectId 执行对象ID +// * @return 执行对象 +// */ +// ExecuteObject getExecuteObjectById(long executeObjectId); +// +// /** +// * 根据执行对象ID查询执行对象 +// * +// * @param executeObjectType 执行对象类型 +// * @param resourceId 执行对象资源ID +// * @return 执行对象 +// */ +// ExecuteObject getExecuteObject(ExecuteObjectTypeEnum executeObjectType, String resourceId); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/AbstractExecuteObjectTaskServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/AbstractExecuteObjectTaskServiceImpl.java index 7c0524a67b..97e9217f0b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/AbstractExecuteObjectTaskServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/AbstractExecuteObjectTaskServiceImpl.java @@ -8,15 +8,14 @@ import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.service.ExecuteObjectTaskService; import com.tencent.bk.job.execute.service.StepInstanceService; -import com.tencent.bk.job.execute.service.TaskInstanceExecuteObjectService; import org.apache.commons.collections4.CollectionUtils; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; +import java.util.Objects; import java.util.stream.Collectors; /** @@ -24,46 +23,28 @@ */ public abstract class AbstractExecuteObjectTaskServiceImpl implements ExecuteObjectTaskService { protected final StepInstanceService stepInstanceService; - protected final TaskInstanceExecuteObjectService taskInstanceExecuteObjectService; - public AbstractExecuteObjectTaskServiceImpl(StepInstanceService stepInstanceService, - TaskInstanceExecuteObjectService taskInstanceExecuteObjectService) { + public AbstractExecuteObjectTaskServiceImpl(StepInstanceService stepInstanceService) { this.stepInstanceService = stepInstanceService; - this.taskInstanceExecuteObjectService = taskInstanceExecuteObjectService; } - protected final List fillExecuteObjectDetail(StepInstanceBaseDTO stepInstance, - List tasks) { + protected final List convertToExecuteObjectDetailList(StepInstanceBaseDTO stepInstance, + List tasks) { if (CollectionUtils.isEmpty(tasks)) { return Collections.emptyList(); } List taskList; - boolean hasExecuteObjectId = tasks.stream().anyMatch(task -> task.getExecuteObjId() != null); - if (hasExecuteObjectId) { + if (stepInstance.isSupportExecuteObject()) { taskList = tasks .stream() .map(ExecuteObjectTaskDetail::new) .collect(Collectors.toList()); - long taskInstanceId = stepInstance.getTaskInstanceId(); - Map executeObjectMap = new HashMap<>(); - List executeObjects; - if (tasks.size() > 1000) { - executeObjects = taskInstanceExecuteObjectService.listExecuteObjectsByJobInstanceId(taskInstanceId); - } else { - // in 查询控制个数在 1000 以内 - Set executeObjectIds = - tasks.stream().map(ExecuteObjectTask::getExecuteObjId).collect(Collectors.toSet()); - executeObjects = taskInstanceExecuteObjectService.listExecuteObjectsByIds(executeObjectIds); - } - if (CollectionUtils.isEmpty(executeObjects)) { - return taskList; - } - executeObjects.forEach(executeObject -> executeObjectMap.put(executeObject.getId(), executeObject)); - + Map executeObjectMap = + stepInstanceService.computeStepExecuteObjects(stepInstance, ExecuteObject::getId); taskList.forEach( - task -> task.setExecuteObject(executeObjectMap.get(task.getExecuteObjId()))); + task -> task.setExecuteObject(executeObjectMap.get(task.getExecuteObjectId()))); } else { // 兼容老版本不支持执行对象的数据 Map hosts = stepInstanceService.computeStepHosts(stepInstance, HostDTO::getHostId); @@ -72,7 +53,7 @@ protected final List fillExecuteObjectDetail(StepInstan .map(task -> { ExecuteObjectTaskDetail taskDetail = new ExecuteObjectTaskDetail(task); HostDTO host = hosts.get(task.getHostId()); - ExecuteObject executeObject = new ExecuteObject(null, host); + ExecuteObject executeObject = new ExecuteObject(host); taskDetail.setExecuteObject(executeObject); return taskDetail; }).collect(Collectors.toList()); @@ -93,5 +74,16 @@ protected final List groupTasks(List ta return resultGroups; } + /** + * 判断任务是否需要按照<执行对象>的方式保存到 DB + * + * @param tasks 需要被保存的任务 + */ + protected boolean isSaveTasksUsingExecuteObjectMode(Collection tasks) { + // 根据执行对象任务模型中的 executeObjectId 参数判断是否支持执行对象 + ExecuteObjectTask anyTask = tasks.stream().findAny().orElse(null); + return Objects.requireNonNull(anyTask).getExecuteObjectId() != null; + } + } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileExecuteObjectTaskServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileExecuteObjectTaskServiceImpl.java index ebc225bd58..b86d481c24 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileExecuteObjectTaskServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileExecuteObjectTaskServiceImpl.java @@ -1,22 +1,18 @@ package com.tencent.bk.job.execute.service.impl; import com.tencent.bk.job.common.constant.Order; -import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.dao.FileAgentTaskDAO; import com.tencent.bk.job.execute.dao.FileExecuteObjectTaskDAO; import com.tencent.bk.job.execute.engine.model.ExecuteObject; import com.tencent.bk.job.execute.model.ExecuteObjectCompositeKey; import com.tencent.bk.job.execute.model.ExecuteObjectTask; import com.tencent.bk.job.execute.model.ExecuteObjectTaskDetail; -import com.tencent.bk.job.execute.model.ExecuteObjectsDTO; -import com.tencent.bk.job.execute.model.FileSourceDTO; import com.tencent.bk.job.execute.model.ResultGroupBaseDTO; import com.tencent.bk.job.execute.model.ResultGroupDTO; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.service.FileExecuteObjectTaskService; import com.tencent.bk.job.execute.service.StepInstanceService; -import com.tencent.bk.job.execute.service.TaskInstanceExecuteObjectService; import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; @@ -26,7 +22,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; -import java.util.Objects; import java.util.stream.Collectors; @Service @@ -39,11 +34,10 @@ public class FileExecuteObjectTaskServiceImpl private final FileAgentTaskDAO fileAgentTaskDAO; @Autowired - public FileExecuteObjectTaskServiceImpl(FileExecuteObjectTaskDAO fileExecuteObjectTaskDAO, - StepInstanceService stepInstanceService, - FileAgentTaskDAO fileAgentTaskDAO, - TaskInstanceExecuteObjectService taskInstanceExecuteObjectService) { - super(stepInstanceService, taskInstanceExecuteObjectService); + public FileExecuteObjectTaskServiceImpl(StepInstanceService stepInstanceService, + FileExecuteObjectTaskDAO fileExecuteObjectTaskDAO, + FileAgentTaskDAO fileAgentTaskDAO) { + super(stepInstanceService); this.fileAgentTaskDAO = fileAgentTaskDAO; this.fileExecuteObjectTaskDAO = fileExecuteObjectTaskDAO; } @@ -53,9 +47,8 @@ public void batchSaveTasks(Collection tasks) { if (CollectionUtils.isEmpty(tasks)) { return; } - ExecuteObjectTask anyTask = tasks.stream().findAny().orElse(null); - if (Objects.requireNonNull(anyTask).getExecuteObjId() != null) { + if (isSaveTasksUsingExecuteObjectMode(tasks)) { fileExecuteObjectTaskDAO.batchSaveTasks(tasks); } else { fileAgentTaskDAO.batchSaveAgentTasks(tasks); @@ -67,8 +60,8 @@ public void batchUpdateTasks(Collection tasks) { if (CollectionUtils.isEmpty(tasks)) { return; } - ExecuteObjectTask anyTask = tasks.stream().findAny().orElse(null); - if (Objects.requireNonNull(anyTask).getExecuteObjId() != null) { + + if (isSaveTasksUsingExecuteObjectMode(tasks)) { fileExecuteObjectTaskDAO.batchUpdateTasks(tasks); } else { fileAgentTaskDAO.batchUpdateAgentTasks(tasks); @@ -89,32 +82,17 @@ public List listTasks(StepInstanceBaseDTO stepInstance, Integ return listTasks(stepInstance, executeCount, batch, null); } - /** - * 判断是否使用执行对象的方式存储 - * - * @param stepInstance 步骤实例 - */ - private boolean isUsingExecuteObject(StepInstanceBaseDTO stepInstance) { - ExecuteObjectsDTO servers = stepInstance.getTargetServers(); - if (CollectionUtils.isNotEmpty(servers.getExecuteObjects())) { - ExecuteObject executeObject = servers.getExecuteObjects().stream().findAny().orElse(null); - return Objects.requireNonNull(executeObject).getId() != null; - } else { - return false; - } - } - @Override - public List listTasksByGseTaskId(StepInstanceBaseDTO stepInstance, Long gseTaskId) { + public List listTasksByGseTaskId(StepInstanceBaseDTO stepInstance, Long gseTaskId) { List executeObjectTasks; - - if (isUsingExecuteObject(stepInstance)) { + if (stepInstance.isSupportExecuteObject()) { executeObjectTasks = fileExecuteObjectTaskDAO.listTasksByGseTaskId(gseTaskId); } else { // 兼容老版本数据 executeObjectTasks = fileAgentTaskDAO.listAgentTasksByGseTaskId(gseTaskId); } - return executeObjectTasks; + + return convertToExecuteObjectDetailList(stepInstance, executeObjectTasks); } @Override @@ -123,84 +101,26 @@ public ExecuteObjectTask getTaskByExecuteObjectCompositeKey(StepInstanceDTO step Integer batch, FileTaskModeEnum fileTaskMode, ExecuteObjectCompositeKey executeObjectCompositeKey) { - ExecuteObjectTask executeObjectTask = null; - long stepInstanceId = stepInstance.getId(); - - if (isUsingExecuteObject(stepInstance)) { - String executeObjectResourceId = executeObjectCompositeKey.getResourceId(); - if (executeObjectResourceId != null) { - ExecuteObject executeObject = taskInstanceExecuteObjectService.getExecuteObject( - executeObjectCompositeKey.getExecuteObjectType(), executeObjectCompositeKey.getResourceId()); - if (executeObject == null) { - return null; - } - executeObjectTask = fileExecuteObjectTaskDAO.getTaskByExecuteObjectId(stepInstanceId, executeCount, - batch, fileTaskMode, executeObject.getId()); - } else { - // 兼容使用<云区域+ip> 的方式查询主机执行任务 - ExecuteObject executeObject = getStepHostExecuteObjectByCloudIp( - stepInstance, executeObjectCompositeKey.getCloudIp()); - executeObjectTask = fileExecuteObjectTaskDAO.getTaskByExecuteObjectId(stepInstance.getId(), - executeCount, batch, fileTaskMode, executeObject.getId()); - } - } else { - // 兼容老版本不使用执行对象的场景(仅支持主机） - Long hostId = executeObjectCompositeKey.getHostId(); - if (hostId != null) { - executeObjectTask = fileAgentTaskDAO.getAgentTaskByHostId(stepInstanceId, executeCount, - batch, fileTaskMode, hostId); - } else { - // 兼容使用<云区域+ip> 的方式查询主机执行任务 - HostDTO host = getStepHostByCloudIp(stepInstance, executeObjectCompositeKey.getCloudIp()); - if (host != null) { - executeObjectTask = fileAgentTaskDAO.getAgentTaskByHostId(stepInstanceId, executeCount, - batch, fileTaskMode, host.getHostId()); - } - } + ExecuteObject executeObject = + stepInstance.findExecuteObjectByCompositeKey(executeObjectCompositeKey); + if (executeObject == null) { + return null; } - return executeObjectTask; - } - - private ExecuteObject getStepHostExecuteObjectByCloudIp(StepInstanceBaseDTO stepInstance, String cloudIp) { - return stepInstance.getTargetServers().getExecuteObjects() - .stream() - .filter(executeObject -> { - HostDTO host = executeObject.getHost(); - if (host == null) { - return false; - } - return cloudIp.equals(host.toCloudIp()); - }) - .findFirst() - .orElse(null); - } - private HostDTO getStepHostByCloudIp(StepInstanceDTO stepInstance, - String cloudIp) { - HostDTO matchHost = stepInstance.getTargetServers().getIpList() - .stream() - .filter(host -> cloudIp.equals(host.toCloudIp())) - .findFirst() - .orElse(null); - if (matchHost == null) { - if (CollectionUtils.isNotEmpty(stepInstance.getFileSourceList())) { - for (FileSourceDTO fileSource : stepInstance.getFileSourceList()) { - if (fileSource.getServers() != null - && CollectionUtils.isNotEmpty(fileSource.getServers().getIpList())) { - matchHost = fileSource.getServers().getIpList().stream() - .filter(sourceHost -> cloudIp.equals(sourceHost.toCloudIp())) - .findFirst() - .orElse(null); - if (matchHost != null) { - break; - } - } - } - } + long stepInstanceId = stepInstance.getId(); + ExecuteObjectTask executeObjectTask; + if (stepInstance.isSupportExecuteObject()) { + executeObjectTask = fileExecuteObjectTaskDAO.getTaskByExecuteObjectId(stepInstanceId, executeCount, + batch, fileTaskMode, executeObject.getId()); + } else { + // 兼容老版本不使用执行对象的场景(仅支持主机） + Long hostId = executeObject.getHost().getHostId(); + executeObjectTask = fileAgentTaskDAO.getAgentTaskByHostId(stepInstanceId, executeCount, + batch, fileTaskMode, hostId); } - return matchHost; + return executeObjectTask; } @Override @@ -213,7 +133,8 @@ public List listAndGroupTasks(StepInstanceBaseDTO stepInstance, return resultGroups; } - List agentTaskDetailList = fillExecuteObjectDetail(stepInstance, executeObjectTasks); + List agentTaskDetailList = convertToExecuteObjectDetailList(stepInstance, + executeObjectTasks); resultGroups = groupTasks(agentTaskDetailList); return resultGroups.stream().sorted().collect(Collectors.toList()); @@ -226,7 +147,7 @@ public List listResultGroups(StepInstanceBaseDTO stepInstanc List resultGroups; long stepInstanceId = stepInstance.getId(); - if (isUsingExecuteObject(stepInstance)) { + if (stepInstance.isSupportExecuteObject()) { resultGroups = fileExecuteObjectTaskDAO.listResultGroups(stepInstanceId, executeCount, batch); } else { // 兼容历史数据 @@ -243,7 +164,7 @@ public List listTaskDetailByResultGroup(StepInstanceBas String tag) { List executeObjectTasks; - if (isUsingExecuteObject(stepInstance)) { + if (stepInstance.isSupportExecuteObject()) { executeObjectTasks = fileExecuteObjectTaskDAO.listTaskByResultGroup(stepInstance.getId(), executeCount, batch, status); } else { @@ -252,7 +173,7 @@ public List listTaskDetailByResultGroup(StepInstanceBas stepInstance.getId(), executeCount, batch, status); } - return fillExecuteObjectDetail(stepInstance, executeObjectTasks); + return convertToExecuteObjectDetailList(stepInstance, executeObjectTasks); } @@ -267,7 +188,7 @@ public List listTaskDetailByResultGroup(StepInstanceBas Order order) { List executeObjectTasks; - if (isUsingExecuteObject(stepInstance)) { + if (stepInstance.isSupportExecuteObject()) { executeObjectTasks = fileExecuteObjectTaskDAO.listTaskByResultGroup(stepInstance.getId(), executeCount, batch, status, limit, orderField, order); } else { @@ -276,7 +197,7 @@ public List listTaskDetailByResultGroup(StepInstanceBas batch, status, limit, orderField, order); } - return fillExecuteObjectDetail(stepInstance, executeObjectTasks); + return convertToExecuteObjectDetailList(stepInstance, executeObjectTasks); } @Override @@ -284,7 +205,7 @@ public List listTaskDetail(StepInstanceBaseDTO stepInst Integer executeCount, Integer batch) { List executeObjectTasks = listTasks(stepInstance, executeCount, batch); - return fillExecuteObjectDetail(stepInstance, executeObjectTasks); + return convertToExecuteObjectDetailList(stepInstance, executeObjectTasks); } private boolean isStepInstanceRecordExist(long stepInstanceId) { @@ -297,7 +218,7 @@ public void updateTaskFields(StepInstanceBaseDTO stepInstance, Integer batch, Integer actualExecuteCount, Long gseTaskId) { - if (isUsingExecuteObject(stepInstance)) { + if (stepInstance.isSupportExecuteObject()) { fileExecuteObjectTaskDAO.updateTaskFields(stepInstance.getId(), executeCount, batch, actualExecuteCount, gseTaskId); } else { @@ -314,7 +235,7 @@ public List listTasks(StepInstanceBaseDTO stepInstance, FileTaskModeEnum fileTaskMode) { List executeObjectTasks; long stepInstanceId = stepInstance.getId(); - if (isUsingExecuteObject(stepInstance)) { + if (stepInstance.isSupportExecuteObject()) { executeObjectTasks = fileExecuteObjectTaskDAO.listTasks(stepInstanceId, executeCount, batch, fileTaskMode); } else { // 兼容老版本数据 diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogExportServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogExportServiceImpl.java index 4fcc38a787..d63b4d1e35 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogExportServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogExportServiceImpl.java @@ -38,7 +38,7 @@ import com.tencent.bk.job.execute.model.ExecuteObjectCompositeKey; import com.tencent.bk.job.execute.model.ExecuteObjectTask; import com.tencent.bk.job.execute.model.LogExportJobInfoDTO; -import com.tencent.bk.job.execute.model.ScriptHostLogContent; +import com.tencent.bk.job.execute.model.ScriptExecuteObjectLogContent; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.service.LogExportService; import com.tencent.bk.job.execute.service.LogService; @@ -276,18 +276,17 @@ private void writeOneBatchIpLogs(PrintWriter out, LogBatchQuery query, List hosts, boolean isGetByHost) { - List scriptHostLogContentList = - logService.batchGetScriptHostLogContent(jobCreateDate, stepInstance.getId(), - query.getExecuteCount(), - null, hosts); - for (ScriptHostLogContent scriptHostLogContent : scriptHostLogContentList) { - if (scriptHostLogContent != null && StringUtils.isNotEmpty(scriptHostLogContent.getContent())) { - String[] logList = scriptHostLogContent.getContent().split("\n"); + List scriptExecuteObjectLogContentList = + logService.batchGetScriptExecuteObjectLogContent(jobCreateDate, stepInstance, + query.getExecuteCount(), null, hosts); + for (ScriptExecuteObjectLogContent scriptExecuteObjectLogContent : scriptExecuteObjectLogContentList) { + if (scriptExecuteObjectLogContent != null && StringUtils.isNotEmpty(scriptExecuteObjectLogContent.getContent())) { + String[] logList = scriptExecuteObjectLogContent.getContent().split("\n"); for (String log : logList) { if (isGetByHost) { out.println(log); } else { - out.println(scriptHostLogContent.getPrimaryIp() + " | " + log); + out.println(scriptExecuteObjectLogContent.getPrimaryIp() + " | " + log); } } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogServiceImpl.java index aa7b4e358e..7161e07971 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogServiceImpl.java @@ -33,25 +33,24 @@ import com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum; import com.tencent.bk.job.execute.engine.model.ExecuteObject; import com.tencent.bk.job.execute.engine.model.JobFile; +import com.tencent.bk.job.execute.model.ExecuteObjectCompositeKey; import com.tencent.bk.job.execute.model.ExecuteObjectTask; -import com.tencent.bk.job.execute.model.FileIpLogContent; -import com.tencent.bk.job.execute.model.ScriptHostLogContent; +import com.tencent.bk.job.execute.model.FileExecuteObjectLogContent; +import com.tencent.bk.job.execute.model.ScriptExecuteObjectLogContent; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.service.FileExecuteObjectTaskService; import com.tencent.bk.job.execute.service.LogService; import com.tencent.bk.job.execute.service.ScriptExecuteObjectTaskService; -import com.tencent.bk.job.execute.service.StepInstanceService; import com.tencent.bk.job.execute.service.TaskInstanceService; import com.tencent.bk.job.logsvr.api.ServiceLogResource; import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; import com.tencent.bk.job.logsvr.consts.LogTypeEnum; import com.tencent.bk.job.logsvr.model.service.ServiceBatchSaveLogRequest; import com.tencent.bk.job.logsvr.model.service.ServiceExecuteObjectLogDTO; -import com.tencent.bk.job.logsvr.model.service.ServiceExecuteObjectLogsDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceExecuteObjectScriptLogDTO; import com.tencent.bk.job.logsvr.model.service.ServiceFileLogQueryRequest; import com.tencent.bk.job.logsvr.model.service.ServiceFileTaskLogDTO; -import com.tencent.bk.job.logsvr.model.service.ServiceScriptLogDTO; import com.tencent.bk.job.logsvr.model.service.ServiceScriptLogQueryRequest; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; @@ -65,37 +64,33 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Objects; -import java.util.Optional; import java.util.stream.Collectors; -@Service("jobExecuteLogService") +@Service @Slf4j public class LogServiceImpl implements LogService { private final ServiceLogResource logResource; private final TaskInstanceService taskInstanceService; - private final ScriptExecuteObjectTaskService scriptAgentTaskService; - private final FileExecuteObjectTaskService fileAgentTaskService; - private final StepInstanceService stepInstanceService; + private final ScriptExecuteObjectTaskService scriptExecuteObjectTaskService; + private final FileExecuteObjectTaskService fileExecuteObjectTaskService; @Autowired public LogServiceImpl(ServiceLogResource logResource, TaskInstanceService taskInstanceService, - ScriptExecuteObjectTaskService scriptAgentTaskService, - FileExecuteObjectTaskService fileAgentTaskService, - StepInstanceService stepInstanceService) { + ScriptExecuteObjectTaskService scriptExecuteObjectTaskService, + FileExecuteObjectTaskService fileExecuteObjectTaskService) { this.logResource = logResource; this.taskInstanceService = taskInstanceService; - this.scriptAgentTaskService = scriptAgentTaskService; - this.fileAgentTaskService = fileAgentTaskService; - this.stepInstanceService = stepInstanceService; + this.scriptExecuteObjectTaskService = scriptExecuteObjectTaskService; + this.fileExecuteObjectTaskService = fileExecuteObjectTaskService; } @Override - public ServiceScriptLogDTO buildSystemScriptLog(ExecuteObject executeObject, - String content, - int offset, - Long logTimeInMillSeconds) { + public ServiceExecuteObjectScriptLogDTO buildSystemScriptLog(StepInstanceBaseDTO stepInstance, + ExecuteObject executeObject, + String content, + int offset, + Long logTimeInMillSeconds) { String logDateTime; if (logTimeInMillSeconds != null) { logDateTime = DateUtils.formatUnixTimestamp(logTimeInMillSeconds, ChronoUnit.MILLIS, @@ -105,13 +100,33 @@ public ServiceScriptLogDTO buildSystemScriptLog(ExecuteObject executeObject, "yyyy-MM-dd HH:mm:ss", ZoneId.systemDefault()); } String logContentWithDateTime = "[" + logDateTime + "] " + content + "\n"; - int length = logContentWithDateTime.getBytes(StandardCharsets.UTF_8).length; - return new ServiceScriptLogDTO(host, offset + length, logContentWithDateTime); + return buildScriptLog(stepInstance, executeObject, logContentWithDateTime, offset); } @Override - public void batchWriteScriptLog(long jobCreateTime, long stepInstanceId, int executeCount, Integer batch, - List scriptLogs) { + public ServiceExecuteObjectScriptLogDTO buildScriptLog(StepInstanceBaseDTO stepInstance, + ExecuteObject executeObject, + String content, + int offset) { + + int length = content.getBytes(StandardCharsets.UTF_8).length; + if (stepInstance.isSupportExecuteObject()) { + return new ServiceExecuteObjectScriptLogDTO( + executeObject.getId(), content, offset + length); + } else { + // 兼容历史版本使用 hostId 的方式；发布完成并全量切换到执行对象特性之后，可删除这里的兼容代码 + HostDTO host = executeObject.getHost(); + return new ServiceExecuteObjectScriptLogDTO( + host.getHostId(), host.toCloudIp(), host.toCloudIpv6(), content, offset + length); + } + } + + @Override + public void batchWriteScriptLog(long jobCreateTime, + long stepInstanceId, + int executeCount, + Integer batch, + List scriptLogs) { if (CollectionUtils.isEmpty(scriptLogs)) { return; } @@ -121,10 +136,10 @@ public void batchWriteScriptLog(long jobCreateTime, long stepInstanceId, int exe request.setJobCreateDate(jobCreateDate); request.setLogType(LogTypeEnum.SCRIPT.getValue()); List logs = scriptLogs.stream() - .map(scriptLog -> buildServiceLogDTO(stepInstanceId, executeCount, batch, scriptLog)) + .map(scriptLog -> buildServiceExecuteObjectLogDTO(stepInstanceId, executeCount, batch, scriptLog)) .collect(Collectors.toList()); request.setLogs(logs); - InternalResponse resp = logResource.saveLogs(request); + InternalResponse resp = logResource.saveLogs(request); if (!resp.isSuccess()) { log.error("Batch write log content fail, stepInstanceId:{}, executeCount:{}, batch: {}", stepInstanceId, executeCount, batch); @@ -132,12 +147,15 @@ public void batchWriteScriptLog(long jobCreateTime, long stepInstanceId, int exe } } - private ServiceExecuteObjectLogDTO buildServiceLogDTO(long stepInstanceId, int executeCount, Integer batch, - ServiceScriptLogDTO scriptLog) { + private ServiceExecuteObjectLogDTO buildServiceExecuteObjectLogDTO(long stepInstanceId, + int executeCount, + Integer batch, + ServiceExecuteObjectScriptLogDTO scriptLog) { ServiceExecuteObjectLogDTO logDTO = new ServiceExecuteObjectLogDTO(); logDTO.setStepInstanceId(stepInstanceId); logDTO.setExecuteCount(executeCount); logDTO.setBatch(batch); + logDTO.setExecuteObjectId(scriptLog.getExecuteObjectId()); logDTO.setHostId(scriptLog.getHostId()); logDTO.setCloudIp(scriptLog.getCloudIp()); logDTO.setCloudIpv6(scriptLog.getCloudIpv6()); @@ -146,166 +164,173 @@ private ServiceExecuteObjectLogDTO buildServiceLogDTO(long stepInstanceId, int e } @Override - public ScriptHostLogContent getScriptHostLogContent(long stepInstanceId, int executeCount, Integer batch, - HostDTO host) { - StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(stepInstanceId); - // 如果存在重试，那么该ip可能是之前已经执行过的，查询日志的时候需要获取到对应的executeCount + public ScriptExecuteObjectLogContent getScriptExecuteObjectLogContent( + StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch, + ExecuteObjectCompositeKey executeObjectCompositeKey + ) { + + // 如果存在重试，那么该任务可能是之前已经执行过的，查询日志的时候需要获取到对应的executeCount int actualExecuteCount = executeCount; - ExecuteObjectTask agentTask = scriptAgentTaskService.getTaskByExecuteObjectCompositeKey(stepInstance, - executeCount, batch, host); - if (agentTask == null) { + ExecuteObjectTask executeObjectTask = + scriptExecuteObjectTaskService.getTaskByExecuteObjectCompositeKey(stepInstance, + executeCount, batch, executeObjectCompositeKey); + if (executeObjectTask == null) { return null; } - if (executeCount > 0 && agentTask.getActualExecuteCount() != null) { - actualExecuteCount = agentTask.getActualExecuteCount(); - } else if (agentTask.getStatus() == ExecuteObjectTaskStatusEnum.LAST_SUCCESS) { - // 兼容历史数据 - actualExecuteCount = scriptAgentTaskService.getActualSuccessExecuteCount(stepInstanceId, host.toCloudIp()); + if (executeCount > 0 && executeObjectTask.getActualExecuteCount() != null) { + actualExecuteCount = executeObjectTask.getActualExecuteCount(); } - String taskCreateDateStr = DateUtils.formatUnixTimestamp(stepInstance.getCreateTime(), ChronoUnit.MILLIS, - "yyyy_MM_dd", ZoneId.of("UTC")); + String taskCreateDateStr = buildTaskCreateDateStr(stepInstance); + long stepInstanceId = stepInstance.getId(); InternalResponse resp; - if (agentTask.getHostId() != null) { - resp = logResource.getScriptHostLogByHostId(taskCreateDateStr, - stepInstanceId, actualExecuteCount, agentTask.getHostId(), batch); + if (stepInstance.isSupportExecuteObject()) { + resp = logResource.getScriptLogByExecuteObjectId(taskCreateDateStr, + stepInstanceId, actualExecuteCount, executeObjectCompositeKey.getExecuteObjectId(), batch); } else { - // 兼容ip查询 - resp = logResource.getScriptHostLogByIp(taskCreateDateStr, - stepInstanceId, actualExecuteCount, agentTask.getCloudIp(), batch); + // 兼容hostId查询 + resp = logResource.getScriptHostLogByHostId(taskCreateDateStr, + stepInstanceId, actualExecuteCount, executeObjectCompositeKey.getHostId(), batch); } if (!resp.isSuccess()) { - log.error("Get script log content by host error, stepInstanceId={}, executeCount={}, batch={}, host={}", - stepInstanceId, actualExecuteCount, batch, host); + log.error("Get script log content by execute object error, stepInstanceId={}, executeCount={}, batch={}" + + "executeObject={}", stepInstanceId, actualExecuteCount, batch, executeObjectCompositeKey); throw new InternalException(resp.getCode()); } - return convertToScriptHostLogContent(stepInstanceId, executeCount, resp.getData(), agentTask); + return convertToScriptExecuteObjectLogContent(executeObjectCompositeKey, stepInstance, executeCount, + resp.getData(), executeObjectTask); + } + + private String buildTaskCreateDateStr(StepInstanceBaseDTO stepInstance) { + return DateUtils.formatUnixTimestamp(stepInstance.getCreateTime(), ChronoUnit.MILLIS, + "yyyy_MM_dd", ZoneId.of("UTC")); } - private ScriptHostLogContent convertToScriptHostLogContent(long stepInstanceId, - int executeCount, - ServiceExecuteObjectLogDTO logDTO, - ExecuteObjectTask agentTask) { + private ScriptExecuteObjectLogContent convertToScriptExecuteObjectLogContent( + ExecuteObjectCompositeKey executeObjectCompositeKey, + StepInstanceBaseDTO stepInstance, + int executeCount, + ServiceExecuteObjectLogDTO executeObjectLog, + ExecuteObjectTask executeObjectTask + ) { + if (executeObjectLog == null) { + return null; + } + ExecuteObject executeObject = + stepInstance.getTargetExecuteObjects().findExecuteObjectByCompositeKey(executeObjectCompositeKey); // 日志是否拉取完成 - boolean isFinished = agentTask.getStatus().isFinished(); - String scriptContent = logDTO != null && logDTO.getScriptLog() != null ? - logDTO.getScriptLog().getContent() : ""; - Long hostId = logDTO != null ? logDTO.getHostId() : null; - String ip = logDTO != null ? logDTO.getCloudIp() : null; - String ipv6 = logDTO != null ? logDTO.getCloudIpv6() : null; - return new ScriptHostLogContent(stepInstanceId, executeCount, hostId, ip, ipv6, scriptContent, isFinished); + boolean isFinished = executeObjectTask.getStatus().isFinished(); + String scriptContent = executeObjectLog.getScriptLog() != null ? + executeObjectLog.getScriptLog().getContent() : ""; + return new ScriptExecuteObjectLogContent(stepInstance.getId(), executeCount, executeObject, scriptContent, + isFinished); } @Override - public List batchGetScriptHostLogContent(String jobCreateDateStr, - long stepInstanceId, - int executeCount, - Integer batch, - List hosts) { - if (CollectionUtils.isEmpty(hosts)) { + public List batchGetScriptExecuteObjectLogContent( + String jobCreateDateStr, + StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch, + List executeObjectCompositeKeys + ) { + if (CollectionUtils.isEmpty(executeObjectCompositeKeys)) { return Collections.emptyList(); } ServiceScriptLogQueryRequest query = new ServiceScriptLogQueryRequest(); query.setBatch(batch); - StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(stepInstanceId); - if (isQueryByHostIdCondition(stepInstance)) { - query.setHostIds(buildHostIdQueryCondition(stepInstance, hosts)); - if (CollectionUtils.isEmpty(query.getHostIds())) { - return Collections.emptyList(); - } + List queryExecuteObjects = + stepInstance.findExecuteObjectByCompositeKeys(executeObjectCompositeKeys); + if (stepInstance.isSupportExecuteObject()) { + List executeObjectId = queryExecuteObjects.stream() + .map(ExecuteObject::getId).collect(Collectors.toList()); + query.setExecuteObjectIds(executeObjectId); } else { - // 兼容历史数据的查询（使用ip，没有hostId) - query.setIps(hosts.stream().map(HostDTO::toCloudIp).distinct().collect(Collectors.toList())); + // 兼容 hostId 查询 + List hostIds = queryExecuteObjects.stream() + .map(executeObject -> executeObject.getHost().getHostId()).collect(Collectors.toList()); + query.setHostIds(hostIds); } + long stepInstanceId = stepInstance.getId(); InternalResponse> resp = logResource.listScriptExecuteObjectLogs(jobCreateDateStr, stepInstanceId, executeCount, query); if (!resp.isSuccess()) { - log.error("Get script log content by ips error, stepInstanceId={}, executeCount={}, batch={}, ips={}", - stepInstanceId, executeCount, batch, hosts); + log.error("Get script log content by execute objects error, stepInstanceId={}, executeCount={}, batch={}," + + "executeObjectCompositeKeys: {}", stepInstanceId, executeCount, batch, executeObjectCompositeKeys); throw new InternalException(resp.getCode()); } if (CollectionUtils.isEmpty(resp.getData())) { return Collections.emptyList(); } - return resp.getData().stream().map(logDTO -> { - String scriptContent = logDTO.getScriptLog() != null ? - logDTO.getScriptLog().getContent() : ""; - return new ScriptHostLogContent(logDTO.getStepInstanceId(), logDTO.getExecuteCount(), logDTO.getHostId(), - logDTO.getCloudIp(), logDTO.getCloudIpv6(), scriptContent, true); - }).collect(Collectors.toList()); - } - - private boolean isQueryByHostIdCondition(StepInstanceBaseDTO stepInstance) { - return stepInstance.getTargetServers().getIpList().stream() - .anyMatch(host -> host.getHostId() != null); - } - - private List buildHostIdQueryCondition(StepInstanceBaseDTO stepInstance, List hosts) { - boolean hostIdParamNull = hosts.get(0).getHostId() == null; - List hostIds; - if (hostIdParamNull) { - // 需要把ip查询参数转换为基于hostId的查询参数 - Map ip2Hosts = stepInstanceService.computeStepHosts(stepInstance, HostDTO::toCloudIp); - hostIds = hosts.stream() - .map(host -> Optional.ofNullable(ip2Hosts.get(host.toCloudIp())) - .map(HostDTO::getHostId) - .orElse(null)) - .filter(Objects::nonNull) - .distinct() - .collect(Collectors.toList()); + if (stepInstance.isSupportExecuteObject()) { + Map executeObjectMap = queryExecuteObjects.stream() + .collect(Collectors.toMap(ExecuteObject::getId, executeObject -> executeObject)); + return resp.getData().stream().map(logDTO -> { + String scriptContent = logDTO.getScriptLog() != null ? + logDTO.getScriptLog().getContent() : ""; + return new ScriptExecuteObjectLogContent(logDTO.getStepInstanceId(), logDTO.getExecuteCount(), + executeObjectMap.get(logDTO.getExecuteObjectId()), scriptContent, true); + }).collect(Collectors.toList()); } else { - hostIds = hosts.stream() - .map(HostDTO::getHostId) - .distinct() - .collect(Collectors.toList()); + Map executeObjectMap = queryExecuteObjects.stream() + .collect(Collectors.toMap( + executeObject -> executeObject.getHost().getHostId(), executeObject -> executeObject)); + return resp.getData().stream().map(logDTO -> { + String scriptContent = logDTO.getScriptLog() != null ? + logDTO.getScriptLog().getContent() : ""; + return new ScriptExecuteObjectLogContent(logDTO.getStepInstanceId(), logDTO.getExecuteCount(), + executeObjectMap.get(logDTO.getHostId()), scriptContent, true); + }).collect(Collectors.toList()); } - - return hostIds; } @Override - public FileIpLogContent getFileIpLogContent(long stepInstanceId, int executeCount, Integer batch, HostDTO host, - Integer mode) { - StepInstanceDTO stepInstance = taskInstanceService.getStepInstanceDetail(stepInstanceId); - // 如果存在重试，那么该ip可能是之前已经执行过的，查询日志的时候需要获取到对应的executeCount + public FileExecuteObjectLogContent getFileExecuteObjectLogContent( + StepInstanceDTO stepInstance, + int executeCount, + Integer batch, + ExecuteObjectCompositeKey executeObjectCompositeKey, + Integer mode + ) { + // 如果存在重试，那么该任务可能是之前已经执行过的，查询日志的时候需要获取到对应的executeCount int actualExecuteCount = executeCount; - ExecuteObjectTask agentTask = fileAgentTaskService.getAgentTaskByHost(stepInstance, executeCount, batch, - FileTaskModeEnum.getFileTaskMode(mode), host); - if (agentTask == null) { + ExecuteObjectTask executeObjectTask = fileExecuteObjectTaskService.getTaskByExecuteObjectCompositeKey( + stepInstance, executeCount, batch, FileTaskModeEnum.getFileTaskMode(mode), executeObjectCompositeKey); + if (executeObjectTask == null) { return null; } - if (executeCount > 0 && agentTask.getActualExecuteCount() != null) { - actualExecuteCount = agentTask.getActualExecuteCount(); - } else if (agentTask.getStatus() == ExecuteObjectTaskStatusEnum.LAST_SUCCESS) { - // 兼容历史数据 - actualExecuteCount = scriptAgentTaskService.getActualSuccessExecuteCount(stepInstanceId, host.toCloudIp()); + if (executeCount > 0 && executeObjectTask.getActualExecuteCount() != null) { + actualExecuteCount = executeObjectTask.getActualExecuteCount(); } - String taskCreateDateStr = DateUtils.formatUnixTimestamp(stepInstance.getCreateTime(), ChronoUnit.MILLIS, - "yyyy_MM_dd", ZoneId.of("UTC")); + String taskCreateDateStr = buildTaskCreateDateStr(stepInstance); + long stepInstanceId = stepInstance.getId(); InternalResponse resp; - if (agentTask.getHostId() != null) { + if (stepInstance.isSupportExecuteObject()) { resp = logResource.getFileLogByExecuteObjectId(taskCreateDateStr, - stepInstanceId, actualExecuteCount, agentTask.getHostId(), mode, batch); + stepInstanceId, actualExecuteCount, executeObjectCompositeKey.getExecuteObjectId(), mode, batch); } else { - // 兼容ip查询 - resp = logResource.getFileHostLogByIp(taskCreateDateStr, - stepInstanceId, actualExecuteCount, agentTask.getCloudIp(), mode, batch); + // 兼容hostId查询 + resp = logResource.getFileHostLogByHostId(taskCreateDateStr, + stepInstanceId, actualExecuteCount, executeObjectCompositeKey.getHostId(), mode, batch); } if (!resp.isSuccess()) { - log.error("Get file log content by ip error, stepInstanceId={}, executeCount={}, batch={}, host={}", - stepInstanceId, actualExecuteCount, batch, host); + log.error("Get file log content by execute object error, stepInstanceId={}, executeCount={}, batch={}" + + "executeObject={}", stepInstanceId, actualExecuteCount, batch, executeObjectCompositeKey); throw new InternalException(resp.getCode()); } List fileTaskLogs = (resp.getData() == null) ? null : resp.getData().getFileTaskLogs(); - ExecuteObjectTaskStatusEnum agentTaskStatus = agentTask.getStatus(); - boolean isFinished = agentTaskStatus.isFinished() || isAllFileTasksFinished(fileTaskLogs); - return new FileIpLogContent(stepInstanceId, executeCount, null, fileTaskLogs, isFinished); + ExecuteObjectTaskStatusEnum executeObjectTaskStatus = executeObjectTask.getStatus(); + boolean isFinished = executeObjectTaskStatus.isFinished() || isAllFileTasksFinished(fileTaskLogs); + ExecuteObject executeObject = stepInstance.findExecuteObjectByCompositeKey(executeObjectCompositeKey); + return new FileExecuteObjectLogContent(stepInstanceId, executeCount, executeObject, fileTaskLogs, isFinished); } private boolean isAllFileTasksFinished(List fileTaskLogs) { @@ -325,11 +350,12 @@ private boolean isFileTaskNotFinished(ServiceFileTaskLogDTO fileTaskLog) { } @Override - public List getFileLogContentByTaskIds(long stepInstanceId, int executeCount, Integer batch, + public List getFileLogContentByTaskIds(long stepInstanceId, + int executeCount, + Integer batch, List taskIds) { StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(stepInstanceId); - String taskCreateDateStr = DateUtils.formatUnixTimestamp(stepInstance.getCreateTime(), ChronoUnit.MILLIS, - "yyyy_MM_dd", ZoneId.of("UTC")); + String taskCreateDateStr = buildTaskCreateDateStr(stepInstance); InternalResponse resp = logResource.listFileLogsByTaskIds( taskCreateDateStr, stepInstanceId, executeCount, batch, taskIds); if (!resp.isSuccess()) { @@ -344,12 +370,11 @@ public List getFileLogContentByTaskIds(long stepInstanceI } @Override - public List batchGetFileSourceIpLogContent(long stepInstanceId, - int executeCount, - Integer batch) { + public List batchGetFileSourceExecuteObjectLogContent(long stepInstanceId, + int executeCount, + Integer batch) { StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(stepInstanceId); - String taskCreateDateStr = DateUtils.formatUnixTimestamp(stepInstance.getCreateTime(), ChronoUnit.MILLIS, - "yyyy_MM_dd", ZoneId.of("UTC")); + String taskCreateDateStr = buildTaskCreateDateStr(stepInstance); InternalResponse> resp = logResource.listFileHostLogs( taskCreateDateStr, stepInstanceId, executeCount, batch, FileDistModeEnum.UPLOAD.getValue(), null, null); if (!resp.isSuccess()) { @@ -361,30 +386,37 @@ public List batchGetFileSourceIpLogContent(long stepInsta } @Override - public ServiceExecuteObjectLogsDTO batchGetFileIpLogContent(long stepInstanceId, int executeCount, Integer batch, - List hosts) { - if (CollectionUtils.isEmpty(hosts)) { + public List batchGetFileExecuteObjectLogContent( + StepInstanceDTO stepInstance, + int executeCount, + Integer batch, + List executeObjectCompositeKeys + ) { + if (CollectionUtils.isEmpty(executeObjectCompositeKeys)) { return null; } - StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(stepInstanceId); - String taskCreateDateStr = DateUtils.formatUnixTimestamp(stepInstance.getCreateTime(), ChronoUnit.MILLIS, - "yyyy_MM_dd", ZoneId.of("UTC")); + String taskCreateDateStr = buildTaskCreateDateStr(stepInstance); ServiceFileLogQueryRequest request = new ServiceFileLogQueryRequest(); - request.setStepInstanceId(stepInstanceId); + request.setStepInstanceId(stepInstance.getId()); request.setExecuteCount(executeCount); request.setBatch(batch); request.setJobCreateDate(taskCreateDateStr); - if (isQueryByHostIdCondition(stepInstance)) { - request.setHostIds(buildHostIdQueryCondition(stepInstance, hosts)); - if (CollectionUtils.isEmpty(request.getHostIds())) { - return null; - } + List queryExecuteObjects = + stepInstance.findExecuteObjectByCompositeKeys(executeObjectCompositeKeys); + if (stepInstance.isSupportExecuteObject()) { + List executeObjectId = queryExecuteObjects.stream() + .map(ExecuteObject::getId).collect(Collectors.toList()); + request.setExecuteObjectIds(executeObjectId); } else { - request.setIps(hosts.stream().map(HostDTO::toCloudIp).distinct().collect(Collectors.toList())); + // 兼容 hostId 查询 + List hostIds = queryExecuteObjects.stream() + .map(executeObject -> executeObject.getHost().getHostId()).collect(Collectors.toList()); + request.setHostIds(hostIds); } - InternalResponse resp = logResource.listFileHostLogs(request); + InternalResponse> resp = logResource.listFileExecuteObjectLogs( + taskCreateDateStr, stepInstance.getId(), executeCount, request); if (!resp.isSuccess()) { log.error("Get file log content error, request={}", request); return null; @@ -393,11 +425,10 @@ public ServiceExecuteObjectLogsDTO batchGetFileIpLogContent(long stepInstanceId, } @Override - public List getIpsByContentKeyword(long stepInstanceId, int executeCount, Integer batch, - String keyword) { + public List getHostsByContentKeyword(long stepInstanceId, int executeCount, Integer batch, + String keyword) { StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(stepInstanceId); - String taskCreateDateStr = DateUtils.formatUnixTimestamp(stepInstance.getCreateTime(), ChronoUnit.MILLIS, - "yyyy_MM_dd", ZoneId.of("UTC")); + String taskCreateDateStr = buildTaskCreateDateStr(stepInstance); InternalResponse> resp = logResource.questHostsByLogKeyword(taskCreateDateStr, stepInstanceId, executeCount, batch, keyword); @@ -444,65 +475,106 @@ public void writeFileLogsWithTimestamp(long jobCreateTime, } @Override - public void writeFileLogs(long jobCreateTime, List hostFileLogs) { - writeFileLogsWithTimestamp(jobCreateTime, hostFileLogs, System.currentTimeMillis()); + public void writeFileLogs(long jobCreateTime, List executeObjectLogs) { + writeFileLogsWithTimestamp(jobCreateTime, executeObjectLogs, System.currentTimeMillis()); } @Override - public ServiceFileTaskLogDTO buildUploadServiceFileTaskLogDTO(JobFile srcFile, + public ServiceFileTaskLogDTO buildUploadServiceFileTaskLogDTO(StepInstanceDTO stepInstance, + JobFile srcFile, FileDistStatusEnum status, String size, String speed, String process, String content) { - return new ServiceFileTaskLogDTO( - FileDistModeEnum.UPLOAD.getValue(), - null, - null, - null, - null, - srcFile.getHost().getHostId(), - srcFile.getHost().toCloudIp(), - srcFile.getHost().toCloudIpv6(), - srcFile.getFileType().getType(), - srcFile.getStandardFilePath(), - srcFile.getDisplayFilePath(), - size, - status.getValue(), - status.getName(), - speed, - process, - content - ); + if (stepInstance.isSupportExecuteObject()) { + return new ServiceFileTaskLogDTO( + FileDistModeEnum.UPLOAD.getValue(), + null, + null, + srcFile.getExecuteObject().getId(), + srcFile.getFileType().getType(), + srcFile.getStandardFilePath(), + srcFile.getDisplayFilePath(), + size, + status.getValue(), + status.getName(), + speed, + process, + content + ); + } else { + HostDTO sourceHost = srcFile.getExecuteObject().getHost(); + return new ServiceFileTaskLogDTO( + FileDistModeEnum.UPLOAD.getValue(), + null, + null, + null, + null, + sourceHost.getHostId(), + sourceHost.toCloudIp(), + sourceHost.toCloudIpv6(), + srcFile.getFileType().getType(), + srcFile.getStandardFilePath(), + srcFile.getDisplayFilePath(), + size, + status.getValue(), + status.getName(), + speed, + process, + content + ); + } } @Override - public ServiceFileTaskLogDTO buildDownloadServiceFileTaskLogDTO(JobFile srcFile, - HostDTO targetHost, + public ServiceFileTaskLogDTO buildDownloadServiceFileTaskLogDTO(StepInstanceDTO stepInstance, + JobFile srcFile, + ExecuteObject targetExecuteObject, String targetPath, FileDistStatusEnum status, String size, String speed, String process, String content) { - return new ServiceFileTaskLogDTO( - FileDistModeEnum.DOWNLOAD.getValue(), - targetHost.getHostId(), - targetHost.toCloudIp(), - targetHost.toCloudIpv6(), - targetPath, - srcFile.getHost().getHostId(), - srcFile.getHost().toCloudIp(), - srcFile.getHost().toCloudIpv6(), - srcFile.getFileType().getType(), - srcFile.getStandardFilePath(), - srcFile.getDisplayFilePath(), - size, - status.getValue(), - status.getName(), - speed, - process, - content - ); + if (stepInstance.isSupportExecuteObject()) { + return new ServiceFileTaskLogDTO( + FileDistModeEnum.DOWNLOAD.getValue(), + targetExecuteObject.getId(), + targetPath, + srcFile.getExecuteObject().getId(), + srcFile.getFileType().getType(), + srcFile.getStandardFilePath(), + srcFile.getDisplayFilePath(), + size, + status.getValue(), + status.getName(), + speed, + process, + content + ); + } else { + HostDTO sourceHost = srcFile.getExecuteObject().getHost(); + HostDTO targetHost = targetExecuteObject.getHost(); + return new ServiceFileTaskLogDTO( + FileDistModeEnum.DOWNLOAD.getValue(), + targetHost.getHostId(), + targetHost.toCloudIp(), + targetHost.toCloudIpv6(), + targetPath, + sourceHost.getHostId(), + sourceHost.toCloudIp(), + sourceHost.toCloudIpv6(), + srcFile.getFileType().getType(), + srcFile.getStandardFilePath(), + srcFile.getDisplayFilePath(), + size, + status.getValue(), + status.getName(), + speed, + process, + content + ); + } } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/NotifyServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/NotifyServiceImpl.java index 5d4137c2d9..17e4b42361 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/NotifyServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/NotifyServiceImpl.java @@ -261,7 +261,7 @@ private Map getTemplateVariablesMap(TaskNotifyDTO taskNotifyDTO, if (taskInstanceDTO.getTotalTime() != null) { variablesMap.put("task.total_duration", "" + taskInstanceDTO.getTotalTime() / 1000.0); } - int totalTargetIpCount = stepInstanceDTO.getTargetServers().getIpList().size(); + int totalTargetIpCount = stepInstanceDTO.getTargetExecuteObjects().getIpList().size(); if (executeStatus == ExecuteStatusEnum.SUCCESS) { variablesMap.put("task.step.success_cnt", "" + totalTargetIpCount); variablesMap.put("task.step.failed_cnt", "" + 0); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/RollingConfigServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/RollingConfigServiceImpl.java index e3095ec810..d1a3985e5a 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/RollingConfigServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/RollingConfigServiceImpl.java @@ -71,14 +71,14 @@ public List getRollingServers(StepInstanceBaseDTO stepInstance, Integer if (rollingConfig.isBatchRollingStep(stepInstanceId)) { if (batch == null || batch == 0) { // 忽略滚动批次，返回当前步骤的所有目标服务器 - return stepInstance.getTargetServers().getIpList(); + return stepInstance.getTargetExecuteObjects().getIpList(); } else { return rollingConfig.getConfigDetail().getHostsBatchList() .stream().filter(serverBatch -> serverBatch.getBatch().equals(batch)) .findFirst().orElseThrow(() -> new InternalException(ErrorCode.INTERNAL_ERROR)).getHosts(); } } else { - return stepInstance.getTargetServers().getIpList(); + return stepInstance.getTargetExecuteObjects().getIpList(); } } @@ -100,7 +100,7 @@ public RollingConfigDTO saveRollingConfigForFastJob(FastTaskDTO fastTask) { rollingConfigDetailDO.setExpr(rollingConfig.getExpr()); RollingBatchServersResolver resolver = - new RollingBatchServersResolver(fastTask.getStepInstance().getTargetServers().getIpList(), + new RollingBatchServersResolver(fastTask.getStepInstance().getTargetExecuteObjects().getIpList(), rollingConfig.getExpr()); List serversBatchList = resolver.resolve(); rollingConfigDetailDO.setHostsBatchList( diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptExecuteObjectTaskServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptExecuteObjectTaskServiceImpl.java index bde530bf9d..aaa00a313f 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptExecuteObjectTaskServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptExecuteObjectTaskServiceImpl.java @@ -1,20 +1,17 @@ package com.tencent.bk.job.execute.service.impl; import com.tencent.bk.job.common.constant.Order; -import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.dao.ScriptAgentTaskDAO; import com.tencent.bk.job.execute.dao.ScriptExecuteObjectTaskDAO; import com.tencent.bk.job.execute.engine.model.ExecuteObject; import com.tencent.bk.job.execute.model.ExecuteObjectCompositeKey; import com.tencent.bk.job.execute.model.ExecuteObjectTask; import com.tencent.bk.job.execute.model.ExecuteObjectTaskDetail; -import com.tencent.bk.job.execute.model.ExecuteObjectsDTO; import com.tencent.bk.job.execute.model.ResultGroupBaseDTO; import com.tencent.bk.job.execute.model.ResultGroupDTO; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.service.ScriptExecuteObjectTaskService; import com.tencent.bk.job.execute.service.StepInstanceService; -import com.tencent.bk.job.execute.service.TaskInstanceExecuteObjectService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -23,7 +20,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; -import java.util.Objects; import java.util.stream.Collectors; @Service @@ -36,11 +32,10 @@ public class ScriptExecuteObjectTaskServiceImpl private final ScriptAgentTaskDAO scriptAgentTaskDAO; @Autowired - public ScriptExecuteObjectTaskServiceImpl(ScriptExecuteObjectTaskDAO scriptExecuteObjectTaskDAO, - StepInstanceService stepInstanceService, - ScriptAgentTaskDAO scriptAgentTaskDAO, - TaskInstanceExecuteObjectService taskInstanceExecuteObjectService) { - super(stepInstanceService, taskInstanceExecuteObjectService); + public ScriptExecuteObjectTaskServiceImpl(StepInstanceService stepInstanceService, + ScriptExecuteObjectTaskDAO scriptExecuteObjectTaskDAO, + ScriptAgentTaskDAO scriptAgentTaskDAO) { + super(stepInstanceService); this.scriptExecuteObjectTaskDAO = scriptExecuteObjectTaskDAO; this.scriptAgentTaskDAO = scriptAgentTaskDAO; } @@ -50,9 +45,8 @@ public void batchSaveTasks(Collection tasks) { if (CollectionUtils.isEmpty(tasks)) { return; } - ExecuteObjectTask anyTask = tasks.stream().findAny().orElse(null); - if (Objects.requireNonNull(anyTask).getExecuteObjId() != null) { + if (isSaveTasksUsingExecuteObjectMode(tasks)) { scriptExecuteObjectTaskDAO.batchSaveTasks(tasks); } else { scriptAgentTaskDAO.batchSaveAgentTasks(tasks); @@ -60,12 +54,11 @@ public void batchSaveTasks(Collection tasks) { } @Override - public void batchUpdateTasks(Collection tasks) { + public void batchUpdateTasks(Collection tasks) { if (CollectionUtils.isEmpty(tasks)) { return; } - ExecuteObjectTask anyTask = tasks.stream().findAny().orElse(null); - if (Objects.requireNonNull(anyTask).getExecuteObjId() != null) { + if (isSaveTasksUsingExecuteObjectMode(tasks)) { scriptExecuteObjectTaskDAO.batchUpdateTasks(tasks); } else { scriptAgentTaskDAO.batchUpdateAgentTasks(tasks); @@ -85,7 +78,7 @@ public int getSuccessTaskCount(long stepInstanceId, int executeCount) { public List listTasks(StepInstanceBaseDTO stepInstance, Integer executeCount, Integer batch) { List executeObjectTasks; long stepInstanceId = stepInstance.getId(); - if (isUsingExecuteObject(stepInstance)) { + if (stepInstance.isSupportExecuteObject()) { executeObjectTasks = scriptExecuteObjectTaskDAO.listTasks(stepInstanceId, executeCount, batch); } else { // 兼容老版本数据 @@ -94,32 +87,17 @@ public List listTasks(StepInstanceBaseDTO stepInstance, Integ return executeObjectTasks; } - /** - * 判断是否使用执行对象的方式存储 - * - * @param stepInstance 步骤实例 - */ - private boolean isUsingExecuteObject(StepInstanceBaseDTO stepInstance) { - ExecuteObjectsDTO servers = stepInstance.getTargetServers(); - if (CollectionUtils.isNotEmpty(servers.getExecuteObjects())) { - ExecuteObject executeObject = servers.getExecuteObjects().stream().findAny().orElse(null); - return Objects.requireNonNull(executeObject).getId() != null; - } else { - return false; - } - } - @Override - public List listTasksByGseTaskId(StepInstanceBaseDTO stepInstance, Long gseTaskId) { + public List listTasksByGseTaskId(StepInstanceBaseDTO stepInstance, Long gseTaskId) { List executeObjectTasks; - if (isUsingExecuteObject(stepInstance)) { + if (stepInstance.isSupportExecuteObject()) { executeObjectTasks = scriptExecuteObjectTaskDAO.listTasksByGseTaskId(gseTaskId); } else { // 兼容老版本数据 executeObjectTasks = scriptAgentTaskDAO.listAgentTasksByGseTaskId(gseTaskId); } - return executeObjectTasks; + return convertToExecuteObjectDetailList(stepInstance, executeObjectTasks); } @Override @@ -127,68 +105,26 @@ public ExecuteObjectTask getTaskByExecuteObjectCompositeKey(StepInstanceBaseDTO Integer executeCount, Integer batch, ExecuteObjectCompositeKey executeObjectCompositeKey) { - ExecuteObjectTask executeObjectTask = null; - long stepInstanceId = stepInstance.getId(); + ExecuteObject executeObject = + stepInstance.findExecuteObjectByCompositeKey(executeObjectCompositeKey); + if (executeObject == null) { + return null; + } - if (isUsingExecuteObject(stepInstance)) { - String executeObjectResourceId = executeObjectCompositeKey.getResourceId(); - if (executeObjectResourceId != null) { - ExecuteObject executeObject = taskInstanceExecuteObjectService.getExecuteObject( - executeObjectCompositeKey.getExecuteObjectType(), executeObjectCompositeKey.getResourceId()); - if (executeObject == null) { - return null; - } - executeObjectTask = scriptExecuteObjectTaskDAO.getTaskByExecuteObjectId(stepInstanceId, executeCount, - batch, executeObject.getId()); - } else { - // 兼容使用<云区域+ip> 的方式查询主机执行任务 - ExecuteObject executeObject = getStepHostExecuteObjectByCloudIp( - stepInstance, executeObjectCompositeKey.getCloudIp()); - executeObjectTask = scriptExecuteObjectTaskDAO.getTaskByExecuteObjectId(stepInstance.getId(), - executeCount, batch, executeObject.getId()); - } + long stepInstanceId = stepInstance.getId(); + ExecuteObjectTask executeObjectTask; + if (stepInstance.isSupportExecuteObject()) { + executeObjectTask = scriptExecuteObjectTaskDAO.getTaskByExecuteObjectId(stepInstanceId, executeCount, + batch, executeObject.getId()); } else { // 兼容老版本不使用执行对象的场景(仅支持主机） - Long hostId = executeObjectCompositeKey.getHostId(); - if (hostId != null) { - executeObjectTask = scriptAgentTaskDAO.getAgentTaskByHostId(stepInstanceId, executeCount, - batch, hostId); - } else { - // 兼容使用<云区域+ip> 的方式查询主机执行任务 - HostDTO host = getStepHostByCloudIp(stepInstance, executeObjectCompositeKey.getCloudIp()); - if (host != null) { - executeObjectTask = scriptAgentTaskDAO.getAgentTaskByHostId(stepInstanceId, executeCount, - batch, host.getHostId()); - } - } - + Long hostId = executeObject.getHost().getHostId(); + executeObjectTask = scriptAgentTaskDAO.getAgentTaskByHostId(stepInstanceId, executeCount, + batch, hostId); } return executeObjectTask; } - private ExecuteObject getStepHostExecuteObjectByCloudIp(StepInstanceBaseDTO stepInstance, String cloudIp) { - return stepInstance.getTargetServers().getExecuteObjects() - .stream() - .filter(executeObject -> { - HostDTO host = executeObject.getHost(); - if (host == null) { - return false; - } - return cloudIp.equals(host.toCloudIp()); - }) - .findFirst() - .orElse(null); - } - - private HostDTO getStepHostByCloudIp(StepInstanceBaseDTO stepInstance, - String cloudIp) { - return stepInstance.getTargetServers().getIpList() - .stream() - .filter(targetHost -> cloudIp.equals(targetHost.toCloudIp())) - .findFirst() - .orElse(null); - } - @Override public List listAndGroupTasks(StepInstanceBaseDTO stepInstance, int executeCount, @@ -199,7 +135,7 @@ public List listAndGroupTasks(StepInstanceBaseDTO stepInstance, return resultGroups; } - List agentTaskDetailList = fillExecuteObjectDetail(stepInstance, executeObjectTasks); + List agentTaskDetailList = convertToExecuteObjectDetailList(stepInstance, executeObjectTasks); resultGroups = groupTasks(agentTaskDetailList); return resultGroups.stream().sorted().collect(Collectors.toList()); @@ -212,7 +148,7 @@ public List listResultGroups(StepInstanceBaseDTO stepInstanc List resultGroups; long stepInstanceId = stepInstance.getId(); - if (isUsingExecuteObject(stepInstance)) { + if (stepInstance.isSupportExecuteObject()) { resultGroups = scriptExecuteObjectTaskDAO.listResultGroups(stepInstanceId, executeCount, batch); } else { // 兼容历史数据 @@ -229,7 +165,7 @@ public List listTaskDetailByResultGroup(StepInstanceBas String tag) { List executeObjectTasks; - if (isUsingExecuteObject(stepInstance)) { + if (stepInstance.isSupportExecuteObject()) { executeObjectTasks = scriptExecuteObjectTaskDAO.listTasksByResultGroup(stepInstance.getId(), executeCount, batch, status, tag); } else { @@ -238,7 +174,7 @@ public List listTaskDetailByResultGroup(StepInstanceBas stepInstance.getId(), executeCount, batch, status, tag); } - return fillExecuteObjectDetail(stepInstance, executeObjectTasks); + return convertToExecuteObjectDetailList(stepInstance, executeObjectTasks); } @@ -253,7 +189,7 @@ public List listTaskDetailByResultGroup(StepInstanceBas Order order) { List executeObjectTasks; - if (isUsingExecuteObject(stepInstance)) { + if (stepInstance.isSupportExecuteObject()) { executeObjectTasks = scriptExecuteObjectTaskDAO.listTasksByResultGroup(stepInstance.getId(), executeCount, batch, status, tag, limit, orderField, order); } else { @@ -262,7 +198,7 @@ public List listTaskDetailByResultGroup(StepInstanceBas batch, status, tag, limit, orderField, order); } - return fillExecuteObjectDetail(stepInstance, executeObjectTasks); + return convertToExecuteObjectDetailList(stepInstance, executeObjectTasks); } @Override @@ -270,7 +206,7 @@ public List listTaskDetail(StepInstanceBaseDTO stepInst Integer executeCount, Integer batch) { List executeObjectTasks = listTasks(stepInstance, executeCount, batch); - return fillExecuteObjectDetail(stepInstance, executeObjectTasks); + return convertToExecuteObjectDetailList(stepInstance, executeObjectTasks); } private boolean isStepInstanceRecordExist(long stepInstanceId) { @@ -283,7 +219,7 @@ public void updateTaskFields(StepInstanceBaseDTO stepInstance, Integer batch, Integer actualExecuteCount, Long gseTaskId) { - if (isUsingExecuteObject(stepInstance)) { + if (stepInstance.isSupportExecuteObject()) { scriptExecuteObjectTaskDAO.updateTaskFields(stepInstance.getId(), executeCount, batch, actualExecuteCount, gseTaskId); } else { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceServiceImpl.java index c9d53efe26..f8251df13a 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceServiceImpl.java @@ -26,6 +26,7 @@ import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.dao.StepInstanceDAO; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; import com.tencent.bk.job.execute.model.FileSourceDTO; import com.tencent.bk.job.execute.model.FileStepInstanceDTO; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; @@ -78,7 +79,7 @@ public Map computeStepHosts(StepInstanceBaseDTO stepInstance, Function keyMapper) { Map hosts = new HashMap<>(); - stepInstance.getTargetServers().getIpList() + stepInstance.getTargetExecuteObjects().getIpList() .forEach(host -> { K key = keyMapper.apply(host); if (key != null) { @@ -122,6 +123,48 @@ public Map computeStepHosts(long stepInstanceId, return computeStepHosts(stepInstance, keyMapper); } + @Override + public Map computeStepExecuteObjects(StepInstanceBaseDTO stepInstance, + Function keyMapper) { + + Map executeObjects = new HashMap<>(); + stepInstance.getTargetExecuteObjects().getExecuteObjects() + .forEach(executeObject -> { + K key = keyMapper.apply(executeObject); + if (key != null) { + executeObjects.put(keyMapper.apply(executeObject), executeObject); + } + }); + + if (stepInstance.isFileStep()) { + List fileSourceList; + if (stepInstance instanceof StepInstanceDTO) { + fileSourceList = ((StepInstanceDTO) stepInstance).getFileSourceList(); + } else { + FileStepInstanceDTO fileStepInstance = stepInstanceDAO.getFileStepInstance(stepInstance.getId()); + fileSourceList = fileStepInstance.getFileSourceList(); + } + + if (CollectionUtils.isNotEmpty(fileSourceList)) { + fileSourceList.forEach( + fileSource -> { + if (fileSource.getServers() != null + && CollectionUtils.isNotEmpty(fileSource.getServers().getExecuteObjects())) { + fileSource.getServers().getExecuteObjects() + .forEach(executeObject -> { + K key = keyMapper.apply(executeObject); + if (key != null) { + executeObjects.put(keyMapper.apply(executeObject), executeObject); + } + }); + } + }); + } + } + + return executeObjects; + } + @Override public StepInstanceBaseDTO getStepInstanceBase(long stepInstanceId) { return stepInstanceDAO.getStepInstanceBase(stepInstanceId); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java index 86c7aa2595..dec927ae1a 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java @@ -683,7 +683,7 @@ private AuthResult authExecuteScript(TaskInstanceDTO taskInstance, , accountId); AuthResult serverAuthResult; - ExecuteObjectsDTO servers = stepInstance.getTargetServers().clone(); + ExecuteObjectsDTO servers = stepInstance.getTargetExecuteObjects().clone(); filterServerDoNotRequireAuth(ActionScopeEnum.SCRIPT_EXECUTE, servers, whiteHostAllowActions); if (servers.isEmpty()) { // 如果主机为空，无需对主机进行鉴权 @@ -750,7 +750,7 @@ private AuthResult authFileTransfer(TaskInstanceDTO taskInstance, AuthResult accountAuthResult = executeAuthService.batchAuthAccountExecutable( username, new AppResourceScope(appId), accounts); - ExecuteObjectsDTO servers = stepInstance.getTargetServers().clone(); + ExecuteObjectsDTO servers = stepInstance.getTargetExecuteObjects().clone(); stepInstance.getFileSourceList().stream() .filter(fileSource -> !fileSource.isLocalUpload() && fileSource.getFileType() != TaskFileTypeEnum.BASE64_FILE.getType() @@ -839,7 +839,7 @@ private void fillDynamicGroupHosts(StopWatch watch, Map> dynamicGroupHosts = hostService.batchGetAndGroupHostsByDynamicGroup(appId, groups); stepInstances.forEach(stepInstance -> { - setHostsForDynamicGroup(stepInstance.getTargetServers(), dynamicGroupHosts); + setHostsForDynamicGroup(stepInstance.getTargetExecuteObjects(), dynamicGroupHosts); if (stepInstance.getExecuteType() == TaskStepTypeEnum.FILE.getValue()) { List fileSources = stepInstance.getFileSourceList(); for (FileSourceDTO fileSource : fileSources) { @@ -872,7 +872,7 @@ private void fillTopoNodeHosts(StopWatch watch, Map> topoNodeHosts = hostService.getAndGroupHostsByTopoNodes(appId, topoNodes); stepInstances.forEach(stepInstance -> { - setHostsForTopoNode(stepInstance.getTargetServers(), topoNodeHosts); + setHostsForTopoNode(stepInstance.getTargetExecuteObjects(), topoNodeHosts); if (stepInstance.getExecuteType() == TaskStepTypeEnum.FILE.getValue()) { List fileSources = stepInstance.getFileSourceList(); for (FileSourceDTO fileSource : fileSources) { @@ -942,7 +942,7 @@ private void checkHostAccessible(List stepInstanceList, } TaskStepTypeEnum stepType = stepInstance.getStepType(); // 检查目标主机 - stepInstance.getTargetServers().getIpList().forEach(host -> { + stepInstance.getTargetExecuteObjects().getIpList().forEach(host -> { if (isHostUnAccessible(stepType, host, notInAppHostMap, whileHostAllowActions)) { invalidHosts.add(host); } @@ -1034,7 +1034,7 @@ private boolean isUsingGseV2(TaskInstanceDTO taskInstance, Collection t private void extractDynamicGroupsAndTopoNodes(StepInstanceDTO stepInstance, Set groups, Set topoNodes) { - extractDynamicGroupsAndTopoNodes(stepInstance.getTargetServers(), groups, topoNodes); + extractDynamicGroupsAndTopoNodes(stepInstance.getTargetExecuteObjects(), groups, topoNodes); if (stepInstance.getExecuteType() == TaskStepTypeEnum.FILE.getValue()) { List fileSources = stepInstance.getFileSourceList(); for (FileSourceDTO fileSource : fileSources) { @@ -1079,7 +1079,7 @@ private void checkStepInstanceHostNonEmpty(StepInstanceDTO stepInstance) { if (!isStepContainsHostProps(stepInstance)) { return; } - ExecuteObjectsDTO targetServers = stepInstance.getTargetServers(); + ExecuteObjectsDTO targetServers = stepInstance.getTargetExecuteObjects(); if (targetServers == null || CollectionUtils.isEmpty(targetServers.getIpList())) { log.warn("Empty target server! stepInstanceName: {}", stepInstance.getName()); throw new FailedPreconditionException(ErrorCode.STEP_TARGET_HOST_EMPTY, @@ -1184,7 +1184,7 @@ private void setHostAgentId(boolean isUsingGseV2, HostDTO host, Set inv } private void fillTargetHostDetail(StepInstanceDTO stepInstance, Map hostMap) { - fillServersDetail(stepInstance.getTargetServers(), hostMap); + fillServersDetail(stepInstance.getTargetExecuteObjects(), hostMap); } private void fillFileSourceHostDetail(StepInstanceDTO stepInstance, Map hostMap) { @@ -1232,8 +1232,8 @@ private Set extractHosts(List stepInstanceList, if (!isStepContainsHostProps(stepInstance)) { continue; } - if (stepInstance.getTargetServers() != null) { - hosts.addAll(stepInstance.getTargetServers().extractHosts()); + if (stepInstance.getTargetExecuteObjects() != null) { + hosts.addAll(stepInstance.getTargetExecuteObjects().extractHosts()); } if (stepInstance.getExecuteType().equals(SEND_FILE.getValue())) { List fileSourceList = stepInstance.getFileSourceList(); @@ -1465,9 +1465,9 @@ private TaskInstanceDTO executeJobPlanInternal(StopWatch watch, TaskExecuteParam private void standardizeStepDynamicGroupId(List stepInstanceList) { for (StepInstanceDTO stepInstance : stepInstanceList) { - if (stepInstance.getTargetServers() != null - && CollectionUtils.isNotEmpty(stepInstance.getTargetServers().getDynamicServerGroups())) { - standardizeServerDynamicGroupId(stepInstance.getTargetServers()); + if (stepInstance.getTargetExecuteObjects() != null + && CollectionUtils.isNotEmpty(stepInstance.getTargetExecuteObjects().getDynamicServerGroups())) { + standardizeServerDynamicGroupId(stepInstance.getTargetExecuteObjects()); } if (CollectionUtils.isNotEmpty(stepInstance.getFileSourceList())) { for (FileSourceDTO fileSource : stepInstance.getFileSourceList()) { @@ -1614,7 +1614,7 @@ private Pair> extractNeedAuthHostsAndAccounts(StepI Set accountIds = new HashSet<>(); accountIds.add(stepInstance.getAccountId()); if (stepInstance.isFileStep()) { - ExecuteObjectsDTO stepTargetServers = stepInstance.getTargetServers().clone(); + ExecuteObjectsDTO stepTargetServers = stepInstance.getTargetExecuteObjects().clone(); filterServerDoNotRequireAuth(ActionScopeEnum.FILE_DISTRIBUTION, stepTargetServers, whiteHostAllowActions); authServers.merge(stepTargetServers); @@ -1632,7 +1632,7 @@ private Pair> extractNeedAuthHostsAndAccounts(StepI ); } } else if (stepInstance.isScriptStep()) { - ExecuteObjectsDTO stepTargetServers = stepInstance.getTargetServers().clone(); + ExecuteObjectsDTO stepTargetServers = stepInstance.getTargetExecuteObjects().clone(); filterServerDoNotRequireAuth(ActionScopeEnum.SCRIPT_EXECUTE, stepTargetServers, whiteHostAllowActions); authServers.merge(stepTargetServers); } @@ -2001,7 +2001,7 @@ private void parseScriptStepInstanceFromPlanStep(StepInstanceDTO stepInstance, S ServiceTaskTargetDTO target = scriptStepInfo.getExecuteTarget(); ExecuteObjectsDTO targetServers = buildFinalTargetServers(target, variableValueMap); - stepInstance.setTargetServers(targetServers); + stepInstance.setTargetExecuteObjects(targetServers); stepInstance.setIgnoreError(scriptStepInfo.getIgnoreError()); } @@ -2064,7 +2064,7 @@ private void parseFileStepInstanceFromPlanStep(StepInstanceDTO stepInstance, Ser ServiceTaskTargetDTO target = fileStepInfo.getExecuteTarget(); ExecuteObjectsDTO targetServers = buildFinalTargetServers(target, variableValueMap); - stepInstance.setTargetServers(targetServers); + stepInstance.setTargetExecuteObjects(targetServers); if (fileStepInfo.getDownloadSpeedLimit() != null) { // MB->KB @@ -2113,8 +2113,8 @@ private void parseScriptStepInstanceFromStepInstance(StepInstanceDTO stepInstanc stepInstance.setAccount(originStepInstance.getAccount()); } - ExecuteObjectsDTO targetServers = buildFinalTargetServers(originStepInstance.getTargetServers(), variableValueMap); - stepInstance.setTargetServers(targetServers); + ExecuteObjectsDTO targetServers = buildFinalTargetServers(originStepInstance.getTargetExecuteObjects(), variableValueMap); + stepInstance.setTargetExecuteObjects(targetServers); } private void parseFileStepInstanceFromStepInstance(StepInstanceDTO stepInstance, @@ -2140,8 +2140,8 @@ private void parseFileStepInstanceFromStepInstance(StepInstanceDTO stepInstance, stepInstance.setFileSourceList(fileSourceList); } - ExecuteObjectsDTO targetServers = buildFinalTargetServers(originStepInstance.getTargetServers(), variableValueMap); - stepInstance.setTargetServers(targetServers); + ExecuteObjectsDTO targetServers = buildFinalTargetServers(originStepInstance.getTargetExecuteObjects(), variableValueMap); + stepInstance.setTargetExecuteObjects(targetServers); } private ExecuteObjectsDTO buildFinalTargetServers(@NotNull ServiceTaskTargetDTO target, diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceExecuteObjectServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceExecuteObjectServiceImpl.java index 2ca05e0061..0b0e4ec417 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceExecuteObjectServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceExecuteObjectServiceImpl.java @@ -24,34 +24,29 @@ package com.tencent.bk.job.execute.service.impl; -import com.tencent.bk.job.common.constant.ExecuteObjectTypeEnum; -import com.tencent.bk.job.execute.engine.model.ExecuteObject; import com.tencent.bk.job.execute.service.TaskInstanceExecuteObjectService; import org.springframework.stereotype.Service; -import java.util.Collection; -import java.util.List; - @Service public class TaskInstanceExecuteObjectServiceImpl implements TaskInstanceExecuteObjectService { - @Override - public void batchSaveExecuteObjects(Collection executeObjects) { - - } - - @Override - public List listExecuteObjectsByJobInstanceId(long jobInstanceId) { - return null; - } - - @Override - public ExecuteObject getExecuteObjectById(long executeObjectId) { - return null; - } - - @Override - public ExecuteObject getExecuteObject(ExecuteObjectTypeEnum executeObjectType, - String resourceId) { - return null; - } +// @Override +// public void batchSaveExecuteObjects(Collection executeObjects) { +// +// } +// +// @Override +// public List listExecuteObjectsByJobInstanceId(long jobInstanceId) { +// return null; +// } +// +// @Override +// public ExecuteObject getExecuteObjectById(long executeObjectId) { +// return null; +// } +// +// @Override +// public ExecuteObject getExecuteObject(ExecuteObjectTypeEnum executeObjectType, +// String resourceId) { +// return null; +// } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java index 79c85e78b0..8365ad5b2c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java @@ -434,7 +434,7 @@ private List filterTargetHostsByBatch(StepInstanceBaseDTO stepInstance, if (stepInstance.isRollingStep()) { targetServers = rollingConfigService.getRollingServers(stepInstance, batch); } else { - targetServers = stepInstance.getTargetServers().getIpList(); + targetServers = stepInstance.getTargetExecuteObjects().getIpList(); } return targetServers; @@ -831,11 +831,11 @@ private void setRollingInfoForStep(StepInstanceBaseDTO stepInstance, private List getHostsByLogContentKeyword(long stepInstanceId, int executeCount, Integer batch, String keyword) { - return logService.getIpsByContentKeyword(stepInstanceId, executeCount, batch, keyword); + return logService.getHostsByContentKeyword(stepInstanceId, executeCount, batch, keyword); } private List fuzzySearchHostsByIp(StepInstanceBaseDTO stepInstance, String searchIp) { - return stepInstance.getTargetServers().getIpList().stream() + return stepInstance.getTargetExecuteObjects().getIpList().stream() .filter(host -> isMatchByIp(host, searchIp)) .collect(Collectors.toList()); } diff --git a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceExecuteObjectLogDTO.java b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceExecuteObjectLogDTO.java index 5c7b912124..a0f6fd819a 100644 --- a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceExecuteObjectLogDTO.java +++ b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceExecuteObjectLogDTO.java @@ -94,7 +94,7 @@ public class ServiceExecuteObjectLogDTO { */ @ApiModelProperty(value = "脚本日志内容，保存日志的时候需要传入,查询日志的时候该字段无效") @JsonProperty("scriptLog") - private ServiceScriptLogDTO scriptLog; + private ServiceExecuteObjectScriptLogDTO scriptLog; /** * 文件任务执行日志 diff --git a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceScriptLogDTO.java b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceExecuteObjectScriptLogDTO.java similarity index 77% rename from src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceScriptLogDTO.java rename to src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceExecuteObjectScriptLogDTO.java index f19d925147..3458229de8 100644 --- a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceScriptLogDTO.java +++ b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceExecuteObjectScriptLogDTO.java @@ -34,13 +34,13 @@ import lombok.NoArgsConstructor; /** - * 脚本日志 + * 执行对象对应的脚本日志 */ @Data @JsonInclude(JsonInclude.Include.NON_NULL) @NoArgsConstructor -@ApiModel("脚本日志") -public class ServiceScriptLogDTO { +@ApiModel("执行对象对应的脚本日志") +public class ServiceExecuteObjectScriptLogDTO { /** * 执行对象ID */ @@ -89,23 +89,38 @@ public class ServiceScriptLogDTO { /** * Constructor * - * @param executeObjectId 执行对象 ID - * @param hostId 主机hostId - * @param cloudIp 主机ipv4,格式: 云区域ID:IPv4 - * @param cloudIpv6 主机ipv6,格式: 云区域ID:IPv6 - * @param content 日志内容 + * @param hostId 主机hostId + * @param cloudIp 主机ipv4,格式: 云区域ID:IPv4 + * @param cloudIpv6 主机ipv6,格式: 云区域ID:IPv6 + * @param content 日志内容 + * @param offset 日志偏移量 */ @Deprecated @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA) - public ServiceScriptLogDTO(String executeObjectId, - Long hostId, - String cloudIp, - String cloudIpv6, - String content) { - this.executeObjectId = executeObjectId; + public ServiceExecuteObjectScriptLogDTO(Long hostId, + String cloudIp, + String cloudIpv6, + String content, + int offset) { this.hostId = hostId; this.cloudIp = cloudIp; this.cloudIpv6 = cloudIpv6; this.content = content; + this.offset = offset; + } + + /** + * Constructor + * + * @param executeObjectId 执行对象 ID + * @param content 日志内容 + * @param offset 日志偏移量 + */ + public ServiceExecuteObjectScriptLogDTO(String executeObjectId, + String content, + int offset) { + this.executeObjectId = executeObjectId; + this.content = content; + this.offset = offset; } } diff --git a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/api/service/impl/ServiceLogResourceImpl.java b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/api/service/impl/ServiceLogResourceImpl.java index 30a835782b..de4b672518 100644 --- a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/api/service/impl/ServiceLogResourceImpl.java +++ b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/api/service/impl/ServiceLogResourceImpl.java @@ -37,9 +37,9 @@ import com.tencent.bk.job.logsvr.model.service.ServiceBatchSaveLogRequest; import com.tencent.bk.job.logsvr.model.service.ServiceExecuteObjectLogDTO; import com.tencent.bk.job.logsvr.model.service.ServiceExecuteObjectLogsDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceExecuteObjectScriptLogDTO; import com.tencent.bk.job.logsvr.model.service.ServiceFileLogQueryRequest; import com.tencent.bk.job.logsvr.model.service.ServiceFileTaskLogDTO; -import com.tencent.bk.job.logsvr.model.service.ServiceScriptLogDTO; import com.tencent.bk.job.logsvr.model.service.ServiceScriptLogQueryRequest; import com.tencent.bk.job.logsvr.service.LogService; import lombok.extern.slf4j.Slf4j; @@ -93,7 +93,7 @@ private TaskExecuteObjectLog convertToTaskExecuteObjectLog(Integer logType, taskExecuteObjectLog.setBatch(log.getBatch()); if (log.getExecuteObjectId() != null) { taskExecuteObjectLog.setExecuteObjectId(log.getExecuteObjectId()); - ServiceScriptLogDTO scriptLog = log.getScriptLog(); + ServiceExecuteObjectScriptLogDTO scriptLog = log.getScriptLog(); if (scriptLog != null) { taskExecuteObjectLog.setScriptTaskLog(new ScriptTaskLogDoc(log.getStepInstanceId(), log.getExecuteCount(), log.getBatch(), log.getExecuteObjectId(), scriptLog.getContent(), @@ -109,7 +109,7 @@ private TaskExecuteObjectLog convertToTaskExecuteObjectLog(Integer logType, taskExecuteObjectLog.setHostId(log.getHostId()); taskExecuteObjectLog.setIp(log.getCloudIp()); taskExecuteObjectLog.setIpv6(log.getCloudIpv6()); - ServiceScriptLogDTO scriptLog = log.getScriptLog(); + ServiceExecuteObjectScriptLogDTO scriptLog = log.getScriptLog(); if (scriptLog != null) { taskExecuteObjectLog.setScriptTaskLog(new ScriptTaskLogDoc(log.getStepInstanceId(), log.getExecuteCount(), log.getBatch(), log.getHostId(), log.getCloudIp(), log.getCloudIpv6(), @@ -140,7 +140,7 @@ private ServiceExecuteObjectLogDTO toServiceLogDTO(TaskExecuteObjectLog taskExec result.setCloudIp(taskExecuteObjectLog.getIp()); result.setCloudIpv6(taskExecuteObjectLog.getIpv6()); if (StringUtils.isNotEmpty(taskExecuteObjectLog.getScriptContent())) { - result.setScriptLog(new ServiceScriptLogDTO( + result.setScriptLog(new ServiceExecuteObjectScriptLogDTO( taskExecuteObjectLog.getExecuteObjectId(), taskExecuteObjectLog.getHostId(), taskExecuteObjectLog.getIp(), diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/task/TaskFileTypeEnum.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/task/TaskFileTypeEnum.java index 74120ea65a..be66aaf7b1 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/task/TaskFileTypeEnum.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/task/TaskFileTypeEnum.java @@ -27,9 +27,6 @@ import lombok.AllArgsConstructor; import lombok.Getter; -/** - * @since 3/10/2019 17:08 - */ @Getter @AllArgsConstructor public enum TaskFileTypeEnum {