Skip to content

Commit

Permalink
perf: 优化查询主机信息内部 API性能 TencentBlueKing#2600
Browse files Browse the repository at this point in the history
  • Loading branch information
@wangyu096
wangyu096 committed Nov 9, 2023
1 parent 50775ae commit 50a309c
Showing 1 changed file with 61 additions and 31 deletions.
92 changes: 61 additions & 31 deletions ...execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -771,52 +771,71 @@ private AuthResult authFileTransfer(TaskInstanceDTO taskInstance,
private ServiceListAppHostResultDTO acquireAndSetHosts(TaskInstanceDTO taskInstance,
List<StepInstanceDTO> stepInstances,
Collection<TaskVariableDTO> variables) {
long appId = taskInstance.getAppId();
StopWatch watch = new StopWatch("AcquireAndSetHosts");
try {
long appId = taskInstance.getAppId();

// 提取动态分组/topo节点
Set<DynamicServerGroupDTO> groups = new HashSet<>();
Set<DynamicServerTopoNodeDTO> topoNodes = new HashSet<>();
stepInstances.forEach(stepInstance -> extractDynamicGroupsAndTopoNodes(stepInstance, groups, topoNodes));
if (CollectionUtils.isNotEmpty(variables)) {
variables.forEach(variable -> {
if (TaskVariableTypeEnum.HOST_LIST.getType() == variable.getType()) {
extractDynamicGroupsAndTopoNodes(variable.getTargetServers(), groups, topoNodes);
}
});
}
// 提取动态分组/topo节点
Set<DynamicServerGroupDTO> groups = new HashSet<>();
Set<DynamicServerTopoNodeDTO> topoNodes = new HashSet<>();
stepInstances.forEach(stepInstance -> extractDynamicGroupsAndTopoNodes(stepInstance, groups, topoNodes));
if (CollectionUtils.isNotEmpty(variables)) {
variables.forEach(variable -> {
if (TaskVariableTypeEnum.HOST_LIST.getType() == variable.getType()) {
extractDynamicGroupsAndTopoNodes(variable.getTargetServers(), groups, topoNodes);
}
});
}

// 获取动态分组的主机并设置
fillDynamicGroupHosts(appId, groups, stepInstances, variables);
// 获取动态分组的主机并设置
fillDynamicGroupHosts(watch, appId, groups, stepInstances, variables);

// 获取topo节点的主机并设置
fillTopoNodeHosts(appId, topoNodes, stepInstances, variables);
// 获取topo节点的主机并设置
fillTopoNodeHosts(watch, appId, topoNodes, stepInstances, variables);

// 提取作业包含的主机列表
Set<HostDTO> queryHosts = extractHosts(stepInstances, variables);
// 提取作业包含的主机列表
watch.start("extractHosts");
Set<HostDTO> queryHosts = extractHosts(stepInstances, variables);
watch.stop();

if (CollectionUtils.isEmpty(queryHosts)) {
return ServiceListAppHostResultDTO.EMPTY;
}
if (CollectionUtils.isEmpty(queryHosts)) {
return ServiceListAppHostResultDTO.EMPTY;
}

ServiceListAppHostResultDTO queryHostsResult = hostService.batchGetAppHosts(appId, queryHosts,
needRefreshHostBkAgentId(taskInstance));
watch.start("batchGetAppHosts");
ServiceListAppHostResultDTO queryHostsResult = hostService.batchGetAppHosts(appId, queryHosts,
needRefreshHostBkAgentId(taskInstance));
watch.stop();

if (CollectionUtils.isNotEmpty(queryHostsResult.getNotExistHosts())) {
// 如果主机在cmdb不存在，直接报错
throwHostInvalidException(queryHostsResult.getNotExistHosts());
}
if (CollectionUtils.isNotEmpty(queryHostsResult.getNotExistHosts())) {
// 如果主机在cmdb不存在，直接报错
throwHostInvalidException(queryHostsResult.getNotExistHosts());
}

fillTaskInstanceHostDetail(taskInstance, stepInstances, variables, queryHostsResult);
watch.start("fillTaskInstanceHostDetail");
fillTaskInstanceHostDetail(taskInstance, stepInstances, variables, queryHostsResult);
watch.stop();

return queryHostsResult;
} finally {
if (watch.isRunning()) {
watch.stop();
}
if (watch.getTotalTimeMillis() > 1000) {
log.warn("AcquireAndSetHosts slow, watch: {}", watch.prettyPrint());
}
}

return queryHostsResult;
}

private void fillDynamicGroupHosts(long appId,
private void fillDynamicGroupHosts(StopWatch watch,
long appId,
Set<DynamicServerGroupDTO> groups,
List<StepInstanceDTO> stepInstances,
Collection<TaskVariableDTO> variables) {
// 获取动态分组的主机并设置
if (CollectionUtils.isNotEmpty(groups)) {
watch.start("fillDynamicGroupHosts");
Map<DynamicServerGroupDTO, List<HostDTO>> dynamicGroupHosts =
hostService.batchGetAndGroupHostsByDynamicGroup(appId, groups);
stepInstances.forEach(stepInstance -> {
Expand All @@ -839,14 +858,17 @@ private void fillDynamicGroupHosts(long appId,
}
});
}
watch.stop();
}
}

private void fillTopoNodeHosts(long appId,
private void fillTopoNodeHosts(StopWatch watch,
long appId,
Set<DynamicServerTopoNodeDTO> topoNodes,
List<StepInstanceDTO> stepInstances,
Collection<TaskVariableDTO> variables) {
if (CollectionUtils.isNotEmpty(topoNodes)) {
watch.start("fillTopoNodeHosts");
Map<DynamicServerTopoNodeDTO, List<HostDTO>> topoNodeHosts =
hostService.getAndGroupHostsByTopoNodes(appId, topoNodes);
stepInstances.forEach(stepInstance -> {
Expand All @@ -869,6 +891,7 @@ private void fillTopoNodeHosts(long appId,
}
});
}
watch.stop();
}
}

Expand Down Expand Up @@ -2217,6 +2240,8 @@ private void setAgentStatus(List<HostDTO> hosts) {
if (CollectionUtils.isEmpty(hosts)) {
return;
}
long start = System.currentTimeMillis();

List<HostAgentStateQuery> hostAgentStateQueryList = new ArrayList<>(hosts.size());
Map<HostDTO, HostAgentStateQuery> hostAgentStateQueryMap = new HashMap<>(hosts.size());
hosts.forEach(hostDTO -> {
Expand All @@ -2242,6 +2267,11 @@ private void setAgentStatus(List<HostDTO> hosts) {
host.setAlive(AgentAliveStatusEnum.NOT_ALIVE.getStatusValue());
}
}

long cost = System.currentTimeMillis() - start;
if (cost > 1000) {
log.warn("SetAgentStatus slow, hostSize: {}, cost:{} ms", hosts.size(), cost);
}
}

@Override
Expand Down

0 comments on commit 50a309c

Please sign in to comment.