From 832d1cd9d184a5ffb47a8f3880a3d1bd631b76ff Mon Sep 17 00:00:00 2001 From: jsonwan Date: Tue, 4 Apr 2023 11:01:45 +0800 Subject: [PATCH 1/3] =?UTF-8?q?bugfix:=20=E5=9C=A8CMDB=E8=B7=A8=E4=B8=9A?= =?UTF-8?q?=E5=8A=A1=E8=BD=AC=E7=A7=BB=E4=B8=BB=E6=9C=BA=E5=90=8E=E4=BB=8E?= =?UTF-8?q?ESB=E6=8E=A5=E5=8F=A3=E7=AB=8B=E5=8D=B3=E4=BD=BF=E7=94=A8?= =?UTF-8?q?=E6=8A=A5=E4=B8=BB=E6=9C=BA=E6=97=A0=E6=95=88=20#1909?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 在其他业务下的主机根据CMDB实时数据进行二次判定 --- .../manage/service/impl/HostServiceImpl.java | 72 +++++++++++++++++-- 1 file changed, 68 insertions(+), 4 deletions(-) diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/HostServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/HostServiceImpl.java index 9406b48f8f..7ea7802cf8 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/HostServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/HostServiceImpl.java @@ -741,7 +741,7 @@ public List findStatusChangedHosts(List hostList) queryAgentStatusClient.batchGetAgentStatus(cloudIpList); for (HostSimpleDTO host : hostList) { QueryAgentStatusClient.AgentStatus agentStatus = agentStatusMap.get(host.getCloudIp()); - if(host.getGseAgentAlive() != agentStatus.status){ + if (host.getGseAgentAlive() != agentStatus.status) { host.setGseAgentAlive(agentStatus.status); statusChangedHosts.add(host); } @@ -1436,7 +1436,8 @@ public int updateHostsStatus(List simpleHostList) { .collect(Collectors.groupingBy(HostSimpleDTO::getGseAgentAlive, Collectors.mapping(HostSimpleDTO::getHostId, Collectors.toList()))); for (Integer status : statusGroupMap.keySet()) { - updateCount += applicationHostDAO.batchUpdateHostStatusByHostIds(status, statusGroupMap.get(status)); + updateCount += applicationHostDAO.batchUpdateHostStatusByHostIds(status, + statusGroupMap.get(status)); } start += batchSize; } while (end < size); @@ -1522,6 +1523,35 @@ public ServiceListAppHostResultDTO listAppHosts(Long appId, } }); + // 对于判定为其他业务下的主机，可能是缓存数据不准确导致，需要根据CMDB实时数据进行二次判定 + if (CollectionUtils.isNotEmpty(notInAppHosts)) { + Pair, List> cmdbHostsPair = listHostsFromCmdb(notInAppHosts); + if (CollectionUtils.isNotEmpty(cmdbHostsPair.getLeft())) { + notExistHosts.addAll(cmdbHostsPair.getLeft()); + } + List cmdbExistHosts = cmdbHostsPair.getRight(); + if (CollectionUtils.isNotEmpty(cmdbExistHosts)) { + notInAppHosts.clear(); + List cmdbValidHosts = new ArrayList<>(); + cmdbExistHosts.forEach(existHost -> { + if (includeBizIds.contains(existHost.getBizId())) { + validHosts.add(existHost.toHostDTO()); + cmdbValidHosts.add(existHost.toHostDTO()); + } else { + notInAppHosts.add(existHost.toHostDTO()); + } + }); + if (!cmdbValidHosts.isEmpty()) { + log.info( + "{} hosts belong to appId {} after check in cmdb, cmdbValidHosts={}", + cmdbValidHosts.size(), + appId, + cmdbValidHosts + ); + } + } + } + if (CollectionUtils.isNotEmpty(notExistHosts) || CollectionUtils.isNotEmpty(notInAppHosts)) { log.info("Contains invalid hosts, appId: {}, notExistHosts: {}, hostsInOtherApp: {}", appId, notExistHosts, notInAppHosts); @@ -1534,9 +1564,34 @@ public ServiceListAppHostResultDTO listAppHosts(Long appId, return result; } - private Pair, List> listHostsFromCacheOrCmdb(Collection hosts) { - List appHosts = new ArrayList<>(); + private Pair, List> listHostsFromCmdb(Collection hosts) { List notExistHosts = new ArrayList<>(); + List appHosts = new ArrayList<>(); + Pair, List> pair = separateByHostIdOrCloudIp(hosts); + List hostIds = pair.getLeft(); + List cloudIps = pair.getRight(); + if (CollectionUtils.isNotEmpty(hostIds)) { + Pair, List> result = new ListHostByHostIdsStrategy().listHostsFromCmdb(hostIds); + appHosts.addAll(result.getRight()); + if (CollectionUtils.isNotEmpty(result.getLeft())) { + result.getLeft().forEach(notExistHostId -> { + notExistHosts.add(HostDTO.fromHostId(notExistHostId)); + }); + } + } + if (CollectionUtils.isNotEmpty(cloudIps)) { + Pair, List> result = new ListHostByIpsStrategy().listHostsFromCmdb(cloudIps); + appHosts.addAll(result.getRight()); + if (CollectionUtils.isNotEmpty(result.getLeft())) { + result.getLeft().forEach(notExistCloudIp -> { + notExistHosts.add(HostDTO.fromCloudIp(notExistCloudIp)); + }); + } + } + return Pair.of(notExistHosts, appHosts); + } + + private Pair, List> separateByHostIdOrCloudIp(Collection hosts) { List hostIds = new ArrayList<>(); List cloudIps = new ArrayList<>(); hosts.forEach(host -> { @@ -1546,6 +1601,15 @@ private Pair, List> listHostsFromCacheOrCmdb(Collect cloudIps.add(host.toCloudIp()); } }); + return Pair.of(hostIds, cloudIps); + } + + private Pair, List> listHostsFromCacheOrCmdb(Collection hosts) { + List appHosts = new ArrayList<>(); + List notExistHosts = new ArrayList<>(); + Pair, List> pair = separateByHostIdOrCloudIp(hosts); + List hostIds = pair.getLeft(); + List cloudIps = pair.getRight(); if (CollectionUtils.isNotEmpty(hostIds)) { Pair, List> result = listHostsByStrategy(hostIds, new ListHostByHostIdsStrategy()); appHosts.addAll(result.getRight()); From c837f63790cdb6b1d590803953c0ffb1533240a3 Mon Sep 17 00:00:00 2001 From: jsonwan Date: Tue, 4 Apr 2023 17:09:50 +0800 Subject: [PATCH 2/3] =?UTF-8?q?bugfix:=20=E5=9C=A8CMDB=E8=B7=A8=E4=B8=9A?= =?UTF-8?q?=E5=8A=A1=E8=BD=AC=E7=A7=BB=E4=B8=BB=E6=9C=BA=E5=90=8E=E4=BB=8E?= =?UTF-8?q?ESB=E6=8E=A5=E5=8F=A3=E7=AB=8B=E5=8D=B3=E4=BD=BF=E7=94=A8?= =?UTF-8?q?=E6=8A=A5=E4=B8=BB=E6=9C=BA=E6=97=A0=E6=95=88=20#1909?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 在其他业务下的主机根据CMDB实时数据进行二次判定 --- .../tencent/bk/job/manage/service/impl/HostServiceImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/HostServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/HostServiceImpl.java index 7ea7802cf8..e8d6abfd7c 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/HostServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/HostServiceImpl.java @@ -1532,11 +1532,11 @@ public ServiceListAppHostResultDTO listAppHosts(Long appId, List cmdbExistHosts = cmdbHostsPair.getRight(); if (CollectionUtils.isNotEmpty(cmdbExistHosts)) { notInAppHosts.clear(); - List cmdbValidHosts = new ArrayList<>(); + List cmdbValidHosts = new ArrayList<>(); cmdbExistHosts.forEach(existHost -> { if (includeBizIds.contains(existHost.getBizId())) { validHosts.add(existHost.toHostDTO()); - cmdbValidHosts.add(existHost.toHostDTO()); + cmdbValidHosts.add(existHost); } else { notInAppHosts.add(existHost.toHostDTO()); } From 99aec7c3b4d870befaac47b45bf7470faa5ecdab Mon Sep 17 00:00:00 2001 From: jsonwan Date: Thu, 6 Apr 2023 09:41:26 +0800 Subject: [PATCH 3/3] =?UTF-8?q?bugfix:=20=E5=9C=A8CMDB=E8=B7=A8=E4=B8=9A?= =?UTF-8?q?=E5=8A=A1=E8=BD=AC=E7=A7=BB=E4=B8=BB=E6=9C=BA=E5=90=8E=E4=BB=8E?= =?UTF-8?q?ESB=E6=8E=A5=E5=8F=A3=E7=AB=8B=E5=8D=B3=E4=BD=BF=E7=94=A8?= =?UTF-8?q?=E6=8A=A5=E4=B8=BB=E6=9C=BA=E6=97=A0=E6=95=88=20#1909?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 重构代码，降低圈复杂度 --- .../manage/service/impl/HostServiceImpl.java | 67 ++++++++++++------- 1 file changed, 42 insertions(+), 25 deletions(-) diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/HostServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/HostServiceImpl.java index e8d6abfd7c..bcefafc269 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/HostServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/HostServiceImpl.java @@ -1525,31 +1525,7 @@ public ServiceListAppHostResultDTO listAppHosts(Long appId, // 对于判定为其他业务下的主机，可能是缓存数据不准确导致，需要根据CMDB实时数据进行二次判定 if (CollectionUtils.isNotEmpty(notInAppHosts)) { - Pair, List> cmdbHostsPair = listHostsFromCmdb(notInAppHosts); - if (CollectionUtils.isNotEmpty(cmdbHostsPair.getLeft())) { - notExistHosts.addAll(cmdbHostsPair.getLeft()); - } - List cmdbExistHosts = cmdbHostsPair.getRight(); - if (CollectionUtils.isNotEmpty(cmdbExistHosts)) { - notInAppHosts.clear(); - List cmdbValidHosts = new ArrayList<>(); - cmdbExistHosts.forEach(existHost -> { - if (includeBizIds.contains(existHost.getBizId())) { - validHosts.add(existHost.toHostDTO()); - cmdbValidHosts.add(existHost); - } else { - notInAppHosts.add(existHost.toHostDTO()); - } - }); - if (!cmdbValidHosts.isEmpty()) { - log.info( - "{} hosts belong to appId {} after check in cmdb, cmdbValidHosts={}", - cmdbValidHosts.size(), - appId, - cmdbValidHosts - ); - } - } + reConfirmNotInAppHostsByCmdb(notInAppHosts, notExistHosts, validHosts, appId, includeBizIds); } if (CollectionUtils.isNotEmpty(notExistHosts) || CollectionUtils.isNotEmpty(notInAppHosts)) { @@ -1564,6 +1540,47 @@ public ServiceListAppHostResultDTO listAppHosts(Long appId, return result; } + /** + * 对于判定为其他业务下的主机，可能是缓存数据不准确导致，根据CMDB实时数据进行二次判定 + * + * @param notInAppHosts 前期判定为在其他业务下的主机，在该方法中数据可能被修改 + * @param notExistHosts 前期判定为不存在的主机，在该方法中数据可能被修改 + * @param validHosts 前期判定为在业务下的主机，在该方法中数据可能被修改 + * @param appId Job内业务ID + * @param includeBizIds Job内业务ID可能对应的多个CMDB业务ID列表 + */ + private void reConfirmNotInAppHostsByCmdb(List notInAppHosts, + List notExistHosts, + List validHosts, + Long appId, + List includeBizIds) { + Pair, List> cmdbHostsPair = listHostsFromCmdb(notInAppHosts); + if (CollectionUtils.isNotEmpty(cmdbHostsPair.getLeft())) { + notExistHosts.addAll(cmdbHostsPair.getLeft()); + } + List cmdbExistHosts = cmdbHostsPair.getRight(); + if (CollectionUtils.isNotEmpty(cmdbExistHosts)) { + notInAppHosts.clear(); + List cmdbValidHosts = new ArrayList<>(); + cmdbExistHosts.forEach(existHost -> { + if (includeBizIds.contains(existHost.getBizId())) { + validHosts.add(existHost.toHostDTO()); + cmdbValidHosts.add(existHost); + } else { + notInAppHosts.add(existHost.toHostDTO()); + } + }); + if (!cmdbValidHosts.isEmpty()) { + log.info( + "{} hosts belong to appId {} after check in cmdb, cmdbValidHosts={}", + cmdbValidHosts.size(), + appId, + cmdbValidHosts + ); + } + } + } + private Pair, List> listHostsFromCmdb(Collection hosts) { List notExistHosts = new ArrayList<>(); List appHosts = new ArrayList<>();