feature: Job 支持容器执行 - 脚本任务 TencentBlueKing#2631

wangyu096 · Dec 30, 2023 · f51c726 · f51c726
1 parent 3c0377a
commit f51c726
Show file tree

Hide file tree

Showing 17 changed files with 553 additions and 127 deletions.
diff --git a/...ackend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/BizCmdbClient.java b/...ackend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/BizCmdbClient.java
@@ -105,6 +105,7 @@
 import com.tencent.bk.job.common.model.error.ErrorType;
 import com.tencent.bk.job.common.util.FlowController;
 import com.tencent.bk.job.common.util.JobContextUtil;
+import com.tencent.bk.job.common.util.PageUtil;
 import com.tencent.bk.job.common.util.ThreadUtils;
 import com.tencent.bk.job.common.util.TimeUtil;
 import com.tencent.bk.job.common.util.Utils;
@@ -1439,4 +1440,40 @@ private void setSupplierAccount(EsbReq esbReq) {
             esbReq.setBkSupplierAccount(cmdbSupplierAccount);
         }
     }
+
+    /**
+     * 根据容器ID批量查询容器
+     *
+     * @param bizId        CMDB 业务 ID
+     * @param containerIds 容器 ID 集合
+     * @return 容器列表
+     */
+    public List<ContainerDetailDTO> listKubeContainerByIds(long bizId, Collection<Long> containerIds) {
+        ListKubeContainerByTopoReq req = makeCmdbBaseReq(ListKubeContainerByTopoReq.class);
+
+        // 查询条件
+        req.setBizId(bizId);
+        PropertyFilterDTO containerFilter = new PropertyFilterDTO();
+        containerFilter.setCondition("AND");
+        containerFilter.addRule(BaseRuleDTO.in(ContainerFields.ID, containerIds));
+        req.setContainerFilter(containerFilter);
+
+        // 返回参数设置
+        req.setContainerFields(ContainerFields.FIELDS);
+        req.setPodFields(PodFields.FIELDS);
+
+        return PageUtil.queryAllWithLoopPageQuery(
+            500,
+            page -> listPageContainersByIds(req, page),
+            pageData -> pageData.getTotal().intValue(),
+            PageData::getData,
+            container -> container
+        );
+    }
+
+    private PageData<ContainerDetailDTO> listPageContainersByIds(ListKubeContainerByTopoReq req,
+                                                                 PageDTO page) {
+        req.setPage(new Page(page.getStart(), page.getLimit()));
+        return listKubeContainerByTopo(req);
+    }
 }
diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message.properties
@@ -179,6 +179,7 @@
 1244027=滚动批次不能大于{0}
 1244028=步骤 [{0}] 的目标主机为空
 1244029=步骤 [{0}] 的源文件主机为空
+1244030=作业引用的执行对象不存在。不存在的执行对象个数:{0},执行对象列表[{1}]
 
 ## 业务错误-定时任务(job-crontab)
 1245006=定时任务执行时间已失效

diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en.properties
@@ -179,6 +179,7 @@
 1244027=Rolling batch can not be greater than {0}
 1244028=Step [{0}] target host is empty
 1244029=Step [{0}] source host is empty
+1244030=Execute object referenced by the job does not exist. Number of non-existent execution objects: {0}, execution object list: [{1}]
 
 ## Business error - job-crontab
 1245006=Cron job execution time already passed

diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en_US.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en_US.properties
@@ -179,6 +179,7 @@
 1244027=Rolling batch can not be greater than {0}
 1244028=Step [{0}] target host is empty
 1244029=Step [{0}] source host is empty
+1244030=Execute object referenced by the job does not exist. Number of non-existent execution objects: {0}, execution object list: [{1}]
 
 ## Business error - job-crontab
 1245006=Cron job execution time already passed

diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh.properties
@@ -179,6 +179,7 @@
 1244027=滚动批次不能大于{0}
 1244028=步骤 [{0}] 的目标主机为空
 1244029=步骤 [{0}] 的源文件主机为空
+1244030=作业引用的执行对象不存在。不存在的执行对象个数:{0},执行对象列表[{1}]
 
 ## 业务错误-定时任务(job-crontab)
 1245006=定时任务执行时间已失效

diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh_CN.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh_CN.properties
@@ -180,6 +180,7 @@
 1244027=滚动批次不能大于{0}
 1244028=步骤 [{0}] 的目标主机为空
 1244029=步骤 [{0}] 的源文件主机为空
+1244030=作业引用的执行对象不存在。不存在的执行对象个数:{0},执行对象列表[{1}]
 
 ## 业务错误-定时任务(job-crontab)
 1245006=定时任务执行时间已失效

diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ErrorCode.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ErrorCode.java
@@ -338,6 +338,8 @@ public class ErrorCode {
     public static final int STEP_TARGET_HOST_EMPTY = 1244028;
     // 步骤：{} 的源文件主机为空
     public static final int STEP_SOURCE_HOST_EMPTY = 1244029;
+    // 执行对象不存在。无效的{0}个执行对象：[{1}]
+    public static final int EXECUTE_OBJECT_NOT_EXIST = 1244030;
     // 作业执行 end
 
     // 定时作业 start

diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/Container.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/Container.java
@@ -34,7 +34,7 @@
 import lombok.extern.slf4j.Slf4j;
 
 import java.util.Map;
-import java.util.StringJoiner;
+import java.util.Objects;
 
 /**
  * 作业执行对象-容器模型
@@ -58,19 +58,22 @@ public class Container implements Cloneable {
     @JsonProperty("containerId")
     private String containerId;
 
+    /**
+     * 容器名称
+     */
+    private String name;
 
     /**
      * 容器所在 Node 对应的主机ID
      */
-    @JsonProperty("hostId")
-    private Long hostId;
+    @JsonProperty("nodeHostId")
+    private Long nodeHostId;
 
     /**
      * 容器所在 Node 对应的 Agent ID
      */
-    @JsonProperty("agentId")
-    private String agentId;
-
+    @JsonProperty("nodeAgentId")
+    private String nodeAgentId;
 
     /**
      * 容器所在集群 ID
@@ -96,43 +99,54 @@ public class Container implements Cloneable {
     @JsonProperty("podLabels")
     private Map<String, String> podLabels;
 
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        Container container = (Container) o;
+        return id.equals(container.id);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id);
+    }
 
     @Override
     @SuppressWarnings("all")
     public Container clone() {
         Container clone = new Container();
         clone.setId(id);
-        clone.setHostId(hostId);
-        clone.setAgentId(agentId);
+        clone.setNodeHostId(nodeHostId);
+        clone.setNodeAgentId(nodeAgentId);
         clone.setContainerId(containerId);
         clone.setPodLabels(podLabels);
         clone.setClusterId(clusterId);
         clone.setNamespace(namespace);
         clone.setPodName(podName);
+        clone.setName(name);
         return clone;
     }
 
-    @Override
-    public String toString() {
-        return new StringJoiner(", ", Container.class.getSimpleName() + "[", "]")
-            .add("id='" + id + "'")
-            .add("containerId='" + containerId + "'")
-            .add("hostId=" + hostId)
-            .add("agentId='" + agentId + "'")
-            .add("clusterId='" + clusterId + "'")
-            .add("namespace='" + namespace + "'")
-            .add("podName='" + podName + "'")
-            .add("podLabels=" + podLabels)
-            .toString();
-    }
-
     public ContainerVO toContainerVO() {
         ContainerVO vo = new ContainerVO();
         vo.setId(id);
+        vo.setName(name);
         vo.setUid(containerId);
-        vo.setNodeHostId(hostId);
+        vo.setNodeHostId(nodeHostId);
         vo.setPodName(podName);
         vo.setPodLabels(podLabels);
         return vo;
     }
+
+    public void updatePropsByContainer(Container container) {
+        this.containerId = container.getContainerId();
+        this.nodeHostId = container.getNodeHostId();
+        this.nodeAgentId = container.getNodeAgentId();
+        this.clusterId = container.getClusterId();
+        this.namespace = container.getNamespace();
+        this.podName = container.getPodName();
+        this.podLabels = container.getPodLabels();
+        this.name = container.getName();
+    }
 }
diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/HostDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/HostDTO.java
@@ -130,6 +130,10 @@ public HostDTO(Long bkCloudId, String ip) {
         this.ip = ip;
     }
 
+    public HostDTO(Long hostId) {
+        this.hostId = hostId;
+    }
+
     public HostDTO(Long hostId, Long bkCloudId, String ip) {
         this.hostId = hostId;
         this.bkCloudId = bkCloudId;
@@ -292,6 +296,9 @@ public String toStringBasic() {
     }
 
     public void updateByHost(HostDTO host) {
+        if (host == null) {
+            return;
+        }
         this.hostId = host.getHostId();
         this.agentId = host.getAgentId();
         this.bkCloudId = host.getBkCloudId();

diff --git a/...sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/AtomicFileTaskResultContent.java b/...sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/AtomicFileTaskResultContent.java
@@ -245,7 +245,7 @@ public ExecuteObjectGseKey getSourceExecuteObjectGseKey() {
             // bk_container_id 不为空，说明是容器执行对象
             sourceExecuteObjectGseKey = ExecuteObjectGseKey.ofContainer(sourceAgentId, sourceContainerId);
         } else {
-            sourceExecuteObjectGseKey = ExecuteObjectGseKey.ofHost(destAgentId);
+            sourceExecuteObjectGseKey = ExecuteObjectGseKey.ofHost(sourceAgentId);
         }
         return sourceExecuteObjectGseKey;
     }

diff --git a/...vice-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/ExecuteObject.java b/...vice-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/ExecuteObject.java
@@ -137,7 +137,7 @@ public ExecuteObjectGseKey toExecuteObjectGseKey() {
         if (isHostExecuteObject()) {
             executeObjectGseKey = ExecuteObjectGseKey.ofHost(host.getAgentId());
         } else {
-            executeObjectGseKey = ExecuteObjectGseKey.ofContainer(container.getAgentId(), container.getContainerId());
+            executeObjectGseKey = ExecuteObjectGseKey.ofContainer(container.getNodeAgentId(), container.getContainerId());
         }
         return executeObjectGseKey;
     }
@@ -147,7 +147,7 @@ public boolean isAgentIdEmpty() {
         if (isHostExecuteObject()) {
             return StringUtils.isEmpty(getHost().getAgentId());
         } else {
-            return StringUtils.isEmpty(getContainer().getAgentId());
+            return StringUtils.isEmpty(getContainer().getNodeAgentId());
         }
     }
 
@@ -156,7 +156,7 @@ public Agent toGseAgent() {
         if (isHostExecuteObject()) {
             agent.setAgentId(host.getAgentId());
         } else {
-            agent.setAgentId(container.getAgentId());
+            agent.setAgentId(container.getNodeAgentId());
             agent.setContainerId(container.getContainerId());
         }
         return agent;

diff --git a/...service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ExecuteObjectsDTO.java b/...service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ExecuteObjectsDTO.java
@@ -426,4 +426,14 @@ public List<ExecuteObject> getDecorateExecuteObjects() {
             return Collections.emptyList();
         }
     }
+
+    /**
+     * 提取所有包含的容器执行对象
+     *
+     * @return 容器执行对象列表
+     */
+    public List<Container> extractContainers() {
+        // 当前只支持静态选择容器
+        return staticContainerList;
+    }
 }
diff --git a/...e/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceDTO.java b/...e/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceDTO.java
@@ -27,12 +27,15 @@
 import com.tencent.bk.job.common.constant.DuplicateHandlerEnum;
 import com.tencent.bk.job.common.constant.ErrorCode;
 import com.tencent.bk.job.common.exception.InternalException;
+import com.tencent.bk.job.common.model.dto.Container;
 import com.tencent.bk.job.execute.engine.model.ExecuteObject;
 import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum;
 import lombok.Getter;
 import lombok.Setter;
 import lombok.ToString;
+import org.apache.commons.collections4.CollectionUtils;
 
+import java.util.ArrayList;
 import java.util.List;
 
 /**
@@ -253,4 +256,21 @@ public ExecuteObject findExecuteObjectByCompositeKey(ExecuteObjectCompositeKey e
             throw new InternalException("Not support method invoke for step", ErrorCode.INTERNAL_ERROR);
         }
     }
+
+    public List<Container> extractStaticContainerList() {
+        List<Container> containers = new ArrayList<>();
+        if (CollectionUtils.isNotEmpty(targetExecuteObjects.getStaticContainerList())) {
+            containers.addAll(targetExecuteObjects.getStaticContainerList());
+        }
+        if (isFileStep()) {
+            for (FileSourceDTO fileSource : fileSourceList) {
+                ExecuteObjectsDTO executeObjectsDTO = fileSource.getServers();
+                if (executeObjectsDTO != null
+                    && CollectionUtils.isNotEmpty(executeObjectsDTO.getStaticContainerList())) {
+                    containers.addAll(executeObjectsDTO.getStaticContainerList());
+                }
+            }
+        }
+        return containers;
+    }
 }
diff --git a/...ob-execute/src/main/java/com/tencent/bk/job/execute/model/TaskInstanceExecuteObjects.java b/...ob-execute/src/main/java/com/tencent/bk/job/execute/model/TaskInstanceExecuteObjects.java
@@ -0,0 +1,75 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available.
+ *
+ * Copyright (C) 2021 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License.
+ *
+ * License for BK-JOB蓝鲸智云作业平台:
+ * --------------------------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+ * to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+ * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
+ * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+package com.tencent.bk.job.execute.model;
+
+import com.tencent.bk.job.common.model.dto.Container;
+import com.tencent.bk.job.common.model.dto.HostDTO;
+import lombok.Data;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * 作业实例中包含的执行对象
+ */
+@Data
+public class TaskInstanceExecuteObjects {
+    /**
+     * 当前作业实例是否包含容器执行对象
+     */
+    private boolean containsAnyContainer;
+    /**
+     * 当前作业实例是否包含主机执行对象
+     */
+    private boolean containsAnyHost;
+
+    /**
+     * 合法的主机（在当前业务下)
+     */
+    private List<HostDTO> validHosts;
+    /**
+     * 不存在的主机
+     */
+    private List<HostDTO> notExistHosts;
+    /**
+     * 在其他业务下的主机
+     */
+    private List<HostDTO> notInAppHosts;
+
+    /**
+     * 合法的容器（在当前业务下)
+     */
+    private List<Container> validContainers;
+    /**
+     * 不存在的容器ID列表
+     */
+    private Set<Long> notExistContainerIds;
+    /**
+     * 主机白名单
+     * key=hostId, value: 允许的操作列表
+     */
+    Map<Long, List<String>> whiteHostAllowActions;
+}