Skip to content

Commit

Permalink
feature: Job 支持容器执行 - 脚本任务 TencentBlueKing#2631
Browse files Browse the repository at this point in the history
  • Loading branch information
@wangyu096
wangyu096 committed Jan 1, 2024
1 parent f51c726 commit e3bdbef
Show file tree
Hide file tree
Showing 11 changed files with 156 additions and 171 deletions.
40 changes: 20 additions & 20 deletions ...rc/test/java/com/tencent/bk/job/execute/dao/impl/RollingConfigDAOImplIntegrationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,21 +81,21 @@ void queryRollingConfig() {
assertThat(stepRollingConfigs.get(101L).isBatch()).isEqualTo(false);
assertThat(stepRollingConfigs.get(102L).isBatch()).isEqualTo(true);
assertThat(stepRollingConfigs.get(103L).isBatch()).isEqualTo(true);
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getDecorateExecuteObjectsBatchList()).hasSize(3);
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getDecorateExecuteObjectsBatchList().get(0).getBatch()).isEqualTo(1);
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getDecorateExecuteObjectsBatchList().get(0).getDecorateExecuteObjects()).hasSize(1);
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getDecorateExecuteObjectsBatchList().get(0).getDecorateExecuteObjects().get(0).getHost().getBkCloudId()).isEqualTo(0L);
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getDecorateExecuteObjectsBatchList().get(0).getDecorateExecuteObjects().get(0).getHost().getIp()).isEqualTo("127.0.0.1");
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getDecorateExecuteObjectsBatchList().get(1).getBatch()).isEqualTo(2);
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getDecorateExecuteObjectsBatchList().get(1).getDecorateExecuteObjects()).hasSize(1);
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getDecorateExecuteObjectsBatchList().get(1).getDecorateExecuteObjects().get(0).getHost().getBkCloudId()).isEqualTo(0L);
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getDecorateExecuteObjectsBatchList().get(1).getDecorateExecuteObjects().get(0).getHost().getIp()).isEqualTo("127.0.0.2");
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getDecorateExecuteObjectsBatchList().get(2).getBatch()).isEqualTo(3);
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getDecorateExecuteObjectsBatchList().get(2).getDecorateExecuteObjects()).hasSize(2);
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getDecorateExecuteObjectsBatchList().get(2).getDecorateExecuteObjects().get(0).getHost().getBkCloudId()).isEqualTo(0L);
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getDecorateExecuteObjectsBatchList().get(2).getDecorateExecuteObjects().get(0).getHost().getIp()).isEqualTo("127.0.0.3");
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getDecorateExecuteObjectsBatchList().get(2).getDecorateExecuteObjects().get(1).getHost().getBkCloudId()).isEqualTo(0L);
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getDecorateExecuteObjectsBatchList().get(2).getDecorateExecuteObjects().get(1).getHost().getIp()).isEqualTo("127.0.0.4");
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getMergedExecuteObjectsBatchList()).hasSize(3);
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getMergedExecuteObjectsBatchList().get(0).getBatch()).isEqualTo(1);
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getMergedExecuteObjectsBatchList().get(0).getMergedExecuteObjects()).hasSize(1);
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getMergedExecuteObjectsBatchList().get(0).getMergedExecuteObjects().get(0).getHost().getBkCloudId()).isEqualTo(0L);
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getMergedExecuteObjectsBatchList().get(0).getMergedExecuteObjects().get(0).getHost().getIp()).isEqualTo("127.0.0.1");
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getMergedExecuteObjectsBatchList().get(1).getBatch()).isEqualTo(2);
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getMergedExecuteObjectsBatchList().get(1).getMergedExecuteObjects()).hasSize(1);
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getMergedExecuteObjectsBatchList().get(1).getMergedExecuteObjects().get(0).getHost().getBkCloudId()).isEqualTo(0L);
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getMergedExecuteObjectsBatchList().get(1).getMergedExecuteObjects().get(0).getHost().getIp()).isEqualTo("127.0.0.2");
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getMergedExecuteObjectsBatchList().get(2).getBatch()).isEqualTo(3);
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getMergedExecuteObjectsBatchList().get(2).getMergedExecuteObjects()).hasSize(2);
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getMergedExecuteObjectsBatchList().get(2).getMergedExecuteObjects().get(0).getHost().getBkCloudId()).isEqualTo(0L);
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getMergedExecuteObjectsBatchList().get(2).getMergedExecuteObjects().get(0).getHost().getIp()).isEqualTo("127.0.0.3");
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getMergedExecuteObjectsBatchList().get(2).getMergedExecuteObjects().get(1).getHost().getBkCloudId()).isEqualTo(0L);
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getMergedExecuteObjectsBatchList().get(2).getMergedExecuteObjects().get(1).getHost().getIp()).isEqualTo("127.0.0.4");
}

@Test
Expand Down Expand Up @@ -148,11 +148,11 @@ void saveRollingConfig() {
assertThat(savedStepRollingConfigs.get(1001L).isBatch()).isEqualTo(false);
assertThat(savedStepRollingConfigs.get(1002L).isBatch()).isEqualTo(true);
assertThat(savedStepRollingConfigs.get(1003L).isBatch()).isEqualTo(true);
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getDecorateExecuteObjectsBatchList()).hasSize(1);
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getDecorateExecuteObjectsBatchList().get(0).getBatch()).isEqualTo(1);
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getDecorateExecuteObjectsBatchList().get(0).getDecorateExecuteObjects().get(0).getHost().getHostId()).isEqualTo(1L);
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getDecorateExecuteObjectsBatchList().get(0).getDecorateExecuteObjects().get(0).getHost().getBkCloudId()).isEqualTo(0L);
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getDecorateExecuteObjectsBatchList().get(0).getDecorateExecuteObjects().get(0).getHost().getIp()).isEqualTo("127.0.0.1");
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getMergedExecuteObjectsBatchList()).hasSize(1);
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getMergedExecuteObjectsBatchList().get(0).getBatch()).isEqualTo(1);
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getMergedExecuteObjectsBatchList().get(0).getMergedExecuteObjects().get(0).getHost().getHostId()).isEqualTo(1L);
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getMergedExecuteObjectsBatchList().get(0).getMergedExecuteObjects().get(0).getHost().getBkCloudId()).isEqualTo(0L);
assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getMergedExecuteObjectsBatchList().get(0).getMergedExecuteObjects().get(0).getHost().getIp()).isEqualTo("127.0.0.1");
}
}

20 changes: 11 additions & 9 deletions ...ob-execute/src/main/java/com/tencent/bk/job/execute/auth/impl/ExecuteAuthServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,9 +88,11 @@ public ExecuteAuthServiceImpl(AuthHelper authHelper,
this.appAuthService.setResourceNameQueryService(resourceNameQueryService);
}

public AuthResult authFastExecuteScript(String username, AppResourceScope appResourceScope, ExecuteObjectsDTO servers) {
public AuthResult authFastExecuteScript(String username,
AppResourceScope appResourceScope,
ExecuteObjectsDTO executeObjects) {

List<InstanceDTO> hostInstanceList = buildHostInstances(appResourceScope, servers);
List<InstanceDTO> hostInstanceList = buildHostInstances(appResourceScope, executeObjects);

log.debug("Auth fast execute script, username:{}, appResourceScope:{}, hostInstances:{}", username,
appResourceScope, hostInstanceList);
Expand All @@ -104,7 +106,7 @@ public AuthResult authFastExecuteScript(String username, AppResourceScope appRes
AuthResult authResult = AuthResult.fail();

List<PermissionResource> hostResources = convertHostsToPermissionResourceList(
appResourceScope, servers);
appResourceScope, executeObjects);
authResult.addRequiredPermissions(ActionId.QUICK_EXECUTE_SCRIPT, hostResources);
log.debug("Auth execute script, authResult:{}", authResult);
return authResult;
Expand Down Expand Up @@ -319,13 +321,13 @@ private List<InstanceDTO> buildBizStaticHostInstances(AppResourceScope appResour
}

private List<InstanceDTO> buildHostInstances(AppResourceScope appResourceScope,
ExecuteObjectsDTO servers) {
ExecuteObjectsDTO executeObjects) {
List<InstanceDTO> hostInstanceList = new ArrayList<>();
// 静态IP
if (!CollectionUtils.isEmpty(servers.getStaticIpList())) {
if (!CollectionUtils.isEmpty(executeObjects.getStaticIpList())) {
switch (appResourceScope.getType()) {
case BIZ:
hostInstanceList.addAll(buildBizStaticHostInstances(appResourceScope, servers));
hostInstanceList.addAll(buildBizStaticHostInstances(appResourceScope, executeObjects));
break;
case BIZ_SET:
InstanceDTO hostInstance = new InstanceDTO();
Expand All @@ -341,13 +343,13 @@ private List<InstanceDTO> buildHostInstances(AppResourceScope appResourceScope,
}
}
// 动态topo节点
if (!CollectionUtils.isEmpty(servers.getTopoNodes())) {
if (!CollectionUtils.isEmpty(executeObjects.getTopoNodes())) {
// CMDB未提供权限中心使用的topo视图，暂时使用“业务”这个topo节点进行鉴权，不细化到集群、模块
hostInstanceList.addAll(buildAppTopoNodeHostInstances(appResourceScope));
}
// 动态分组
if (!CollectionUtils.isEmpty(servers.getDynamicServerGroups())) {
servers.getDynamicServerGroups().forEach(serverGroup -> {
if (!CollectionUtils.isEmpty(executeObjects.getDynamicServerGroups())) {
executeObjects.getDynamicServerGroups().forEach(serverGroup -> {
InstanceDTO serverGroupInstance = new InstanceDTO();
serverGroupInstance.setType(ResourceTypeEnum.HOST.getId());
serverGroupInstance.setSystem(SystemId.CMDB);
Expand Down
6 changes: 3 additions & 3 deletions ...execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -267,7 +267,7 @@ private void saveExecuteObjectTasksForStartStep(Long gseTaskId,
// 普通步骤，启动的时候需要初始化所有ExecuteObjectTask
List<ExecuteObjectTask> executeObjectTasks = new ArrayList<>(
buildInitialExecuteObjectTasks(stepInstanceId, executeCount, executeCount, batch,
gseTaskId, stepInstance.getTargetExecuteObjects().getDecorateExecuteObjects()));
gseTaskId, stepInstance.getTargetExecuteObjects().getMergedExecuteObjects()));
saveExecuteObjectTasks(stepInstance, executeObjectTasks);
}
}
Expand All @@ -290,7 +290,7 @@ private void saveGseExecuteObjectTasksForStartRollingStep(Long gseTaskId,
List<ExecuteObjectTask> executeObjectTasks = new ArrayList<>();
if (rollingConfig.isBatchRollingStep(stepInstanceId)) {
List<RollingExecuteObjectsBatchDO> executeObjectsBatchList =
rollingConfig.getConfigDetail().getDecorateExecuteObjectsBatchList();
rollingConfig.getConfigDetail().getMergedExecuteObjectsBatchList();
executeObjectsBatchList.forEach(executeObjectsBatch -> {
executeObjectTasks.addAll(
buildInitialExecuteObjectTasks(
Expand All @@ -299,7 +299,7 @@ private void saveGseExecuteObjectTasksForStartRollingStep(Long gseTaskId,
executeObjectsBatch.getBatch() == 1 ? executeCount : null,
executeObjectsBatch.getBatch(),
executeObjectsBatch.getBatch() == 1 ? gseTaskId : 0,
executeObjectsBatch.getDecorateExecuteObjects()
executeObjectsBatch.getMergedExecuteObjects()
)
);
});
Expand Down
8 changes: 4 additions & 4 deletions ...ice-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/JobSrcFileUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ public static Set<JobFile> parseSrcFiles(StepInstanceDTO stepInstance,
Pair<String, String> fileNameAndPath = FilePathUtils.parseDirAndFileName(filePath);
String dir = fileNameAndPath.getLeft();
String fileName = fileNameAndPath.getRight();
List<ExecuteObject> sourceExecuteObjects = fileSource.getServers().getDecorateExecuteObjects();
List<ExecuteObject> sourceExecuteObjects = fileSource.getServers().getMergedExecuteObjects();
for (ExecuteObject sourceExecuteObject : sourceExecuteObjects) {
// 第三方源文件的displayName不同
if (isThirdFile) {
Expand All @@ -154,8 +154,8 @@ public static Set<JobFile> parseSrcFiles(StepInstanceDTO stepInstance,
+ fileNameAndPath.getLeft();
String fileName = fileNameAndPath.getRight();
ExecuteObjectsDTO servers = fileSource.getServers();
if (servers != null && CollectionUtils.isNotEmpty(servers.getDecorateExecuteObjects())) {
List<ExecuteObject> executeObjects = servers.getDecorateExecuteObjects();
if (servers != null && CollectionUtils.isNotEmpty(servers.getMergedExecuteObjects())) {
List<ExecuteObject> executeObjects = servers.getMergedExecuteObjects();
for (ExecuteObject executeObject : executeObjects) {
sendFiles.add(new JobFile(TaskFileTypeEnum.LOCAL, executeObject, file.getFilePath(), dir,
fileName, "root", null,
Expand All @@ -170,7 +170,7 @@ public static Set<JobFile> parseSrcFiles(StepInstanceDTO stepInstance,
String dir = NFSUtils.getFileDir(jobStorageRootDir, FileDirTypeConf.UPLOAD_FILE_DIR)
+ fileNameAndPath.getLeft();
String fileName = fileNameAndPath.getRight();
List<ExecuteObject> executeObjects = fileSource.getServers().getDecorateExecuteObjects();
List<ExecuteObject> executeObjects = fileSource.getServers().getMergedExecuteObjects();
for (ExecuteObject executeObject : executeObjects) {
sendFiles.add(new JobFile(TaskFileTypeEnum.BASE64_FILE, executeObject, file.getFilePath(), dir,
fileName, "root", null,
Expand Down
46 changes: 24 additions & 22 deletions ...service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ExecuteObjectsDTO.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,37 +103,37 @@ public static ExecuteObjectsDTO emptyInstance() {
}

public ExecuteObjectsDTO clone() {
ExecuteObjectsDTO cloneExecuteObjectsDTO = new ExecuteObjectsDTO();
cloneExecuteObjectsDTO.setVariable(variable);
ExecuteObjectsDTO clone = new ExecuteObjectsDTO();
clone.setVariable(variable);
if (CollectionUtils.isNotEmpty(staticIpList)) {
List<HostDTO> cloneStaticIpList = new ArrayList<>(staticIpList.size());
staticIpList.forEach(staticIp -> cloneStaticIpList.add(staticIp.clone()));
cloneExecuteObjectsDTO.setStaticIpList(cloneStaticIpList);
clone.setStaticIpList(cloneStaticIpList);
}
if (CollectionUtils.isNotEmpty(dynamicServerGroups)) {
List<DynamicServerGroupDTO> cloneServerGroups = new ArrayList<>(dynamicServerGroups.size());
dynamicServerGroups.forEach(serverGroup -> cloneServerGroups.add(serverGroup.clone()));
cloneExecuteObjectsDTO.setDynamicServerGroups(cloneServerGroups);
clone.setDynamicServerGroups(cloneServerGroups);
}
if (CollectionUtils.isNotEmpty(topoNodes)) {
cloneExecuteObjectsDTO.setTopoNodes(topoNodes);
clone.setTopoNodes(topoNodes);
}
if (CollectionUtils.isNotEmpty(staticContainerList)) {
List<Container> cloneContainerList = new ArrayList<>(staticContainerList.size());
staticContainerList.forEach(container -> cloneContainerList.add(container.clone()));
cloneExecuteObjectsDTO.setStaticContainerList(cloneContainerList);
clone.setStaticContainerList(cloneContainerList);
}
if (CollectionUtils.isNotEmpty(ipList)) {
List<HostDTO> cloneIpList = new ArrayList<>(ipList.size());
ipList.forEach(ip -> cloneIpList.add(ip.clone()));
cloneExecuteObjectsDTO.setIpList(cloneIpList);
clone.setIpList(cloneIpList);
}
if (CollectionUtils.isNotEmpty(executeObjects)) {
List<ExecuteObject> cloneExecuteObjectList = new ArrayList<>(executeObjects.size());
executeObjects.forEach(executeObject -> cloneExecuteObjectList.add(executeObject.clone()));
cloneExecuteObjectsDTO.setExecuteObjects(cloneExecuteObjectList);
clone.setExecuteObjects(cloneExecuteObjectList);
}
return cloneExecuteObjectsDTO;
return clone;
}

public ExecuteObjectsDTO merge(ExecuteObjectsDTO executeObjects) {
Expand Down Expand Up @@ -173,6 +173,17 @@ public ExecuteObjectsDTO merge(ExecuteObjectsDTO executeObjects) {
});
}
}
if (executeObjects.getStaticContainerList() != null) {
if (this.staticContainerList == null) {
this.staticContainerList = new ArrayList<>(executeObjects.getStaticContainerList());
} else {
executeObjects.getStaticContainerList().forEach(container -> {
if (!this.staticContainerList.contains(container)) {
this.staticContainerList.add(container);
}
});
}
}
return this;
}

Expand All @@ -189,7 +200,8 @@ public void addStaticHosts(Collection<HostDTO> hosts) {
public boolean isEmpty() {
return CollectionUtils.isEmpty(this.staticIpList)
&& CollectionUtils.isEmpty(this.topoNodes)
&& CollectionUtils.isEmpty(this.dynamicServerGroups);
&& CollectionUtils.isEmpty(this.dynamicServerGroups)
&& CollectionUtils.isEmpty(this.staticContainerList);
}

/**
Expand Down Expand Up @@ -414,10 +426,10 @@ public static ExecuteObjectsDTO fromTaskTargetVO(TaskTargetVO target) {
}

/**
* 获取包装过的执行对象列表
* 获取合并之后的所有执行对象列表
*/
@JsonIgnore
public List<ExecuteObject> getDecorateExecuteObjects() {
public List<ExecuteObject> getMergedExecuteObjects() {
if (executeObjects != null) {
return executeObjects;
} else if (ipList != null) {
Expand All @@ -426,14 +438,4 @@ public List<ExecuteObject> getDecorateExecuteObjects() {
return Collections.emptyList();
}
}

/**
* 提取所有包含的容器执行对象
*
* @return 容器执行对象列表
*/
public List<Container> extractContainers() {
// 当前只支持静态选择容器
return staticContainerList;
}
}
4 changes: 2 additions & 2 deletions ...-job-execute/src/main/java/com/tencent/bk/job/execute/model/db/RollingConfigDetailDO.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,10 +113,10 @@ public boolean isLastRollingStep(long stepInstanceId) {
}

/**
* 获取包装过的执行对象列表
* 获取合并后的执行对象滚动批次
*/
@JsonIgnore
public List<RollingExecuteObjectsBatchDO> getDecorateExecuteObjectsBatchList() {
public List<RollingExecuteObjectsBatchDO> getMergedExecuteObjectsBatchList() {
if (executeObjectsBatchList != null) {
return executeObjectsBatchList;
} else if (hostsBatchList != null) {
Expand Down
Loading

0 comments on commit e3bdbef

Please sign in to comment.