diff --git a/src/backend/k8s-startup-controller/build.gradle b/src/backend/k8s-startup-controller/build.gradle new file mode 100644 index 0000000000..aa046b5488 --- /dev/null +++ b/src/backend/k8s-startup-controller/build.gradle @@ -0,0 +1,65 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ +plugins { + id "com.github.johnrengelman.shadow" version "7.1.1" +} +ext { + if (System.getProperty("jobVersion")) { + set("jobVersion", System.getProperty("jobVersion")) + } else if (System.getProperty("bkjobVersion")) { + set("jobVersion", System.getProperty("bkjobVersion")) + } else { + set("jobVersion", "1.0.0") + } +} +version "${jobVersion}" +dependencies { + api project(":commons:common") + api 'org.springframework.cloud:spring-cloud-starter-kubernetes-client-all' + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' + implementation 'ch.qos.logback:logback-core' + implementation 'ch.qos.logback:logback-classic' + testImplementation 'org.junit.jupiter:junit-jupiter' +} +apply plugin: "com.github.johnrengelman.shadow" +apply plugin: "application" + +// 固定入口类 不要改 +mainClassName = "com.tencent.bk.job.k8s.StartupController" + +shadowJar { + classifier = null + zip64 true +} +task copyToRelease(type: Copy) { + from("build/libs") { + include("**/k8s-startup-controller-*.jar") + } + into "${rootDir}/release" + outputs.upToDateWhen { false } +} + +copyToRelease.dependsOn shadowJar +build.dependsOn copyToRelease diff --git a/src/backend/k8s-startup-controller/src/main/java/com/tencent/bk/job/k8s/Consts.java b/src/backend/k8s-startup-controller/src/main/java/com/tencent/bk/job/k8s/Consts.java new file mode 100644 index 0000000000..611f5360d7 --- /dev/null +++ b/src/backend/k8s-startup-controller/src/main/java/com/tencent/bk/job/k8s/Consts.java @@ -0,0 +1,31 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.k8s; + +public class Consts { + public static String KEY_KUBERNETES_NAMESPACE = "KUBERNETES_NAMESPACE"; + public static String KEY_STARTUP_DEPENDENCIES_STR = "BK_JOB_STARTUP_DEPENDENCIES_STR"; + public static String KEY_CURRENT_SERVICE = "BK_JOB_APP_NAME"; +} diff --git a/src/backend/k8s-startup-controller/src/main/java/com/tencent/bk/job/k8s/StartupController.java b/src/backend/k8s-startup-controller/src/main/java/com/tencent/bk/job/k8s/StartupController.java new file mode 100644 index 0000000000..ad4638df86 --- /dev/null +++ b/src/backend/k8s-startup-controller/src/main/java/com/tencent/bk/job/k8s/StartupController.java @@ -0,0 +1,225 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.k8s; + +import com.tencent.bk.job.common.util.StringUtil; +import com.tencent.bk.job.common.util.ThreadUtils; +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.Configuration; +import io.kubernetes.client.openapi.apis.CoreV1Api; +import io.kubernetes.client.openapi.models.V1Pod; +import io.kubernetes.client.openapi.models.V1PodList; +import io.kubernetes.client.openapi.models.V1PodStatus; +import io.kubernetes.client.util.Config; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * 启动控制器，用于控制在K8s部署时各微服务的启动顺序 + */ +@Slf4j +public class StartupController { + + private final static CoreV1Api api; + + static { + ApiClient client = null; + try { + client = Config.defaultClient(); + } catch (IOException e) { + log.error("Fail to get defaultClient", e); + } + Configuration.setDefaultApiClient(client); + api = new CoreV1Api(); + } + + public static void main(String[] args) { + String namespace = System.getenv(Consts.KEY_KUBERNETES_NAMESPACE); + String dependenciesStr = System.getenv(Consts.KEY_STARTUP_DEPENDENCIES_STR); + String currentService = System.getenv(Consts.KEY_CURRENT_SERVICE); + log.info("namespace={}", namespace); + log.info("dependenciesStr={}", dependenciesStr); + log.info("currentService={}", currentService); + Map> dependencyMap = parseDependencyMap(dependenciesStr); + printDependencyMap(dependencyMap); + while (!isAllDependServiceReady(dependencyMap, namespace, currentService)) { + ThreadUtils.sleep(3000); + } + log.info("all depend services are ready, it`s time for {} to start", currentService); + } + + /** + * 根据依赖定义字符串解析出依赖关系Map + * + * @param dependenciesStr 依赖定义字符串，模式：(service1:service2,service3),(service2:service4),... + * 含义：service1依赖于service2、service3，service2依赖于service4 + * @return 服务间依赖关系Map<服务名 ， 依赖的服务列表> + */ + public static Map> parseDependencyMap(String dependenciesStr) { + if (StringUtils.isBlank(dependenciesStr)) { + return Collections.emptyMap(); + } + dependenciesStr = dependenciesStr.trim(); + dependenciesStr = dependenciesStr.replace(" ", ""); + Map> dependencyMap = new HashMap<>(); + String separator = "\\),\\("; + String[] dependencyArr = dependenciesStr.split(separator); + for (String serviceDepStr : dependencyArr) { + serviceDepStr = StringUtil.removePrefix(serviceDepStr, "("); + serviceDepStr = StringUtil.removeSuffix(serviceDepStr, ")"); + String[] parts = serviceDepStr.split(":"); + if (parts.length != 2) { + log.warn("illegal dependency:{}", serviceDepStr); + continue; + } + String serviceName = parts[0]; + String depServiceStr = parts[1]; + List dependServiceList = new ArrayList<>(Arrays.asList(depServiceStr.split(","))); + if (dependencyMap.containsKey(serviceName)) { + dependencyMap.get(serviceName).addAll(dependServiceList); + } else { + dependencyMap.put(serviceName, dependServiceList); + } + } + return dependencyMap; + } + + /** + * 检查当前服务的所有依赖服务是否准备好 + * + * @param dependencyMap 依赖关系Map + * @param namespace 命名空间 + * @param currentService 当前服务名称 + * @return 所有依赖服务是否准备好 + */ + private static boolean isAllDependServiceReady(Map> dependencyMap, + String namespace, + String currentService) { + if (StringUtils.isBlank(currentService)) { + log.warn("currentService is blank, ignore dependency check"); + return true; + } + List dependServiceList = dependencyMap.get(currentService); + if (CollectionUtils.isEmpty(dependServiceList)) { + log.info("There is no depend service for {}", currentService); + return true; + } + log.info("{} depend service found for {}:{}", dependServiceList.size(), currentService, dependServiceList); + for (String dependService : dependServiceList) { + if (!isServiceReady(namespace, dependService)) { + return false; + } + } + return true; + } + + private static String buildServiceLabelSelector(String serviceName) { + return "bk.job.scope=backend,app.kubernetes.io/component=" + serviceName; + } + + /** + * 根据服务名称获取Pod列表 + * + * @param namespace 命名空间 + * @param serviceName 服务名称 + * @return Pod列表 + */ + private static List listPodByServiceName(String namespace, String serviceName) { + String labelSelector = buildServiceLabelSelector(serviceName); + try { + V1PodList podList = api.listNamespacedPod( + namespace, null, null, null, + null, labelSelector, null, null, + null, null, null + ); + return podList.getItems(); + } catch (ApiException e) { + log.error("Fail to list pod", e); + return Collections.emptyList(); + } + } + + private static void printDependencyMap(Map> dependencyMap) { + dependencyMap.forEach( + (serviceName, dependServiceList) -> log.info("{} depends on {}", serviceName, dependServiceList) + ); + } + + /** + * 根据服务名称对应的Pod状态判断服务是否准备好，依据：所有Pod均准备好 + * + * @param namespace 命名空间 + * @param serviceName 服务名称 + * @return 服务是否准备好布尔值 + */ + private static boolean isServiceReady(String namespace, String serviceName) { + List servicePodList = listPodByServiceName(namespace, serviceName); + if (CollectionUtils.isEmpty(servicePodList)) { + log.info("no pod found by service {}", serviceName); + return true; + } + int readyPodNum = 0; + for (V1Pod v1Pod : servicePodList) { + if (isPodReady(v1Pod)) { + readyPodNum++; + } + } + log.info("{}/{} pod ready for service {}", readyPodNum, servicePodList.size(), serviceName); + return readyPodNum == servicePodList.size(); + } + + /** + * 判断Pod是否准备好，判断依据：状态数据中的phase字段值 + * + * @param v1Pod Pod实例信息 + * @return Pod是否准备好布尔值 + */ + private static boolean isPodReady(V1Pod v1Pod) { + V1PodStatus v1PodStatus = v1Pod.getStatus(); + if (log.isDebugEnabled()) { + if (v1Pod.getMetadata() == null) { + log.debug("unexpected pod:{}", v1Pod); + return false; + } + if (v1PodStatus == null) { + log.debug("status of pod {} is null", v1Pod.getMetadata().getName()); + return false; + } + log.debug("phase of pod {}:{}", v1Pod.getMetadata().getName(), v1PodStatus.getPhase()); + } + return v1PodStatus != null && "Running".equalsIgnoreCase(v1PodStatus.getPhase()); + } + +} diff --git a/src/backend/k8s-startup-controller/src/main/resources/logback.xml b/src/backend/k8s-startup-controller/src/main/resources/logback.xml new file mode 100644 index 0000000000..c7f7636cc1 --- /dev/null +++ b/src/backend/k8s-startup-controller/src/main/resources/logback.xml @@ -0,0 +1,32 @@ + + + + + + + logback + + + ${CONTROLLER_LOG_FILE} + + ${CONTROLLER_LOG_FILE}-%d{yyyyMMdd_HH}.log.%i + 1GB + 30 + 10GB + + + ${LOG_PATTERN} + UTF-8 + + + + + ${LOG_PATTERN} + + + + + + + diff --git a/src/backend/k8s-startup-controller/src/test/java/com/tencent/bk/job/k8s/StartupControllerTest.java b/src/backend/k8s-startup-controller/src/test/java/com/tencent/bk/job/k8s/StartupControllerTest.java new file mode 100644 index 0000000000..ea3c5d6f9b --- /dev/null +++ b/src/backend/k8s-startup-controller/src/test/java/com/tencent/bk/job/k8s/StartupControllerTest.java @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.k8s; + +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class StartupControllerTest { + + @Test + void testParseDependencyMap() { + String dependenciesStr = "(job-execute:job-manage,job-logsvr)," + + "(job-crontab:job-execute),(job-analysis:job-crontab),(job-file-worker:job-file-gateway)"; + Map> dependencyMap = StartupController.parseDependencyMap(dependenciesStr); + assertEquals(4, dependencyMap.size()); + assertTrue(dependencyMap.containsKey("job-execute")); + assertTrue(dependencyMap.containsKey("job-crontab")); + assertTrue(dependencyMap.containsKey("job-analysis")); + assertTrue(dependencyMap.containsKey("job-file-worker")); + assertEquals(2, dependencyMap.get("job-execute").size()); + } +} diff --git a/src/backend/settings.gradle b/src/backend/settings.gradle index 90307d0dab..2fcee4c21c 100644 --- a/src/backend/settings.gradle +++ b/src/backend/settings.gradle @@ -89,5 +89,6 @@ include 'job-analysis:api-job-analysis' include 'job-analysis:service-job-analysis' include 'job-analysis:boot-job-analysis' include 'job-analysis:model-job-analysis' +include 'k8s-startup-controller' include 'upgrader' diff --git a/src/backend/upgrader/build.gradle b/src/backend/upgrader/build.gradle index 6e4587c9ad..99086efd57 100644 --- a/src/backend/upgrader/build.gradle +++ b/src/backend/upgrader/build.gradle @@ -60,9 +60,6 @@ apply plugin: "application" mainClassName = "com.tencent.bk.job.upgrader.Upgrader" shadowJar { - // baseName为插件默认打包名+".jar"，bkdevops-plugin.jar - // 如果修改，则要一同修改插件task.json中的target启动命令 - // 为了省事，建议不用修改 classifier = null zip64 true }