perf: 容器化-支持各微服务按顺序更新 TencentBlueKing#919

chart优化，探针信息查看
jsonwan · Nov 30, 2022 · d2dcdcd · d2dcdcd
1 parent d8f7be6
commit d2dcdcd
Show file tree

Hide file tree

Showing 13 changed files with 48 additions and 37 deletions.
diff --git a/...-tools/k8s-startup-controller/src/main/java/com/tencent/bk/job/k8s/StartupController.java b/...-tools/k8s-startup-controller/src/main/java/com/tencent/bk/job/k8s/StartupController.java
@@ -30,9 +30,11 @@
 import io.kubernetes.client.openapi.ApiException;
 import io.kubernetes.client.openapi.Configuration;
 import io.kubernetes.client.openapi.apis.CoreV1Api;
+import io.kubernetes.client.openapi.models.V1ObjectMeta;
 import io.kubernetes.client.openapi.models.V1Pod;
 import io.kubernetes.client.openapi.models.V1PodList;
 import io.kubernetes.client.openapi.models.V1PodStatus;
+import io.kubernetes.client.openapi.models.V1Probe;
 import io.kubernetes.client.util.Config;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
@@ -74,7 +76,17 @@ public static void main(String[] args) {
         log.info("currentService={}", currentService);
         Map<String, List<String>> dependencyMap = parseDependencyMap(dependenciesStr);
         printDependencyMap(dependencyMap);
-        while (!isAllDependServiceReady(dependencyMap, namespace, currentService)) {
+        if (StringUtils.isBlank(currentService)) {
+            log.warn("currentService is blank, ignore dependency check");
+            return;
+        }
+        List<String> dependServiceList = dependencyMap.get(currentService);
+        if (CollectionUtils.isEmpty(dependServiceList)) {
+            log.info("There is no depend service for {}", currentService);
+            return;
+        }
+        log.info("{} depend service found for {}:{}", dependServiceList.size(), currentService, dependServiceList);
+        while (!isAllDependServiceReady(namespace, dependServiceList)) {
             ThreadUtils.sleep(3000);
         }
         log.info("all depend services are ready, it`s time for {} to start", currentService);
@@ -117,28 +129,17 @@ public static Map<String, List<String>> parseDependencyMap(String dependenciesSt
     }
 
     /**
-     * 检查当前服务的所有依赖服务是否准备好
+     * 检查所有依赖服务是否准备好
      *
-     * @param dependencyMap  依赖关系Map
-     * @param namespace      命名空间
-     * @param currentService 当前服务名称
+     * @param namespace         命名空间
+     * @param dependServiceList 依赖服务列表
      * @return 所有依赖服务是否准备好
      */
-    private static boolean isAllDependServiceReady(Map<String, List<String>> dependencyMap,
-                                                   String namespace,
-                                                   String currentService) {
-        if (StringUtils.isBlank(currentService)) {
-            log.warn("currentService is blank, ignore dependency check");
-            return true;
-        }
-        List<String> dependServiceList = dependencyMap.get(currentService);
-        if (CollectionUtils.isEmpty(dependServiceList)) {
-            log.info("There is no depend service for {}", currentService);
-            return true;
-        }
-        log.info("{} depend service found for {}:{}", dependServiceList.size(), currentService, dependServiceList);
+    private static boolean isAllDependServiceReady(String namespace,
+                                                   List<String> dependServiceList) {
         for (String dependService : dependServiceList) {
             if (!isServiceReady(namespace, dependService)) {
+                log.info("{} is not ready, waiting...", dependService);
                 return false;
             }
         }
@@ -194,9 +195,16 @@ private static boolean isServiceReady(String namespace, String serviceName) {
         for (V1Pod v1Pod : servicePodList) {
             if (isPodReady(v1Pod)) {
                 readyPodNum++;
+            } else {
+                V1ObjectMeta metaData = v1Pod.getMetadata();
+                if (metaData != null) {
+                    log.info("pod {} is not ready", metaData.getName());
+                } else {
+                    log.warn("pod {} is not ready", v1Pod);
+                }
             }
         }
-        log.info("{}/{} pod ready for service {}", readyPodNum, servicePodList.size(), serviceName);
+        log.info("{}: {}/{} pod ready", serviceName, readyPodNum, servicePodList.size());
         return readyPodNum == servicePodList.size();
     }
 
@@ -219,7 +227,10 @@ private static boolean isPodReady(V1Pod v1Pod) {
             }
             log.debug("phase of pod {}:{}", v1Pod.getMetadata().getName(), v1PodStatus.getPhase());
         }
-        return v1PodStatus != null && "Running".equalsIgnoreCase(v1PodStatus.getPhase());
+        V1Probe readinessProbe = v1Pod.getSpec().getContainers().get(0).getReadinessProbe();
+        log.info("readinessProbe={}", readinessProbe.getHttpGet());
+        return v1PodStatus != null
+            && "Running".equalsIgnoreCase(v1PodStatus.getPhase());
     }
 
 }
diff --git a/support-files/kubernetes/charts/bk-job/templates/_helpers.tpl b/support-files/kubernetes/charts/bk-job/templates/_helpers.tpl
@@ -476,20 +476,20 @@ Return the Job InitContainer WaitForMigration Content
 
 {{/*
 Return the Job InitContainer WaitForDependServices Content
-{{ include "job.initContainer.waitForDependServices" ( dict "appName" "${appName}" "global" $) }}
+{{ include "job.initContainer.waitForDependServices" ( dict "appName" "${appName}" "context" $) }}
 */}}
 {{- define "job.initContainer.waitForDependServices" -}}
-{{- if .global.waitForDependServices.enabled -}}
+{{- if .context.Values.waitForDependServices.enabled -}}
 - name: "wait-for-depend-services"
-  image: {{ include "common.images.image" (dict "imageRoot" .global.waitForDependServices.image "global" .global.global) }}
-  imagePullPolicy: {{ .global.waitForDependServices.image.pullPolicy }}
+  image: {{ include "common.images.image" (dict "imageRoot" .context.Values.waitForDependServices.image "global" .context.Values.global) }}
+  imagePullPolicy: {{ .context.Values.waitForDependServices.image.pullPolicy }}
   env:
     - name: KUBERNETES_NAMESPACE
-      value: {{ .global.Release.Namespace }}
+      value: {{ .context.Release.Namespace }}
     - name: BK_JOB_APP_NAME
       value: {{ .appName }}
     - name: BK_JOB_STARTUP_DEPENDENCIES_STR
-      value: {{ .global.waitForDependServices.dependencies }}
+      value: {{ .context.Values.waitForDependServices.dependencies }}
 {{- end -}}
 {{- end -}}
 

diff --git a/support-files/kubernetes/charts/bk-job/templates/job-analysis/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-analysis/deployment.yaml
@@ -58,7 +58,7 @@ spec:
       {{- end }}
       initContainers:
       {{- include "job.initContainer.waitForMigration" . | nindent 8 }}
-      {{- include "job.initContainer.waitForDependServices"  ( dict "appName" "job-analysis" "global" $) | nindent 8 }}
+      {{- include "job.initContainer.waitForDependServices"  ( dict "appName" "job-analysis" "context" $) | nindent 8 }}
       containers:
         - name: "job-analysis"
           {{- if .Values.analysisConfig.containerSecurityContext.enabled }}

diff --git a/support-files/kubernetes/charts/bk-job/templates/job-backup/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-backup/deployment.yaml
@@ -58,7 +58,7 @@ spec:
       {{- end }}
       initContainers:
       {{- include "job.initContainer.waitForMigration" . | nindent 8 }}
-      {{- include "job.initContainer.waitForDependServices"   ( dict "appName" "job-backup" "global" $) | nindent 8 }}
+      {{- include "job.initContainer.waitForDependServices"   ( dict "appName" "job-backup" "context" $) | nindent 8 }}
       containers:
         - name: "job-backup"
           {{- if .Values.backupConfig.containerSecurityContext.enabled }}

diff --git a/support-files/kubernetes/charts/bk-job/templates/job-crontab/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-crontab/deployment.yaml
@@ -58,7 +58,7 @@ spec:
       {{- end }}
       initContainers:
       {{- include "job.initContainer.waitForMigration" . | nindent 8 }}
-      {{- include "job.initContainer.waitForDependServices"   ( dict "appName" "job-crontab" "global" $) | nindent 8 }}
+      {{- include "job.initContainer.waitForDependServices"   ( dict "appName" "job-crontab" "context" $) | nindent 8 }}
       containers:
         - name: "job-crontab"
           {{- if .Values.crontabConfig.containerSecurityContext.enabled }}

diff --git a/support-files/kubernetes/charts/bk-job/templates/job-execute/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-execute/deployment.yaml
@@ -58,7 +58,7 @@ spec:
       {{- end }}
       initContainers:
       {{- include "job.initContainer.waitForMigration" . | nindent 8 }}
-      {{- include "job.initContainer.waitForDependServices"   ( dict "appName" "job-execute" "global" $) | nindent 8 }}
+      {{- include "job.initContainer.waitForDependServices"   ( dict "appName" "job-execute" "context" $) | nindent 8 }}
       containers:
         - name: "job-execute"
           {{- if .Values.executeConfig.containerSecurityContext.enabled }}

diff --git a/support-files/kubernetes/charts/bk-job/templates/job-file-gateway/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-file-gateway/deployment.yaml
@@ -58,7 +58,7 @@ spec:
       {{- end }}
       initContainers:
       {{- include "job.initContainer.waitForMigration" . | nindent 8 }}
-      {{- include "job.initContainer.waitForDependServices"   ( dict "appName" "job-file-gateway" "global" $) | nindent 8 }}
+      {{- include "job.initContainer.waitForDependServices"   ( dict "appName" "job-file-gateway" "context" $) | nindent 8 }}
       containers:
         - name: "job-file-gateway"
           {{- if .Values.fileGatewayConfig.containerSecurityContext.enabled }}

diff --git a/support-files/kubernetes/charts/bk-job/templates/job-file-worker/statefulset.yaml b/support-files/kubernetes/charts/bk-job/templates/job-file-worker/statefulset.yaml
@@ -58,7 +58,7 @@ spec:
       securityContext: {{- omit .Values.fileWorkerConfig.podSecurityContext "enabled" | toYaml | nindent 8 }}
       {{- end }}
       initContainers:
-      {{- include "job.initContainer.waitForDependServices"   ( dict "appName" "job-file-worker" "global" $) | nindent 8 }}
+      {{- include "job.initContainer.waitForDependServices"   ( dict "appName" "job-file-worker" "context" $) | nindent 8 }}
       containers:
         - name: "job-file-worker"
           {{- if .Values.fileWorkerConfig.containerSecurityContext.enabled }}

diff --git a/support-files/kubernetes/charts/bk-job/templates/job-frontend/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-frontend/deployment.yaml
@@ -55,7 +55,7 @@ spec:
       securityContext: {{- omit .Values.frontendConfig.podSecurityContext "enabled" | toYaml | nindent 8 }}
       {{- end }}
       initContainers:
-      {{- include "job.initContainer.waitForDependServices"   ( dict "appName" "job-frontend" "global" $) | nindent 8 }}
+      {{- include "job.initContainer.waitForDependServices"   ( dict "appName" "job-frontend" "context" $) | nindent 8 }}
       containers:
         - name: "job-frontend"
           {{- if .Values.frontendConfig.containerSecurityContext.enabled }}

diff --git a/support-files/kubernetes/charts/bk-job/templates/job-gateway/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-gateway/deployment.yaml
@@ -57,7 +57,7 @@ spec:
       securityContext: {{- omit .Values.gatewayConfig.podSecurityContext "enabled" | toYaml | nindent 8 }}
       {{- end }}
       initContainers:
-      {{- include "job.initContainer.waitForDependServices"   ( dict "appName" "job-gateway" "global" $) | nindent 8 }}
+      {{- include "job.initContainer.waitForDependServices"   ( dict "appName" "job-gateway" "context" $) | nindent 8 }}
       containers:
         - name: "job-gateway"
           {{- if .Values.gatewayConfig.containerSecurityContext.enabled }}

diff --git a/support-files/kubernetes/charts/bk-job/templates/job-logsvr/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-logsvr/deployment.yaml
@@ -57,7 +57,7 @@ spec:
       securityContext: {{- omit .Values.logsvrConfig.podSecurityContext "enabled" | toYaml | nindent 8 }}
       {{- end }}
       initContainers:
-      {{- include "job.initContainer.waitForDependServices"   ( dict "appName" "job-logsvr" "global" $) | nindent 8 }}
+      {{- include "job.initContainer.waitForDependServices"   ( dict "appName" "job-logsvr" "context" $) | nindent 8 }}
       containers:
         - name: "job-logsvr"
           {{- if .Values.logsvrConfig.containerSecurityContext.enabled }}

diff --git a/support-files/kubernetes/charts/bk-job/templates/job-manage/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-manage/deployment.yaml
@@ -58,7 +58,7 @@ spec:
       {{- end }}
       initContainers:
       {{- include "job.initContainer.waitForMigration" . | nindent 8 }}
-      {{- include "job.initContainer.waitForDependServices"   ( dict "appName" "job-manage" "global" $) | nindent 8 }}
+      {{- include "job.initContainer.waitForDependServices"   ( dict "appName" "job-manage" "context" $) | nindent 8 }}
       containers:
         - name: "job-manage"
           {{- if .Values.manageConfig.containerSecurityContext.enabled }}

diff --git a/support-files/kubernetes/images/build.sh b/support-files/kubernetes/images/build.sh
@@ -228,7 +228,7 @@ build_migration_image(){
 build_startup_controller_image(){
     log "Building startup-controller image, version: ${VERSION}..."
     TOOL_NAME="k8s-startup-controller"
-    $BACKEND_DIR/gradlew -p $BACKEND_DIR clean :job-tools:$TOOL_NAME:shadowJar -DmavenRepoUrl=$MAVEN_REPO_URL -DbkjobVersion=$VERSION
+    $BACKEND_DIR/gradlew -p $BACKEND_DIR clean :job-tools:$TOOL_NAME:build -DmavenRepoUrl=$MAVEN_REPO_URL -DbkjobVersion=$VERSION
     rm -rf tmp/*
     cp $BACKEND_DIR/release/$TOOL_NAME-$VERSION.jar tmp/$TOOL_NAME.jar
     cp startup-controller/startup.sh tmp/