From d2dcdcd7f487955ed317e32ec80d1bb83f290fd6 Mon Sep 17 00:00:00 2001 From: jsonwan Date: Wed, 30 Nov 2022 10:35:20 +0800 Subject: [PATCH] =?UTF-8?q?perf:=20=E5=AE=B9=E5=99=A8=E5=8C=96-=E6=94=AF?= =?UTF-8?q?=E6=8C=81=E5=90=84=E5=BE=AE=E6=9C=8D=E5=8A=A1=E6=8C=89=E9=A1=BA?= =?UTF-8?q?=E5=BA=8F=E6=9B=B4=E6=96=B0=20#919?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit chart优化，探针信息查看 --- .../tencent/bk/job/k8s/StartupController.java | 51 +++++++++++-------- .../charts/bk-job/templates/_helpers.tpl | 12 ++--- .../templates/job-analysis/deployment.yaml | 2 +- .../templates/job-backup/deployment.yaml | 2 +- .../templates/job-crontab/deployment.yaml | 2 +- .../templates/job-execute/deployment.yaml | 2 +- .../job-file-gateway/deployment.yaml | 2 +- .../job-file-worker/statefulset.yaml | 2 +- .../templates/job-frontend/deployment.yaml | 2 +- .../templates/job-gateway/deployment.yaml | 2 +- .../templates/job-logsvr/deployment.yaml | 2 +- .../templates/job-manage/deployment.yaml | 2 +- support-files/kubernetes/images/build.sh | 2 +- 13 files changed, 48 insertions(+), 37 deletions(-) diff --git a/src/backend/job-tools/k8s-startup-controller/src/main/java/com/tencent/bk/job/k8s/StartupController.java b/src/backend/job-tools/k8s-startup-controller/src/main/java/com/tencent/bk/job/k8s/StartupController.java index ad4638df86..6d1a0763cb 100644 --- a/src/backend/job-tools/k8s-startup-controller/src/main/java/com/tencent/bk/job/k8s/StartupController.java +++ b/src/backend/job-tools/k8s-startup-controller/src/main/java/com/tencent/bk/job/k8s/StartupController.java @@ -30,9 +30,11 @@ import io.kubernetes.client.openapi.ApiException; import io.kubernetes.client.openapi.Configuration; import io.kubernetes.client.openapi.apis.CoreV1Api; +import io.kubernetes.client.openapi.models.V1ObjectMeta; import io.kubernetes.client.openapi.models.V1Pod; import io.kubernetes.client.openapi.models.V1PodList; import io.kubernetes.client.openapi.models.V1PodStatus; +import io.kubernetes.client.openapi.models.V1Probe; import io.kubernetes.client.util.Config; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; @@ -74,7 +76,17 @@ public static void main(String[] args) { log.info("currentService={}", currentService); Map> dependencyMap = parseDependencyMap(dependenciesStr); printDependencyMap(dependencyMap); - while (!isAllDependServiceReady(dependencyMap, namespace, currentService)) { + if (StringUtils.isBlank(currentService)) { + log.warn("currentService is blank, ignore dependency check"); + return; + } + List dependServiceList = dependencyMap.get(currentService); + if (CollectionUtils.isEmpty(dependServiceList)) { + log.info("There is no depend service for {}", currentService); + return; + } + log.info("{} depend service found for {}:{}", dependServiceList.size(), currentService, dependServiceList); + while (!isAllDependServiceReady(namespace, dependServiceList)) { ThreadUtils.sleep(3000); } log.info("all depend services are ready, it`s time for {} to start", currentService); @@ -117,28 +129,17 @@ public static Map> parseDependencyMap(String dependenciesSt } /** - * 检查当前服务的所有依赖服务是否准备好 + * 检查所有依赖服务是否准备好 * - * @param dependencyMap 依赖关系Map - * @param namespace 命名空间 - * @param currentService 当前服务名称 + * @param namespace 命名空间 + * @param dependServiceList 依赖服务列表 * @return 所有依赖服务是否准备好 */ - private static boolean isAllDependServiceReady(Map> dependencyMap, - String namespace, - String currentService) { - if (StringUtils.isBlank(currentService)) { - log.warn("currentService is blank, ignore dependency check"); - return true; - } - List dependServiceList = dependencyMap.get(currentService); - if (CollectionUtils.isEmpty(dependServiceList)) { - log.info("There is no depend service for {}", currentService); - return true; - } - log.info("{} depend service found for {}:{}", dependServiceList.size(), currentService, dependServiceList); + private static boolean isAllDependServiceReady(String namespace, + List dependServiceList) { for (String dependService : dependServiceList) { if (!isServiceReady(namespace, dependService)) { + log.info("{} is not ready, waiting...", dependService); return false; } } @@ -194,9 +195,16 @@ private static boolean isServiceReady(String namespace, String serviceName) { for (V1Pod v1Pod : servicePodList) { if (isPodReady(v1Pod)) { readyPodNum++; + } else { + V1ObjectMeta metaData = v1Pod.getMetadata(); + if (metaData != null) { + log.info("pod {} is not ready", metaData.getName()); + } else { + log.warn("pod {} is not ready", v1Pod); + } } } - log.info("{}/{} pod ready for service {}", readyPodNum, servicePodList.size(), serviceName); + log.info("{}: {}/{} pod ready", serviceName, readyPodNum, servicePodList.size()); return readyPodNum == servicePodList.size(); } @@ -219,7 +227,10 @@ private static boolean isPodReady(V1Pod v1Pod) { } log.debug("phase of pod {}:{}", v1Pod.getMetadata().getName(), v1PodStatus.getPhase()); } - return v1PodStatus != null && "Running".equalsIgnoreCase(v1PodStatus.getPhase()); + V1Probe readinessProbe = v1Pod.getSpec().getContainers().get(0).getReadinessProbe(); + log.info("readinessProbe={}", readinessProbe.getHttpGet()); + return v1PodStatus != null + && "Running".equalsIgnoreCase(v1PodStatus.getPhase()); } } diff --git a/support-files/kubernetes/charts/bk-job/templates/_helpers.tpl b/support-files/kubernetes/charts/bk-job/templates/_helpers.tpl index 050ce96002..d8392cadeb 100644 --- a/support-files/kubernetes/charts/bk-job/templates/_helpers.tpl +++ b/support-files/kubernetes/charts/bk-job/templates/_helpers.tpl @@ -476,20 +476,20 @@ Return the Job InitContainer WaitForMigration Content {{/* Return the Job InitContainer WaitForDependServices Content -{{ include "job.initContainer.waitForDependServices" ( dict "appName" "${appName}" "global" $) }} +{{ include "job.initContainer.waitForDependServices" ( dict "appName" "${appName}" "context" $) }} */}} {{- define "job.initContainer.waitForDependServices" -}} -{{- if .global.waitForDependServices.enabled -}} +{{- if .context.Values.waitForDependServices.enabled -}} - name: "wait-for-depend-services" - image: {{ include "common.images.image" (dict "imageRoot" .global.waitForDependServices.image "global" .global.global) }} - imagePullPolicy: {{ .global.waitForDependServices.image.pullPolicy }} + image: {{ include "common.images.image" (dict "imageRoot" .context.Values.waitForDependServices.image "global" .context.Values.global) }} + imagePullPolicy: {{ .context.Values.waitForDependServices.image.pullPolicy }} env: - name: KUBERNETES_NAMESPACE - value: {{ .global.Release.Namespace }} + value: {{ .context.Release.Namespace }} - name: BK_JOB_APP_NAME value: {{ .appName }} - name: BK_JOB_STARTUP_DEPENDENCIES_STR - value: {{ .global.waitForDependServices.dependencies }} + value: {{ .context.Values.waitForDependServices.dependencies }} {{- end -}} {{- end -}} diff --git a/support-files/kubernetes/charts/bk-job/templates/job-analysis/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-analysis/deployment.yaml index 8b79b719b5..bc601e460f 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-analysis/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-analysis/deployment.yaml @@ -58,7 +58,7 @@ spec: {{- end }} initContainers: {{- include "job.initContainer.waitForMigration" . | nindent 8 }} - {{- include "job.initContainer.waitForDependServices" ( dict "appName" "job-analysis" "global" $) | nindent 8 }} + {{- include "job.initContainer.waitForDependServices" ( dict "appName" "job-analysis" "context" $) | nindent 8 }} containers: - name: "job-analysis" {{- if .Values.analysisConfig.containerSecurityContext.enabled }} diff --git a/support-files/kubernetes/charts/bk-job/templates/job-backup/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-backup/deployment.yaml index 4386f3e1f9..f0bf341f0c 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-backup/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-backup/deployment.yaml @@ -58,7 +58,7 @@ spec: {{- end }} initContainers: {{- include "job.initContainer.waitForMigration" . | nindent 8 }} - {{- include "job.initContainer.waitForDependServices" ( dict "appName" "job-backup" "global" $) | nindent 8 }} + {{- include "job.initContainer.waitForDependServices" ( dict "appName" "job-backup" "context" $) | nindent 8 }} containers: - name: "job-backup" {{- if .Values.backupConfig.containerSecurityContext.enabled }} diff --git a/support-files/kubernetes/charts/bk-job/templates/job-crontab/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-crontab/deployment.yaml index ceca8a6e3a..eeb080d7ec 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-crontab/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-crontab/deployment.yaml @@ -58,7 +58,7 @@ spec: {{- end }} initContainers: {{- include "job.initContainer.waitForMigration" . | nindent 8 }} - {{- include "job.initContainer.waitForDependServices" ( dict "appName" "job-crontab" "global" $) | nindent 8 }} + {{- include "job.initContainer.waitForDependServices" ( dict "appName" "job-crontab" "context" $) | nindent 8 }} containers: - name: "job-crontab" {{- if .Values.crontabConfig.containerSecurityContext.enabled }} diff --git a/support-files/kubernetes/charts/bk-job/templates/job-execute/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-execute/deployment.yaml index e891e48533..cf3b9e42e0 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-execute/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-execute/deployment.yaml @@ -58,7 +58,7 @@ spec: {{- end }} initContainers: {{- include "job.initContainer.waitForMigration" . | nindent 8 }} - {{- include "job.initContainer.waitForDependServices" ( dict "appName" "job-execute" "global" $) | nindent 8 }} + {{- include "job.initContainer.waitForDependServices" ( dict "appName" "job-execute" "context" $) | nindent 8 }} containers: - name: "job-execute" {{- if .Values.executeConfig.containerSecurityContext.enabled }} diff --git a/support-files/kubernetes/charts/bk-job/templates/job-file-gateway/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-file-gateway/deployment.yaml index bea00a9726..5b09f072d3 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-file-gateway/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-file-gateway/deployment.yaml @@ -58,7 +58,7 @@ spec: {{- end }} initContainers: {{- include "job.initContainer.waitForMigration" . | nindent 8 }} - {{- include "job.initContainer.waitForDependServices" ( dict "appName" "job-file-gateway" "global" $) | nindent 8 }} + {{- include "job.initContainer.waitForDependServices" ( dict "appName" "job-file-gateway" "context" $) | nindent 8 }} containers: - name: "job-file-gateway" {{- if .Values.fileGatewayConfig.containerSecurityContext.enabled }} diff --git a/support-files/kubernetes/charts/bk-job/templates/job-file-worker/statefulset.yaml b/support-files/kubernetes/charts/bk-job/templates/job-file-worker/statefulset.yaml index f46e8ceb9e..3c61af33c3 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-file-worker/statefulset.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-file-worker/statefulset.yaml @@ -58,7 +58,7 @@ spec: securityContext: {{- omit .Values.fileWorkerConfig.podSecurityContext "enabled" | toYaml | nindent 8 }} {{- end }} initContainers: - {{- include "job.initContainer.waitForDependServices" ( dict "appName" "job-file-worker" "global" $) | nindent 8 }} + {{- include "job.initContainer.waitForDependServices" ( dict "appName" "job-file-worker" "context" $) | nindent 8 }} containers: - name: "job-file-worker" {{- if .Values.fileWorkerConfig.containerSecurityContext.enabled }} diff --git a/support-files/kubernetes/charts/bk-job/templates/job-frontend/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-frontend/deployment.yaml index 949df35f2a..41206f523b 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-frontend/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-frontend/deployment.yaml @@ -55,7 +55,7 @@ spec: securityContext: {{- omit .Values.frontendConfig.podSecurityContext "enabled" | toYaml | nindent 8 }} {{- end }} initContainers: - {{- include "job.initContainer.waitForDependServices" ( dict "appName" "job-frontend" "global" $) | nindent 8 }} + {{- include "job.initContainer.waitForDependServices" ( dict "appName" "job-frontend" "context" $) | nindent 8 }} containers: - name: "job-frontend" {{- if .Values.frontendConfig.containerSecurityContext.enabled }} diff --git a/support-files/kubernetes/charts/bk-job/templates/job-gateway/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-gateway/deployment.yaml index 14999e79e9..32f7bd430d 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-gateway/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-gateway/deployment.yaml @@ -57,7 +57,7 @@ spec: securityContext: {{- omit .Values.gatewayConfig.podSecurityContext "enabled" | toYaml | nindent 8 }} {{- end }} initContainers: - {{- include "job.initContainer.waitForDependServices" ( dict "appName" "job-gateway" "global" $) | nindent 8 }} + {{- include "job.initContainer.waitForDependServices" ( dict "appName" "job-gateway" "context" $) | nindent 8 }} containers: - name: "job-gateway" {{- if .Values.gatewayConfig.containerSecurityContext.enabled }} diff --git a/support-files/kubernetes/charts/bk-job/templates/job-logsvr/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-logsvr/deployment.yaml index f85f542511..8a82c4451b 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-logsvr/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-logsvr/deployment.yaml @@ -57,7 +57,7 @@ spec: securityContext: {{- omit .Values.logsvrConfig.podSecurityContext "enabled" | toYaml | nindent 8 }} {{- end }} initContainers: - {{- include "job.initContainer.waitForDependServices" ( dict "appName" "job-logsvr" "global" $) | nindent 8 }} + {{- include "job.initContainer.waitForDependServices" ( dict "appName" "job-logsvr" "context" $) | nindent 8 }} containers: - name: "job-logsvr" {{- if .Values.logsvrConfig.containerSecurityContext.enabled }} diff --git a/support-files/kubernetes/charts/bk-job/templates/job-manage/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-manage/deployment.yaml index 3ac82324c6..ae586133c2 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-manage/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-manage/deployment.yaml @@ -58,7 +58,7 @@ spec: {{- end }} initContainers: {{- include "job.initContainer.waitForMigration" . | nindent 8 }} - {{- include "job.initContainer.waitForDependServices" ( dict "appName" "job-manage" "global" $) | nindent 8 }} + {{- include "job.initContainer.waitForDependServices" ( dict "appName" "job-manage" "context" $) | nindent 8 }} containers: - name: "job-manage" {{- if .Values.manageConfig.containerSecurityContext.enabled }} diff --git a/support-files/kubernetes/images/build.sh b/support-files/kubernetes/images/build.sh index b70705a2fd..4f2e7dedce 100755 --- a/support-files/kubernetes/images/build.sh +++ b/support-files/kubernetes/images/build.sh @@ -228,7 +228,7 @@ build_migration_image(){ build_startup_controller_image(){ log "Building startup-controller image, version: ${VERSION}..." TOOL_NAME="k8s-startup-controller" - $BACKEND_DIR/gradlew -p $BACKEND_DIR clean :job-tools:$TOOL_NAME:shadowJar -DmavenRepoUrl=$MAVEN_REPO_URL -DbkjobVersion=$VERSION + $BACKEND_DIR/gradlew -p $BACKEND_DIR clean :job-tools:$TOOL_NAME:build -DmavenRepoUrl=$MAVEN_REPO_URL -DbkjobVersion=$VERSION rm -rf tmp/* cp $BACKEND_DIR/release/$TOOL_NAME-$VERSION.jar tmp/$TOOL_NAME.jar cp startup-controller/startup.sh tmp/