bug:构建机网络有问题或者僵死等情况，用户无法立马取消构建 TencentBlueKing#5780

carlyin0801 · Dec 24, 2021 · 6d9682f · 6d9682f
1 parent dead04f
commit 6d9682f
Show file tree

Hide file tree

Showing 6 changed files with 16 additions and 25 deletions.
diff --git a/...s/api-process/src/main/kotlin/com/tencent/devops/process/engine/common/EngineConstants.kt b/...s/api-process/src/main/kotlin/com/tencent/devops/process/engine/common/EngineConstants.kt
@@ -45,5 +45,6 @@ const val BS_QUALITY_ABORT_STAGE = "quality_abort_stage"
 const val BS_CONTAINER_END_SOURCE_PREFIX = "CONTAINER_END_"
 const val BS_MANUAL_STOP_PAUSE_ATOM = "taskCancel_"
 const val BS_STAGE_CANCELED_END_SOURCE = "STAGE_CANCELED_END"
+const val BS_CANCEL_BUILD_SOURCE = "cancel_build"
 
 const val BS_PAUSE_TASK = "_bkTaskPauseTag_"
diff --git a/...-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRuntimeService.kt b/...-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRuntimeService.kt
@@ -65,6 +65,14 @@ import com.tencent.devops.common.pipeline.utils.SkipElementUtils
 import com.tencent.devops.common.redis.RedisLock
 import com.tencent.devops.common.redis.RedisOperation
 import com.tencent.devops.common.service.trace.TraceTag
+import com.tencent.devops.common.webhook.pojo.code.BK_REPO_GIT_WEBHOOK_EVENT_TYPE
+import com.tencent.devops.common.webhook.pojo.code.BK_REPO_GIT_WEBHOOK_MR_MERGE_COMMIT_SHA
+import com.tencent.devops.common.webhook.pojo.code.BK_REPO_WEBHOOK_REPO_URL
+import com.tencent.devops.common.webhook.pojo.code.PIPELINE_WEBHOOK_BRANCH
+import com.tencent.devops.common.webhook.pojo.code.PIPELINE_WEBHOOK_COMMIT_MESSAGE
+import com.tencent.devops.common.webhook.pojo.code.PIPELINE_WEBHOOK_EVENT_TYPE
+import com.tencent.devops.common.webhook.pojo.code.PIPELINE_WEBHOOK_REVISION
+import com.tencent.devops.common.webhook.pojo.code.PIPELINE_WEBHOOK_TYPE
 import com.tencent.devops.common.websocket.enum.RefreshType
 import com.tencent.devops.model.process.tables.records.TPipelineBuildContainerRecord
 import com.tencent.devops.model.process.tables.records.TPipelineBuildHistoryRecord
@@ -73,6 +81,7 @@ import com.tencent.devops.model.process.tables.records.TPipelineBuildSummaryReco
 import com.tencent.devops.model.process.tables.records.TPipelineBuildTaskRecord
 import com.tencent.devops.process.dao.BuildDetailDao
 import com.tencent.devops.process.engine.cfg.BuildIdGenerator
+import com.tencent.devops.process.engine.common.BS_CANCEL_BUILD_SOURCE
 import com.tencent.devops.process.engine.common.BS_MANUAL_ACTION
 import com.tencent.devops.process.engine.common.BS_MANUAL_ACTION_DESC
 import com.tencent.devops.process.engine.common.BS_MANUAL_ACTION_PARAMS
@@ -95,9 +104,11 @@ import com.tencent.devops.process.engine.pojo.PipelineInfo
 import com.tencent.devops.process.engine.pojo.builds.CompleteTask
 import com.tencent.devops.process.engine.pojo.event.PipelineBuildAtomTaskEvent
 import com.tencent.devops.process.engine.pojo.event.PipelineBuildCancelEvent
+import com.tencent.devops.process.engine.pojo.event.PipelineBuildContainerEvent
 import com.tencent.devops.process.engine.pojo.event.PipelineBuildMonitorEvent
 import com.tencent.devops.process.engine.pojo.event.PipelineBuildStartEvent
 import com.tencent.devops.process.engine.pojo.event.PipelineBuildWebSocketPushEvent
+import com.tencent.devops.process.engine.pojo.event.PipelineContainerAgentHeartBeatEvent
 import com.tencent.devops.process.engine.service.rule.PipelineRuleService
 import com.tencent.devops.process.engine.utils.ContainerUtils
 import com.tencent.devops.process.pojo.BuildBasicInfo
@@ -107,7 +118,6 @@ import com.tencent.devops.process.pojo.PipelineBuildMaterial
 import com.tencent.devops.process.pojo.PipelineSortType
 import com.tencent.devops.process.pojo.ReviewParam
 import com.tencent.devops.process.pojo.code.WebhookInfo
-import com.tencent.devops.process.engine.pojo.event.PipelineBuildContainerEvent
 import com.tencent.devops.process.pojo.pipeline.PipelineLatestBuild
 import com.tencent.devops.process.pojo.pipeline.enums.PipelineRuleBusCodeEnum
 import com.tencent.devops.process.service.BuildVariableService
@@ -128,16 +138,6 @@ import com.tencent.devops.process.utils.PIPELINE_START_TYPE
 import com.tencent.devops.process.utils.PIPELINE_VERSION
 import com.tencent.devops.process.utils.PROJECT_NAME
 import com.tencent.devops.process.utils.PROJECT_NAME_CHINESE
-import com.tencent.devops.common.webhook.pojo.code.BK_REPO_GIT_WEBHOOK_EVENT_TYPE
-import com.tencent.devops.common.webhook.pojo.code.BK_REPO_GIT_WEBHOOK_MR_MERGE_COMMIT_SHA
-import com.tencent.devops.common.webhook.pojo.code.BK_REPO_WEBHOOK_REPO_URL
-import com.tencent.devops.common.webhook.pojo.code.PIPELINE_WEBHOOK_BRANCH
-import com.tencent.devops.common.webhook.pojo.code.PIPELINE_WEBHOOK_COMMIT_MESSAGE
-import com.tencent.devops.common.webhook.pojo.code.PIPELINE_WEBHOOK_EVENT_TYPE
-import com.tencent.devops.common.webhook.pojo.code.PIPELINE_WEBHOOK_REVISION
-import com.tencent.devops.common.webhook.pojo.code.PIPELINE_WEBHOOK_TYPE
-import com.tencent.devops.process.engine.pojo.event.PipelineContainerAgentHeartBeatEvent
-import com.tencent.devops.process.engine.utils.BuildUtils
 import org.jooq.DSLContext
 import org.jooq.Record
 import org.jooq.Result
@@ -150,7 +150,6 @@ import java.time.Duration
 import java.time.LocalDateTime
 import java.time.format.DateTimeFormatter
 import java.time.temporal.TemporalAccessor
-import java.util.concurrent.TimeUnit
 
 /**
  * 流水线运行时相关的服务
@@ -617,8 +616,6 @@ class PipelineRuntimeService @Autowired constructor(
         buildStatus: BuildStatus
     ): Boolean {
         logger.info("[$buildId]|SHUTDOWN_BUILD|userId=$userId|status=$buildStatus")
-        // 往redis中设置当前构建已取消标识
-        redisOperation.set(BuildUtils.getCancelBuildKey(buildId), "true", TimeUnit.DAYS.toSeconds(1))
         // 发送取消事件
         pipelineEventDispatcher.dispatch(
             PipelineBuildCancelEvent(
@@ -646,7 +643,7 @@ class PipelineRuntimeService @Autowired constructor(
         containers.forEach { container ->
             pipelineEventDispatcher.dispatch(
                 PipelineContainerAgentHeartBeatEvent(
-                    source = "cancelBuild",
+                    source = BS_CANCEL_BUILD_SOURCE,
                     projectId = projectId,
                     pipelineId = pipelineId,
                     userId = userId,

diff --git a/...re/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/utils/BuildUtils.kt b/...re/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/utils/BuildUtils.kt
@@ -29,7 +29,5 @@ package com.tencent.devops.process.engine.utils
 
 object BuildUtils {
 
-    fun getCancelBuildKey(buildId: String) = "CANCEL_BUILD_$buildId"
-
     fun getCancelActionBuildKey(buildId: String) = "CANCEL_ACTION_$buildId"
 }
diff --git a/...iz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/BuildCancelControl.kt b/...iz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/BuildCancelControl.kt
@@ -259,8 +259,6 @@ class BuildCancelControl @Autowired constructor(
                 unlockMutexGroup(variables = variables, container = container,
                     buildId = event.buildId, projectId = event.projectId, stageId = stageId
                 )
-                // 删除redis中当前构建已取消标识
-                redisOperation.delete(BuildUtils.getCancelBuildKey(buildId))
                 // 构建机关机
                 if (container is VMBuildContainer) {
                     container.shutdown(event = event, executeCount = executeCount)

diff --git a/...s/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/BuildEndControl.kt b/...s/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/BuildEndControl.kt
@@ -61,7 +61,6 @@ import com.tencent.devops.process.engine.service.PipelineBuildDetailService
 import com.tencent.devops.process.engine.service.PipelineRuntimeExtService
 import com.tencent.devops.process.engine.service.PipelineRuntimeService
 import com.tencent.devops.process.engine.service.PipelineTaskService
-import com.tencent.devops.process.engine.utils.BuildUtils
 import com.tencent.devops.process.utils.PIPELINE_MESSAGE_STRING_LENGTH_MAX
 import com.tencent.devops.process.utils.PIPELINE_TASK_MESSAGE_STRING_LENGTH_MAX
 import org.slf4j.LoggerFactory
@@ -164,9 +163,6 @@ class BuildEndControl @Autowired constructor(
 
         pipelineRuntimeService.updateBuildHistoryStageState(buildId, allStageStatus)
 
-        // 删除redis中当前构建已取消标识
-        redisOperation.delete(BuildUtils.getCancelBuildKey(buildId))
-
         // 广播结束事件
         pipelineEventDispatcher.dispatch(
             PipelineBuildFinishBroadCastEvent(

diff --git a/.../biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/HeartbeatControl.kt b/.../biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/HeartbeatControl.kt
@@ -34,6 +34,7 @@ import com.tencent.devops.common.event.enums.ActionType
 import com.tencent.devops.common.log.utils.BuildLogPrinter
 import com.tencent.devops.common.pipeline.utils.HeartBeatUtils
 import com.tencent.devops.common.redis.RedisOperation
+import com.tencent.devops.process.engine.common.BS_CANCEL_BUILD_SOURCE
 import com.tencent.devops.process.engine.common.VMUtils
 import com.tencent.devops.process.engine.pojo.event.PipelineBuildContainerEvent
 import com.tencent.devops.process.engine.pojo.event.PipelineContainerAgentHeartBeatEvent
@@ -72,8 +73,8 @@ class HeartbeatControl @Autowired constructor(
             }
 
         val elapse = System.currentTimeMillis() - lastUpdate.toLong()
-        val cancelBuildFlag = redisOperation.get(BuildUtils.getCancelBuildKey(buildId))?.toBoolean()
-        val timeOutLimit = if (cancelBuildFlag == true) CANCEL_TIMEOUT_IN_MS else TIMEOUT_IN_MS
+        // 如果消息事件来源是取消构建操作，超时时间为CANCEL_TIMEOUT_IN_MS
+        val timeOutLimit = if (event.source == BS_CANCEL_BUILD_SOURCE) CANCEL_TIMEOUT_IN_MS else TIMEOUT_IN_MS
         if (elapse > timeOutLimit) {
             timeout(event, elapse)
         } else {