Merge pull request TencentBlueKing#1047 from zacYL/issue_1046

bug: 分发增加存储一致性校验TencentBlueKing#1046
gujunling · Aug 9, 2023 · c33046a · c33046a
2 parents 33e26f8 + f537c52
commit c33046a
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 6 deletions.
diff --git a/...cation/src/main/kotlin/com/tencent/bkrepo/replication/pojo/task/setting/ReplicaSetting.kt b/...cation/src/main/kotlin/com/tencent/bkrepo/replication/pojo/task/setting/ReplicaSetting.kt
@@ -54,5 +54,9 @@ data class ReplicaSetting(
     /**
      * 执行计划
      */
-    val executionPlan: ExecutionPlan = ExecutionPlan()
+    val executionPlan: ExecutionPlan = ExecutionPlan(),
+    /**
+     * 是否校验文件存储一致性
+     */
+    val storageConsistencyCheck: Boolean = false
 )
diff --git a/...c/main/kotlin/com/tencent/bkrepo/replication/replica/base/replicator/ClusterReplicator.kt b/...c/main/kotlin/com/tencent/bkrepo/replication/replica/base/replicator/ClusterReplicator.kt
@@ -262,20 +262,25 @@ class ClusterReplicator(
     }
 
     private fun doubleCheck(context: ReplicaContext, sha256: String) {
-        var count = 0
+        if (!context.task.setting.storageConsistencyCheck) return
+        logger.info("will check the storage consistency for $sha256")
+        var checkResult = false
+        var costTime = 0
         val remoteRepositoryType = context.remoteRepoType
-        while (count < FILE_EXIST_CHECK_RETRY_COUNT) {
+        while (!checkResult && costTime < STORAGE_CONSISTENCY_CHECK_TIME_OUT) {
             if (context.blobReplicaClient!!.check(
                     sha256,
                     context.remoteRepo?.storageCredentials?.key,
                     remoteRepositoryType
                 ).data == true
             ) {
-                break
+                checkResult = true
             } else {
                 TimeUnit.SECONDS.sleep(1)
-                count++
+                costTime++
             }
+            logger.info("the result of storage consistency check for $sha256 is $checkResult," +
+                            " costTime: $costTime seconds")
         }
     }
 
@@ -325,7 +330,7 @@ class ClusterReplicator(
 
     companion object {
         private val logger = LoggerFactory.getLogger(ClusterReplicator::class.java)
-        private const val FILE_EXIST_CHECK_RETRY_COUNT = 10
+        private const val STORAGE_CONSISTENCY_CHECK_TIME_OUT = 600
 
         fun buildRemoteRepoCacheKey(clusterInfo: ClusterInfo, projectId: String, repoName: String): String {
             return "$projectId/$repoName/${clusterInfo.hashCode()}"