feat: 仓库清理任务只在特定场景统计已删除个数和容量TencentBlueKing#2534

* feat: 仓库清理任务只在特定场景统计已删除个数和容量TencentBlueKing#2534 * feat: 代码调整TencentBlueKing#2534 * feat: 代码调整TencentBlueKing#2534
zacYL · Oct 31, 2024 · ab557ce · ab557ce
1 parent eb08c3f
commit ab557ce
Show file tree

Hide file tree

Showing 8 changed files with 75 additions and 34 deletions.
diff --git a/...ce/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeDeleteOperation.kt b/...ce/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeDeleteOperation.kt
@@ -77,7 +77,11 @@ interface NodeDeleteOperation {
      * 根据最后访问时间删除[date]之前的历史数据
      */
     fun deleteBeforeDate(
-        projectId: String, repoName: String,
-        date: LocalDateTime, operator: String, path: String
+        projectId: String,
+        repoName: String,
+        date: LocalDateTime,
+        operator: String,
+        path: String,
+        decreaseVolume: Boolean = true
     ): NodeDeleteResult
 }
diff --git a/...src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt b/...src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt
@@ -165,7 +165,8 @@ open class NodeDeleteSupport(
         repoName: String,
         date: LocalDateTime,
         operator: String,
-        path: String
+        path: String,
+        decreaseVolume: Boolean
     ): NodeDeleteResult {
         val option = NodeListOption(includeFolder = false, deep = true)
         val timeCriteria = Criteria().orOperator(
@@ -175,7 +176,7 @@ open class NodeDeleteSupport(
         val criteria = NodeQueryHelper.nodeListCriteria(projectId, repoName, path, option)
             .andOperator(timeCriteria)
         val query = Query(criteria)
-        val nodeDeleteResult = delete(query, operator, criteria, projectId, repoName)
+        val nodeDeleteResult = delete(query, operator, criteria, projectId, repoName, decreaseVolume = decreaseVolume)
         publishEvent(
             buildNodeCleanEvent(
                 projectId, repoName, path, operator, nodeDeleteResult.deletedTime.toString()

diff --git a/...e/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeServiceImpl.kt b/...e/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeServiceImpl.kt
@@ -161,8 +161,9 @@ class NodeServiceImpl(
         date: LocalDateTime,
         operator: String,
         path: String,
+        decreaseVolume: Boolean
     ): NodeDeleteResult {
-        return NodeDeleteSupport(this).deleteBeforeDate(projectId, repoName, date, operator, path)
+        return NodeDeleteSupport(this).deleteBeforeDate(projectId, repoName, date, operator, path, decreaseVolume)
     }
 
     @Transactional(rollbackFor = [Throwable::class])

diff --git a/...in/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeDeleteSupport.kt b/...in/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeDeleteSupport.kt
@@ -104,11 +104,12 @@ class CenterNodeDeleteSupport(
         repoName: String,
         date: LocalDateTime,
         operator: String,
-        path: String
+        path: String,
+        decreaseVolume: Boolean
     ): NodeDeleteResult {
         val clusterName = SecurityUtils.getClusterName()
         if (clusterName.isNullOrEmpty()) {
-            return super.deleteBeforeDate(projectId, repoName, date, operator, path)
+            return super.deleteBeforeDate(projectId, repoName, date, operator, path, decreaseVolume)
         }
         var deletedSize = 0L
         var deletedNum = 0L

diff --git a/...tlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt b/...tlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt
@@ -240,14 +240,16 @@ class CenterNodeServiceImpl(
         repoName: String,
         date: LocalDateTime,
         operator: String,
-        path: String
+        path: String,
+        decreaseVolume: Boolean
     ): NodeDeleteResult {
         return CenterNodeDeleteSupport(this, clusterProperties).deleteBeforeDate(
             projectId,
             repoName,
             date,
             operator,
-            path
+            path,
+            decreaseVolume
         )
     }
 

diff --git a/...n/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/edge/EdgeNodeServiceImpl.kt b/...n/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/edge/EdgeNodeServiceImpl.kt
@@ -189,6 +189,7 @@ class EdgeNodeServiceImpl(
         date: LocalDateTime,
         operator: String,
         path: String,
+        decreaseVolume: Boolean
     ): NodeDeleteResult {
         ignoreException(
             projectId = projectId,
@@ -197,7 +198,7 @@ class EdgeNodeServiceImpl(
         ) {
             centerNodeClient.deleteNodeLastModifiedBeforeDate(projectId, repoName, path, date, operator)
         }
-        return NodeDeleteSupport(this).deleteBeforeDate(projectId, repoName, date, operator, path)
+        return NodeDeleteSupport(this).deleteBeforeDate(projectId, repoName, date, operator, path, decreaseVolume)
     }
 
     @Transactional(rollbackFor = [Throwable::class])

diff --git a/...main/kotlin/com/tencent/bkrepo/common/metadata/service/repo/impl/RepositoryServiceImpl.kt b/...main/kotlin/com/tencent/bkrepo/common/metadata/service/repo/impl/RepositoryServiceImpl.kt
@@ -282,6 +282,7 @@ class RepositoryServiceImpl(
             quota?.let {
                 Preconditions.checkArgument(it >= (repository.used ?: 0), this::quota.name)
                 repository.quota = it
+                repository.used = repository.used ?: 0
             }
             val oldConfiguration = repository.configuration.readJsonString<RepositoryConfiguration>()
             repository.public = public ?: repository.public

diff --git a/...job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/ArtifactCleanupJob.kt b/...job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/ArtifactCleanupJob.kt
@@ -33,7 +33,10 @@ import com.tencent.bkrepo.common.api.util.readJsonString
 import com.tencent.bkrepo.common.artifact.path.PathUtils
 import com.tencent.bkrepo.common.artifact.pojo.RepositoryType
 import com.tencent.bkrepo.common.artifact.pojo.configuration.RepositoryConfiguration
+import com.tencent.bkrepo.common.metadata.model.TNode
 import com.tencent.bkrepo.common.metadata.service.node.NodeService
+import com.tencent.bkrepo.common.metadata.service.repo.QuotaService
+import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper
 import com.tencent.bkrepo.common.security.constant.MS_AUTH_HEADER_SECURITY_TOKEN
 import com.tencent.bkrepo.common.security.service.ServiceAuthManager
 import com.tencent.bkrepo.common.service.log.LoggerHolder
@@ -45,12 +48,14 @@ import com.tencent.bkrepo.job.batch.base.JobContext
 import com.tencent.bkrepo.job.config.properties.ArtifactCleanupJobProperties
 import com.tencent.bkrepo.job.exception.JobExecuteException
 import com.tencent.bkrepo.repository.constant.SYSTEM_USER
+import com.tencent.bkrepo.repository.pojo.node.NodeListOption
 import org.springframework.beans.factory.annotation.Value
 import org.springframework.boot.context.properties.EnableConfigurationProperties
 import org.springframework.cloud.client.discovery.DiscoveryClient
 import org.springframework.data.mongodb.core.find
 import org.springframework.data.mongodb.core.query.Criteria
 import org.springframework.data.mongodb.core.query.Query
+import org.springframework.data.mongodb.core.query.and
 import org.springframework.data.mongodb.core.query.isEqualTo
 import org.springframework.http.HttpEntity
 import org.springframework.http.HttpHeaders
@@ -61,6 +66,7 @@ import org.springframework.web.client.RestTemplate
 import java.time.Duration
 import java.time.LocalDateTime
 import java.time.format.DateTimeFormatter
+import java.util.Locale
 import kotlin.reflect.KClass
 
 /**
@@ -72,7 +78,8 @@ class ArtifactCleanupJob(
     private val properties: ArtifactCleanupJobProperties,
     private val nodeService: NodeService,
     private val discoveryClient: DiscoveryClient,
-    private val serviceAuthManager: ServiceAuthManager
+    private val serviceAuthManager: ServiceAuthManager,
+    private val quotaService: QuotaService,
 ) : DefaultContextMongoDbJob<ArtifactCleanupJob.RepoData>(properties) {
 
     private val restTemplate = RestTemplate()
@@ -104,17 +111,17 @@ class ArtifactCleanupJob(
             if (filterConfig(row.projectId, row.name, cleanupStrategy)) return
             logger.info(
                 "Will clean the artifacts in repo ${row.projectId}|${row.name} " +
-                        "with cleanup strategy $cleanupStrategy"
+                    "with cleanup strategy $cleanupStrategy"
             )
             when (row.type) {
                 RepositoryType.GENERIC.name -> {
                     // 清理generic制品
-                    deleteNodes(row.projectId, row.name, cleanupStrategy)
+                    deleteNodes(row, cleanupStrategy)
                 }
                 RepositoryType.DOCKER.name,
                 RepositoryType.OCI.name,
                 RepositoryType.HELM.name
-                    -> {
+                -> {
                     // 清理镜像制品
                     deletePackages(
                         projectId = row.projectId,
@@ -128,7 +135,7 @@ class ArtifactCleanupJob(
         } catch (e: Exception) {
             throw JobExecuteException(
                 "Failed to send cleanup repository for " +
-                        "repo ${row.projectId}|${row.name}, error: ${e.message}", e
+                    "repo ${row.projectId}|${row.name}, error: ${e.message}", e
             )
         }
     }
@@ -159,7 +166,7 @@ class ArtifactCleanupJob(
     }
 
 
-    private fun deleteNodes(projectId: String, repoName: String, cleanupStrategy: CleanupStrategy) {
+    private fun deleteNodes(row: RepoData, cleanupStrategy: CleanupStrategy) {
         val cleanupDate = when (cleanupStrategy.cleanupType) {
             // 只保留距离当前时间天数以内的制品
             CleanupStrategyEnum.RETENTION_DAYS.value -> {
@@ -171,37 +178,55 @@ class ArtifactCleanupJob(
             }
             else -> return
         }
-        doNodeCleanup(projectId, repoName, cleanupDate, cleanupStrategy.cleanTargets)
+        doNodeCleanup(row, cleanupDate, cleanupStrategy.cleanTargets)
     }
 
-    private fun doNodeCleanup(
-        projectId: String, repoName: String, cleanupDate: LocalDateTime,
-        cleanupFolders: List<String>? = null
-    ) {
+    private fun doNodeCleanup(row: RepoData, cleanupDate: LocalDateTime, cleanupFolders: List<String>? = null) {
         val folders = if (cleanupFolders.isNullOrEmpty()) {
             listOf(PathUtils.ROOT)
         } else {
             cleanupFolders
         }
         folders.forEach {
             try {
-                nodeService.deleteBeforeDate(
-                    projectId = projectId,
-                    repoName = repoName,
-                    path = PathUtils.toPath(it),
+                val path = PathUtils.toPath(it)
+                val result = nodeService.deleteBeforeDate(
+                    projectId = row.projectId,
+                    repoName = row.name,
+                    path = path,
                     date = cleanupDate,
-                    operator = SYSTEM_USER
+                    operator = SYSTEM_USER,
+                    decreaseVolume = false
                 )
+                decreaseVolume(row, cleanupDate, path, result.deletedTime)
             } catch (e: Exception) {
                 logger.warn(
-                    "Request of clean nodes $it in repo $projectId|$repoName failed, error is ${e.message}," +
-                            " cause is ${e.cause?.message}!, will sleep ${properties.sleepSeconds} seconds"
+                    "Request of clean nodes $it in repo ${row.projectId}|${row.name} failed, error is ${e.message}," +
+                        " cause is ${e.cause?.message}!"
                 )
-                Thread.sleep(properties.sleepSeconds * 1000)
             }
         }
     }
 
+    /**
+     * 清理容量，只针对配置了容量限制的才实时更新
+     */
+    private fun decreaseVolume(row: RepoData, cleanupDate: LocalDateTime, path: String, deletedTime: LocalDateTime) {
+        if (row.quota == null) return
+        val option = NodeListOption(includeFolder = false, deep = true)
+        val timeCriteria = Criteria().orOperator(
+            Criteria().and(TNode::lastAccessDate).lt(cleanupDate).and(TNode::lastModifiedDate).lt(cleanupDate),
+            Criteria().and(TNode::lastAccessDate).`is`(null).and(TNode::lastModifiedDate).lt(cleanupDate),
+        )
+        val criteria = NodeQueryHelper.nodeListCriteria(row.projectId, row.name, path, option)
+            .andOperator(timeCriteria)
+
+        val deletedCriteria = criteria.and(TNode::deleted).isEqualTo(deletedTime)
+        val deletedSize = nodeService.aggregateComputeSize(deletedCriteria)
+        logger.info("decrease volume $deletedSize for repo ${row.projectId}|${row.name}")
+        quotaService.decreaseUsedVolume(row.projectId, row.name, deletedSize)
+    }
+
     private fun deletePackages(
         projectId: String, repoName: String,
         cleanupStrategy: CleanupStrategy,
@@ -237,9 +262,12 @@ class ArtifactCleanupJob(
 
 
     private fun doPackageVersionCleanup(
-        projectId: String, repoName: String,
-        cleanupStrategy: CleanupStrategy, packageName: String,
-        versionList: List<PackageVersionData>, repoType: String
+        projectId: String,
+        repoName: String,
+        cleanupStrategy: CleanupStrategy,
+        packageName: String,
+        versionList: List<PackageVersionData>,
+        repoType: String
     ) {
         when (cleanupStrategy.cleanupType) {
             // 只保留距离当前时间天数以内的制品
@@ -283,15 +311,15 @@ class ArtifactCleanupJob(
     ) {
         val (urlPath, serviceInstance) = when (repoType) {
             RepositoryType.DOCKER.name, RepositoryType.OCI.name -> {
-                val dockerServiceId = buildServiceName(RepositoryType.DOCKER.name.toLowerCase())
+                val dockerServiceId = buildServiceName(RepositoryType.DOCKER.name.lowercase(Locale.getDefault()))
                 val instance = discoveryClient.getInstances(dockerServiceId).firstOrNull()
                 Pair(
                     "/service/third/version/delete/$projectId/$repoName",
                     instance
                 )
             }
             RepositoryType.HELM.name -> {
-                val dockerServiceId = buildServiceName(RepositoryType.HELM.name.toLowerCase())
+                val dockerServiceId = buildServiceName(RepositoryType.HELM.name.lowercase(Locale.getDefault()))
                 val instance = discoveryClient.getInstances(dockerServiceId).firstOrNull()
                 Pair(
                     "/service/index/version/delete/$projectId/$repoName",
@@ -345,6 +373,8 @@ class ArtifactCleanupJob(
         val projectId: String by map
         val type: String by map
         val configuration: String by map
+        val quota: Long? = map["quota"] as Long?
+        val used: Long? = map["used"] as Long?
     }
 
     data class CleanupStrategy(