feat: 目录内的第一层节点发生改变时,更新该目录的最后修改信息 #756

src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/metadata/impl/MetadataServiceImpl.kt

@@ -36,6 +36,7 @@ import com.tencent.bkrepo.common.api.exception.ErrorCodeException
 import com.tencent.bkrepo.common.api.message.CommonMessageCode
 import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException
 import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode
+import com.tencent.bkrepo.common.artifact.path.PathUtils
 import com.tencent.bkrepo.common.artifact.path.PathUtils.normalizeFullPath
 import com.tencent.bkrepo.common.artifact.util.ClusterUtils
 import com.tencent.bkrepo.common.security.exception.PermissionException
@@ -48,6 +49,7 @@ import com.tencent.bkrepo.repository.model.TNode
 import com.tencent.bkrepo.repository.pojo.metadata.MetadataDeleteRequest
 import com.tencent.bkrepo.repository.pojo.metadata.MetadataSaveRequest
 import com.tencent.bkrepo.repository.service.metadata.MetadataService
+import com.tencent.bkrepo.repository.service.node.impl.NodeBaseService
 import com.tencent.bkrepo.repository.util.MetadataUtils
 import com.tencent.bkrepo.repository.util.NodeEventFactory.buildMetadataDeletedEvent
 import com.tencent.bkrepo.repository.util.NodeEventFactory.buildMetadataSavedEvent
@@ -60,17 +62,20 @@ import org.springframework.data.mongodb.core.query.inValues
 import org.springframework.data.mongodb.core.query.where
 import org.springframework.stereotype.Service
 import org.springframework.transaction.annotation.Transactional
+import java.time.LocalDateTime
 
 /**
  * 元数据服务实现类
  */
 @Service
 @Conditional(DefaultCondition::class)
 class MetadataServiceImpl(
-    private val nodeDao: NodeDao,
+    private val nodeBaseService: NodeBaseService,
     private val repositoryProperties: RepositoryProperties
 ) : MetadataService {
 
+    private val nodeDao: NodeDao = nodeBaseService.nodeDao
+
     override fun listMetadata(projectId: String, repoName: String, fullPath: String): Map<String, Any> {
         return MetadataUtils.toMap(nodeDao.findOne(NodeQueryHelper.nodeQuery(projectId, repoName, fullPath))?.metadata)
     }
@@ -98,7 +103,13 @@ class MetadataServiceImpl(
                 MetadataUtils.merge(oldMetadata, newMetadata)
             }
 
+            val currentTime = LocalDateTime.now()
+            node.lastModifiedBy = operator
+            node.lastModifiedDate = currentTime
             nodeDao.save(node)
+            // 更新父目录的修改时间
+            val parentFullPath = PathUtils.toFullPath(PathUtils.resolveParent(fullPath))
+            nodeBaseService.updateModifiedInfo(projectId, repoName, parentFullPath, operator, currentTime)
             publishEvent(buildMetadataSavedEvent(request))
             logger.info("Save metadata[$newMetadata] on node[/$projectId/$repoName$fullPath] success.")
         }
@@ -135,11 +146,17 @@ class MetadataServiceImpl(
                 }
             }
 
+            val currentTime = LocalDateTime.now()
             val update = Update().pull(
                 TNode::metadata.name,
                 Query.query(where(TMetadata::key).inValues(keyList))
-            )
-            nodeDao.updateMulti(query, update)
+            ).set(TNode::lastModifiedDate.name, currentTime).set(TNode::lastModifiedBy.name, operator)
+            val modifiedCount = nodeDao.updateMulti(query, update).modifiedCount
+            if (modifiedCount == 1L) {
+                // 更新父目录的修改时间
+                val parentFullPath = PathUtils.toFullPath(PathUtils.resolveParent(fullPath))
+                nodeBaseService.updateModifiedInfo(projectId, repoName, parentFullPath, operator, currentTime)
+            }
             publishEvent(buildMetadataDeletedEvent(this))
             logger.info("Delete metadata[$keyList] on node[/$projectId/$repoName$fullPath] success.")
         }

src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/metadata/impl/center/CommitEdgeCenterMetadataServiceImpl.kt

@@ -29,17 +29,17 @@ package com.tencent.bkrepo.repository.service.metadata.impl.center
 
 import com.tencent.bkrepo.common.service.cluster.CommitEdgeCenterCondition
 import com.tencent.bkrepo.repository.config.RepositoryProperties
-import com.tencent.bkrepo.repository.dao.NodeDao
 import com.tencent.bkrepo.repository.service.metadata.impl.MetadataServiceImpl
+import com.tencent.bkrepo.repository.service.node.impl.NodeBaseService
 import org.springframework.context.annotation.Conditional
 import org.springframework.stereotype.Service
 
 @Service
 @Conditional(CommitEdgeCenterCondition::class)
 class CommitEdgeCenterMetadataServiceImpl(
-    nodeDao: NodeDao,
+    nodeBaseService: NodeBaseService,
     repositoryProperties: RepositoryProperties
 ) : MetadataServiceImpl(
-    nodeDao,
+    nodeBaseService,
     repositoryProperties
 )

src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/metadata/impl/edge/EdgeMetadataServiceImpl.kt

@@ -32,21 +32,21 @@ import com.tencent.bkrepo.common.service.cluster.CommitEdgeEdgeCondition
 import com.tencent.bkrepo.common.service.feign.FeignClientFactory
 import com.tencent.bkrepo.repository.api.cluster.ClusterMetadataClient
 import com.tencent.bkrepo.repository.config.RepositoryProperties
-import com.tencent.bkrepo.repository.dao.NodeDao
 import com.tencent.bkrepo.repository.pojo.metadata.MetadataDeleteRequest
 import com.tencent.bkrepo.repository.pojo.metadata.MetadataSaveRequest
 import com.tencent.bkrepo.repository.service.metadata.impl.MetadataServiceImpl
+import com.tencent.bkrepo.repository.service.node.impl.NodeBaseService
 import org.springframework.context.annotation.Conditional
 import org.springframework.stereotype.Service
 
 @Service
 @Conditional(CommitEdgeEdgeCondition::class)
 class EdgeMetadataServiceImpl(
-    nodeDao: NodeDao,
+    nodeBaseService: NodeBaseService,
     repositoryProperties: RepositoryProperties,
     clusterProperties: ClusterProperties
 ) : MetadataServiceImpl(
-    nodeDao,
+    nodeBaseService,
     repositoryProperties
 ) {
 

src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeBaseService.kt

@@ -27,6 +27,10 @@
 
 package com.tencent.bkrepo.repository.service.node.impl
 
+import com.google.common.cache.CacheBuilder
+import com.google.common.cache.RemovalCause
+import com.google.common.cache.RemovalListeners
+import com.google.common.util.concurrent.ThreadFactoryBuilder
 import com.tencent.bkrepo.auth.api.ServicePermissionClient
 import com.tencent.bkrepo.auth.pojo.enums.PermissionAction
 import com.tencent.bkrepo.common.api.exception.BadRequestException
@@ -87,6 +91,10 @@ import org.springframework.data.mongodb.core.query.where
 import org.springframework.transaction.annotation.Transactional
 import java.time.LocalDateTime
 import java.time.format.DateTimeFormatter
+import java.util.concurrent.ArrayBlockingQueue
+import java.util.concurrent.Executors
+import java.util.concurrent.ThreadPoolExecutor
+import java.util.concurrent.TimeUnit
 
 /**
  * 节点基础服务，实现了CRUD基本操作
@@ -103,6 +111,37 @@ abstract class NodeBaseService(
     open val servicePermissionClient: ServicePermissionClient,
 ) : NodeService {
 
+    init {
+        // 定时清理过期缓存, 否则写入不频繁时可能很长时间也不触发数据库更新
+        scheduler.scheduleWithFixedDelay(
+            { lastModifiedInfoUpdateCache.cleanUp() },
+            UPDATE_LAST_MODIFIED_INFO_INTERVAL,
+            UPDATE_LAST_MODIFIED_INFO_INTERVAL,
+            TimeUnit.MINUTES
+        )
+    }
+
+    // 目录最后修改信息更新缓存, 缓存过期后才可能写入数据库
+    private val lastModifiedInfoUpdateCache = CacheBuilder.newBuilder()
+        .maximumSize(1000)
+        .expireAfterWrite(UPDATE_LAST_MODIFIED_INFO_INTERVAL, TimeUnit.MINUTES)
+        .removalListener(
+            RemovalListeners.asynchronous<Triple<String, String, String>, Pair<String, LocalDateTime>> (
+                {
+                    if (it.cause == RemovalCause.EXPIRED || it.cause == RemovalCause.SIZE) {
+                        val (projectId, repoName, fullPath) = it.key!!
+                        val (lastModifiedBy, lastModifiedDate) = it.value!!
+                        val query = NodeQueryHelper.nodeQuery(projectId, repoName, fullPath)
+                        query.addCriteria(where(TNode::createdDate).lt(lastModifiedDate))
+                        val update = NodeQueryHelper.update(lastModifiedBy, lastModifiedDate)
+                        nodeDao.updateFirst(query, update)
+                    }
+                },
+                updateLastModifiedInfoExecutor
+            )
+        )
+        .build<Triple<String, String, String>, Pair<String, LocalDateTime>>()
+
     @Autowired
     @Lazy
     protected lateinit var permissionManager: PermissionManager
@@ -411,7 +450,13 @@ abstract class NodeBaseService(
     /**
      * 递归创建目录
      */
-    fun mkdirs(projectId: String, repoName: String, path: String, createdBy: String): List<TNode> {
+    fun mkdirs(
+        projectId: String,
+        repoName: String,
+        path: String,
+        createdBy: String,
+        currentTime: LocalDateTime = LocalDateTime.now()
+    ): List<TNode> {
         val nodes = mutableListOf<TNode>()
         // 格式化
         val fullPath = PathUtils.toFullPath(path)
@@ -422,7 +467,7 @@ abstract class NodeBaseService(
         if (creatingNode == null) {
             val parentPath = PathUtils.resolveParent(fullPath)
             val name = PathUtils.resolveName(fullPath)
-            val creates = mkdirs(projectId, repoName, parentPath, createdBy)
+            val creates = mkdirs(projectId, repoName, parentPath, createdBy, currentTime)
             val node = TNode(
                 folder = true,
                 path = parentPath,
@@ -434,17 +479,42 @@ abstract class NodeBaseService(
                 projectId = projectId,
                 repoName = repoName,
                 createdBy = createdBy,
-                createdDate = LocalDateTime.now(),
+                createdDate = currentTime,
                 lastModifiedBy = createdBy,
-                lastModifiedDate = LocalDateTime.now(),
+                lastModifiedDate = currentTime
             )
             doCreate(node)
             nodes.addAll(creates)
             nodes.add(node)
+        } else {
+            // 更新已存在的最近父目录的最后修改信息
+            updateModifiedInfo(projectId, repoName, fullPath, createdBy, currentTime)
         }
         return nodes
     }
 
+    fun updateModifiedInfo(
+        projectId: String,
+        repoName: String,
+        fullPath: String,
+        modifiedBy: String,
+        modifiedDate: LocalDateTime = LocalDateTime.now()
+    ) {
+        if (!PathUtils.isRoot(fullPath)) {
+            lastModifiedInfoUpdateCache.put(Triple(projectId, repoName, fullPath), Pair(modifiedBy, modifiedDate))
+        }
+    }
+
+    fun cancelUpdateModifiedInfo(projectId: String, repoName: String, fullPathList: List<String>) {
+        val keys = lastModifiedInfoUpdateCache.asMap().keys.filter { key ->
+            key.first == projectId && key.second == repoName &&
+                fullPathList.any { key.third == it || key.third.startsWith(PathUtils.toPath(it)) }
+        }
+        if (keys.isNotEmpty()) {
+            lastModifiedInfoUpdateCache.invalidateAll(keys)
+        }
+    }
+
     open fun checkConflictAndQuota(createRequest: NodeCreateRequest, fullPath: String): LocalDateTime? {
         with(createRequest) {
             val existNode = nodeDao.findNode(projectId, repoName, fullPath)
@@ -500,6 +570,19 @@ abstract class NodeBaseService(
     companion object {
         private val logger = LoggerFactory.getLogger(NodeBaseService::class.java)
         private const val TOPIC = "bkbase_bkrepo_artifact_node_created"
+        private const val UPDATE_LAST_MODIFIED_INFO_INTERVAL = 5L
+        private val scheduler = Executors.newSingleThreadScheduledExecutor()
+
+        // 更新目录最后修改信息任务线程池
+        private val updateLastModifiedInfoExecutor = ThreadPoolExecutor(
+            4,
+            8,
+            60,
+            TimeUnit.SECONDS,
+            ArrayBlockingQueue(1024),
+            ThreadFactoryBuilder().setNameFormat("update-lastModified-info-%d").build(),
+            ThreadPoolExecutor.CallerRunsPolicy()
+        )
 
         private fun convert(tNode: TNode?): NodeInfo? {
             return tNode?.let {

src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeDeleteSupport.kt

@@ -181,19 +181,33 @@ open class NodeDeleteSupport(
         var deletedNum = 0L
         var deletedSize = 0L
         val deleteTime = LocalDateTime.now()
+        var existFullPaths: List<String>? = null
         val resourceKey = if (fullPaths == null) {
             "/$projectId/$repoName"
         } else if (fullPaths.size == 1) {
             "/$projectId/$repoName${fullPaths[0]}"
         } else {
-            "/$projectId/$repoName$fullPaths"
+            existFullPaths = nodeBaseService.listExistFullPath(projectId, repoName, fullPaths)
+            "/$projectId/$repoName$existFullPaths"
         }
         try {
             val updateResult = nodeDao.updateMulti(query, NodeQueryHelper.nodeDeleteUpdate(operator, deleteTime))
             deletedNum = updateResult.modifiedCount
             if (deletedNum == 0L) {
                 return NodeDeleteResult(deletedNum, deletedSize, deleteTime)
             }
+            // 获取被删除节点的父目录并更新修改信息
+            val parentFullPaths = if (fullPaths?.size == 1) {
+                nodeBaseService.cancelUpdateModifiedInfo(projectId, repoName, fullPaths)
+                listOf(PathUtils.toFullPath(PathUtils.resolveParent(fullPaths[0])))
+            } else {
+                existFullPaths?.map { PathUtils.toFullPath(PathUtils.resolveParent(it)) }
+                    ?.distinct()?.filterNot { PathUtils.isRoot(it) }
+                    ?.also { nodeBaseService.cancelUpdateModifiedInfo(projectId, repoName, it) }
+            }
+            parentFullPaths?.forEach {
+                nodeBaseService.updateModifiedInfo(projectId, repoName, it, operator, deleteTime)
+            }
             deletedSize = nodeBaseService.aggregateComputeSize(criteria.and(TNode::deleted).isEqualTo(deleteTime))
             quotaService.decreaseUsedVolume(projectId, repoName, deletedSize)
             fullPaths?.forEach { publishEvent(buildDeletedEvent(projectId, repoName, it, operator)) }

src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeMoveCopySupport.kt

@@ -105,6 +105,8 @@ open class NodeMoveCopySupport(
                 moveCopyFile(this)
             }
             if (move) {
+                val srcParentFullPath = PathUtils.toFullPath(resolveParent(srcNode.fullPath))
+                nodeBaseService.updateModifiedInfo(srcRepo.projectId, srcRepo.name, srcParentFullPath, operator)
                 publishEvent(NodeEventFactory.buildMovedEvent(request))
             } else {
                 publishEvent(NodeEventFactory.buildCopiedEvent(request))
@@ -322,6 +324,8 @@ open class NodeMoveCopySupport(
                 val name = srcNode.name
                 // 操作节点
                 doMoveCopy(this, srcNode, path, name)
+                // 更新dst目录的修改信息
+                nodeBaseService.updateModifiedInfo(dstProjectId, dstRepoName, dstNode.fullPath, operator)
                 PathUtils.combinePath(path, name)
             }
             val srcRootNodePath = toPath(srcNode.fullPath)

src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRenameSupport.kt

@@ -73,6 +73,7 @@ open class NodeRenameSupport(
         val repoName = node.repoName
         val newPath = PathUtils.resolveParent(newFullPath)
         val newName = PathUtils.resolveName(newFullPath)
+        var modifiedCount = 0L
 
         // 检查新路径是否被占用
         if (nodeDao.exists(projectId, repoName, newFullPath)) {
@@ -93,12 +94,17 @@ open class NodeRenameSupport(
             val query = NodeQueryHelper.nodeListQuery(projectId, repoName, node.fullPath, listOption)
             nodeDao.find(query).forEach { doRename(it, newParentPath + it.name, operator) }
             // 删除自己
-            nodeDao.remove(NodeQueryHelper.nodeQuery(projectId, repoName, node.fullPath))
+            modifiedCount = nodeDao.remove(NodeQueryHelper.nodeQuery(projectId, repoName, node.fullPath)).deletedCount
         } else {
             // 修改自己
             val selfQuery = NodeQueryHelper.nodeQuery(projectId, repoName, node.fullPath)
             val selfUpdate = NodeQueryHelper.nodePathUpdate(newPath, newName, operator)
-            nodeDao.updateFirst(selfQuery, selfUpdate)
+            modifiedCount = nodeDao.updateFirst(selfQuery, selfUpdate).modifiedCount
+        }
+        if (modifiedCount == 1L) {
+            // 更新父目录的最后修改信息
+            val parentFullPath = PathUtils.toFullPath(PathUtils.resolveParent(node.fullPath))
+            nodeBaseService.updateModifiedInfo(projectId, repoName, parentFullPath, operator)
         }
     }
 

src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt

@@ -35,6 +35,8 @@ import com.tencent.bkrepo.common.api.exception.ErrorCodeException
 import com.tencent.bkrepo.common.artifact.api.ArtifactInfo
 import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode
 import com.tencent.bkrepo.common.artifact.path.PathUtils.isRoot
+import com.tencent.bkrepo.common.artifact.path.PathUtils.resolveParent
+import com.tencent.bkrepo.common.artifact.path.PathUtils.toFullPath
 import com.tencent.bkrepo.common.security.util.SecurityUtils
 import com.tencent.bkrepo.repository.dao.NodeDao
 import com.tencent.bkrepo.repository.model.TNode
@@ -65,7 +67,7 @@ import java.time.ZoneId
  * 节点统计接口实现
  */
 open class NodeRestoreSupport(
-    nodeBaseService: NodeBaseService
+    val nodeBaseService: NodeBaseService
 ) : NodeRestoreOperation {
 
     val nodeDao: NodeDao = nodeBaseService.nodeDao
@@ -164,7 +166,11 @@ open class NodeRestoreSupport(
                 }
             }
             val query = nodeDeletedPointQuery(projectId, repoName, fullPath, deletedTime)
-            restoreCount += nodeDao.updateFirst(query, nodeRestoreUpdate()).modifiedCount
+            val modifiedCount = nodeDao.updateFirst(query, nodeRestoreUpdate()).modifiedCount
+            restoreCount += modifiedCount
+            if (modifiedCount == 1L) {
+                nodeBaseService.updateModifiedInfo(projectId, repoName, toFullPath(resolveParent(fullPath)), operator)
+            }
         }
     }
 

src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeRestoreSupport.kt

@@ -29,6 +29,7 @@ package com.tencent.bkrepo.repository.service.node.impl.center
 
 import com.tencent.bkrepo.common.api.exception.ErrorCodeException
 import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode
+import com.tencent.bkrepo.common.artifact.path.PathUtils
 import com.tencent.bkrepo.common.artifact.util.ClusterUtils
 import com.tencent.bkrepo.repository.model.TNode
 import com.tencent.bkrepo.repository.pojo.node.ConflictStrategy
@@ -66,6 +67,11 @@ class CommitEdgeCenterNodeRestoreSupport(
             fullPath = fullPath,
             deleted = context.deletedTime
         )
-        context.restoreCount += nodeDao.updateFirst(query, NodeQueryHelper.nodeRestoreUpdate()).modifiedCount
+        val modifiedCount = nodeDao.updateFirst(query, NodeQueryHelper.nodeRestoreUpdate()).modifiedCount
+        context.restoreCount += modifiedCount
+        if (modifiedCount == 1L) {
+            val parentFullPath = PathUtils.toFullPath(PathUtils.resolveParent(fullPath))
+            nodeBaseService.updateModifiedInfo(context.projectId, context.repoName, parentFullPath, context.operator)
+        }
     }
 }