feat: 优化系统GC采样效率 TencentBlueKing#1600 (TencentBlueKing#1627)

felixncheng · Jan 8, 2024 · 2702b08 · 2702b08
1 parent 47e22c1
commit 2702b08
Show file tree

Hide file tree

Showing 11 changed files with 168 additions and 79 deletions.
diff --git a/...ckend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/api/ArchiveClient.kt b/...ckend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/api/ArchiveClient.kt
@@ -1,6 +1,7 @@
 package com.tencent.bkrepo.archive.api
 
 import com.tencent.bkrepo.archive.pojo.ArchiveFile
+import com.tencent.bkrepo.archive.pojo.CompressFile
 import com.tencent.bkrepo.archive.request.ArchiveFileRequest
 import com.tencent.bkrepo.archive.request.CompleteCompressRequest
 import com.tencent.bkrepo.archive.request.CompressFileRequest
@@ -43,7 +44,7 @@ interface ArchiveClient {
     ): Response<ArchiveFile?>
 
     @PutMapping("/compress")
-    fun compress(@RequestBody request: CompressFileRequest): Response<Void>
+    fun compress(@RequestBody request: CompressFileRequest): Response<Int>
 
     @PutMapping("/uncompress")
     fun uncompress(@RequestBody request: UncompressFileRequest): Response<Void>
@@ -54,6 +55,12 @@ interface ArchiveClient {
     @PutMapping("/compress/complete")
     fun completeCompress(@RequestBody request: CompleteCompressRequest): Response<Void>
 
+    @GetMapping("/compress")
+    fun getCompressInfo(
+        @RequestParam sha256: String,
+        @RequestParam(required = false) storageCredentialsKey: String?,
+    ): Response<CompressFile?>
+
     @DeleteMapping("/deleteAll")
     fun deleteAll(@RequestBody request: ArchiveFileRequest): Response<Void>
 }
diff --git a/...ackend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/pojo/ArchiveFile.kt b/...ackend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/pojo/ArchiveFile.kt
@@ -4,7 +4,6 @@ import com.tencent.bkrepo.archive.ArchiveStatus
 import java.time.LocalDateTime
 
 data class ArchiveFile(
-    var id: String? = null,
     var createdBy: String,
     var createdDate: LocalDateTime,
     var lastModifiedBy: String,

diff --git a/...ckend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/pojo/CompressFile.kt b/...ckend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/pojo/CompressFile.kt
@@ -0,0 +1,17 @@
+package com.tencent.bkrepo.archive.pojo
+
+import com.tencent.bkrepo.archive.CompressStatus
+import java.time.LocalDateTime
+
+data class CompressFile(
+    var createdBy: String,
+    var createdDate: LocalDateTime,
+    var lastModifiedBy: String,
+    var lastModifiedDate: LocalDateTime,
+    val sha256: String,
+    val baseSha256: String,
+    val uncompressedSize: Long,
+    var compressedSize: Long = -1,
+    val storageCredentialsKey: String?,
+    var status: CompressStatus,
+)
diff --git a/...end/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/pojo/CompressedInfo.kt b/...end/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/pojo/CompressedInfo.kt
diff --git a/...rchive/src/main/kotlin/com/tencent/bkrepo/archive/controller/service/ArchiveController.kt b/...rchive/src/main/kotlin/com/tencent/bkrepo/archive/controller/service/ArchiveController.kt
@@ -2,6 +2,7 @@ package com.tencent.bkrepo.archive.controller.service
 
 import com.tencent.bkrepo.archive.api.ArchiveClient
 import com.tencent.bkrepo.archive.pojo.ArchiveFile
+import com.tencent.bkrepo.archive.pojo.CompressFile
 import com.tencent.bkrepo.archive.request.ArchiveFileRequest
 import com.tencent.bkrepo.archive.request.CompleteCompressRequest
 import com.tencent.bkrepo.archive.request.CompressFileRequest
@@ -46,9 +47,8 @@ class ArchiveController(
         return ResponseBuilder.success()
     }
 
-    override fun compress(request: CompressFileRequest): Response<Void> {
-        compressService.compress(request)
-        return ResponseBuilder.success()
+    override fun compress(request: CompressFileRequest): Response<Int> {
+        return ResponseBuilder.success(compressService.compress(request))
     }
 
     override fun uncompress(request: UncompressFileRequest): Response<Void> {
@@ -66,6 +66,10 @@ class ArchiveController(
         return ResponseBuilder.success()
     }
 
+    override fun getCompressInfo(sha256: String, storageCredentialsKey: String?): Response<CompressFile?> {
+        return ResponseBuilder.success(compressService.getCompressInfo(sha256, storageCredentialsKey))
+    }
+
     override fun deleteAll(request: ArchiveFileRequest): Response<Void> {
         archiveService.delete(request)
         val deleteCompressRequest = DeleteCompressRequest(

diff --git a/...hive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/ArchiveServiceImpl.kt b/...hive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/ArchiveServiceImpl.kt
@@ -109,7 +109,6 @@ class ArchiveServiceImpl(
         val af = archiveFileRepository.findBySha256AndStorageCredentialsKey(sha256, storageCredentialsKey)
             ?: return null
         return ArchiveFile(
-            id = af.id,
             createdBy = af.createdBy,
             createdDate = af.createdDate,
             lastModifiedBy = af.lastModifiedBy,

diff --git a/...archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/CompressService.kt b/...archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/CompressService.kt
@@ -1,5 +1,6 @@
 package com.tencent.bkrepo.archive.service
 
+import com.tencent.bkrepo.archive.pojo.CompressFile
 import com.tencent.bkrepo.archive.request.CompleteCompressRequest
 import com.tencent.bkrepo.archive.request.CompressFileRequest
 import com.tencent.bkrepo.archive.request.DeleteCompressRequest
@@ -11,8 +12,9 @@ import com.tencent.bkrepo.archive.request.UncompressFileRequest
 interface CompressService {
     /**
      * 压缩文件
+     * @return 1表示压缩成功，0表示未压缩
      * */
-    fun compress(request: CompressFileRequest)
+    fun compress(request: CompressFileRequest): Int
 
     /**
      * 解压文件
@@ -30,4 +32,12 @@ interface CompressService {
      * 完成压缩
      * */
     fun complete(request: CompleteCompressRequest)
+
+    /**
+     * 获取压缩信息
+     * */
+    fun getCompressInfo(
+        sha256: String,
+        storageCredentialsKey: String?,
+    ): CompressFile?
 }
diff --git a/...ive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/CompressServiceImpl.kt b/...ive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/CompressServiceImpl.kt
@@ -6,6 +6,7 @@ import com.tencent.bkrepo.archive.CompressStatus
 import com.tencent.bkrepo.archive.job.compress.CompressSubscriber
 import com.tencent.bkrepo.archive.job.compress.UncompressSubscriber
 import com.tencent.bkrepo.archive.model.TCompressFile
+import com.tencent.bkrepo.archive.pojo.CompressFile
 import com.tencent.bkrepo.archive.repository.CompressFileDao
 import com.tencent.bkrepo.archive.repository.CompressFileRepository
 import com.tencent.bkrepo.archive.request.CompleteCompressRequest
@@ -47,7 +48,7 @@ class CompressServiceImpl(
         executor,
     )
 
-    override fun compress(request: CompressFileRequest) {
+    override fun compress(request: CompressFileRequest): Int {
         with(request) {
             // 队头元素
             val head = compressFileRepository.findBySha256AndStorageCredentialsKey(sha256, storageCredentialsKey)
@@ -61,7 +62,7 @@ class CompressServiceImpl(
                     head.status = CompressStatus.COMPRESSED
                     compressFileRepository.save(head)
                 }
-                return
+                return 1
             }
             var currentChainLength = 0
             // 这是队头
@@ -71,7 +72,7 @@ class CompressServiceImpl(
                 if (currentChainLength > MAX_CHAIN_LENGTH) {
                     // 超出队列长度
                     logger.info("Exceed max chain length,ignore it.")
-                    return
+                    return 0
                 }
                 // 删除旧头
                 compressFileRepository.delete(head)
@@ -116,6 +117,7 @@ class CompressServiceImpl(
             if (sync) {
                 compressor.doOnNext(compressFile)
             }
+            return 1
         }
     }
 
@@ -174,6 +176,27 @@ class CompressServiceImpl(
         }
     }
 
+    override fun getCompressInfo(sha256: String, storageCredentialsKey: String?): CompressFile? {
+        val file = compressFileRepository.findBySha256AndStorageCredentialsKey(sha256, storageCredentialsKey)
+        if (file == null || file.status == CompressStatus.NONE) {
+            return null
+        }
+        with(file) {
+            return CompressFile(
+                createdBy = createdBy,
+                createdDate = createdDate,
+                lastModifiedBy = lastModifiedBy,
+                lastModifiedDate = lastModifiedDate,
+                sha256 = sha256,
+                baseSha256 = baseSha256,
+                status = status,
+                compressedSize = compressedSize,
+                uncompressedSize = uncompressedSize,
+                storageCredentialsKey = storageCredentialsKey,
+            )
+        }
+    }
+
     companion object {
         private val logger = LoggerFactory.getLogger(CompressServiceImpl::class.java)
         private const val MAX_CHAIN_LENGTH = 10

diff --git a/...mmon-api/src/main/kotlin/com/tencent/bkrepo/common/api/collection/CollectionExtensions.kt b/...mmon-api/src/main/kotlin/com/tencent/bkrepo/common/api/collection/CollectionExtensions.kt
@@ -26,9 +26,10 @@ fun <E> Collection<E>.groupBySimilar(key: (x1: E) -> String, similar: (x1: E, x2
         * 2. 中间存在干扰数据，加入之前的组
         * 3. 新建一个组
         * */
-        if (similar(group.last(), str2)) {
+        // 判断新加入的元素与一头一尾是否相似，控制单个分类不要过于离散
+        if (similar(group.first(), str2) && similar(group.last(), str2)) {
             group.add(str2)
-        } else if (previousGroup != null && similar(previousGroup.last(), str2)) {
+        } else if (previousGroup != null && similar(group.first(), str2) && similar(previousGroup.last(), str2)) {
             previousGroup.add(str2)
         } else {
             if (group.size > 1) {