From 1c67d48ab55076fe9acaa66709e803eb74f46093 Mon Sep 17 00:00:00 2001 From: felixncheng Date: Fri, 5 Jan 2024 16:28:46 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E4=BC=98=E5=8C=96=E7=B3=BB=E7=BB=9FGC?= =?UTF-8?q?=E9=87=87=E6=A0=B7=E6=95=88=E7=8E=87=20#1600?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/archive/api/ArchiveClient.kt | 9 +- .../bkrepo/archive/pojo/ArchiveFile.kt | 1 - .../bkrepo/archive/pojo/CompressFile.kt | 17 ++ .../bkrepo/archive/pojo/CompressedInfo.kt | 10 -- .../controller/service/ArchiveController.kt | 10 +- .../archive/service/ArchiveServiceImpl.kt | 1 - .../bkrepo/archive/service/CompressService.kt | 12 +- .../archive/service/CompressServiceImpl.kt | 29 +++- .../api/collection/CollectionExtensions.kt | 5 +- .../tencent/bkrepo/job/batch/SystemGcJob.kt | 147 +++++++++++------- .../properties/SystemGcJobProperties.kt | 6 +- 11 files changed, 168 insertions(+), 79 deletions(-) create mode 100644 src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/pojo/CompressFile.kt delete mode 100644 src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/pojo/CompressedInfo.kt diff --git a/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/api/ArchiveClient.kt b/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/api/ArchiveClient.kt index c119f065b9..cdda705e9b 100644 --- a/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/api/ArchiveClient.kt +++ b/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/api/ArchiveClient.kt @@ -1,6 +1,7 @@ package com.tencent.bkrepo.archive.api import com.tencent.bkrepo.archive.pojo.ArchiveFile +import com.tencent.bkrepo.archive.pojo.CompressFile import com.tencent.bkrepo.archive.request.ArchiveFileRequest import com.tencent.bkrepo.archive.request.CompleteCompressRequest import com.tencent.bkrepo.archive.request.CompressFileRequest @@ -43,7 +44,7 @@ interface ArchiveClient { ): Response @PutMapping("/compress") - fun compress(@RequestBody request: CompressFileRequest): Response + fun compress(@RequestBody request: CompressFileRequest): Response @PutMapping("/uncompress") fun uncompress(@RequestBody request: UncompressFileRequest): Response @@ -54,6 +55,12 @@ interface ArchiveClient { @PutMapping("/compress/complete") fun completeCompress(@RequestBody request: CompleteCompressRequest): Response + @GetMapping("/compress") + fun getCompressInfo( + @RequestParam sha256: String, + @RequestParam(required = false) storageCredentialsKey: String?, + ): Response + @DeleteMapping("/deleteAll") fun deleteAll(@RequestBody request: ArchiveFileRequest): Response } diff --git a/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/pojo/ArchiveFile.kt b/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/pojo/ArchiveFile.kt index 13efc4fe7b..7495ecacb3 100644 --- a/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/pojo/ArchiveFile.kt +++ b/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/pojo/ArchiveFile.kt @@ -4,7 +4,6 @@ import com.tencent.bkrepo.archive.ArchiveStatus import java.time.LocalDateTime data class ArchiveFile( - var id: String? = null, var createdBy: String, var createdDate: LocalDateTime, var lastModifiedBy: String, diff --git a/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/pojo/CompressFile.kt b/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/pojo/CompressFile.kt new file mode 100644 index 0000000000..b6ec890549 --- /dev/null +++ b/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/pojo/CompressFile.kt @@ -0,0 +1,17 @@ +package com.tencent.bkrepo.archive.pojo + +import com.tencent.bkrepo.archive.CompressStatus +import java.time.LocalDateTime + +data class CompressFile( + var createdBy: String, + var createdDate: LocalDateTime, + var lastModifiedBy: String, + var lastModifiedDate: LocalDateTime, + val sha256: String, + val baseSha256: String, + val uncompressedSize: Long, + var compressedSize: Long = -1, + val storageCredentialsKey: String?, + var status: CompressStatus, +) diff --git a/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/pojo/CompressedInfo.kt b/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/pojo/CompressedInfo.kt deleted file mode 100644 index 4461f8c022..0000000000 --- a/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/pojo/CompressedInfo.kt +++ /dev/null @@ -1,10 +0,0 @@ -package com.tencent.bkrepo.archive.pojo - -import com.tencent.bkrepo.common.api.constant.StringPool - -class CompressedInfo( - val status: Int, // 0 压缩中， 1 压缩成功 - val uncompressedSize: Long, - val compressedSize: Long, - val ratio: String = StringPool.calculateRatio(uncompressedSize, compressedSize), -) diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/controller/service/ArchiveController.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/controller/service/ArchiveController.kt index cbc070c56a..641e5185be 100644 --- a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/controller/service/ArchiveController.kt +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/controller/service/ArchiveController.kt @@ -2,6 +2,7 @@ package com.tencent.bkrepo.archive.controller.service import com.tencent.bkrepo.archive.api.ArchiveClient import com.tencent.bkrepo.archive.pojo.ArchiveFile +import com.tencent.bkrepo.archive.pojo.CompressFile import com.tencent.bkrepo.archive.request.ArchiveFileRequest import com.tencent.bkrepo.archive.request.CompleteCompressRequest import com.tencent.bkrepo.archive.request.CompressFileRequest @@ -46,9 +47,8 @@ class ArchiveController( return ResponseBuilder.success() } - override fun compress(request: CompressFileRequest): Response { - compressService.compress(request) - return ResponseBuilder.success() + override fun compress(request: CompressFileRequest): Response { + return ResponseBuilder.success(compressService.compress(request)) } override fun uncompress(request: UncompressFileRequest): Response { @@ -66,6 +66,10 @@ class ArchiveController( return ResponseBuilder.success() } + override fun getCompressInfo(sha256: String, storageCredentialsKey: String?): Response { + return ResponseBuilder.success(compressService.getCompressInfo(sha256, storageCredentialsKey)) + } + override fun deleteAll(request: ArchiveFileRequest): Response { archiveService.delete(request) val deleteCompressRequest = DeleteCompressRequest( diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/ArchiveServiceImpl.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/ArchiveServiceImpl.kt index 1db89d4c51..28822770ba 100644 --- a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/ArchiveServiceImpl.kt +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/ArchiveServiceImpl.kt @@ -109,7 +109,6 @@ class ArchiveServiceImpl( val af = archiveFileRepository.findBySha256AndStorageCredentialsKey(sha256, storageCredentialsKey) ?: return null return ArchiveFile( - id = af.id, createdBy = af.createdBy, createdDate = af.createdDate, lastModifiedBy = af.lastModifiedBy, diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/CompressService.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/CompressService.kt index fb682b6b34..64e9713123 100644 --- a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/CompressService.kt +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/CompressService.kt @@ -1,5 +1,6 @@ package com.tencent.bkrepo.archive.service +import com.tencent.bkrepo.archive.pojo.CompressFile import com.tencent.bkrepo.archive.request.CompleteCompressRequest import com.tencent.bkrepo.archive.request.CompressFileRequest import com.tencent.bkrepo.archive.request.DeleteCompressRequest @@ -11,8 +12,9 @@ import com.tencent.bkrepo.archive.request.UncompressFileRequest interface CompressService { /** * 压缩文件 + * @return 1表示压缩成功，0表示未压缩 * */ - fun compress(request: CompressFileRequest) + fun compress(request: CompressFileRequest): Int /** * 解压文件 @@ -30,4 +32,12 @@ interface CompressService { * 完成压缩 * */ fun complete(request: CompleteCompressRequest) + + /** + * 获取压缩信息 + * */ + fun getCompressInfo( + sha256: String, + storageCredentialsKey: String?, + ): CompressFile? } diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/CompressServiceImpl.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/CompressServiceImpl.kt index 03a17d008f..77c33f5c6d 100644 --- a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/CompressServiceImpl.kt +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/CompressServiceImpl.kt @@ -6,6 +6,7 @@ import com.tencent.bkrepo.archive.CompressStatus import com.tencent.bkrepo.archive.job.compress.CompressSubscriber import com.tencent.bkrepo.archive.job.compress.UncompressSubscriber import com.tencent.bkrepo.archive.model.TCompressFile +import com.tencent.bkrepo.archive.pojo.CompressFile import com.tencent.bkrepo.archive.repository.CompressFileDao import com.tencent.bkrepo.archive.repository.CompressFileRepository import com.tencent.bkrepo.archive.request.CompleteCompressRequest @@ -47,7 +48,7 @@ class CompressServiceImpl( executor, ) - override fun compress(request: CompressFileRequest) { + override fun compress(request: CompressFileRequest): Int { with(request) { // 队头元素 val head = compressFileRepository.findBySha256AndStorageCredentialsKey(sha256, storageCredentialsKey) @@ -61,7 +62,7 @@ class CompressServiceImpl( head.status = CompressStatus.COMPRESSED compressFileRepository.save(head) } - return + return 1 } var currentChainLength = 0 // 这是队头 @@ -71,7 +72,7 @@ class CompressServiceImpl( if (currentChainLength > MAX_CHAIN_LENGTH) { // 超出队列长度 logger.info("Exceed max chain length,ignore it.") - return + return 0 } // 删除旧头 compressFileRepository.delete(head) @@ -116,6 +117,7 @@ class CompressServiceImpl( if (sync) { compressor.doOnNext(compressFile) } + return 1 } } @@ -174,6 +176,27 @@ class CompressServiceImpl( } } + override fun getCompressInfo(sha256: String, storageCredentialsKey: String?): CompressFile? { + val file = compressFileRepository.findBySha256AndStorageCredentialsKey(sha256, storageCredentialsKey) + if (file == null || file.status == CompressStatus.NONE) { + return null + } + with(file) { + return CompressFile( + createdBy = createdBy, + createdDate = createdDate, + lastModifiedBy = lastModifiedBy, + lastModifiedDate = lastModifiedDate, + sha256 = sha256, + baseSha256 = baseSha256, + status = status, + compressedSize = compressedSize, + uncompressedSize = uncompressedSize, + storageCredentialsKey = storageCredentialsKey, + ) + } + } + companion object { private val logger = LoggerFactory.getLogger(CompressServiceImpl::class.java) private const val MAX_CHAIN_LENGTH = 10 diff --git a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/collection/CollectionExtensions.kt b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/collection/CollectionExtensions.kt index 3aed90d413..ccb45da203 100644 --- a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/collection/CollectionExtensions.kt +++ b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/collection/CollectionExtensions.kt @@ -26,9 +26,10 @@ fun Collection.groupBySimilar(key: (x1: E) -> String, similar: (x1: E, x2 * 2. 中间存在干扰数据，加入之前的组 * 3. 新建一个组 * */ - if (similar(group.last(), str2)) { + // 判断新加入的元素与一头一尾是否相似，控制单个分类不要过于离散 + if (similar(group.first(), str2) && similar(group.last(), str2)) { group.add(str2) - } else if (previousGroup != null && similar(previousGroup.last(), str2)) { + } else if (previousGroup != null && similar(group.first(), str2) && similar(previousGroup.last(), str2)) { previousGroup.add(str2) } else { if (group.size > 1) { diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/SystemGcJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/SystemGcJob.kt index d985b025c4..31c5dcf448 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/SystemGcJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/SystemGcJob.kt @@ -1,20 +1,14 @@ package com.tencent.bkrepo.job.batch +import com.tencent.bkrepo.archive.CompressStatus import com.tencent.bkrepo.archive.api.ArchiveClient import com.tencent.bkrepo.archive.request.CompressFileRequest import com.tencent.bkrepo.common.api.collection.groupBySimilar -import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.api.util.HumanReadable -import com.tencent.bkrepo.common.api.util.StreamUtils -import com.tencent.bkrepo.common.bksync.BkSync -import com.tencent.bkrepo.common.bksync.DiffResult import com.tencent.bkrepo.common.mongo.constant.ID import com.tencent.bkrepo.common.mongo.constant.MIN_OBJECT_ID import com.tencent.bkrepo.common.mongo.dao.util.sharding.HashShardingUtils -import com.tencent.bkrepo.common.storage.core.StorageProperties -import com.tencent.bkrepo.common.storage.core.locator.FileLocator import com.tencent.bkrepo.common.storage.credentials.StorageCredentials -import com.tencent.bkrepo.common.storage.util.StorageUtils import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 import com.tencent.bkrepo.job.SHARDING_COUNT import com.tencent.bkrepo.job.batch.base.DefaultContextJob @@ -34,9 +28,6 @@ import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.Query import org.springframework.data.mongodb.core.query.isEqualTo import org.springframework.stereotype.Component -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.Paths import kotlin.reflect.full.declaredMemberProperties import kotlin.system.measureNanoTime @@ -50,14 +41,12 @@ class SystemGcJob( val properties: SystemGcJobProperties, private val mongoTemplate: MongoTemplate, private val archiveClient: ArchiveClient, - private val storageProperties: StorageProperties, - private val fileLocator: FileLocator, ) : DefaultContextJob(properties) { private var lastId = MIN_OBJECT_ID private var lastCutoffTimeMap = mutableMapOf() private var curCutoffTime = LocalDateTime.MIN - private var bksync = BkSync() + private val sampleNodesMap = mutableMapOf>() override fun doStart0(jobContext: JobContext) { curCutoffTime = LocalDateTime.now().minus(Duration.ofDays(properties.idleDays.toLong())) properties.repos.forEach { @@ -103,12 +92,16 @@ class SystemGcJob( } fun isSimilar(node1: Node, node2: Node): Boolean { + val name1 = node1.name + val name2 = node2.name // 大小差异过大 - if (abs(node1.size - node2.size) * 2 / (node1.size + node2.size) > 0.5) { + if (name1.length != name2.length || abs(node1.size - node2.size).toDouble() / maxOf( + node1.size, + node2.size, + ) > 0.5 + ) { return false } - val name1 = node1.name - val name2 = node2.name val editDistance = HAMMING_DISTANCE_INSTANCE.apply(name1, name2) val ratio = editDistance.toDouble() * 2 / (name1.length + name2.length) if (logger.isDebugEnabled) { @@ -159,64 +152,106 @@ class SystemGcJob( // 没有新的节点，表示节点已经gc过一轮了 val repoKey = nodes.first().let { "${it.projectId}/${it.repoName}" } val lastCutoffTime = lastCutoffTimeMap[repoKey] - if (lastCutoffTime != null && sortedNodes.last().createdDate < lastCutoffTime) { + val sampleNodes = sampleNodesMap.getOrPut(repoKey) { mutableListOf() } + // 有采样节点存在，表示上次gc并没有完成 + if (lastCutoffTime != null && sortedNodes.last().createdDate < lastCutoffTime && sampleNodes.isEmpty()) { logger.info("There are no new nodes, gc is skipped.") return } val gcNodes = sortedNodes.subList(0, sortedNodes.size - properties.retain) // 保留最新的 val newest = sortedNodes.last() - if (samplingSurvey(gcNodes, newest)) { - val repo = RepositoryCommonUtils.getRepositoryDetail(newest.projectId, newest.repoName) - val credentials = repo.storageCredentials - gcNodes.forEach { - val compressedRequest = CompressFileRequest(it.sha256, it.size, newest.sha256, credentials?.key) - archiveClient.compress(compressedRequest) - logger.info("Compress node ${it.name} by node ${newest.name}.") + val repo = RepositoryCommonUtils.getRepositoryDetail(newest.projectId, newest.repoName) + val credentials = repo.storageCredentials + logger.info("Start gc ${gcNodes.size} nodes.") + if (gcNodes.size < MIN_SAMPLING_GROUP_SIZE) { + // 直接压缩 + gcNodes.forEach { compressNode(it, newest, credentials) } + } else { + // 从采样节点中找到相同的组 + val fileType = nodes.first().name.substringAfterLast(".") + val samplingNode = sampleNodes + .firstOrNull { + it.name.substringAfterLast(".") == fileType && + isSimilar(it, nodes.first()) && isSimilar(it, nodes.last()) + } + if (samplingNode != null) { + gcBySample(samplingNode, credentials, newest, gcNodes, sampleNodes) + } else { + createNewSample(gcNodes, newest, credentials, sampleNodes) } } } /** - * 抽样调查，快速判断一组节点的实际数据是否相似 + * 创建采样节点 * */ - private fun samplingSurvey(nodes: List, base: Node): Boolean { - val src = nodes.first() - val repo = RepositoryCommonUtils.getRepositoryDetail(src.projectId, src.repoName) - val storageCredentials = repo.storageCredentials ?: storageProperties.defaultStorageCredentials() - val dir = Paths.get(storageCredentials.upload.localPath, GC_DIR, StringPool.randomStringByLongValue()) - val baseFilePath = createTempFile(dir) - val srcFilePath = createTempFile(dir) - val signFilePath = createTempFile(dir) - val deltaFilePath = createTempFile(dir) - try { - downloadFile(base.sha256, storageCredentials, baseFilePath) - downloadFile(src.sha256, storageCredentials, srcFilePath) - Files.newOutputStream(signFilePath).use { - bksync.checksum(baseFilePath.toFile(), it) - } - var actual = DiffResult(0, 1) - StreamUtils.use(Files.newInputStream(signFilePath), Files.newOutputStream(deltaFilePath)) { input, output -> - actual = bksync.diff(srcFilePath.toFile(), input, output, storageCredentials.compress.ratio) + private fun createNewSample( + gcNodes: List, + newest: Node, + credentials: StorageCredentials?, + sampleNodes: MutableList, + ) { + if (sampleNodes.size > properties.maxSampleNum) { + val remove = sampleNodes.removeFirst() + logger.info("Sample list is full,remove first node [$remove].") + } + for (node in gcNodes) { + val resp = compressNode(node, newest, credentials) + if (resp == 1) { + sampleNodes.add(node) + logger.info("Create a new sample [$node].") + return + } else { + logger.info("Node [$node] is root and reach maximum chain length.") } - logger.info("Sampling (${src.name},${base.name}),reuse: ${actual.hitRate}.") - return actual.hitRate >= storageCredentials.compress.ratio - } finally { - dir.toFile().deleteRecursively() } } - private fun createTempFile(dir: Path): Path { - val fileName = StringPool.randomStringByLongValue(prefix = "gc", suffix = ".temp") - return dir.resolve(fileName) + /** + * 通过已有的采样节点进行gc + * */ + private fun gcBySample( + sampleNode: Node, + credentials: StorageCredentials?, + newest: Node, + gcNodes: List, + sampleNodes: MutableList, + ) { + val compressFile = archiveClient.getCompressInfo(sampleNode.sha256, credentials?.key).data + val status = compressFile?.status + when (status) { + // 压缩信息丢失 + null -> { + logger.info("Lost sample information.") + compressNode(sampleNode, newest, credentials) + } + // 压缩中 + CompressStatus.CREATED, + CompressStatus.COMPRESSING, + -> { + logger.info("Sample [$sampleNode] in process") + } + // 压缩失败，放弃此次gc nodes,移除采样节点后，如果有gc node有新的group形成，则会进行新一轮的gc。 + CompressStatus.COMPRESS_FAILED -> { + sampleNodes.remove(sampleNode) + logger.info("Sample survey [$sampleNode] fails, gc is skipped.") + } + // 压缩成功，压缩gc nodes + else -> { + logger.info("Sample survey [$sampleNode] success, start gc.") + gcNodes.forEach { compressNode(it, newest, credentials) } + sampleNodes.remove(sampleNode) + } + } } - private fun downloadFile(sha256: String, storageCredentials: StorageCredentials, filePath: Path) { - val path = fileLocator.locate(sha256) - if (!Files.isDirectory(filePath.parent)) { - Files.createDirectories(filePath.parent) + private fun compressNode(node: Node, baseNode: Node, storageCredentials: StorageCredentials?): Int { + with(node) { + logger.info("Compress node $name by node ${baseNode.name}.") + val compressedRequest = CompressFileRequest(sha256, size, baseNode.sha256, storageCredentials?.key) + return archiveClient.compress(compressedRequest).data ?: 0 } - StorageUtils.download(path, sha256, storageCredentials, filePath) } data class Node( @@ -237,6 +272,6 @@ class SystemGcJob( companion object { private val logger = LoggerFactory.getLogger(SystemGcJob::class.java) private val HAMMING_DISTANCE_INSTANCE = HammingDistance() - private const val GC_DIR = "gc" + private const val MIN_SAMPLING_GROUP_SIZE = 5 } } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/SystemGcJobProperties.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/SystemGcJobProperties.kt index 55362e7d2f..f1bb23f493 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/SystemGcJobProperties.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/SystemGcJobProperties.kt @@ -59,10 +59,14 @@ data class SystemGcJobProperties( /** * 最大批处理数量 * */ - var maxBatchSize: Int = 100000, + var maxBatchSize: Int = 10000, /** * 只有超过该节点数量，才会进行gc * */ var nodeLimit: Int = 1000, + /** + * 最大的采样数量 + * */ + var maxSampleNum: Int = 10000, ) : MongodbJobProperties()