diff --git a/src/backend/analysis-executor/biz-analysis-executor/src/main/kotlin/com/tencent/bkrepo/analysis/executor/component/FileLoader.kt b/src/backend/analysis-executor/biz-analysis-executor/src/main/kotlin/com/tencent/bkrepo/analysis/executor/component/FileLoader.kt index 9ee80dd589..fe12b30bcd 100644 --- a/src/backend/analysis-executor/biz-analysis-executor/src/main/kotlin/com/tencent/bkrepo/analysis/executor/component/FileLoader.kt +++ b/src/backend/analysis-executor/biz-analysis-executor/src/main/kotlin/com/tencent/bkrepo/analysis/executor/component/FileLoader.kt @@ -39,11 +39,13 @@ import com.tencent.bkrepo.common.api.exception.SystemErrorException import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.common.api.util.readJsonString import com.tencent.bkrepo.common.api.util.toJsonString +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.manager.StorageManager import com.tencent.bkrepo.common.artifact.pojo.RepositoryType +import com.tencent.bkrepo.common.metadata.service.node.NodeSearchService +import com.tencent.bkrepo.common.metadata.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService import com.tencent.bkrepo.common.storage.credentials.StorageCredentials -import com.tencent.bkrepo.repository.api.NodeClient -import com.tencent.bkrepo.repository.api.StorageCredentialsClient import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.search.NodeQueryBuilder import org.apache.commons.codec.binary.Hex @@ -64,9 +66,10 @@ import java.security.MessageDigest @Component class FileLoader( private val executorProperties: ScannerExecutorProperties, - private val nodeClient: NodeClient, + private val nodeService: NodeService, + private val nodeSearchService: NodeSearchService, private val storageManager: StorageManager, - private val storageCredentialsClient: StorageCredentialsClient, + private val storageCredentialService: StorageCredentialService, ) { /** * 加载[subtask]要扫描的制品 @@ -82,8 +85,8 @@ class FileLoader( } // 获取存储凭据 - val storageCredentials = credentialsKey?.let { storageCredentialsClient.findByKey(it).data!! } - val node = nodeClient.getNodeDetail(projectId, repoName, fullPath).data + val storageCredentials = credentialsKey?.let { storageCredentialService.findByKey(it)!! } + val node = nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, fullPath)) // 获取文件 val file = File(tempDir, fileName(taskId, fileName(), repoType)) val fos = DigestOutputStream(file.outputStream(), MessageDigest.getInstance("SHA-256")) @@ -173,7 +176,7 @@ class FileLoader( } private fun getNode(projectId: String, repoName: String, sha256: String): NodeDetail? { - val nodes = nodeClient.queryWithoutCount( + val nodes = nodeSearchService.searchWithoutCount( NodeQueryBuilder() .projectId(projectId) .repoName(repoName) @@ -182,11 +185,11 @@ class FileLoader( .page(1, 1) .build() ) - if (nodes.isNotOk() || nodes.data!!.records.isEmpty()) { + if (nodes.records.isEmpty()) { throw SystemErrorException(CommonMessageCode.RESOURCE_NOT_FOUND, sha256) } - val fullPath = nodes.data!!.records[0][NodeDetail::fullPath.name].toString() - return nodeClient.getNodeDetail(projectId, repoName, fullPath).data + val fullPath = nodes.records[0][NodeDetail::fullPath.name].toString() + return nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, fullPath)) } /** diff --git a/src/backend/analysis-executor/biz-analysis-executor/src/main/kotlin/com/tencent/bkrepo/analysis/executor/trivy/TrivyScanExecutor.kt b/src/backend/analysis-executor/biz-analysis-executor/src/main/kotlin/com/tencent/bkrepo/analysis/executor/trivy/TrivyScanExecutor.kt index a04fe3d2f8..eeff5c63db 100644 --- a/src/backend/analysis-executor/biz-analysis-executor/src/main/kotlin/com/tencent/bkrepo/analysis/executor/trivy/TrivyScanExecutor.kt +++ b/src/backend/analysis-executor/biz-analysis-executor/src/main/kotlin/com/tencent/bkrepo/analysis/executor/trivy/TrivyScanExecutor.kt @@ -50,12 +50,13 @@ import com.tencent.bkrepo.common.api.exception.NotFoundException import com.tencent.bkrepo.common.api.exception.SystemErrorException import com.tencent.bkrepo.common.api.message.CommonMessageCode.RESOURCE_NOT_FOUND import com.tencent.bkrepo.common.api.message.CommonMessageCode.SYSTEM_ERROR +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.hash.md5 import com.tencent.bkrepo.common.artifact.manager.StorageManager import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream +import com.tencent.bkrepo.common.metadata.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.query.model.Sort -import com.tencent.bkrepo.repository.api.NodeClient -import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.NodeInfo import com.tencent.bkrepo.repository.pojo.node.NodeListOption @@ -71,9 +72,9 @@ import java.io.FileOutputStream class TrivyScanExecutor @Autowired constructor( dockerClient: DockerClient, private val scannerExecutorProperties: ScannerExecutorProperties, - private val repositoryClient: RepositoryClient, + private val repositoryService: RepositoryService, private val storageManager: StorageManager, - private val nodeClient: NodeClient + private val nodeService: NodeService ) : CommonScanExecutor() { private val dockerScanHelper = DockerScanHelper(scannerExecutorProperties, dockerClient) @@ -169,19 +170,11 @@ class TrivyScanExecutor @Autowired constructor( val scanner = task.scanner require(scanner is TrivyScanner) // 获取trivy默认仓库信息 - val repoRes = repositoryClient.getRepoDetail(scanner.vulDbConfig.projectId, scanner.vulDbConfig.repo) - if (repoRes.isNotOk()) { - logger.error( - "Get repo info failed: code[${repoRes.code}], message[${repoRes.message}]," + - " projectId[${scanner.vulDbConfig.projectId}], repoName[${scanner.vulDbConfig.repo}]" - ) - throw SystemErrorException(SYSTEM_ERROR, repoRes.message ?: "") - } - val repositoryDetail = repoRes.data + val repositoryDetail = repositoryService.getRepoDetail(scanner.vulDbConfig.projectId, scanner.vulDbConfig.repo) ?: throw NotFoundException(RESOURCE_NOT_FOUND, scanner.vulDbConfig.repo) return storageManager.loadFullArtifactInputStream(NodeDetail(dbNode), repositoryDetail.storageCredentials) - ?: throw SystemErrorException(SYSTEM_ERROR, "load trivy.db file failed: res: ${repoRes.message}") + ?: throw SystemErrorException(SYSTEM_ERROR, "load trivy.db file failed") } private fun getNewestNode(projectId: String, repo: String): NodeInfo { @@ -193,16 +186,9 @@ class TrivyScanExecutor @Autowired constructor( sortProperty = listOf("lastModifiedDate", "createdDate"), direction = listOf(Sort.Direction.DESC.name, Sort.Direction.DESC.name) ) - val nodeRes = nodeClient.listNodePage(projectId, repo, "/trivy", option) - if (nodeRes.isNotOk()) { - logger.error( - "Get node info failed: code[${nodeRes.code}], message[${nodeRes.message}]," + - " projectId[$projectId], repoName[$repo]" - ) - throw SystemErrorException(SYSTEM_ERROR, nodeRes.message ?: "") - } + val nodeRes = nodeService.listNodePage(ArtifactInfo(projectId, repo, "/trivy"), option) // 获取最新的trivy.db - val newestDB = nodeRes.data!!.records.firstOrNull() + val newestDB = nodeRes.records.firstOrNull() if (newestDB == null) { logger.error("Get trivy.db file failed") throw SystemErrorException(SYSTEM_ERROR, "Get trivy.db file failed") diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/CacheableRepositoryClient.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/CacheableRepositoryClient.kt index 6138b49433..814eead444 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/CacheableRepositoryClient.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/CacheableRepositoryClient.kt @@ -31,16 +31,15 @@ import com.google.common.cache.CacheBuilder import com.google.common.cache.CacheLoader import com.google.common.cache.LoadingCache import com.tencent.bkrepo.common.api.exception.NotFoundException -import com.tencent.bkrepo.common.api.exception.SystemErrorException import com.tencent.bkrepo.common.api.message.CommonMessageCode -import com.tencent.bkrepo.repository.api.RepositoryClient +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.repository.pojo.repo.RepositoryInfo import org.slf4j.LoggerFactory import org.springframework.stereotype.Component import java.util.concurrent.TimeUnit @Component -class CacheableRepositoryClient(private val repositoryClient: RepositoryClient) { +class CacheableRepositoryClient(private val repositoryService: RepositoryService) { private val repoInfoCache: LoadingCache = CacheBuilder.newBuilder() .maximumSize(DEFAULT_REPO_INFO_CACHE_SIZE) .expireAfterWrite(DEFAULT_REPO_INFO_CACHE_DURATION_MINUTES, TimeUnit.MINUTES) @@ -52,15 +51,8 @@ class CacheableRepositoryClient(private val repositoryClient: RepositoryClient) private fun loadRepoInfo(key: String): RepositoryInfo { val (projectId, repoName) = fromKey(key) - val repoRes = repositoryClient.getRepoInfo(projectId, repoName) - if (repoRes.isNotOk()) { - logger.error( - "Get repo info failed: code[${repoRes.code}], message[${repoRes.message}]," + - " projectId[$projectId], repoName[$repoName]" - ) - throw SystemErrorException(CommonMessageCode.SYSTEM_ERROR, repoRes.message ?: "") - } - return repoRes.data ?: throw NotFoundException(CommonMessageCode.RESOURCE_NOT_FOUND, key) + val repo = repositoryService.getRepoInfo(projectId, repoName) + return repo ?: throw NotFoundException(CommonMessageCode.RESOURCE_NOT_FOUND, key) } private fun fromKey(key: String): Pair { diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/ScannerPermissionCheckHandler.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/ScannerPermissionCheckHandler.kt index 383bda3669..5e747a7583 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/ScannerPermissionCheckHandler.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/ScannerPermissionCheckHandler.kt @@ -39,6 +39,7 @@ import com.tencent.bkrepo.common.security.permission.Principal import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.analyst.model.SubScanTaskDefinition +import com.tencent.bkrepo.common.artifact.pojo.RepositoryId import com.tencent.bkrepo.common.security.permission.PrincipalType import org.springframework.context.annotation.Primary import org.springframework.stereotype.Component @@ -153,5 +154,5 @@ class ScannerPermissionCheckHandler( } private fun repoDetail(projectId: String, repoName: String) = - ArtifactContextHolder.getRepoDetail(ArtifactContextHolder.RepositoryId(projectId, repoName)) + ArtifactContextHolder.getRepoDetail(RepositoryId(projectId, repoName)) } diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/manager/arrowhead/ArrowheadConverter.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/manager/arrowhead/ArrowheadConverter.kt index 1ab488f4a0..896c8edda2 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/manager/arrowhead/ArrowheadConverter.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/manager/arrowhead/ArrowheadConverter.kt @@ -48,6 +48,7 @@ import com.tencent.bkrepo.common.api.pojo.Page import com.tencent.bkrepo.common.mongo.dao.util.Pages import com.tencent.bkrepo.common.query.model.PageLimit import org.springframework.stereotype.Component +import java.util.Locale @Component("${ArrowheadScanner.TYPE}Converter") class ArrowheadConverter(private val licenseService: SpdxLicenseService) : ScannerConverter { @@ -57,10 +58,10 @@ class ArrowheadConverter(private val licenseService: SpdxLicenseService) : Scann result as Page // 查询数据库中存放的applicationItem时已经过滤了只存在license的项，license一定存在 val licenseIds = result.records.map { it.license!!.name }.distinct() - val licenses = licenseService.listLicenseByIds(licenseIds).mapKeys { it.key.toLowerCase() } + val licenses = licenseService.listLicenseByIds(licenseIds).mapKeys { it.key.lowercase(Locale.getDefault()) } val reports = result.records.map { - val detail = licenses[it.license!!.name.toLowerCase()] + val detail = licenses[it.license!!.name.lowercase(Locale.getDefault())] FileLicensesResultDetail( licenseId = it.license!!.name, fullName = detail?.name ?: "", @@ -144,9 +145,10 @@ class ArrowheadConverter(private val licenseService: SpdxLicenseService) : Scann overview[LicenseOverviewKey.overviewKeyOf(LicenseOverviewKey.TOTAL)] = licenses.size.toLong() // 获取许可证详情 - val licenseInfo = licenseService.listLicenseByIds(licenseIds.toList()).mapKeys { it.key.toLowerCase() } + val licenseInfo = + licenseService.listLicenseByIds(licenseIds.toList()).mapKeys { it.key.lowercase(Locale.getDefault()) } for (license in licenses) { - val detail = licenseInfo[license.license!!.name.toLowerCase()] + val detail = licenseInfo[license.license!!.name.lowercase(Locale.getDefault())] if (detail == null) { incLicenseOverview(overview, LicenseNature.UNKNOWN.natureName) continue diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/manager/arrowhead/dao/CveSecItemDao.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/manager/arrowhead/dao/CveSecItemDao.kt index 9d4a401018..caaf1db44b 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/manager/arrowhead/dao/CveSecItemDao.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/manager/arrowhead/dao/CveSecItemDao.kt @@ -36,6 +36,7 @@ import com.tencent.bkrepo.analyst.pojo.request.LoadResultArguments import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.inValues import org.springframework.stereotype.Repository +import java.util.Locale @Repository class CveSecItemDao : ResultItemDao() { @@ -63,7 +64,7 @@ class CveSecItemDao : ResultItemDao() { continue } - val prefix = vulId.substring(0, indexOfDash).toLowerCase() + val prefix = vulId.substring(0, indexOfDash).lowercase(Locale.getDefault()) when (prefix) { "cve" -> cveIds.add(vulId) "cnnvd" -> cnnvdIds.add(vulId) diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/manager/dependencycheck/Converter.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/manager/dependencycheck/Converter.kt index e5dc7d85de..6f911c42a8 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/manager/dependencycheck/Converter.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/manager/dependencycheck/Converter.kt @@ -38,6 +38,7 @@ import com.tencent.bkrepo.analyst.component.manager.dependencycheck.model.TDepen import com.tencent.bkrepo.analyst.component.manager.dependencycheck.model.TDependencyItemData import com.tencent.bkrepo.analyst.component.manager.knowledgebase.TCve import java.time.LocalDateTime +import java.util.Locale object Converter { /** @@ -171,5 +172,5 @@ object Converter { } } - fun pocIdOf(cveId: String) = "${DependencyScanner.TYPE.toLowerCase()}-$cveId" + fun pocIdOf(cveId: String) = "${DependencyScanner.TYPE.lowercase(Locale.getDefault())}-$cveId" } diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/manager/standard/StandardConverter.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/manager/standard/StandardConverter.kt index b5cd1c30f0..4f49f4ddcc 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/manager/standard/StandardConverter.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/manager/standard/StandardConverter.kt @@ -54,6 +54,7 @@ import com.tencent.bkrepo.common.api.pojo.Page import com.tencent.bkrepo.common.mongo.dao.util.Pages import com.tencent.bkrepo.common.query.model.PageLimit import org.springframework.stereotype.Component +import java.util.Locale @Component("${StandardScanner.TYPE}Converter") class StandardConverter(private val licenseService: SpdxLicenseService) : ScannerConverter { @@ -61,10 +62,10 @@ class StandardConverter(private val licenseService: SpdxLicenseService) : Scanne override fun convertLicenseResult(result: Any): Page { result as Page val licenseIds = result.records.map { it.licenseName }.distinct() - val licenses = licenseService.listLicenseByIds(licenseIds).mapKeys { it.key.toLowerCase() } + val licenses = licenseService.listLicenseByIds(licenseIds).mapKeys { it.key.lowercase(Locale.getDefault()) } val reports = result.records.map { - val detail = licenses[it.licenseName.toLowerCase()] + val detail = licenses[it.licenseName.lowercase(Locale.getDefault())] FileLicensesResultDetail( licenseId = it.licenseName, fullName = detail?.name ?: "", @@ -140,7 +141,7 @@ class StandardConverter(private val licenseService: SpdxLicenseService) : Scanne // security统计 securityResults?.forEach { securityResult -> - val severityLevel = Level.valueOf(securityResult.severity.toUpperCase()).level + val severityLevel = Level.valueOf(securityResult.severity.uppercase(Locale.getDefault())).level val shouldIgnore = filterRule?.shouldIgnore( securityResult.vulId, securityResult.cveId, @@ -164,8 +165,8 @@ class StandardConverter(private val licenseService: SpdxLicenseService) : Scanne return overview } - val licenseIds = licenseResults.map { it.licenseName.toLowerCase() }.distinct() - val licensesInfo = licenseService.listLicenseByIds(licenseIds).mapKeys { it.key.toLowerCase() } + val licenseIds = licenseResults.map { it.licenseName.lowercase(Locale.getDefault()) }.distinct() + val licensesInfo = licenseService.listLicenseByIds(licenseIds).mapKeys { it.key.lowercase(Locale.getDefault()) } overview[LicenseOverviewKey.overviewKeyOf(TOTAL)] = licenseResults.size.toLong() for (licenseResult in licenseResults) { @@ -175,7 +176,7 @@ class StandardConverter(private val licenseService: SpdxLicenseService) : Scanne continue } - val detail = licensesInfo[licenseResult.licenseName.toLowerCase()] + val detail = licensesInfo[licenseResult.licenseName.lowercase(Locale.getDefault())] if (detail == null) { incLicenseOverview(overview, LicenseNature.UNKNOWN.natureName) continue diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/manager/trivy/Converter.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/manager/trivy/Converter.kt index 7a701212ca..f26d027867 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/manager/trivy/Converter.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/manager/trivy/Converter.kt @@ -32,6 +32,7 @@ import com.tencent.bkrepo.common.analysis.pojo.scanner.trivy.VulnerabilityItem import com.tencent.bkrepo.repository.constant.SYSTEM_USER import com.tencent.bkrepo.analyst.component.manager.knowledgebase.TCve import java.time.LocalDateTime +import java.util.Locale object Converter { /** @@ -63,5 +64,6 @@ object Converter { } } - private fun pocIdOf(vulnerabilityId: String) = "${TrivyScanner.TYPE.toLowerCase()}-$vulnerabilityId" + private fun pocIdOf(vulnerabilityId: String) = + "${TrivyScanner.TYPE.lowercase(Locale.getDefault())}-$vulnerabilityId" } diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/manager/trivy/TrivyConverter.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/manager/trivy/TrivyConverter.kt index 7c0825a274..3c5fa56001 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/manager/trivy/TrivyConverter.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/manager/trivy/TrivyConverter.kt @@ -44,6 +44,7 @@ import com.tencent.bkrepo.analyst.pojo.response.ArtifactVulnerabilityInfo import com.tencent.bkrepo.analyst.utils.ScanPlanConverter import org.springframework.data.domain.PageRequest import org.springframework.stereotype.Component +import java.util.Locale @Component("${TrivyScanner.TYPE}Converter") class TrivyConverter : ScannerConverter { @@ -54,7 +55,7 @@ class TrivyConverter : ScannerConverter { val reports = result.records.mapTo(HashSet(result.records.size)) { ArtifactVulnerabilityInfo( vulId = it.data.vulnerabilityId, - severity = ScanPlanConverter.convertToLeakLevel(it.data.severity.toLowerCase()), + severity = ScanPlanConverter.convertToLeakLevel(it.data.severity.lowercase(Locale.getDefault())), pkgName = it.data.pkgName, installedVersion = setOf(it.data.installedVersion), title = it.data.title, @@ -82,9 +83,9 @@ class TrivyConverter : ScannerConverter { // cve count scanExecutorResult.vulnerabilityItems.forEach { if (it.severity == "UNKNOWN") { - it.severity = Level.CRITICAL.levelName.toUpperCase() + it.severity = Level.CRITICAL.levelName.uppercase(Locale.getDefault()) } - val overviewKey = CveOverviewKey.overviewKeyOf(it.severity.toLowerCase()) + val overviewKey = CveOverviewKey.overviewKeyOf(it.severity.lowercase(Locale.getDefault())) overview[overviewKey] = overview.getOrDefault(overviewKey, 0L) + 1L } return overview diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/manager/trivy/dao/VulnerabilityItemDao.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/manager/trivy/dao/VulnerabilityItemDao.kt index 0ab46fe538..722ec7f875 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/manager/trivy/dao/VulnerabilityItemDao.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/manager/trivy/dao/VulnerabilityItemDao.kt @@ -35,13 +35,15 @@ import com.tencent.bkrepo.analyst.pojo.request.trivy.TrivyLoadResultArguments import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.inValues import org.springframework.stereotype.Repository +import java.util.Locale @Repository class VulnerabilityItemDao : ResultItemDao() { override fun customizePageBy(criteria: Criteria, arguments: LoadResultArguments): Criteria { require(arguments is TrivyLoadResultArguments) if (arguments.vulnerabilityLevels.isNotEmpty()) { - val levels = arguments.vulnerabilityLevels + arguments.vulnerabilityLevels.map { it.toUpperCase() } + val levels = + arguments.vulnerabilityLevels + arguments.vulnerabilityLevels.map { it.uppercase(Locale.getDefault()) } criteria.and(dataKey(VulnerabilityItem::severity.name)).inValues(levels) } if (arguments.vulIds.isNotEmpty()) { diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/ScanServiceImpl.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/ScanServiceImpl.kt index bda158df41..d47fb15083 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/ScanServiceImpl.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/ScanServiceImpl.kt @@ -73,9 +73,9 @@ import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.exception.NotFoundException import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.common.lock.service.LockOperation +import com.tencent.bkrepo.common.metadata.service.project.ProjectService import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.HttpContextHolder -import com.tencent.bkrepo.repository.api.ProjectClient import com.tencent.bkrepo.repository.pojo.project.ProjectRangeQueryRequest import com.tencent.bkrepo.statemachine.Event import com.tencent.bkrepo.statemachine.StateMachine @@ -102,7 +102,7 @@ class ScanServiceImpl @Autowired constructor( private val redisTemplate: RedisTemplate, private val reportExporter: ReportExporter, private val scannerProperties: ScannerProperties, - private val projectClient: ProjectClient, + private val projectService: ProjectService, private val lockOperation: LockOperation, ) : ScanService { @@ -116,9 +116,9 @@ class ScanServiceImpl @Autowired constructor( // projectId需要在第一层 var projectIds = RuleUtil.getProjectIds(rule) if (projectMetadata.isNotEmpty()) { - val metadataProjectIds = projectClient.rangeQuery( + val metadataProjectIds = projectService.rangeQuery( ProjectRangeQueryRequest(emptyList(), projectMetadata = projectMetadata) - ).data?.records?.map { it?.name!! } ?: emptyList() + ).records.map { it?.name!! } projectIds = projectIds + metadataProjectIds } val scanRequest = ScanRequest( diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/ScanTaskServiceImpl.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/ScanTaskServiceImpl.kt index a164ca3d52..3a7673b7be 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/ScanTaskServiceImpl.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/ScanTaskServiceImpl.kt @@ -76,12 +76,12 @@ import com.tencent.bkrepo.common.api.pojo.Page import com.tencent.bkrepo.common.api.util.readJsonString import com.tencent.bkrepo.common.api.util.toJsonString import com.tencent.bkrepo.common.artifact.exception.RepoNotFoundException +import com.tencent.bkrepo.common.metadata.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.mongo.dao.util.Pages import com.tencent.bkrepo.common.query.model.PageLimit import com.tencent.bkrepo.common.security.permission.PrincipalType import com.tencent.bkrepo.common.security.util.SecurityUtils -import com.tencent.bkrepo.repository.api.NodeClient -import com.tencent.bkrepo.repository.api.RepositoryClient import org.slf4j.LoggerFactory import org.springframework.stereotype.Service import java.time.format.DateTimeFormatter @@ -96,8 +96,8 @@ class ScanTaskServiceImpl( private val archiveSubScanTaskDao: ArchiveSubScanTaskDao, private val planArtifactLatestSubScanTaskDao: PlanArtifactLatestSubScanTaskDao, private val fileScanResultDao: FileScanResultDao, - private val nodeClient: NodeClient, - private val repositoryClient: RepositoryClient, + private val nodeService: NodeService, + private val repositoryService: RepositoryService, private val resultManagers: Map, private val scannerConverters: Map, private val filterRuleService: FilterRuleService @@ -221,14 +221,14 @@ class ScanTaskServiceImpl( override fun resultDetail(request: FileScanResultDetailRequest): FileScanResultDetail { with(request) { val node = artifactInfo!!.run { - nodeClient.getNodeDetail(projectId, repoName, getArtifactFullPath()).data + nodeService.getNodeDetail(this) ?: throw NotFoundException(CommonMessageCode.RESOURCE_NOT_FOUND, getArtifactFullPath()) } if (node.folder) { throw ParameterInvalidException(node.fullPath) } - val repo = repositoryClient.getRepoInfo(node.projectId, node.repoName).data!! + val repo = repositoryService.getRepoInfo(node.projectId, node.repoName)!! val scanner = scannerService.get(scanner) val matchFilterRuleRequest = MatchFilterRuleRequest( diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/SpdxLicenseServiceImpl.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/SpdxLicenseServiceImpl.kt index 7d5df29232..0f03fb68ef 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/SpdxLicenseServiceImpl.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/SpdxLicenseServiceImpl.kt @@ -38,12 +38,13 @@ import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.common.api.pojo.Page import com.tencent.bkrepo.common.api.util.JsonUtils import com.tencent.bkrepo.common.api.util.readJsonString +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.manager.StorageManager +import com.tencent.bkrepo.common.metadata.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService +import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService import com.tencent.bkrepo.common.mongo.dao.util.Pages import com.tencent.bkrepo.common.security.util.SecurityUtils -import com.tencent.bkrepo.repository.api.NodeClient -import com.tencent.bkrepo.repository.api.RepositoryClient -import com.tencent.bkrepo.repository.api.StorageCredentialsClient import org.slf4j.LoggerFactory import org.springframework.data.domain.Sort import org.springframework.data.mongodb.core.query.Criteria @@ -58,9 +59,9 @@ import java.time.format.DateTimeFormatter @Service class SpdxLicenseServiceImpl( private val licenseDao: SpdxLicenseDao, - private val nodeClient: NodeClient, - private val repositoryClient: RepositoryClient, - private val storageCredentialsClient: StorageCredentialsClient, + private val nodeService: NodeService, + private val repositoryService: RepositoryService, + private val storageCredentialService: StorageCredentialService, private val storageManager: StorageManager, ) : SpdxLicenseService { override fun importLicense(path: String): Boolean { @@ -78,10 +79,10 @@ class SpdxLicenseServiceImpl( } override fun importLicense(projectId: String, repoName: String, fullPath: String): Boolean { - val repo = repositoryClient.getRepoInfo(projectId, repoName).data + val repo = repositoryService.getRepoInfo(projectId, repoName) ?: throw NotFoundException(CommonMessageCode.RESOURCE_NOT_FOUND, projectId, repoName) - val storageCredentials = repo.storageCredentialsKey?.let { storageCredentialsClient.findByKey(it).data } - val node = nodeClient.getNodeDetail(projectId, repoName, fullPath).data + val storageCredentials = repo.storageCredentialsKey?.let { storageCredentialService.findByKey(it) } + val node = nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, fullPath)) ?: throw NotFoundException(CommonMessageCode.RESOURCE_NOT_FOUND, projectId, repoName, fullPath) storageManager.loadFullArtifactInputStream(node, storageCredentials)?.use { importLicense(it.readJsonString()) diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/TemporaryScanTokenServiceImpl.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/TemporaryScanTokenServiceImpl.kt index 468e2b5877..aaae9fe479 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/TemporaryScanTokenServiceImpl.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/TemporaryScanTokenServiceImpl.kt @@ -50,14 +50,16 @@ import com.tencent.bkrepo.common.api.exception.SystemErrorException import com.tencent.bkrepo.common.api.message.CommonMessageCode.RESOURCE_NOT_FOUND import com.tencent.bkrepo.common.api.message.CommonMessageCode.SYSTEM_ERROR import com.tencent.bkrepo.common.api.util.StreamUtils.readText +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.manager.StorageManager import com.tencent.bkrepo.common.artifact.pojo.RepositoryType +import com.tencent.bkrepo.common.metadata.service.node.NodeSearchService +import com.tencent.bkrepo.common.metadata.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService import com.tencent.bkrepo.common.query.enums.OperationType import com.tencent.bkrepo.common.security.exception.AuthenticationException import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.oci.util.OciUtils -import com.tencent.bkrepo.repository.api.NodeClient -import com.tencent.bkrepo.repository.api.StorageCredentialsClient import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.search.NodeQueryBuilder import org.slf4j.LoggerFactory @@ -75,8 +77,9 @@ class TemporaryScanTokenServiceImpl( private val redisTemplate: RedisTemplate, private val scannerProperties: ScannerProperties, private val storageManager: StorageManager, - private val storageCredentialsClient: StorageCredentialsClient, - private val nodeClient: NodeClient + private val storageCredentialService: StorageCredentialService, + private val nodeService: NodeService, + private val nodeSearchService: NodeSearchService ) : TemporaryScanTokenService { private val baseUrl get() = scannerProperties.baseUrl.removeSuffix(SLASH) @@ -220,8 +223,8 @@ class TemporaryScanTokenServiceImpl( } private fun readManifest(projectId: String, repoName: String, sha256: String, credentialsKey: String?): String { - val storageCredentials = credentialsKey?.let { storageCredentialsClient.findByKey(it).data!! } - val nodes = nodeClient.queryWithoutCount( + val storageCredentials = credentialsKey?.let { storageCredentialService.findByKey(it)!! } + val nodes = nodeSearchService.searchWithoutCount( NodeQueryBuilder() .projectId(projectId) .repoName(repoName) @@ -230,11 +233,11 @@ class TemporaryScanTokenServiceImpl( .page(1, 1) .build() ) - if (nodes.isNotOk() || nodes.data!!.records.isEmpty()) { + if (nodes.records.isEmpty()) { throw SystemErrorException(RESOURCE_NOT_FOUND, sha256) } - val fullPath = nodes.data!!.records[0][NodeDetail::fullPath.name].toString() - return nodeClient.getNodeDetail(projectId, repoName, fullPath).data?.let { node -> + val fullPath = nodes.records[0][NodeDetail::fullPath.name].toString() + return nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, fullPath))?.let { node -> storageManager.loadFullArtifactInputStream(node, storageCredentials)?.readText() } ?: throw ErrorCodeException(RESOURCE_NOT_FOUND, "file [$projectId:$repoName:$fullPath] not found") } @@ -243,7 +246,7 @@ class TemporaryScanTokenServiceImpl( private fun getNodes(projectId: String, repoName: String, sha256: List): List> { return sha256.toSet().map { - val res = nodeClient.queryWithoutCount( + val res = nodeSearchService.searchWithoutCount( NodeQueryBuilder() .projectId(projectId) .repoName(repoName) @@ -253,16 +256,11 @@ class TemporaryScanTokenServiceImpl( .build() ) - if (res.isNotOk()) { - logger.error("get node of layer[$it] failed, msg[${res.message}]") - throw SystemErrorException() - } - - if (res.data!!.records.isEmpty()) { + if (res.records.isEmpty()) { throw ArtifactDeletedException(it) } - res.data!!.records.first() + res.records.first() } } diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/statemachine/iterator/IteratorManager.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/statemachine/iterator/IteratorManager.kt index 85848e00f9..50a53fdf83 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/statemachine/iterator/IteratorManager.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/statemachine/iterator/IteratorManager.kt @@ -30,19 +30,18 @@ package com.tencent.bkrepo.analyst.statemachine.iterator import com.tencent.bkrepo.analyst.pojo.Node import com.tencent.bkrepo.analyst.pojo.ScanPlan import com.tencent.bkrepo.analyst.pojo.ScanTask +import com.tencent.bkrepo.analyst.pojo.rule.RuleArtifact +import com.tencent.bkrepo.analyst.utils.RuleUtil +import com.tencent.bkrepo.common.analysis.pojo.scanner.Scanner import com.tencent.bkrepo.common.artifact.pojo.RepositoryType +import com.tencent.bkrepo.common.metadata.service.node.NodeSearchService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.query.enums.OperationType import com.tencent.bkrepo.common.query.model.Rule -import com.tencent.bkrepo.common.analysis.pojo.scanner.Scanner -import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.PackageClient -import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.NodeInfo import com.tencent.bkrepo.repository.pojo.packages.PackageSummary -import com.tencent.bkrepo.analyst.pojo.rule.RuleArtifact -import com.tencent.bkrepo.analyst.utils.Request -import com.tencent.bkrepo.analyst.utils.RuleUtil import org.springframework.stereotype.Component /** @@ -50,8 +49,8 @@ import org.springframework.stereotype.Component */ @Component class IteratorManager( - private val nodeClient: NodeClient, - private val repositoryClient: RepositoryClient, + private val nodeSearchService: NodeSearchService, + private val repositoryService: RepositoryService, private val packageClient: PackageClient ) { /** @@ -76,9 +75,9 @@ class IteratorManager( val isPackageScanPlanType = scanTask.scanPlan != null && scanTask.scanPlan!!.type != RepositoryType.GENERIC.name return if (isPackageScanPlanType || packageRule(rule)) { - PackageIterator(packageClient, nodeClient, PackageIterator.PackageIteratePosition(rule)) + PackageIterator(packageClient, nodeSearchService, PackageIterator.PackageIteratePosition(rule)) } else { - NodeIterator(projectIdIterator, nodeClient, NodeIterator.NodeIteratePosition(rule)) + NodeIterator(projectIdIterator, nodeSearchService, NodeIterator.NodeIteratePosition(rule)) } } @@ -122,10 +121,8 @@ class IteratorManager( } private fun addRepoNames(rule: Rule, projectId: String): Rule { - val repoNames = Request - .request { repositoryClient.listRepo(projectId, null, RepositoryType.GENERIC.name) } - ?.map { it.name } - ?: return rule + val repoNames = repositoryService.listRepo(projectId, null, RepositoryType.GENERIC.name) + .map { it.name } val repoRule = Rule.QueryRule(NodeInfo::repoName.name, repoNames, OperationType.IN) return if (rule is Rule.NestedRule && rule.relation == Rule.NestedRule.RelationType.AND) { diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/statemachine/iterator/NodeIterator.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/statemachine/iterator/NodeIterator.kt index f6686ce562..a0e0333137 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/statemachine/iterator/NodeIterator.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/statemachine/iterator/NodeIterator.kt @@ -28,24 +28,23 @@ package com.tencent.bkrepo.analyst.statemachine.iterator import com.tencent.bkrepo.analyst.pojo.Node +import com.tencent.bkrepo.analyst.utils.Request import com.tencent.bkrepo.common.api.constant.DEFAULT_PAGE_SIZE +import com.tencent.bkrepo.common.metadata.service.node.NodeSearchService import com.tencent.bkrepo.common.query.enums.OperationType import com.tencent.bkrepo.common.query.model.Rule -import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.pojo.node.NodeDetail -import com.tencent.bkrepo.analyst.utils.Request import org.slf4j.LoggerFactory /** * 文件迭代器 * * @param projectIdIterator 用于提供需要遍历的Node所属的项目 - * @param nodeClient nodeClient * @param position 当前文件遍历到的位置 */ class NodeIterator( private val projectIdIterator: Iterator, - private val nodeClient: NodeClient, + private val nodeSearchService: NodeSearchService, override val position: NodeIteratePosition = NodeIteratePosition() ) : PageableIterator(position) { @@ -89,7 +88,7 @@ class NodeIterator( val projectIdRule = modifyRule(projectId, rule) // 获取下一页需要扫描的文件 return try { - Request.requestNodes(nodeClient, projectIdRule, page, pageSize) + Request.requestNodes(nodeSearchService, projectIdRule, page, pageSize) } catch (e: Exception) { logger.error("iterate node failed, projectId[$projectId], page[$page], pageSize[$pageSize], rule[$rule]", e) emptyList() diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/statemachine/iterator/PackageIterator.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/statemachine/iterator/PackageIterator.kt index 3057b35ac7..14aae43203 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/statemachine/iterator/PackageIterator.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/statemachine/iterator/PackageIterator.kt @@ -28,20 +28,20 @@ package com.tencent.bkrepo.analyst.statemachine.iterator import com.tencent.bkrepo.analyst.pojo.Node +import com.tencent.bkrepo.analyst.pojo.rule.RuleArtifact +import com.tencent.bkrepo.analyst.pojo.rule.RuleArtifact.Companion.RULE_FIELD_LATEST_VERSION +import com.tencent.bkrepo.analyst.utils.Request import com.tencent.bkrepo.common.api.constant.DEFAULT_PAGE_NUMBER import com.tencent.bkrepo.common.api.constant.DEFAULT_PAGE_SIZE +import com.tencent.bkrepo.common.metadata.service.node.NodeSearchService import com.tencent.bkrepo.common.query.enums.OperationType import com.tencent.bkrepo.common.query.matcher.RuleMatcher import com.tencent.bkrepo.common.query.model.PageLimit import com.tencent.bkrepo.common.query.model.QueryModel import com.tencent.bkrepo.common.query.model.Rule -import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.PackageClient import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.packages.PackageSummary -import com.tencent.bkrepo.analyst.pojo.rule.RuleArtifact -import com.tencent.bkrepo.analyst.pojo.rule.RuleArtifact.Companion.RULE_FIELD_LATEST_VERSION -import com.tencent.bkrepo.analyst.utils.Request import org.slf4j.LoggerFactory import kotlin.math.min @@ -50,7 +50,7 @@ import kotlin.math.min */ class PackageIterator( private val packageClient: PackageClient, - private val nodeClient: NodeClient, + private val nodeSearchService: NodeSearchService, override val position: PackageIteratePosition ) : PageableIterator() { override fun nextPageData(page: Int, pageSize: Int): List { @@ -235,7 +235,7 @@ class PackageIterator( return emptyList() } val nodeQueryRule = nodeQueryRule(packages) - val nodes = Request.requestNodes(nodeClient, nodeQueryRule, DEFAULT_PAGE_NUMBER, position.pageSize) + val nodes = Request.requestNodes(nodeSearchService, nodeQueryRule, DEFAULT_PAGE_NUMBER, position.pageSize) val packageMap = packages.associateBy { it.fullPath } nodes.forEach { val pkg = packageMap[it.fullPath]!! diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/utils/Request.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/utils/Request.kt index a407ed484a..a1fc6212cc 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/utils/Request.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/utils/Request.kt @@ -31,11 +31,11 @@ import com.tencent.bkrepo.analyst.pojo.Node import com.tencent.bkrepo.common.api.exception.SystemErrorException import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.common.api.pojo.Response +import com.tencent.bkrepo.common.metadata.service.node.NodeSearchService import com.tencent.bkrepo.common.query.model.PageLimit import com.tencent.bkrepo.common.query.model.QueryModel import com.tencent.bkrepo.common.query.model.Rule import com.tencent.bkrepo.common.query.model.Sort -import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.pojo.node.NodeDetail object Request { @@ -63,7 +63,7 @@ object Request { /** * 请求node数据并解析成[Node] */ - fun requestNodes(nodeClient: NodeClient, rule: Rule, page: Int, pageSize: Int): List { + fun requestNodes(nodeSearchService: NodeSearchService, rule: Rule, page: Int, pageSize: Int): List { // 通常根据projectId,repoName等字段搜索Node，此时如果结果数量较多时用_id排序会有性能问题导致查询超时 // 使用projectId、repoName、fullPath字段有建立唯一索引，因此使用这些字段进行排序 val sort = Sort( @@ -76,7 +76,7 @@ object Request { select = nodeSelected, rule = rule ) - return request { nodeClient.queryWithoutCount(queryModel) }!!.records.map { + return nodeSearchService.searchWithoutCount(queryModel).records.map { val projectId = it[NodeDetail::projectId.name]!! as String val repoName = it[NodeDetail::repoName.name]!! as String val sha256 = it[NodeDetail::sha256.name]!! as String diff --git a/src/backend/analyst/biz-analyst/src/test/kotlin/com/tencent/bkrepo/analyst/utils/RequestTest.kt b/src/backend/analyst/biz-analyst/src/test/kotlin/com/tencent/bkrepo/analyst/utils/RequestTest.kt index 70481967d2..582fbcc4ec 100644 --- a/src/backend/analyst/biz-analyst/src/test/kotlin/com/tencent/bkrepo/analyst/utils/RequestTest.kt +++ b/src/backend/analyst/biz-analyst/src/test/kotlin/com/tencent/bkrepo/analyst/utils/RequestTest.kt @@ -27,18 +27,16 @@ package com.tencent.bkrepo.analyst.utils -import com.tencent.bkrepo.common.mongo.dao.util.Pages -import com.tencent.bkrepo.common.query.model.Rule -import com.tencent.bkrepo.repository.api.NodeClient -import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.analyst.NODE_FULL_PATH import com.tencent.bkrepo.analyst.NODE_NAME import com.tencent.bkrepo.analyst.NODE_SHA256 import com.tencent.bkrepo.analyst.NODE_SIZE import com.tencent.bkrepo.analyst.PROJECT_ID import com.tencent.bkrepo.analyst.REPO -import com.tencent.bkrepo.common.api.message.CommonMessageCode -import com.tencent.bkrepo.common.api.pojo.Response +import com.tencent.bkrepo.common.metadata.service.node.NodeSearchService +import com.tencent.bkrepo.common.mongo.dao.util.Pages +import com.tencent.bkrepo.common.query.model.Rule +import com.tencent.bkrepo.repository.pojo.node.NodeDetail import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Test import org.mockito.kotlin.any @@ -54,7 +52,7 @@ class RequestTest { Assertions.assertEquals(nodes.first().sha256, NODE_SHA256) } - private fun mockNodeClient(): NodeClient { + private fun mockNodeClient(): NodeSearchService { val node = mapOf( NodeDetail::sha256.name to NODE_SHA256, NodeDetail::size.name to NODE_SIZE, @@ -64,10 +62,9 @@ class RequestTest { NodeDetail::name.name to NODE_NAME, NodeDetail::lastModifiedBy.name to "user", ) - val response = Response(CommonMessageCode.SUCCESS.getCode(), null, Pages.buildPage(listOf(node), 0, 1), null) return mock { - on { queryWithoutCount(any()) }.doReturn(response) + on { searchWithoutCount(any()) }.doReturn(Pages.buildPage(listOf(node), 0, 1)) } } } diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/utils/ArchiveUtils.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/utils/ArchiveUtils.kt index f797e7bd8b..e5bb36b93f 100644 --- a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/utils/ArchiveUtils.kt +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/utils/ArchiveUtils.kt @@ -7,7 +7,6 @@ import com.tencent.bkrepo.archive.config.ArchiveProperties import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService import com.tencent.bkrepo.common.storage.config.StorageProperties import com.tencent.bkrepo.common.storage.credentials.StorageCredentials -import com.tencent.bkrepo.repository.api.RepositoryClient import org.slf4j.LoggerFactory import org.springframework.stereotype.Component import java.util.concurrent.ArrayBlockingQueue @@ -20,13 +19,11 @@ import java.util.concurrent.TimeUnit class ArchiveUtils( storageCredentialService: StorageCredentialService, storageProperties: StorageProperties, - repositoryClient: RepositoryClient, archiveProperties: ArchiveProperties, ) { init { Companion.storageCredentialService = storageCredentialService - Companion.repositoryClient = repositoryClient Companion.archiveProperties = archiveProperties defaultStorageCredentials = storageProperties.defaultStorageCredentials() } @@ -34,7 +31,6 @@ class ArchiveUtils( companion object { private lateinit var storageCredentialService: StorageCredentialService private lateinit var defaultStorageCredentials: StorageCredentials - private lateinit var repositoryClient: RepositoryClient private lateinit var archiveProperties: ArchiveProperties private val logger = LoggerFactory.getLogger(ArchiveUtils::class.java) private val storageCredentialsCache: LoadingCache = CacheBuilder.newBuilder() diff --git a/src/backend/archive/biz-archive/src/test/kotlin/com/tencent/bkrepo/archive/core/FileCoreProcessorTest.kt b/src/backend/archive/biz-archive/src/test/kotlin/com/tencent/bkrepo/archive/core/FileCoreProcessorTest.kt index ab43c744ad..f70029159e 100644 --- a/src/backend/archive/biz-archive/src/test/kotlin/com/tencent/bkrepo/archive/core/FileCoreProcessorTest.kt +++ b/src/backend/archive/biz-archive/src/test/kotlin/com/tencent/bkrepo/archive/core/FileCoreProcessorTest.kt @@ -8,8 +8,6 @@ import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.api.FileSystemArtifactFile import com.tencent.bkrepo.common.storage.StorageAutoConfiguration import com.tencent.bkrepo.common.storage.core.StorageService -import com.tencent.bkrepo.repository.api.FileReferenceClient -import com.tencent.bkrepo.repository.api.RepositoryClient import org.junit.jupiter.api.AfterAll import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.BeforeAll @@ -19,7 +17,6 @@ import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.autoconfigure.ImportAutoConfiguration import org.springframework.boot.autoconfigure.task.TaskExecutionAutoConfiguration import org.springframework.boot.test.autoconfigure.data.mongo.DataMongoTest -import org.springframework.boot.test.mock.mockito.MockBean import org.springframework.test.context.TestPropertySource import java.time.LocalDateTime import kotlin.random.Random @@ -34,12 +31,6 @@ class FileCoreProcessorTest @Autowired constructor( private val compressFileRepository: CompressFileRepository, ) : BaseTest() { - @MockBean - lateinit var fileReferenceClient: FileReferenceClient - - @MockBean - lateinit var repositoryClient: RepositoryClient - @BeforeAll fun beforeAll() { initMock() diff --git a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/config/AuthServiceConfig.kt b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/config/AuthServiceConfig.kt index 477cfa0c10..85261a6d3d 100644 --- a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/config/AuthServiceConfig.kt +++ b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/config/AuthServiceConfig.kt @@ -55,7 +55,7 @@ import com.tencent.bkrepo.auth.service.local.PermissionServiceImpl import com.tencent.bkrepo.auth.service.local.RoleServiceImpl import com.tencent.bkrepo.auth.service.local.UserServiceImpl import com.tencent.bkrepo.common.metadata.service.project.ProjectService -import com.tencent.bkrepo.repository.api.RepositoryClient +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.autoconfigure.AutoConfigureOrder import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean @@ -71,7 +71,7 @@ class AuthServiceConfig { @Autowired @Lazy - private lateinit var repositoryClient: RepositoryClient + private lateinit var repositoryService: RepositoryService @Autowired @Lazy @@ -102,7 +102,7 @@ class AuthServiceConfig { userDao, personalPathDao, repoAuthConfigDao, - repositoryClient, + repositoryService, projectService ) } @@ -116,7 +116,6 @@ class AuthServiceConfig { roleRepository: RoleRepository, accountDao: AccountDao, permissionDao: PermissionDao, - repoClient: RepositoryClient ): PermissionService { return BkIamV3PermissionServiceImpl( bkiamV3Service, @@ -126,7 +125,7 @@ class AuthServiceConfig { permissionDao, personalPathDao, repoAuthConfigDao, - repoClient, + repositoryService, projectService ) } @@ -155,7 +154,7 @@ class AuthServiceConfig { bkAuthConfig, bkAuthPipelineService, bkAuthProjectService, - repositoryClient, + repositoryService, projectService, bkiamV3Service ) diff --git a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/bkdevops/DevopsPermissionServiceImpl.kt b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/bkdevops/DevopsPermissionServiceImpl.kt index 04d4d6f9c2..973fbe2c14 100644 --- a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/bkdevops/DevopsPermissionServiceImpl.kt +++ b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/bkdevops/DevopsPermissionServiceImpl.kt @@ -58,7 +58,7 @@ import com.tencent.bkrepo.auth.service.bkiamv3.BkIamV3PermissionServiceImpl import com.tencent.bkrepo.auth.service.bkiamv3.BkIamV3Service import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.common.metadata.service.project.ProjectService -import com.tencent.bkrepo.repository.api.RepositoryClient +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import org.slf4j.LoggerFactory /** @@ -74,7 +74,7 @@ class DevopsPermissionServiceImpl constructor( private val devopsAuthConfig: DevopsAuthConfig, private val devopsPipelineService: DevopsPipelineService, private val devopsProjectService: DevopsProjectService, - repoClient: RepositoryClient, + repositoryService: RepositoryService, projectService: ProjectService, bkIamV3Service: BkIamV3Service ) : BkIamV3PermissionServiceImpl( @@ -85,7 +85,7 @@ class DevopsPermissionServiceImpl constructor( permissionDao, personalPathDao, repoAuthConfigDao, - repoClient, + repositoryService, projectService, ) { diff --git a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/bkiamv3/BkIamV3PermissionServiceImpl.kt b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/bkiamv3/BkIamV3PermissionServiceImpl.kt index c7c7a0ec5d..405af91642 100644 --- a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/bkiamv3/BkIamV3PermissionServiceImpl.kt +++ b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/bkiamv3/BkIamV3PermissionServiceImpl.kt @@ -44,8 +44,9 @@ import com.tencent.bkrepo.auth.service.local.PermissionServiceImpl import com.tencent.bkrepo.auth.util.BkIamV3Utils.convertActionType import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.metadata.service.project.ProjectService -import com.tencent.bkrepo.repository.api.RepositoryClient +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import org.slf4j.LoggerFactory +import java.util.Locale /** * 对接蓝鲸权限中心V3 RBAC @@ -58,7 +59,7 @@ open class BkIamV3PermissionServiceImpl( permissionDao: PermissionDao, personalPathDao: PersonalPathDao, repoAuthConfigDao: RepoAuthConfigDao, - repoClient: RepositoryClient, + repositoryService: RepositoryService, projectService: ProjectService ) : PermissionServiceImpl( roleRepository, @@ -67,7 +68,7 @@ open class BkIamV3PermissionServiceImpl( userDao, personalPathDao, repoAuthConfigDao, - repoClient, + repositoryService, projectService ) { override fun checkPermission(request: CheckPermissionRequest): Boolean { @@ -114,7 +115,7 @@ open class BkIamV3PermissionServiceImpl( userId = uid, projectId = projectId!!, repoName = repoName, - resourceType = resourceType.toLowerCase(), + resourceType = resourceType.lowercase(Locale.getDefault()), action = convertActionType(resourceType, action), resourceId = resourceId, appId = appId @@ -165,7 +166,7 @@ open class BkIamV3PermissionServiceImpl( action = ActionTypeMapping.REPO_VIEW.id() ) return if (pList.contains(StringPool.POUND)) { - repoClient.listRepo(projectId).data?.map { it.name } ?: emptyList() + repositoryService.listRepo(projectId).map { it.name } } else { pList } diff --git a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/bkiamv3/BkIamV3ServiceImpl.kt b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/bkiamv3/BkIamV3ServiceImpl.kt index d0671a012f..70489b6955 100644 --- a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/bkiamv3/BkIamV3ServiceImpl.kt +++ b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/bkiamv3/BkIamV3ServiceImpl.kt @@ -68,11 +68,12 @@ import com.tencent.bkrepo.auth.util.IamGroupUtils import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.message.CommonMessageCode +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.metadata.service.project.ProjectService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.mongo.dao.util.sharding.HashShardingUtils import com.tencent.bkrepo.common.security.util.SecurityUtils -import com.tencent.bkrepo.repository.api.NodeClient -import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.pojo.repo.RepositoryInfo import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired @@ -82,6 +83,7 @@ import org.springframework.context.annotation.Lazy import org.springframework.data.mongodb.core.MongoTemplate import org.springframework.stereotype.Service import java.time.LocalDateTime +import java.util.Locale import java.util.concurrent.TimeUnit @Service @@ -102,7 +104,7 @@ class BkIamV3ServiceImpl( @Autowired @Lazy - private lateinit var nodeClient: NodeClient + private lateinit var nodeService: NodeService @Autowired @Lazy @@ -114,7 +116,7 @@ class BkIamV3ServiceImpl( @Autowired @Lazy - private lateinit var repositoryClient: RepositoryClient + private lateinit var repositoryService: RepositoryService @Value("\${$AUTH_CONFIG_PREFIX.$AUTH_CONFIG_TYPE_NAME}") private var ciAuthServer: String = "" @@ -179,7 +181,7 @@ class BkIamV3ServiceImpl( resourceType, projectId, repoName, path ) val action = BkIamV3Utils.convertActionType(request.resourceType, request.action) - val resourceType = request.resourceType.lowercase() + val resourceType = request.resourceType.lowercase(Locale.getDefault()) if (repoName != null && !checkBkiamv3Config(projectId, repoName)) return null authManagerRepository.findByTypeAndResourceIdAndParentResId( ResourceType.PROJECT, projectId!!, null @@ -299,12 +301,12 @@ class BkIamV3ServiceImpl( } override fun convertRepoResourceId(projectId: String, repoName: String): String? { - return repositoryClient.getRepoInfo(projectId, repoName).data?.id + return repositoryService.getRepoInfo(projectId, repoName)?.id } override fun convertNodeResourceId(projectId: String, repoName: String, fullPath: String): String? { val index = HashShardingUtils.shardingSequenceFor(projectId, 256).toString() - val nodeId = nodeClient.getNodeDetail(projectId, repoName, fullPath).data?.nodeInfo?.id ?: return null + val nodeId = nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, fullPath))?.nodeInfo?.id ?: return null return buildId(nodeId, index) } @@ -488,7 +490,7 @@ class BkIamV3ServiceImpl( repoName: String ): String? { val projectInfo = projectService.getProjectInfo(projectId)!! - val repoDetail = repositoryClient.getRepoInfo(projectId, repoName).data!! + val repoDetail = repositoryService.getRepoInfo(projectId, repoName)!! // 如果已经创建repo管理员，则返回 var repoManagerId = authManagerRepository.findByTypeAndResourceIdAndParentResId( ResourceType.REPO, repoName, projectId @@ -682,7 +684,7 @@ class BkIamV3ServiceImpl( // 赋予权限 try { createRoleGroupMember(defaultGroupType, roleId, members) - val actions = DefaultGroupTypeAndActions.get(defaultGroupType.name.lowercase()).actions + val actions = DefaultGroupTypeAndActions.get(defaultGroupType.name.lowercase(Locale.getDefault())).actions grantGroupPermission(projectResInfo, repoResInfo, roleId, actions) } catch (e: Exception) { managerService.deleteRoleGroupV2(roleId) diff --git a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/bkiamv3/callback/BkiamNodeResourceService.kt b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/bkiamv3/callback/BkiamNodeResourceService.kt index 528e879d6c..1dbb8a603d 100644 --- a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/bkiamv3/callback/BkiamNodeResourceService.kt +++ b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/bkiamv3/callback/BkiamNodeResourceService.kt @@ -35,8 +35,9 @@ import com.tencent.bk.sdk.iam.dto.callback.response.InstanceInfoDTO import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.auth.util.BkIamV3Utils.buildId import com.tencent.bkrepo.auth.util.BkIamV3Utils.splitId +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.mongo.dao.util.sharding.HashShardingUtils -import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.pojo.node.NodeInfo import com.tencent.bkrepo.repository.pojo.node.NodeListOption import com.tencent.bkrepo.repository.pojo.repo.RepositoryInfo @@ -53,7 +54,7 @@ import org.springframework.stereotype.Component */ @Component class BkiamNodeResourceService( - private val nodeClient: NodeClient, + private val nodeService: NodeService, private val mongoTemplate: MongoTemplate ): BkiamResourceBaseService { override fun resourceType(): ResourceType { @@ -126,9 +127,9 @@ class BkiamNodeResourceService( offset = page.offset.toInt() limit = page.limit.toInt() } - val nodePage = nodeClient.listNodePage( - projectId, repoName, path, NodeListOption(pageNumber = offset, pageSize = limit, deep = true) - ).data!! + val nodePage = nodeService.listNodePage( + ArtifactInfo(projectId, repoName, path), NodeListOption(pageNumber = offset, pageSize = limit, deep = true) + ) val nodes = nodePage.records.map { val entity = InstanceInfoDTO() entity.id = buildId(it.id!!, index) diff --git a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/bkiamv3/callback/BkiamRepoResourceService.kt b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/bkiamv3/callback/BkiamRepoResourceService.kt index fa2fcac48f..6caad9e681 100644 --- a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/bkiamv3/callback/BkiamRepoResourceService.kt +++ b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/bkiamv3/callback/BkiamRepoResourceService.kt @@ -34,7 +34,7 @@ import com.tencent.bk.sdk.iam.dto.callback.response.CallbackBaseResponseDTO import com.tencent.bk.sdk.iam.dto.callback.response.InstanceInfoDTO import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.auth.service.bkiamv3.BkiamV3BaseService -import com.tencent.bkrepo.repository.api.RepositoryClient +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.repository.pojo.project.RepoRangeQueryRequest import com.tencent.bkrepo.repository.pojo.repo.RepositoryInfo import org.slf4j.LoggerFactory @@ -46,7 +46,7 @@ import org.springframework.stereotype.Component */ @Component class BkiamRepoResourceService( - private val repositoryClient: RepositoryClient, + private val repositoryService: RepositoryService, val mongoTemplate: MongoTemplate ): BkiamResourceBaseService, BkiamV3BaseService(mongoTemplate) { override fun resourceType(): ResourceType { @@ -97,7 +97,7 @@ class BkiamRepoResourceService( offset = page.offset limit = page.limit.toInt() } - val repoPage = repositoryClient.rangeQuery(RepoRangeQueryRequest(idList, projectId, offset, limit)).data!! + val repoPage = repositoryService.rangeQuery(RepoRangeQueryRequest(idList, projectId, offset, limit)) val repos = repoPage.records.map { val entity = InstanceInfoDTO() entity.id = it!!.id diff --git a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/impl/RepoModeServiceImpl.kt b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/impl/RepoModeServiceImpl.kt index be541f23fc..9d62940279 100644 --- a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/impl/RepoModeServiceImpl.kt +++ b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/impl/RepoModeServiceImpl.kt @@ -37,14 +37,14 @@ import com.tencent.bkrepo.auth.pojo.enums.AccessControlMode import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.auth.pojo.permission.RepoModeStatus import com.tencent.bkrepo.auth.service.RepoModeService -import com.tencent.bkrepo.repository.api.RepositoryClient +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import org.springframework.stereotype.Service @Service class RepoModeServiceImpl( private val repoAuthDao: RepoAuthConfigDao, private val permissionDao: PermissionDao, - private val repoClient: RepositoryClient, + private val repositoryService: RepositoryService, ) : RepoModeService { override fun createOrUpdateConfig( @@ -54,7 +54,7 @@ class RepoModeServiceImpl( officeDenyGroupSet: Set, bkiamv3Check: Boolean ): RepoModeStatus? { - val repoDetail = repoClient.getRepoDetail(projectId, repoName).data ?: return null + val repoDetail = repositoryService.getRepoDetail(projectId, repoName) ?: return null if (repoDetail.public) return null var controlMode = accessControlMode if (accessControlMode == null) { @@ -95,4 +95,4 @@ class RepoModeServiceImpl( ) } -} \ No newline at end of file +} diff --git a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/local/PermissionServiceImpl.kt b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/local/PermissionServiceImpl.kt index 63c9e3dc79..5a9c078f5b 100644 --- a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/local/PermissionServiceImpl.kt +++ b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/local/PermissionServiceImpl.kt @@ -75,7 +75,7 @@ import com.tencent.bkrepo.common.api.constant.ANONYMOUS_USER import com.tencent.bkrepo.common.api.constant.DEVX_ACCESS_FROM_OFFICE import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.metadata.service.project.ProjectService -import com.tencent.bkrepo.repository.api.RepositoryClient +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import org.slf4j.LoggerFactory open class PermissionServiceImpl constructor( @@ -85,7 +85,7 @@ open class PermissionServiceImpl constructor( private val userDao: UserDao, private val personalPathDao: PersonalPathDao, private val repoAuthConfigDao: RepoAuthConfigDao, - val repoClient: RepositoryClient, + val repositoryService: RepositoryService, val projectService: ProjectService ) : PermissionService { @@ -350,7 +350,7 @@ open class PermissionServiceImpl constructor( } fun getAllRepoByProjectId(projectId: String): List { - return repoClient.listRepo(projectId).data?.map { it.name } ?: emptyList() + return repositoryService.listRepo(projectId).map { it.name } } override fun checkPlatformPermission(request: CheckPermissionRequest): Boolean { diff --git a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/local/UserServiceImpl.kt b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/local/UserServiceImpl.kt index e17e4c037c..59bb0ec346 100644 --- a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/local/UserServiceImpl.kt +++ b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/local/UserServiceImpl.kt @@ -53,9 +53,9 @@ import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.common.api.pojo.Page import com.tencent.bkrepo.common.metadata.service.project.ProjectService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.mongo.dao.util.Pages import com.tencent.bkrepo.common.metadata.util.DesensitizedUtils -import com.tencent.bkrepo.repository.api.RepositoryClient import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.dao.DuplicateKeyException @@ -69,7 +69,7 @@ class UserServiceImpl constructor( ) : UserService { @Autowired - lateinit var repoClient: RepositoryClient + lateinit var repositoryService: RepositoryService @Autowired lateinit var projectService: ProjectService @@ -122,7 +122,7 @@ class UserServiceImpl constructor( override fun createUserToRepo(request: CreateUserToRepoRequest): Boolean { logger.info("create user to repo request : [${DesensitizedUtils.toString(request)}]") - repoClient.getRepoInfo(request.projectId, request.repoName).data ?: run { + repositoryService.getRepoInfo(request.projectId, request.repoName) ?: run { logger.warn("repo [${request.projectId}/${request.repoName}] not exist.") throw ErrorCodeException(AuthMessageCode.AUTH_REPO_NOT_EXIST) } diff --git a/src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/pojo/RepositoryId.kt b/src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/pojo/RepositoryId.kt new file mode 100644 index 0000000000..5ebf44a709 --- /dev/null +++ b/src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/pojo/RepositoryId.kt @@ -0,0 +1,12 @@ +package com.tencent.bkrepo.common.artifact.pojo + +import com.tencent.bkrepo.common.api.constant.CharPool + +/** + * 仓库标识类 + */ +data class RepositoryId(val projectId: String, val repoName: String) { + override fun toString(): String { + return StringBuilder(projectId).append(CharPool.SLASH).append(repoName).toString() + } +} diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/ArtifactEventProperties.kt b/src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/properties/ArtifactEventProperties.kt similarity index 97% rename from src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/ArtifactEventProperties.kt rename to src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/properties/ArtifactEventProperties.kt index d285305011..87bc214ed9 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/ArtifactEventProperties.kt +++ b/src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/properties/ArtifactEventProperties.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.common.artifact.event +package com.tencent.bkrepo.common.artifact.properties import org.springframework.boot.context.properties.ConfigurationProperties import java.time.Duration diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/router/RouterControllerProperties.kt b/src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/properties/RouterControllerProperties.kt similarity index 97% rename from src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/router/RouterControllerProperties.kt rename to src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/properties/RouterControllerProperties.kt index 724040cd31..a93024d75d 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/router/RouterControllerProperties.kt +++ b/src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/properties/RouterControllerProperties.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.common.artifact.router +package com.tencent.bkrepo.common.artifact.properties import org.springframework.boot.context.properties.ConfigurationProperties diff --git a/src/backend/common/common-artifact/artifact-cache/src/main/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/ArtifactPreloadPlanServiceImpl.kt b/src/backend/common/common-artifact/artifact-cache/src/main/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/ArtifactPreloadPlanServiceImpl.kt index 941120055b..55f0ced68b 100644 --- a/src/backend/common/common-artifact/artifact-cache/src/main/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/ArtifactPreloadPlanServiceImpl.kt +++ b/src/backend/common/common-artifact/artifact-cache/src/main/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/ArtifactPreloadPlanServiceImpl.kt @@ -33,6 +33,7 @@ import com.tencent.bkrepo.common.api.exception.BadRequestException import com.tencent.bkrepo.common.api.exception.NotFoundException import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.common.api.pojo.Page +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.cache.config.ArtifactPreloadProperties import com.tencent.bkrepo.common.artifact.cache.dao.ArtifactPreloadPlanDao import com.tencent.bkrepo.common.artifact.cache.model.TArtifactPreloadPlan @@ -49,9 +50,9 @@ import com.tencent.bkrepo.common.artifact.cache.service.PreloadPlanExecutor import com.tencent.bkrepo.common.artifact.exception.ArtifactNotFoundException import com.tencent.bkrepo.common.artifact.metrics.ArtifactCacheMetrics import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 +import com.tencent.bkrepo.common.metadata.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.mongo.dao.util.Pages -import com.tencent.bkrepo.repository.api.NodeClient -import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.NodeInfo import com.tencent.bkrepo.repository.pojo.node.NodeListOption @@ -64,8 +65,8 @@ import java.time.format.DateTimeFormatter import java.util.concurrent.TimeUnit class ArtifactPreloadPlanServiceImpl( - private val nodeClient: NodeClient, - private val repositoryClient: RepositoryClient, + private val nodeService: NodeService, + private val repositoryService: RepositoryService, private val strategyService: ArtifactPreloadStrategyService, private val preloadPlanDao: ArtifactPreloadPlanDao, private val preloadStrategies: Map, @@ -112,8 +113,8 @@ class ArtifactPreloadPlanServiceImpl( return } val option = NodeListOption(pageSize = properties.maxNodes, includeFolder = false) - val res = nodeClient.listPageNodeBySha256(sha256, option) - val nodes = res.data?.records ?: return + val res = nodeService.listNodePageBySha256(sha256, option) + val nodes = res.records if (nodes.size >= properties.maxNodes) { // 限制查询出来的最大node数量，避免预加载计划创建时间过久 logger.warn("nodes of sha256[$sha256] exceed max page size[${properties.maxNodes}]") @@ -207,14 +208,14 @@ class ArtifactPreloadPlanServiceImpl( private fun getRepo(key: String): RepositoryInfo { val repoId = key.split(REPO_ID_DELIMITERS) require(repoId.size == 2) - return repositoryClient.getRepoInfo(repoId[0], repoId[1]).data + return repositoryService.getRepoInfo(repoId[0], repoId[1]) ?: throw RuntimeException("repo[$key] was not exists") } private fun buildRepoId(projectId: String, repoName: String) = "${projectId}$REPO_ID_DELIMITERS$repoName" private fun getNode(projectId: String, repoName: String, fullPath: String): NodeDetail { - val node = nodeClient.getNodeDetail(projectId, repoName, fullPath).data + val node = nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, fullPath)) ?: throw ArtifactNotFoundException("$projectId/$repoName$fullPath not found") if (node.folder) { throw BadRequestException(CommonMessageCode.PARAMETER_INVALID, "folder is unsupported") diff --git a/src/backend/common/common-artifact/artifact-cache/src/test/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/ArtifactPreloadBaseServiceTest.kt b/src/backend/common/common-artifact/artifact-cache/src/test/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/ArtifactPreloadBaseServiceTest.kt index 7d46e044c4..5030a769a5 100644 --- a/src/backend/common/common-artifact/artifact-cache/src/test/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/ArtifactPreloadBaseServiceTest.kt +++ b/src/backend/common/common-artifact/artifact-cache/src/test/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/ArtifactPreloadBaseServiceTest.kt @@ -32,6 +32,8 @@ import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.api.FileSystemArtifactFile import com.tencent.bkrepo.common.artifact.cache.config.ArtifactPreloadConfiguration import com.tencent.bkrepo.common.artifact.cache.config.ArtifactPreloadProperties +import com.tencent.bkrepo.common.metadata.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService import com.tencent.bkrepo.common.storage.StorageAutoConfiguration import com.tencent.bkrepo.common.storage.config.StorageProperties @@ -39,8 +41,6 @@ import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.storage.core.cache.CacheStorageService import com.tencent.bkrepo.common.storage.core.locator.FileLocator import com.tencent.bkrepo.common.storage.credentials.StorageCredentials -import com.tencent.bkrepo.repository.api.NodeClient -import com.tencent.bkrepo.repository.api.RepositoryClient import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.TestInstance import org.springframework.boot.autoconfigure.ImportAutoConfiguration @@ -66,11 +66,12 @@ open class ArtifactPreloadBaseServiceTest( protected val fileLocator: FileLocator, protected val storageProperties: StorageProperties, ) { + @MockBean - protected lateinit var nodeClient: NodeClient + protected lateinit var nodeService: NodeService @MockBean - protected lateinit var repositoryClient: RepositoryClient + protected lateinit var repositoryService: RepositoryService @MockBean protected lateinit var storageCredentialService: StorageCredentialService diff --git a/src/backend/common/common-artifact/artifact-cache/src/test/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/ArtifactPreloadPlanServiceImplTest.kt b/src/backend/common/common-artifact/artifact-cache/src/test/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/ArtifactPreloadPlanServiceImplTest.kt index 105c5d2112..a08bdddc42 100644 --- a/src/backend/common/common-artifact/artifact-cache/src/test/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/ArtifactPreloadPlanServiceImplTest.kt +++ b/src/backend/common/common-artifact/artifact-cache/src/test/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/ArtifactPreloadPlanServiceImplTest.kt @@ -27,7 +27,6 @@ package com.tencent.bkrepo.common.artifact.cache.service.impl -import com.tencent.bkrepo.common.api.pojo.Response import com.tencent.bkrepo.common.artifact.cache.UT_PROJECT_ID import com.tencent.bkrepo.common.artifact.cache.UT_REPO_NAME import com.tencent.bkrepo.common.artifact.cache.UT_SHA256 @@ -58,6 +57,7 @@ import org.junit.jupiter.api.DisplayName import org.junit.jupiter.api.Test import org.mockito.ArgumentMatchers.anyString import org.mockito.kotlin.any +import org.mockito.kotlin.anyOrNull import org.mockito.kotlin.whenever import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.mock.mockito.MockBean @@ -99,9 +99,7 @@ class ArtifactPreloadPlanServiceImplTest @Autowired constructor( @Test fun testCreatePlan() { val node = NodeDetail(buildNodeInfo(UT_PROJECT_ID, UT_REPO_NAME)) - whenever(nodeClient.getNodeDetail(anyString(), anyString(), anyString())).thenReturn( - Response(0, data = node) - ) + whenever(nodeService.getNodeDetail(any(), anyOrNull())).thenReturn(node) val executeTime = System.currentTimeMillis() val request = ArtifactPreloadPlanCreateRequest( projectId = UT_PROJECT_ID, @@ -137,12 +135,8 @@ class ArtifactPreloadPlanServiceImplTest @Autowired constructor( node.copy(fullPath = "test.txt"), node.copy(fullPath = "test2.txt"), ) - whenever(nodeClient.listPageNodeBySha256(anyString(), any())).thenReturn( - Response( - 0, - "", - Pages.ofResponse(Pages.ofRequest(0, 2000), nodes.size.toLong(), nodes) - ) + whenever(nodeService.listNodePageBySha256(anyString(), any())).thenReturn( + Pages.ofResponse(Pages.ofRequest(0, 2000), nodes.size.toLong(), nodes) ) preloadPlanService.generatePlan(null, UT_SHA256) @@ -183,12 +177,8 @@ class ArtifactPreloadPlanServiceImplTest @Autowired constructor( for (i in 0..1000) { nodes.add(buildNodeInfo()) } - whenever(nodeClient.listPageNodeBySha256(anyString(), any())).thenReturn( - Response( - 0, - "", - Pages.ofResponse(Pages.ofRequest(0, 2000), nodes.size.toLong(), nodes) - ) + whenever(nodeService.listNodePageBySha256(anyString(), any())).thenReturn( + Pages.ofResponse(Pages.ofRequest(0, 2000), nodes.size.toLong(), nodes) ) preloadPlanService.generatePlan(null, UT_SHA256) val plans = preloadPlanService.plans(UT_PROJECT_ID, UT_REPO_NAME, Pages.ofRequest(0, 10)).records @@ -224,12 +214,12 @@ class ArtifactPreloadPlanServiceImplTest @Autowired constructor( } private fun resetMock(projectId: String = UT_PROJECT_ID, repoName: String = UT_REPO_NAME) { - whenever(repositoryClient.getRepoInfo(anyString(), anyString())).thenReturn( - Response(0, "", buildRepo(projectId = projectId, repoName = repoName)) + whenever(repositoryService.getRepoInfo(anyString(), anyString(), anyOrNull())).thenReturn( + buildRepo(projectId = projectId, repoName = repoName) ) val nodes = listOf(buildNodeInfo(projectId, repoName)) - whenever(nodeClient.listPageNodeBySha256(anyString(), any())).thenReturn( - Response(0, "", Pages.ofResponse(Pages.ofRequest(0, 20), 1L, nodes)) + whenever(nodeService.listNodePageBySha256(anyString(), any())).thenReturn( + Pages.ofResponse(Pages.ofRequest(0, 20), 1L, nodes) ) } diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/ArtifactEventConfiguration.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/ArtifactEventConfiguration.kt index b96d5dd072..ccbba9fe20 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/ArtifactEventConfiguration.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/ArtifactEventConfiguration.kt @@ -34,6 +34,7 @@ package com.tencent.bkrepo.common.artifact.event import com.tencent.bkrepo.common.artifact.event.listener.ArtifactDownloadListener import com.tencent.bkrepo.common.artifact.event.listener.ArtifactTransferListener import com.tencent.bkrepo.common.artifact.event.listener.ArtifactWebHookListener +import com.tencent.bkrepo.common.artifact.properties.ArtifactEventProperties import com.tencent.bkrepo.common.artifact.webhook.WebHookService import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.context.annotation.Configuration diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/listener/ArtifactDownloadListener.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/listener/ArtifactDownloadListener.kt index c0b6b6240a..1268b7f2ef 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/listener/ArtifactDownloadListener.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/listener/ArtifactDownloadListener.kt @@ -33,7 +33,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder import com.tencent.bkrepo.common.api.constant.HttpHeaders import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.artifact.event.ArtifactDownloadedEvent -import com.tencent.bkrepo.common.artifact.event.ArtifactEventProperties +import com.tencent.bkrepo.common.artifact.properties.ArtifactEventProperties import com.tencent.bkrepo.common.artifact.event.node.NodeDownloadedEvent import com.tencent.bkrepo.common.artifact.event.node.NodeUpdateAccessDateEvent import com.tencent.bkrepo.common.artifact.repository.context.ArtifactClient diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManager.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManager.kt index 312be98fca..3993f58136 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManager.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManager.kt @@ -36,12 +36,12 @@ import com.tencent.bkrepo.common.artifact.stream.EmptyInputStream import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.artifact.util.http.HttpRangeUtils.resolveRange import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.HttpContextHolder.getRequestOrNull import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.innercos.http.HttpMethod -import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.NodeInfo import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest @@ -54,7 +54,7 @@ import org.slf4j.LoggerFactory * * 虽然[StorageService]提供了构件存储服务，但保存一个文件节点需要两步操作: * 1. [StorageService]保存文件数据 - * 2. [NodeClient]微服务调用创建文件节点 + * 2. [NodeService]创建文件节点 * 这样会存在几个问题: * 1. 每个地方都会进行同样的操作，增加代码重复率 * 2. 不支持事务，如果文件保存成功，但节点创建失败，会导致产生垃圾文件并且无法清理 @@ -65,7 +65,7 @@ import org.slf4j.LoggerFactory @Suppress("TooGenericExceptionCaught") class StorageManager( private val storageService: StorageService, - private val nodeClient: NodeClient, + private val nodeService: NodeService, private val fileReferenceService: FileReferenceService, private val nodeResourceFactory: NodeResourceFactory, private val pluginManager: PluginManager, @@ -82,7 +82,7 @@ class StorageManager( ): NodeDetail { val affectedCount = storageService.store(request.sha256!!, artifactFile, storageCredentials) try { - return nodeClient.createNode(request).data!! + return nodeService.createNode(request) } catch (exception: Exception) { if (affectedCount == 1) { try { diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/resource/LocalNodeResource.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/resource/LocalNodeResource.kt index 9ec4c5d55b..7234a3ffa9 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/resource/LocalNodeResource.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/resource/LocalNodeResource.kt @@ -32,6 +32,7 @@ import com.tencent.bkrepo.archive.request.ArchiveFileRequest import com.tencent.bkrepo.archive.request.UncompressFileRequest import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.message.CommonMessageCode +import com.tencent.bkrepo.common.artifact.pojo.RepositoryId import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream import com.tencent.bkrepo.common.artifact.stream.Range @@ -137,7 +138,7 @@ class LocalNodeResource( } } // 如果是异步或者请求上下文找不到，则通过查询，并进行缓存 - val repositoryId = ArtifactContextHolder.RepositoryId( + val repositoryId = RepositoryId( projectId = projectId, repoName = repoName, ) diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/context/ArtifactClient.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/context/ArtifactClient.kt index adf2218a6b..b310c608c1 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/context/ArtifactClient.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/context/ArtifactClient.kt @@ -27,10 +27,12 @@ package com.tencent.bkrepo.common.artifact.repository.context -import com.tencent.bkrepo.repository.api.NodeClient -import com.tencent.bkrepo.repository.api.RepositoryClient +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo +import com.tencent.bkrepo.common.metadata.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.NodeInfo +import com.tencent.bkrepo.repository.pojo.node.NodeListOption import com.tencent.bkrepo.repository.pojo.node.service.NodeUpdateAccessDateRequest import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail import org.springframework.beans.factory.annotation.Autowired @@ -39,21 +41,21 @@ import org.springframework.beans.factory.annotation.Autowired open class ArtifactClient { @Autowired - private lateinit var repositoryClient: RepositoryClient + private lateinit var repositoryService: RepositoryService @Autowired - private lateinit var nodeClient: NodeClient + private lateinit var nodeService: NodeService open fun getRepositoryDetailOrNull( projectId: String, repoName: String, repoType: String ): RepositoryDetail? { - return repositoryClient.getRepoDetail(projectId, repoName, repoType).data + return repositoryService.getRepoDetail(projectId, repoName, repoType) } open fun getNodeDetailOrNull(projectId: String, repoName: String, fullPath: String): NodeDetail? { - return nodeClient.getNodeDetail(projectId, repoName, fullPath).data + return nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, fullPath)) } open fun listNode( @@ -63,10 +65,13 @@ open class ArtifactClient { includeFolder: Boolean, deep: Boolean ): List? { - return nodeClient.listNode(projectId, repoName, fullPath, includeFolder, deep).data + return nodeService.listNode( + ArtifactInfo(projectId, repoName, fullPath), + NodeListOption(includeFolder = includeFolder, deep = deep) + ) } open fun updateAccessDate(request: NodeUpdateAccessDateRequest) { - nodeClient.updateNodeAccessDate(request) + nodeService.updateNodeAccessDate(request) } } diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/context/ArtifactContextHolder.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/context/ArtifactContextHolder.kt index 414b71f9a5..e2bf61c7a9 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/context/ArtifactContextHolder.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/context/ArtifactContextHolder.kt @@ -44,6 +44,7 @@ import com.tencent.bkrepo.common.artifact.constant.REPO_NAME import com.tencent.bkrepo.common.artifact.constant.REPO_RATE_LIMIT_KEY import com.tencent.bkrepo.common.artifact.exception.RepoNotFoundException import com.tencent.bkrepo.common.artifact.pojo.RepositoryCategory +import com.tencent.bkrepo.common.artifact.pojo.RepositoryId import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.repository.composite.CompositeRepository import com.tencent.bkrepo.common.artifact.repository.core.ArtifactRepository @@ -66,7 +67,7 @@ class ArtifactContextHolder( compositeRepository: CompositeRepository, proxyRepository: ProxyRepository, artifactClient: ArtifactClient, - private val httpAuthSecurity: ObjectProvider + httpAuthSecurity: ObjectProvider ) { init { @@ -315,13 +316,4 @@ class ArtifactContextHolder( } } } - - /** - * 仓库标识类 - */ - data class RepositoryId(val projectId: String, val repoName: String) { - override fun toString(): String { - return StringBuilder(projectId).append(CharPool.SLASH).append(repoName).toString() - } - } } diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/AbstractArtifactRepository.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/AbstractArtifactRepository.kt index 008cb7738e..3d8bfb7770 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/AbstractArtifactRepository.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/AbstractArtifactRepository.kt @@ -54,6 +54,9 @@ import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactChannel import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResourceWriter import com.tencent.bkrepo.common.artifact.util.PackageKeys +import com.tencent.bkrepo.common.metadata.service.node.NodeSearchService +import com.tencent.bkrepo.common.metadata.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.HeaderUtils import com.tencent.bkrepo.common.service.util.HttpContextHolder @@ -61,15 +64,14 @@ import com.tencent.bkrepo.common.service.util.LocaleMessageUtils import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.storage.monitor.Throughput import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier -import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.PackageClient import com.tencent.bkrepo.repository.api.PackageDownloadsClient -import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.pojo.download.PackageDownloadRecord import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.ApplicationEventPublisher import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor +import java.util.Locale /** * 构件仓库抽象类 @@ -80,10 +82,13 @@ import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor abstract class AbstractArtifactRepository : ArtifactRepository { @Autowired - lateinit var nodeClient: NodeClient + lateinit var nodeService: NodeService @Autowired - lateinit var repositoryClient: RepositoryClient + lateinit var nodeSearchService: NodeSearchService + + @Autowired + lateinit var repositoryService: RepositoryService @Autowired lateinit var packageClient: PackageClient @@ -329,7 +334,7 @@ abstract class AbstractArtifactRepository : ArtifactRepository { private fun publishPackageDownloadEvent(context: ArtifactDownloadContext, record: PackageDownloadRecord) { if (context.repositoryDetail.type != RepositoryType.GENERIC) { val packageType = context.repositoryDetail.type.name - val packageName = PackageKeys.resolveName(packageType.toLowerCase(), record.packageKey) + val packageName = PackageKeys.resolveName(packageType.lowercase(Locale.getDefault()), record.packageKey) publisher.publishEvent( VersionDownloadEvent( projectId = record.projectId, diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/redirect/LinkNodeRedirectService.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/redirect/LinkNodeRedirectService.kt index b96d7a479e..7c646329d3 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/redirect/LinkNodeRedirectService.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/redirect/LinkNodeRedirectService.kt @@ -35,8 +35,8 @@ import com.tencent.bkrepo.common.api.constant.HttpStatus import com.tencent.bkrepo.common.artifact.constant.METADATA_KEY_LINK_FULL_PATH import com.tencent.bkrepo.common.artifact.constant.METADATA_KEY_LINK_PROJECT import com.tencent.bkrepo.common.artifact.constant.METADATA_KEY_LINK_REPO +import com.tencent.bkrepo.common.artifact.pojo.RepositoryId import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder -import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder.RepositoryId import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext import com.tencent.bkrepo.common.storage.config.StorageProperties import com.tencent.bkrepo.repository.pojo.node.NodeDetail diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/remote/RemoteRepository.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/remote/RemoteRepository.kt index 54b4844a99..d0fffb8caa 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/remote/RemoteRepository.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/remote/RemoteRepository.kt @@ -143,7 +143,7 @@ abstract class RemoteRepository : AbstractArtifactRepository() { */ open fun findCacheNodeDetail(context: ArtifactDownloadContext): NodeDetail? { with(context) { - return nodeClient.getNodeDetail(projectId, repoName, artifactInfo.getArtifactFullPath()).data + return nodeService.getNodeDetail(artifactInfo) } } diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/virtual/VirtualRepository.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/virtual/VirtualRepository.kt index c64e0171fc..ba2a9ab3a7 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/virtual/VirtualRepository.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/virtual/VirtualRepository.kt @@ -112,7 +112,7 @@ abstract class VirtualRepository : AbstractArtifactRepository() { traversedList.add(repoIdentify) try { permissionManager.checkRepoPermission(PermissionAction.READ, repoIdentify.projectId, repoIdentify.name) - val subRepoDetail = repositoryClient.getRepoDetail(repoIdentify.projectId, repoIdentify.name).data!! + val subRepoDetail = repositoryService.getRepoDetail(repoIdentify.projectId, repoIdentify.name)!! val repository = ArtifactContextHolder.getRepository(subRepoDetail.category) val subContext = context.copy(subRepoDetail) action(subContext, repository)?.let { return it } diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/router/ArtifactRouterControllerConfiguration.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/router/ArtifactRouterControllerConfiguration.kt index b5e39712ed..ecc107c9aa 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/router/ArtifactRouterControllerConfiguration.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/router/ArtifactRouterControllerConfiguration.kt @@ -27,6 +27,7 @@ package com.tencent.bkrepo.common.artifact.router +import com.tencent.bkrepo.common.artifact.properties.RouterControllerProperties import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.context.annotation.Configuration import org.springframework.context.annotation.Import diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/router/RouterControllerAspect.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/router/RouterControllerAspect.kt index fe84a2830c..c655d022c5 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/router/RouterControllerAspect.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/router/RouterControllerAspect.kt @@ -28,6 +28,7 @@ package com.tencent.bkrepo.common.artifact.router import com.tencent.bkrepo.common.artifact.api.ArtifactInfo +import com.tencent.bkrepo.common.artifact.properties.RouterControllerProperties import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.router.api.RouterControllerClient diff --git a/src/backend/common/common-artifact/artifact-service/src/test/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManagerTest.kt b/src/backend/common/common-artifact/artifact-service/src/test/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManagerTest.kt index a6b75c7239..10805b49ea 100644 --- a/src/backend/common/common-artifact/artifact-service/src/test/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManagerTest.kt +++ b/src/backend/common/common-artifact/artifact-service/src/test/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManagerTest.kt @@ -1,7 +1,6 @@ package com.tencent.bkrepo.common.artifact.manager import com.tencent.bkrepo.common.api.constant.StringPool -import com.tencent.bkrepo.common.api.pojo.Response import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.api.FileSystemArtifactFile import com.tencent.bkrepo.common.artifact.util.Constant.UT_PROJECT_ID @@ -9,8 +8,8 @@ import com.tencent.bkrepo.common.artifact.util.Constant.UT_REPO_NAME import com.tencent.bkrepo.common.artifact.util.Constant.UT_SHA256 import com.tencent.bkrepo.common.artifact.util.Constant.UT_USER import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.storage.core.StorageService -import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.NodeInfo import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest @@ -53,7 +52,7 @@ class StorageManagerTest @Autowired constructor( private lateinit var fileReferenceService: FileReferenceService @MockBean - private lateinit var nodeClient: NodeClient + private lateinit var nodeService: NodeService @MockBean private lateinit var pluginManager: PluginManager @@ -68,8 +67,8 @@ class StorageManagerTest @Autowired constructor( fun beforeEach() { whenever(storageService.store(anyString(), any(), anyOrNull(), anyOrNull())) .thenReturn(1) - whenever(nodeClient.createNode(any())) - .thenReturn(Response(code = 0, data = buildNodeDetail(UT_SHA256))) + whenever(nodeService.createNode(any())) + .thenReturn(buildNodeDetail(UT_SHA256)) whenever(fileReferenceService.increment(anyString(), anyOrNull(), any())) .thenReturn(true) } @@ -77,7 +76,7 @@ class StorageManagerTest @Autowired constructor( @Test fun testStoreSuccess() { store() - verify(nodeClient, times(1)).createNode(any()) + verify(nodeService, times(1)).createNode(any()) verify(fileReferenceService, times(0)).increment(anyString(), anyOrNull(), any()) } @@ -93,7 +92,7 @@ class StorageManagerTest @Autowired constructor( // store failed assertThrows { store() } - verify(nodeClient, times(0)).createNode(any()) + verify(nodeService, times(0)).createNode(any()) verify(fileReferenceService, times(0)).increment(anyString(), anyOrNull(), any()) // reset mock @@ -103,11 +102,11 @@ class StorageManagerTest @Autowired constructor( @Test fun `test create node failed`() { // mock - whenever(nodeClient.createNode(any())).then { throw RuntimeException() } + whenever(nodeService.createNode(any())).then { throw RuntimeException() } // store failed assertThrows { store() } - verify(nodeClient, times(1)).createNode(any()) + verify(nodeService, times(1)).createNode(any()) verify(fileReferenceService, times(1)).increment(anyString(), anyOrNull(), any()) } diff --git a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/pojo/node/NodeRestoreOption.kt b/src/backend/common/common-metadata/metadata-api/src/main/kotlin/com/tencent/bkrepo/common/metadata/pojo/node/NodeRestoreOption.kt similarity index 97% rename from src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/pojo/node/NodeRestoreOption.kt rename to src/backend/common/common-metadata/metadata-api/src/main/kotlin/com/tencent/bkrepo/common/metadata/pojo/node/NodeRestoreOption.kt index c4ea5dfc1b..a4ee23d4c7 100644 --- a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/pojo/node/NodeRestoreOption.kt +++ b/src/backend/common/common-metadata/metadata-api/src/main/kotlin/com/tencent/bkrepo/common/metadata/pojo/node/NodeRestoreOption.kt @@ -29,7 +29,7 @@ * SOFTWARE. */ -package com.tencent.bkrepo.repository.pojo.node +package com.tencent.bkrepo.common.metadata.pojo.node import io.swagger.annotations.ApiModel import io.swagger.annotations.ApiModelProperty diff --git a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/pojo/node/service/NodeRestoreRequest.kt b/src/backend/common/common-metadata/metadata-api/src/main/kotlin/com/tencent/bkrepo/common/metadata/pojo/node/NodeRestoreRequest.kt similarity index 93% rename from src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/pojo/node/service/NodeRestoreRequest.kt rename to src/backend/common/common-metadata/metadata-api/src/main/kotlin/com/tencent/bkrepo/common/metadata/pojo/node/NodeRestoreRequest.kt index 902d1f819a..b2be5c2a84 100644 --- a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/pojo/node/service/NodeRestoreRequest.kt +++ b/src/backend/common/common-metadata/metadata-api/src/main/kotlin/com/tencent/bkrepo/common/metadata/pojo/node/NodeRestoreRequest.kt @@ -25,10 +25,9 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.repository.pojo.node.service +package com.tencent.bkrepo.common.metadata.pojo.node import com.tencent.bkrepo.common.artifact.api.ArtifactInfo -import com.tencent.bkrepo.repository.pojo.node.NodeRestoreOption /** * 节点恢复请求 diff --git a/src/backend/common/common-metadata/metadata-api/src/main/kotlin/com/tencent/bkrepo/common/metadata/pojo/node/RestoreContext.kt b/src/backend/common/common-metadata/metadata-api/src/main/kotlin/com/tencent/bkrepo/common/metadata/pojo/node/RestoreContext.kt new file mode 100644 index 0000000000..04debe38e4 --- /dev/null +++ b/src/backend/common/common-metadata/metadata-api/src/main/kotlin/com/tencent/bkrepo/common/metadata/pojo/node/RestoreContext.kt @@ -0,0 +1,15 @@ +package com.tencent.bkrepo.common.metadata.pojo.node + +import java.time.LocalDateTime + +data class RestoreContext( + val projectId: String, + val repoName: String, + val rootFullPath: String, + val deletedTime: LocalDateTime, + val conflictStrategy: ConflictStrategy, + val operator: String, + var restoreCount: Long = 0L, + var conflictCount: Long = 0L, + var skipCount: Long = 0L +) diff --git a/src/backend/common/common-metadata/metadata-service/build.gradle.kts b/src/backend/common/common-metadata/metadata-service/build.gradle.kts index ee66dc1f72..a706c744f2 100644 --- a/src/backend/common/common-metadata/metadata-service/build.gradle.kts +++ b/src/backend/common/common-metadata/metadata-service/build.gradle.kts @@ -38,6 +38,10 @@ dependencies { } api(project(":common:common-service:service-base")) api(project(":common:common-stream")) + api(project(":common:common-query:query-mongo")) + api(project(":archive:api-archive")) + api(project(":router-controller:api-router-controller")) + api(project(":fs:api-fs-server")) compileOnly(project(":common:common-mongo-reactive")) compileOnly(project(":common:common-mongo")) diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/MetadataAutoConfiguration.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/MetadataAutoConfiguration.kt index 3d52bd5787..971602d22e 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/MetadataAutoConfiguration.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/MetadataAutoConfiguration.kt @@ -27,6 +27,8 @@ package com.tencent.bkrepo.common.metadata +import com.tencent.bkrepo.common.artifact.properties.ArtifactEventProperties +import com.tencent.bkrepo.common.artifact.properties.RouterControllerProperties import com.tencent.bkrepo.common.metadata.config.RepositoryProperties import com.tencent.bkrepo.common.metadata.properties.OperateProperties import com.tencent.bkrepo.common.metadata.properties.ProjectUsageStatisticsProperties @@ -43,6 +45,8 @@ import org.springframework.context.annotation.Configuration StorageProperties::class, OperateProperties::class, ProjectUsageStatisticsProperties::class, + RouterControllerProperties::class, + ArtifactEventProperties::class, RepositoryProperties::class, ) class MetadataAutoConfiguration diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/client/RAuthClient.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/client/RAuthClient.kt index 84cce81e57..c58e70fffc 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/client/RAuthClient.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/client/RAuthClient.kt @@ -29,6 +29,7 @@ package com.tencent.bkrepo.common.metadata.client import com.tencent.bkrepo.auth.pojo.oauth.AuthorizationGrantType import com.tencent.bkrepo.auth.pojo.permission.CheckPermissionRequest +import com.tencent.bkrepo.auth.pojo.permission.ListPathResult import com.tencent.bkrepo.auth.pojo.user.CreateUserRequest import com.tencent.bkrepo.auth.pojo.user.CreateUserToProjectRequest import com.tencent.bkrepo.auth.pojo.user.User @@ -113,6 +114,16 @@ interface RAuthClient { @RequestParam userId: String ): Mono>> + @GetMapping("/permission/path/list") + fun listPermissionPath( + @ApiParam(value = "用户ID") + @RequestParam userId: String, + @ApiParam(value = "项目ID") + @RequestParam projectId: String, + @ApiParam(value = "仓库名称") + @RequestParam repoName: String + ): Mono> + @PostMapping("/bkiamv3/rbac/group/check") fun getExistRbacDefaultGroupProjectIds( @ApiParam(value = "项目ID列表") diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/client/RRepositoryClient.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/client/RRepositoryClient.kt deleted file mode 100644 index 530e4d9afa..0000000000 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/client/RRepositoryClient.kt +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. - * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. - * - * A copy of the MIT License is included in this file. - * - * - * Terms of the MIT License: - * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT - * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN - * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package com.tencent.bkrepo.common.metadata.client - -import com.tencent.bkrepo.common.api.constant.REPOSITORY_SERVICE_NAME -import com.tencent.bkrepo.common.api.pojo.Page -import com.tencent.bkrepo.common.api.pojo.Response -import com.tencent.bkrepo.repository.pojo.node.NodeDeleteResult -import com.tencent.bkrepo.repository.pojo.node.NodeDetail -import com.tencent.bkrepo.repository.pojo.node.NodeInfo -import com.tencent.bkrepo.repository.pojo.node.NodeListOption -import com.tencent.bkrepo.repository.pojo.node.NodeSizeInfo -import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest -import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest -import com.tencent.bkrepo.repository.pojo.node.service.NodeLinkRequest -import com.tencent.bkrepo.repository.pojo.node.service.NodeRenameRequest -import com.tencent.bkrepo.repository.pojo.node.service.NodeSetLengthRequest -import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail -import org.springframework.web.bind.annotation.DeleteMapping -import org.springframework.web.bind.annotation.GetMapping -import org.springframework.web.bind.annotation.PathVariable -import org.springframework.web.bind.annotation.PostMapping -import org.springframework.web.bind.annotation.PutMapping -import org.springframework.web.bind.annotation.RequestBody -import org.springframework.web.bind.annotation.RequestMapping -import org.springframework.web.bind.annotation.RequestParam -import reactivefeign.spring.config.ReactiveFeignClient -import reactor.core.publisher.Mono - -@ReactiveFeignClient(REPOSITORY_SERVICE_NAME) -@RequestMapping("/service") -interface RRepositoryClient { - - @GetMapping("/node/detail/{projectId}/{repoName}") - fun getNodeDetail( - @PathVariable projectId: String, - @PathVariable repoName: String, - @RequestParam fullPath: String - ): Mono> - - @PostMapping("/node/page/{projectId}/{repoName}") - fun listNodePage( - @PathVariable projectId: String, - @PathVariable repoName: String, - @RequestParam path: String, - @RequestBody option: NodeListOption = NodeListOption() - ): Mono>> - - @DeleteMapping("/node/delete") - fun deleteNode(@RequestBody nodeDeleteRequest: NodeDeleteRequest): Mono> - - @PostMapping("/node/rename") - fun renameNode(@RequestBody nodeRenameRequest: NodeRenameRequest): Mono> - - @PostMapping("/node/create") - fun createNode(@RequestBody nodeCreateRequest: NodeCreateRequest): Mono> - - @PostMapping("/node/fs/create") - fun createFsNode(@RequestBody nodeCreateRequest: NodeCreateRequest): Mono> - - @PutMapping("/node/fs/length") - fun setLength(@RequestBody nodeSetLengthRequest: NodeSetLengthRequest): Mono> - - @GetMapping("/node/size/{projectId}/{repoName}") - fun computeSize( - @PathVariable projectId: String, - @PathVariable repoName: String, - @RequestParam fullPath: String, - @RequestParam estimated: Boolean = false - ): Mono> - - @PostMapping("/node/link") - fun link(@RequestBody nodeLinkRequest: NodeLinkRequest): Mono> - - @GetMapping("/repo/detail/{projectId}/{repoName}") - fun getRepoDetail( - @PathVariable projectId: String, - @PathVariable repoName: String, - @RequestParam type: String? = null - ): Mono> - - @GetMapping("/repo/stat/{projectId}/{repoName}") - fun statRepo( - @PathVariable projectId: String, - @PathVariable repoName: String, - ): Mono> -} diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/EventAuditListener.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/listener/EventAuditListener.kt similarity index 92% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/EventAuditListener.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/listener/EventAuditListener.kt index da6601d352..7335360131 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/EventAuditListener.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/listener/EventAuditListener.kt @@ -25,11 +25,13 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.repository.listener +package com.tencent.bkrepo.common.metadata.listener import com.tencent.bkrepo.common.artifact.event.base.ArtifactEvent +import com.tencent.bkrepo.common.metadata.condition.SyncCondition import com.tencent.bkrepo.common.metadata.service.log.OperateLogService import com.tencent.bkrepo.common.service.util.HttpContextHolder +import org.springframework.context.annotation.Conditional import org.springframework.context.event.EventListener import org.springframework.stereotype.Component @@ -37,6 +39,7 @@ import org.springframework.stereotype.Component * 事件审计记录监听器 */ @Component +@Conditional(SyncCondition::class) class EventAuditListener( private val operateLogService: OperateLogService ) { diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/EventStreamListener.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/listener/EventStreamListener.kt similarity index 97% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/EventStreamListener.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/listener/EventStreamListener.kt index fe109b06ca..51092aaf8f 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/EventStreamListener.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/listener/EventStreamListener.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.repository.listener +package com.tencent.bkrepo.common.metadata.listener import com.tencent.bkrepo.common.artifact.event.base.ArtifactEvent import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/NodeModifyEventListener.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/listener/NodeModifyEventListener.kt similarity index 98% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/NodeModifyEventListener.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/listener/NodeModifyEventListener.kt index 4c156e9751..bd84c0ee80 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/NodeModifyEventListener.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/listener/NodeModifyEventListener.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.repository.listener +package com.tencent.bkrepo.common.metadata.listener import com.google.common.cache.CacheBuilder import com.google.common.cache.CacheLoader @@ -45,12 +45,14 @@ import com.tencent.bkrepo.common.artifact.event.node.NodeMovedEvent import com.tencent.bkrepo.common.artifact.event.node.NodeRenamedEvent import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.common.artifact.path.PathUtils.combineFullPath +import com.tencent.bkrepo.common.metadata.condition.SyncCondition import com.tencent.bkrepo.common.mongo.dao.AbstractMongoDao import com.tencent.bkrepo.common.mongo.dao.util.Pages import com.tencent.bkrepo.common.metadata.dao.node.NodeDao import com.tencent.bkrepo.common.metadata.model.TNode -import com.tencent.bkrepo.repository.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.node.NodeService import org.slf4j.LoggerFactory +import org.springframework.context.annotation.Conditional import org.springframework.context.event.EventListener import org.springframework.data.domain.Sort import org.springframework.data.mongodb.core.query.Query @@ -69,6 +71,7 @@ import java.util.concurrent.atomic.LongAdder * 节点事件监听，用户统计目录size以及目录下文件个数 */ @Component +@Conditional(SyncCondition::class) class NodeModifyEventListener( private val nodeService: NodeService, private val nodeDao: NodeDao, @@ -446,3 +449,4 @@ class NodeModifyEventListener( private val IGNORE_PROJECT_PREFIX_LIST = listOf(CODE_PROJECT_PREFIX, CLOSED_SOURCE_PREFIX) } } + diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/NodeUpdateAccessDateEventListener.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/listener/NodeUpdateAccessDateEventListener.kt similarity index 94% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/NodeUpdateAccessDateEventListener.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/listener/NodeUpdateAccessDateEventListener.kt index dfa8a9b565..7c20bce953 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/NodeUpdateAccessDateEventListener.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/listener/NodeUpdateAccessDateEventListener.kt @@ -25,15 +25,17 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.repository.listener +package com.tencent.bkrepo.common.metadata.listener -import com.tencent.bkrepo.common.artifact.event.ArtifactEventProperties import com.tencent.bkrepo.common.artifact.event.base.ArtifactEvent import com.tencent.bkrepo.common.artifact.event.base.EventType -import com.tencent.bkrepo.common.mongo.constant.ID +import com.tencent.bkrepo.common.artifact.properties.ArtifactEventProperties +import com.tencent.bkrepo.common.metadata.condition.SyncCondition import com.tencent.bkrepo.common.metadata.dao.node.NodeDao import com.tencent.bkrepo.common.metadata.model.TNode +import com.tencent.bkrepo.common.mongo.constant.ID import org.slf4j.LoggerFactory +import org.springframework.context.annotation.Conditional import org.springframework.data.mongodb.core.query.Query import org.springframework.data.mongodb.core.query.Update import org.springframework.data.mongodb.core.query.isEqualTo @@ -48,6 +50,7 @@ import java.time.format.DateTimeFormatter * 消费基于MQ传递的事件去更新对应access date */ @Component +@Conditional(SyncCondition::class) class NodeUpdateAccessDateEventListener( private val nodeDao: NodeDao, private val artifactEventProperties: ArtifactEventProperties, diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/common/CommonQueryContext.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/common/CommonQueryContext.kt similarity index 98% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/common/CommonQueryContext.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/common/CommonQueryContext.kt index 870a6a81d1..6882e2c87b 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/common/CommonQueryContext.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/common/CommonQueryContext.kt @@ -29,7 +29,7 @@ * SOFTWARE. */ -package com.tencent.bkrepo.repository.search.common +package com.tencent.bkrepo.common.metadata.search.common import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.message.CommonMessageCode diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/common/CommonQueryInterpreter.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/common/CommonQueryInterpreter.kt similarity index 97% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/common/CommonQueryInterpreter.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/common/CommonQueryInterpreter.kt index 341b2c16ac..19f931f166 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/common/CommonQueryInterpreter.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/common/CommonQueryInterpreter.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.repository.search.common +package com.tencent.bkrepo.common.metadata.search.common import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.common.query.builder.MongoQueryInterpreter diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/common/LocalDatetimeRuleInterceptor.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/common/LocalDatetimeRuleInterceptor.kt similarity index 95% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/common/LocalDatetimeRuleInterceptor.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/common/LocalDatetimeRuleInterceptor.kt index 0bc50484ec..19e9fe570c 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/common/LocalDatetimeRuleInterceptor.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/common/LocalDatetimeRuleInterceptor.kt @@ -25,14 +25,16 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.repository.search.common +package com.tencent.bkrepo.common.metadata.search.common +import com.tencent.bkrepo.common.metadata.condition.SyncCondition import com.tencent.bkrepo.common.query.enums.OperationType import com.tencent.bkrepo.common.query.interceptor.QueryContext import com.tencent.bkrepo.common.query.interceptor.QueryRuleInterceptor import com.tencent.bkrepo.common.query.model.Rule import com.tencent.bkrepo.common.metadata.model.TNode import org.slf4j.LoggerFactory +import org.springframework.context.annotation.Conditional import org.springframework.data.mongodb.core.query.Criteria import org.springframework.stereotype.Component import java.time.LocalDateTime @@ -43,6 +45,7 @@ import java.time.format.DateTimeParseException * 拦截TNode LocalDatetime类型的字段查询，将字符串转换为LocalDatetime */ @Component +@Conditional(SyncCondition::class) class LocalDatetimeRuleInterceptor : QueryRuleInterceptor { override fun match(rule: Rule): Boolean { return rule is Rule.QueryRule && isSupportRule(rule) diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/common/MetadataRuleInterceptor.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/common/MetadataRuleInterceptor.kt similarity index 98% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/common/MetadataRuleInterceptor.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/common/MetadataRuleInterceptor.kt index 7151d5f7cd..337b8698fa 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/common/MetadataRuleInterceptor.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/common/MetadataRuleInterceptor.kt @@ -29,7 +29,7 @@ * SOFTWARE. */ -package com.tencent.bkrepo.repository.search.common +package com.tencent.bkrepo.common.metadata.search.common import com.tencent.bkrepo.common.query.enums.OperationType import com.tencent.bkrepo.common.query.interceptor.QueryContext diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/common/ModelValidateInterceptor.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/common/ModelValidateInterceptor.kt similarity index 98% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/common/ModelValidateInterceptor.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/common/ModelValidateInterceptor.kt index e88d6cf0d5..d570db393a 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/common/ModelValidateInterceptor.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/common/ModelValidateInterceptor.kt @@ -29,7 +29,7 @@ * SOFTWARE. */ -package com.tencent.bkrepo.repository.search.common +package com.tencent.bkrepo.common.metadata.search.common import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.message.CommonMessageCode diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/common/RepoNameRuleInterceptor.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/common/RepoNameRuleInterceptor.kt similarity index 97% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/common/RepoNameRuleInterceptor.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/common/RepoNameRuleInterceptor.kt index 1110f3e4c0..4e3572d0f8 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/common/RepoNameRuleInterceptor.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/common/RepoNameRuleInterceptor.kt @@ -29,13 +29,14 @@ * SOFTWARE. */ -package com.tencent.bkrepo.repository.search.common +package com.tencent.bkrepo.common.metadata.search.common import com.tencent.bkrepo.auth.api.ServicePermissionClient import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.common.api.constant.ensureSuffix import com.tencent.bkrepo.common.artifact.exception.RepoNotFoundException import com.tencent.bkrepo.common.artifact.path.PathUtils +import com.tencent.bkrepo.common.metadata.condition.SyncCondition import com.tencent.bkrepo.common.query.enums.OperationType import com.tencent.bkrepo.common.query.interceptor.QueryContext import com.tencent.bkrepo.common.query.interceptor.QueryRuleInterceptor @@ -48,6 +49,7 @@ import com.tencent.bkrepo.repository.pojo.repo.RepoListOption import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper.listPermissionPaths import org.slf4j.LoggerFactory +import org.springframework.context.annotation.Conditional import org.springframework.data.mongodb.core.query.Criteria import org.springframework.stereotype.Component @@ -57,6 +59,7 @@ import org.springframework.stereotype.Component * 条件构造器中传入条件是`repoName`，过滤无权限的仓库 */ @Component +@Conditional(SyncCondition::class) class RepoNameRuleInterceptor( private val permissionManager: PermissionManager, private val repositoryService: RepositoryService, diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/common/RepoTypeRuleInterceptor.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/common/RepoTypeRuleInterceptor.kt similarity index 93% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/common/RepoTypeRuleInterceptor.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/common/RepoTypeRuleInterceptor.kt index cb20ab47e0..be2627701b 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/common/RepoTypeRuleInterceptor.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/common/RepoTypeRuleInterceptor.kt @@ -29,14 +29,16 @@ * SOFTWARE. */ -package com.tencent.bkrepo.repository.search.common +package com.tencent.bkrepo.common.metadata.search.common +import com.tencent.bkrepo.common.metadata.condition.SyncCondition import com.tencent.bkrepo.common.query.enums.OperationType import com.tencent.bkrepo.common.query.interceptor.QueryContext import com.tencent.bkrepo.common.query.interceptor.QueryRuleInterceptor import com.tencent.bkrepo.common.query.model.Rule import com.tencent.bkrepo.repository.pojo.node.NodeInfo import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService +import org.springframework.context.annotation.Conditional import org.springframework.data.mongodb.core.query.Criteria import org.springframework.stereotype.Component @@ -46,6 +48,7 @@ import org.springframework.stereotype.Component * 条件构造器中传入条件是`repoType`，需要转换成对应的仓库列表 */ @Component +@Conditional(SyncCondition::class) class RepoTypeRuleInterceptor( private val repositoryService: RepositoryService ) : QueryRuleInterceptor { diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/common/SelectFieldInterceptor.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/common/SelectFieldInterceptor.kt similarity index 97% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/common/SelectFieldInterceptor.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/common/SelectFieldInterceptor.kt index 690bfae2c5..2ba9d06463 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/common/SelectFieldInterceptor.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/common/SelectFieldInterceptor.kt @@ -29,7 +29,7 @@ * SOFTWARE. */ -package com.tencent.bkrepo.repository.search.common +package com.tencent.bkrepo.common.metadata.search.common import com.tencent.bkrepo.common.query.interceptor.QueryContext import com.tencent.bkrepo.common.query.interceptor.QueryModelInterceptor diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/node/NodeModelInterceptor.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/node/NodeModelInterceptor.kt similarity index 95% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/node/NodeModelInterceptor.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/node/NodeModelInterceptor.kt index d448bd3e31..46e75d4ff7 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/node/NodeModelInterceptor.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/node/NodeModelInterceptor.kt @@ -29,7 +29,7 @@ * SOFTWARE. */ -package com.tencent.bkrepo.repository.search.node +package com.tencent.bkrepo.common.metadata.search.node import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.query.enums.OperationType @@ -40,7 +40,7 @@ import com.tencent.bkrepo.common.security.manager.PermissionManager import com.tencent.bkrepo.common.security.permission.PrincipalType import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.metadata.model.TNode -import com.tencent.bkrepo.repository.search.common.ModelValidateInterceptor +import com.tencent.bkrepo.common.metadata.search.common.ModelValidateInterceptor import org.slf4j.LoggerFactory /** diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/node/NodeQueryContext.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/node/NodeQueryContext.kt similarity index 93% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/node/NodeQueryContext.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/node/NodeQueryContext.kt index d343dfbed2..93876db3e3 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/node/NodeQueryContext.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/node/NodeQueryContext.kt @@ -29,11 +29,11 @@ * SOFTWARE. */ -package com.tencent.bkrepo.repository.search.node +package com.tencent.bkrepo.common.metadata.search.node import com.tencent.bkrepo.common.query.builder.MongoQueryInterpreter import com.tencent.bkrepo.common.query.model.QueryModel -import com.tencent.bkrepo.repository.search.common.CommonQueryContext +import com.tencent.bkrepo.common.metadata.search.common.CommonQueryContext import org.springframework.data.mongodb.core.query.Query class NodeQueryContext( diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/node/NodeQueryInterpreter.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/node/NodeQueryInterpreter.kt similarity index 80% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/node/NodeQueryInterpreter.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/node/NodeQueryInterpreter.kt index 591e0124ef..61b46a300e 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/node/NodeQueryInterpreter.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/node/NodeQueryInterpreter.kt @@ -29,24 +29,27 @@ * SOFTWARE. */ -package com.tencent.bkrepo.repository.search.node +package com.tencent.bkrepo.common.metadata.search.node +import com.tencent.bkrepo.common.metadata.condition.SyncCondition import com.tencent.bkrepo.common.query.interceptor.QueryContext import com.tencent.bkrepo.common.query.model.QueryModel import com.tencent.bkrepo.common.security.manager.PermissionManager -import com.tencent.bkrepo.repository.search.common.CommonQueryInterpreter -import com.tencent.bkrepo.repository.search.common.LocalDatetimeRuleInterceptor -import com.tencent.bkrepo.repository.search.common.MetadataRuleInterceptor -import com.tencent.bkrepo.repository.search.common.RepoNameRuleInterceptor -import com.tencent.bkrepo.repository.search.common.RepoTypeRuleInterceptor -import com.tencent.bkrepo.repository.search.common.SelectFieldInterceptor +import com.tencent.bkrepo.common.metadata.search.common.CommonQueryInterpreter +import com.tencent.bkrepo.common.metadata.search.common.LocalDatetimeRuleInterceptor +import com.tencent.bkrepo.common.metadata.search.common.MetadataRuleInterceptor +import com.tencent.bkrepo.common.metadata.search.common.RepoNameRuleInterceptor +import com.tencent.bkrepo.common.metadata.search.common.RepoTypeRuleInterceptor +import com.tencent.bkrepo.common.metadata.search.common.SelectFieldInterceptor import org.springframework.beans.factory.annotation.Autowired +import org.springframework.context.annotation.Conditional import org.springframework.context.annotation.Lazy import org.springframework.data.mongodb.core.query.Query import org.springframework.stereotype.Component import javax.annotation.PostConstruct @Component +@Conditional(SyncCondition::class) class NodeQueryInterpreter @Autowired @Lazy constructor( private val permissionManager: PermissionManager, private val repoNameRuleInterceptor: RepoNameRuleInterceptor, diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/AbstractBlockNodeService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt similarity index 89% rename from src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/AbstractBlockNodeService.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt index aded3986f7..eea4787b5f 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/AbstractBlockNodeService.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt @@ -27,10 +27,13 @@ package com.tencent.bkrepo.common.metadata.service.blocknode.impl +import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.metadata.condition.SyncCondition import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.metadata.constant.ID import com.tencent.bkrepo.common.metadata.dao.blocknode.BlockNodeDao +import com.tencent.bkrepo.common.metadata.dao.node.NodeDao import com.tencent.bkrepo.common.metadata.model.TBlockNode import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService @@ -39,16 +42,21 @@ import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.pojo.RegionResource import com.tencent.bkrepo.repository.pojo.node.NodeDetail import org.slf4j.LoggerFactory +import org.springframework.context.annotation.Conditional import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.Query import org.springframework.data.mongodb.core.query.Update import org.springframework.data.mongodb.core.query.and import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.stereotype.Service import java.time.LocalDateTime -abstract class AbstractBlockNodeService( +@Service +@Conditional(SyncCondition::class) +class BlockNodeServiceImpl( private val blockNodeDao: BlockNodeDao, - private val fileReferenceService: FileReferenceService + private val fileReferenceService: FileReferenceService, + private val nodeDao: NodeDao ) : BlockNodeService { override fun createBlock(blockNode: TBlockNode, storageCredentials: StorageCredentials?): TBlockNode { @@ -83,8 +91,9 @@ abstract class AbstractBlockNodeService( } override fun moveBlocks(projectId: String, repoName: String, fullPath: String, dstFullPath: String) { - val nodeDetail = getNodeDetail(projectId, repoName, dstFullPath) - if (nodeDetail.folder) { + val node = nodeDao.findNode(projectId, repoName, dstFullPath) + ?: throw NodeNotFoundException(dstFullPath) + if (node.folder) { val criteria = BlockNodeQueryHelper.fullPathCriteria(projectId, repoName, fullPath, true) val blocks = blockNodeDao.find(Query(criteria)) blocks.forEach { @@ -133,9 +142,7 @@ abstract class AbstractBlockNodeService( } } - abstract fun getNodeDetail(projectId: String, repoName: String, fullPath: String): NodeDetail - companion object { - private val logger = LoggerFactory.getLogger(AbstractBlockNodeService::class.java) + private val logger = LoggerFactory.getLogger(BlockNodeServiceImpl::class.java) } } diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/DefaultBlockNodeServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/DefaultBlockNodeServiceImpl.kt deleted file mode 100644 index 402590b0ad..0000000000 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/DefaultBlockNodeServiceImpl.kt +++ /dev/null @@ -1,23 +0,0 @@ -package com.tencent.bkrepo.common.metadata.service.blocknode.impl - -import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException -import com.tencent.bkrepo.common.metadata.condition.SyncCondition -import com.tencent.bkrepo.common.metadata.dao.blocknode.BlockNodeDao -import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService -import com.tencent.bkrepo.repository.api.NodeClient -import com.tencent.bkrepo.repository.pojo.node.NodeDetail -import org.springframework.context.annotation.Conditional -import org.springframework.stereotype.Service - -@Service -@Conditional(SyncCondition::class) -class DefaultBlockNodeServiceImpl( - blockNodeDao: BlockNodeDao, - fileReferenceService: FileReferenceService, - private val nodeClient: NodeClient -) : AbstractBlockNodeService(blockNodeDao, fileReferenceService) { - - override fun getNodeDetail(projectId: String, repoName: String, fullPath: String): NodeDetail { - return nodeClient.getNodeDetail(projectId, repoName, fullPath).data ?: throw NodeNotFoundException(fullPath) - } -} diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/RAbstractBlockNodeService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/RBlockNodeServiceImpl.kt similarity index 90% rename from src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/RAbstractBlockNodeService.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/RBlockNodeServiceImpl.kt index c844f21911..4203f2236a 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/RAbstractBlockNodeService.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/RBlockNodeServiceImpl.kt @@ -27,10 +27,13 @@ package com.tencent.bkrepo.common.metadata.service.blocknode.impl +import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.metadata.condition.ReactiveCondition import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.metadata.constant.ID import com.tencent.bkrepo.common.metadata.dao.blocknode.RBlockNodeDao +import com.tencent.bkrepo.common.metadata.dao.node.RNodeDao import com.tencent.bkrepo.common.metadata.model.TBlockNode import com.tencent.bkrepo.common.metadata.service.blocknode.RBlockNodeService import com.tencent.bkrepo.common.metadata.service.file.RFileReferenceService @@ -39,15 +42,20 @@ import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.pojo.RegionResource import com.tencent.bkrepo.repository.pojo.node.NodeDetail import org.slf4j.LoggerFactory +import org.springframework.context.annotation.Conditional import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.Query import org.springframework.data.mongodb.core.query.and import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.stereotype.Service import java.time.LocalDateTime -abstract class RAbstractBlockNodeService( +@Service +@Conditional(ReactiveCondition::class) +class RBlockNodeServiceImpl( private val rBlockNodeDao: RBlockNodeDao, - private val rFileReferenceService: RFileReferenceService + private val rFileReferenceService: RFileReferenceService, + private val rNodeDao: RNodeDao ) : RBlockNodeService { override suspend fun createBlock(blockNode: TBlockNode, storageCredentials: StorageCredentials?): TBlockNode { @@ -82,7 +90,7 @@ abstract class RAbstractBlockNodeService( } override suspend fun moveBlocks(projectId: String, repoName: String, fullPath: String, dstFullPath: String) { - val nodeDetail = getNodeDetail(projectId, repoName, dstFullPath) + val nodeDetail = rNodeDao.findNode(projectId, repoName, dstFullPath) ?: throw NodeNotFoundException(dstFullPath) if (nodeDetail.folder) { val criteria = BlockNodeQueryHelper.fullPathCriteria(projectId, repoName, fullPath, true) val blocks = rBlockNodeDao.find(Query(criteria)) @@ -133,9 +141,7 @@ abstract class RAbstractBlockNodeService( } } - abstract suspend fun getNodeDetail(projectId: String, repoName: String, fullPath: String): NodeDetail - companion object { - private val logger = LoggerFactory.getLogger(RAbstractBlockNodeService::class.java) + private val logger = LoggerFactory.getLogger(RBlockNodeService::class.java) } } diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/fs/FsService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/fs/FsService.kt similarity index 89% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/fs/FsService.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/fs/FsService.kt index a2d2d5ee7b..2251e2d606 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/fs/FsService.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/fs/FsService.kt @@ -25,14 +25,14 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.repository.service.fs +package com.tencent.bkrepo.common.metadata.service.fs import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeSetLengthRequest interface FsService { - fun createNode(createRequest: NodeCreateRequest): NodeDetail + suspend fun createNode(createRequest: NodeCreateRequest): NodeDetail - fun setLength(setLengthRequest: NodeSetLengthRequest) + suspend fun setLength(setLengthRequest: NodeSetLengthRequest) } diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/fs/impl/FsServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/fs/impl/FsServiceImpl.kt similarity index 85% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/fs/impl/FsServiceImpl.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/fs/impl/FsServiceImpl.kt index fd76def54a..ce7a7a3148 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/fs/impl/FsServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/fs/impl/FsServiceImpl.kt @@ -25,26 +25,27 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.repository.service.fs.impl +package com.tencent.bkrepo.common.metadata.service.fs.impl import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.util.Preconditions import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.artifact.path.PathUtils +import com.tencent.bkrepo.common.metadata.condition.ReactiveCondition import com.tencent.bkrepo.common.metadata.constant.FAKE_MD5 import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 -import com.tencent.bkrepo.common.service.util.SpringContextUtils -import com.tencent.bkrepo.common.service.cluster.condition.DefaultCondition -import com.tencent.bkrepo.common.metadata.dao.node.NodeDao +import com.tencent.bkrepo.common.metadata.dao.node.RNodeDao import com.tencent.bkrepo.common.metadata.model.TNode -import com.tencent.bkrepo.repository.pojo.node.NodeDetail -import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest -import com.tencent.bkrepo.repository.pojo.node.service.NodeSetLengthRequest -import com.tencent.bkrepo.repository.service.fs.FsService -import com.tencent.bkrepo.repository.service.node.impl.NodeBaseService +import com.tencent.bkrepo.common.metadata.service.fs.FsService import com.tencent.bkrepo.common.metadata.util.MetadataUtils +import com.tencent.bkrepo.common.metadata.util.NodeBaseServiceHelper.convertToDetail +import com.tencent.bkrepo.common.metadata.util.NodeBaseServiceHelper.parseExpireDate import com.tencent.bkrepo.common.metadata.util.NodeEventFactory import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper +import com.tencent.bkrepo.common.service.util.SpringContextUtils +import com.tencent.bkrepo.repository.pojo.node.NodeDetail +import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest +import com.tencent.bkrepo.repository.pojo.node.service.NodeSetLengthRequest import org.slf4j.LoggerFactory import org.springframework.context.annotation.Conditional import org.springframework.dao.DuplicateKeyException @@ -52,11 +53,11 @@ import org.springframework.stereotype.Service import java.time.LocalDateTime @Service -@Conditional(DefaultCondition::class) +@Conditional(ReactiveCondition::class) class FsServiceImpl( - private val nodeDao: NodeDao, + private val nodeDao: RNodeDao, ) : FsService { - override fun createNode(createRequest: NodeCreateRequest): NodeDetail { + override suspend fun createNode(createRequest: NodeCreateRequest): NodeDetail { with(createRequest) { val fullPath = PathUtils.normalizeFullPath(fullPath) Preconditions.checkArgument(!PathUtils.isRoot(fullPath), this::fullPath.name) @@ -71,11 +72,11 @@ class FsServiceImpl( } SpringContextUtils.publishEvent(NodeEventFactory.buildCreatedEvent(node)) logger.info("Create node[/$projectId/$repoName$fullPath], sha256[$sha256] success.") - return NodeBaseService.convertToDetail(node)!! + return convertToDetail(node)!! } } - open fun buildTNode(request: NodeCreateRequest): TNode { + fun buildTNode(request: NodeCreateRequest): TNode { with(request) { val fullPath = PathUtils.normalizeFullPath(fullPath) return TNode( @@ -85,7 +86,7 @@ class FsServiceImpl( name = PathUtils.resolveName(fullPath), fullPath = fullPath, folder = folder, - expireDate = if (folder) null else NodeBaseService.parseExpireDate(expires), + expireDate = if (folder) null else parseExpireDate(expires), size = if (folder) 0 else size ?: 0, sha256 = if (folder) null else sha256, md5 = if (folder) null else md5, @@ -99,7 +100,7 @@ class FsServiceImpl( } } - override fun setLength(setLengthRequest: NodeSetLengthRequest) { + override suspend fun setLength(setLengthRequest: NodeSetLengthRequest) { with(setLengthRequest) { val fullPath = PathUtils.normalizeFullPath(fullPath) val node = nodeDao.findNode(projectId, repoName, fullPath) diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/metadata/impl/MetadataServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/metadata/impl/MetadataServiceImpl.kt index e9916a1450..3bdc8736ba 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/metadata/impl/MetadataServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/metadata/impl/MetadataServiceImpl.kt @@ -40,23 +40,23 @@ import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.artifact.path.PathUtils.normalizeFullPath import com.tencent.bkrepo.common.metadata.condition.SyncCondition -import com.tencent.bkrepo.common.metadata.util.ClusterUtils -import com.tencent.bkrepo.common.security.exception.PermissionException -import com.tencent.bkrepo.common.security.manager.ci.CIPermissionManager -import com.tencent.bkrepo.common.service.cluster.condition.DefaultCondition -import com.tencent.bkrepo.common.service.util.SpringContextUtils.Companion.publishEvent import com.tencent.bkrepo.common.metadata.config.RepositoryProperties import com.tencent.bkrepo.common.metadata.dao.node.NodeDao -import com.tencent.bkrepo.repository.message.RepositoryMessageCode import com.tencent.bkrepo.common.metadata.model.TMetadata import com.tencent.bkrepo.common.metadata.model.TNode import com.tencent.bkrepo.common.metadata.service.metadata.MetadataService -import com.tencent.bkrepo.repository.pojo.metadata.MetadataDeleteRequest -import com.tencent.bkrepo.repository.pojo.metadata.MetadataSaveRequest +import com.tencent.bkrepo.common.metadata.util.ClusterUtils import com.tencent.bkrepo.common.metadata.util.MetadataUtils import com.tencent.bkrepo.common.metadata.util.NodeEventFactory.buildMetadataDeletedEvent import com.tencent.bkrepo.common.metadata.util.NodeEventFactory.buildMetadataSavedEvent import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper +import com.tencent.bkrepo.common.security.exception.PermissionException +import com.tencent.bkrepo.common.security.manager.ci.CIPermissionManager +import com.tencent.bkrepo.common.service.cluster.condition.DefaultCondition +import com.tencent.bkrepo.common.service.util.SpringContextUtils.Companion.publishEvent +import com.tencent.bkrepo.repository.message.RepositoryMessageCode +import com.tencent.bkrepo.repository.pojo.metadata.MetadataDeleteRequest +import com.tencent.bkrepo.repository.pojo.metadata.MetadataSaveRequest import org.slf4j.LoggerFactory import org.springframework.context.annotation.Conditional import org.springframework.data.mongodb.core.query.Query diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeArchiveOperation.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeArchiveOperation.kt similarity index 92% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeArchiveOperation.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeArchiveOperation.kt index 3fdb2d3a42..39f44ac4fb 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeArchiveOperation.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeArchiveOperation.kt @@ -1,4 +1,4 @@ -package com.tencent.bkrepo.repository.service.node +package com.tencent.bkrepo.common.metadata.service.node import com.tencent.bkrepo.repository.pojo.node.service.NodeArchiveRestoreRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeArchiveRequest diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeBaseOperation.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeBaseOperation.kt similarity index 98% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeBaseOperation.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeBaseOperation.kt index f6b6493ebb..8b33618ed1 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeBaseOperation.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeBaseOperation.kt @@ -29,7 +29,7 @@ * SOFTWARE. */ -package com.tencent.bkrepo.repository.service.node +package com.tencent.bkrepo.common.metadata.service.node import com.tencent.bkrepo.common.api.pojo.Page import com.tencent.bkrepo.common.artifact.api.ArtifactInfo diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeCompressOperation.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeCompressOperation.kt similarity index 88% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeCompressOperation.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeCompressOperation.kt index 9fd0b4c9e2..39bd5d7364 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeCompressOperation.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeCompressOperation.kt @@ -1,4 +1,4 @@ -package com.tencent.bkrepo.repository.service.node +package com.tencent.bkrepo.common.metadata.service.node import com.tencent.bkrepo.repository.pojo.node.service.NodeCompressedRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeUnCompressedRequest diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeDeleteOperation.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeDeleteOperation.kt similarity index 98% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeDeleteOperation.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeDeleteOperation.kt index d156522974..b9c78e8907 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeDeleteOperation.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeDeleteOperation.kt @@ -29,7 +29,7 @@ * SOFTWARE. */ -package com.tencent.bkrepo.repository.service.node +package com.tencent.bkrepo.common.metadata.service.node import com.tencent.bkrepo.repository.pojo.node.NodeDeleteResult import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeMoveCopyOperation.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeMoveCopyOperation.kt similarity index 98% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeMoveCopyOperation.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeMoveCopyOperation.kt index cb467dfd1f..43f1087b6c 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeMoveCopyOperation.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeMoveCopyOperation.kt @@ -29,7 +29,7 @@ * SOFTWARE. */ -package com.tencent.bkrepo.repository.service.node +package com.tencent.bkrepo.common.metadata.service.node import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.service.NodeMoveCopyRequest diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeRenameOperation.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeRenameOperation.kt similarity index 97% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeRenameOperation.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeRenameOperation.kt index b6466b44bd..5effb3b6b7 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeRenameOperation.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeRenameOperation.kt @@ -29,7 +29,7 @@ * SOFTWARE. */ -package com.tencent.bkrepo.repository.service.node +package com.tencent.bkrepo.common.metadata.service.node import com.tencent.bkrepo.repository.pojo.node.service.NodeRenameRequest diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeRestoreOperation.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeRestoreOperation.kt similarity index 89% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeRestoreOperation.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeRestoreOperation.kt index 617603e44e..cf6d9e31e5 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeRestoreOperation.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeRestoreOperation.kt @@ -29,14 +29,14 @@ * SOFTWARE. */ -package com.tencent.bkrepo.repository.service.node +package com.tencent.bkrepo.common.metadata.service.node import com.tencent.bkrepo.common.artifact.api.ArtifactInfo +import com.tencent.bkrepo.common.metadata.pojo.node.NodeRestoreOption +import com.tencent.bkrepo.common.metadata.pojo.node.RestoreContext import com.tencent.bkrepo.repository.pojo.node.NodeDeletedPoint import com.tencent.bkrepo.repository.pojo.node.NodeDetail -import com.tencent.bkrepo.repository.pojo.node.NodeRestoreOption import com.tencent.bkrepo.repository.pojo.node.NodeRestoreResult -import com.tencent.bkrepo.repository.service.node.impl.NodeRestoreSupport /** * 节点恢复接口 @@ -66,5 +66,5 @@ interface NodeRestoreOperation { /** * 根据上下文恢复被删除节点 * */ - fun restoreNode(restoreContext: NodeRestoreSupport.RestoreContext): NodeRestoreResult + fun restoreNode(restoreContext: RestoreContext): NodeRestoreResult } diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeSearchService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeSearchService.kt similarity index 97% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeSearchService.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeSearchService.kt index a454b9a8e6..b4798884f8 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeSearchService.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeSearchService.kt @@ -29,7 +29,7 @@ * SOFTWARE. */ -package com.tencent.bkrepo.repository.service.node +package com.tencent.bkrepo.common.metadata.service.node import com.tencent.bkrepo.common.api.pojo.Page import com.tencent.bkrepo.common.query.model.QueryModel diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeService.kt similarity index 96% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeService.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeService.kt index abc9e16e8c..d8e45f42bb 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeService.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeService.kt @@ -29,7 +29,7 @@ * SOFTWARE. */ -package com.tencent.bkrepo.repository.service.node +package com.tencent.bkrepo.common.metadata.service.node /** * 节点服务接口 diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeStatsOperation.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeStatsOperation.kt similarity index 97% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeStatsOperation.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeStatsOperation.kt index 8d02512ae5..b76e777baf 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeStatsOperation.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeStatsOperation.kt @@ -29,7 +29,7 @@ * SOFTWARE. */ -package com.tencent.bkrepo.repository.service.node +package com.tencent.bkrepo.common.metadata.service.node import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.repository.pojo.node.NodeSizeInfo diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeArchiveSupport.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeArchiveSupport.kt similarity index 88% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeArchiveSupport.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeArchiveSupport.kt index 6cd40472e7..f5974dd740 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeArchiveSupport.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeArchiveSupport.kt @@ -1,29 +1,31 @@ -package com.tencent.bkrepo.repository.service.node.impl +package com.tencent.bkrepo.common.metadata.service.node.impl import com.tencent.bkrepo.archive.api.ArchiveClient import com.tencent.bkrepo.archive.request.ArchiveFileRequest import com.tencent.bkrepo.archive.request.UncompressFileRequest -import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder +import com.tencent.bkrepo.common.artifact.exception.RepoNotFoundException import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.metadata.dao.node.NodeDao +import com.tencent.bkrepo.common.metadata.dao.repo.RepositoryDao import com.tencent.bkrepo.common.metadata.model.TNode -import com.tencent.bkrepo.repository.pojo.node.service.NodeArchiveRestoreRequest -import com.tencent.bkrepo.repository.pojo.node.service.NodeArchiveRequest -import com.tencent.bkrepo.repository.service.node.NodeArchiveOperation +import com.tencent.bkrepo.common.metadata.service.node.NodeArchiveOperation import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper -import java.time.LocalDateTime +import com.tencent.bkrepo.repository.pojo.node.service.NodeArchiveRequest +import com.tencent.bkrepo.repository.pojo.node.service.NodeArchiveRestoreRequest import org.slf4j.LoggerFactory import org.springframework.data.mongodb.core.query.Update import org.springframework.data.mongodb.core.query.and import org.springframework.data.mongodb.core.query.isEqualTo import org.springframework.data.mongodb.core.query.where import java.time.Duration +import java.time.LocalDateTime class NodeArchiveSupport( private val nodeBaseService: NodeBaseService, private val archiveClient: ArchiveClient, ) : NodeArchiveOperation { val nodeDao: NodeDao = nodeBaseService.nodeDao + val repositoryDao: RepositoryDao = nodeBaseService.repositoryDao override fun archiveNode(nodeArchiveRequest: NodeArchiveRequest) { with(nodeArchiveRequest) { @@ -53,9 +55,8 @@ class NodeArchiveSupport( if (nodes.isEmpty()) { return emptyList() } - val repoId = ArtifactContextHolder.RepositoryId(projectId, repoName) - val repo = ArtifactContextHolder.getRepoDetail(repoId) - val storageCredentialsKey = repo.storageCredentials?.key + val repo = repositoryDao.findByNameAndType(projectId, repoName) ?: throw RepoNotFoundException(repoName) + val storageCredentialsKey = repo.credentialsKey return nodes.map { val sha256 = it.sha256!! if (it.archived == true) { diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeBaseService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt similarity index 79% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeBaseService.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt index eae2aebe53..7ff238397f 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeBaseService.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt @@ -25,10 +25,9 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.repository.service.node.impl +package com.tencent.bkrepo.common.metadata.service.node.impl import com.tencent.bkrepo.auth.api.ServicePermissionClient -import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.common.api.constant.PROXY_HEADER_NAME import com.tencent.bkrepo.common.api.exception.BadRequestException import com.tencent.bkrepo.common.api.exception.ErrorCodeException @@ -41,28 +40,37 @@ import com.tencent.bkrepo.common.artifact.constant.METADATA_KEY_LINK_REPO import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.common.artifact.pojo.RepositoryType -import com.tencent.bkrepo.common.artifact.router.RouterControllerProperties +import com.tencent.bkrepo.common.artifact.properties.RouterControllerProperties +import com.tencent.bkrepo.common.metadata.config.RepositoryProperties import com.tencent.bkrepo.common.metadata.constant.FAKE_MD5 import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 +import com.tencent.bkrepo.common.metadata.dao.node.NodeDao import com.tencent.bkrepo.common.metadata.dao.repo.RepositoryDao +import com.tencent.bkrepo.common.metadata.model.TNode import com.tencent.bkrepo.common.metadata.model.TRepository import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.metadata.service.project.ProjectService +import com.tencent.bkrepo.common.metadata.service.repo.QuotaService import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService +import com.tencent.bkrepo.common.metadata.util.NodeBaseServiceHelper +import com.tencent.bkrepo.common.metadata.util.NodeBaseServiceHelper.TOPIC +import com.tencent.bkrepo.common.metadata.util.NodeBaseServiceHelper.checkNodeListOption +import com.tencent.bkrepo.common.metadata.util.NodeBaseServiceHelper.convert +import com.tencent.bkrepo.common.metadata.util.NodeBaseServiceHelper.convertToDetail +import com.tencent.bkrepo.common.metadata.util.NodeBaseServiceHelper.parseExpireDate +import com.tencent.bkrepo.common.metadata.util.NodeBaseServiceHelper.validateParameter +import com.tencent.bkrepo.common.metadata.util.NodeEventFactory.buildCreatedEvent +import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper +import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper.listPermissionPaths import com.tencent.bkrepo.common.mongo.dao.util.Pages -import com.tencent.bkrepo.common.query.model.Sort -import com.tencent.bkrepo.common.security.manager.PermissionManager import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.HeaderUtils import com.tencent.bkrepo.common.service.util.SpringContextUtils.Companion.publishEvent -import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.stream.constant.BinderType import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier -import com.tencent.bkrepo.common.metadata.config.RepositoryProperties import com.tencent.bkrepo.repository.constant.SYSTEM_USER -import com.tencent.bkrepo.common.metadata.dao.node.NodeDao -import com.tencent.bkrepo.common.metadata.model.TNode import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.NodeInfo @@ -71,16 +79,8 @@ import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeLinkRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeUpdateAccessDateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeUpdateRequest -import com.tencent.bkrepo.repository.service.node.NodeService -import com.tencent.bkrepo.common.metadata.service.repo.QuotaService -import com.tencent.bkrepo.common.metadata.util.MetadataUtils -import com.tencent.bkrepo.common.metadata.util.NodeEventFactory.buildCreatedEvent -import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper -import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper.listPermissionPaths import com.tencent.bkrepo.router.api.RouterControllerClient import org.slf4j.LoggerFactory -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.context.annotation.Lazy import org.springframework.dao.DuplicateKeyException import org.springframework.data.mongodb.core.query.Query import org.springframework.data.mongodb.core.query.Update @@ -89,7 +89,6 @@ import org.springframework.data.mongodb.core.query.isEqualTo import org.springframework.data.mongodb.core.query.where import org.springframework.transaction.annotation.Transactional import java.time.LocalDateTime -import java.time.format.DateTimeFormatter /** * 节点基础服务，实现了CRUD基本操作 @@ -99,7 +98,6 @@ abstract class NodeBaseService( open val repositoryDao: RepositoryDao, open val fileReferenceService: FileReferenceService, open val storageCredentialService: StorageCredentialService, - open val storageService: StorageService, open val quotaService: QuotaService, open val repositoryProperties: RepositoryProperties, open val messageSupplier: MessageSupplier, @@ -110,10 +108,6 @@ abstract class NodeBaseService( open val projectService: ProjectService, ) : NodeService { - @Autowired - @Lazy - protected lateinit var permissionManager: PermissionManager - override fun getNodeDetail(artifact: ArtifactInfo, repoType: String?): NodeDetail? { with(artifact) { val node = nodeDao.findNode(projectId, repoName, getArtifactFullPath()) @@ -201,12 +195,6 @@ abstract class NodeBaseService( @Transactional(rollbackFor = [Throwable::class]) override fun link(request: NodeLinkRequest): NodeDetail { with(request) { - // 校验源仓库与目标节点权限 - permissionManager.checkRepoPermission(PermissionAction.WRITE, projectId, repoName, userId = operator) - permissionManager.checkNodePermission( - PermissionAction.READ, targetProjectId, targetRepoName, targetFullPath, userId = operator - ) - val targetArtifact = "/$targetProjectId/$targetRepoName/$targetFullPath" if (checkTargetExist) { val targetNode = nodeDao.findNode(targetProjectId, targetRepoName, targetFullPath) @@ -240,31 +228,7 @@ abstract class NodeBaseService( } open fun buildTNode(request: NodeCreateRequest): TNode { - with(request) { - val normalizeFullPath = PathUtils.normalizeFullPath(fullPath) - return TNode( - projectId = projectId, - repoName = repoName, - path = PathUtils.resolveParent(normalizeFullPath), - name = PathUtils.resolveName(normalizeFullPath), - fullPath = normalizeFullPath, - folder = folder, - expireDate = if (folder) null else parseExpireDate(expires), - size = if (folder) 0 else size ?: 0, - sha256 = if (folder) null else sha256, - md5 = if (folder) null else md5, - nodeNum = null, - metadata = MetadataUtils.compatibleConvertAndCheck( - metadata, - MetadataUtils.changeSystem(nodeMetadata, repositoryProperties.allowUserAddSystemMetadata), - ), - createdBy = createdBy ?: operator, - createdDate = createdDate ?: LocalDateTime.now(), - lastModifiedBy = createdBy ?: operator, - lastModifiedDate = lastModifiedDate ?: LocalDateTime.now(), - lastAccessDate = LocalDateTime.now(), - ) - } + return NodeBaseServiceHelper.buildTNode(request, repositoryProperties.allowUserAddSystemMetadata) } private fun getTotalNodeNum(artifact: ArtifactInfo, query: Query): Long { @@ -455,17 +419,6 @@ abstract class NodeBaseService( } } - private fun checkNodeListOption(option: NodeListOption) { - Preconditions.checkArgument( - option.sortProperty.none { !TNode::class.java.declaredFields.map { f -> f.name }.contains(it) }, - "sortProperty", - ) - Preconditions.checkArgument( - option.direction.none { it != Sort.Direction.DESC.name && it != Sort.Direction.ASC.name }, - "direction", - ) - } - private fun incrementFileReference(node: TNode, repository: TRepository?): Boolean { if (!validateParameter(node)) return false return try { @@ -486,15 +439,6 @@ abstract class NodeBaseService( return tRepository.credentialsKey } - private fun validateParameter(node: TNode): Boolean { - if (node.folder) return false - if (node.sha256.isNullOrBlank()) { - logger.warn("Failed to change file reference, node[$node] sha256 is null or blank.") - return false - } - return true - } - /** * 获取用户无权限路径列表 */ @@ -511,51 +455,5 @@ abstract class NodeBaseService( companion object { private val logger = LoggerFactory.getLogger(NodeBaseService::class.java) - private const val TOPIC = "bkbase_bkrepo_artifact_node_created" - - private fun convert(tNode: TNode?): NodeInfo? { - return tNode?.let { - val metadata = MetadataUtils.toMap(it.metadata) - NodeInfo( - id = it.id, - createdBy = it.createdBy, - createdDate = it.createdDate.format(DateTimeFormatter.ISO_DATE_TIME), - lastModifiedBy = it.lastModifiedBy, - lastModifiedDate = it.lastModifiedDate.format(DateTimeFormatter.ISO_DATE_TIME), - projectId = it.projectId, - repoName = it.repoName, - folder = it.folder, - path = it.path, - name = it.name, - fullPath = it.fullPath, - size = if (it.size < 0L) 0L else it.size, - nodeNum = it.nodeNum?.let { nodeNum -> - if (nodeNum < 0L) 0L else nodeNum - }, - sha256 = it.sha256, - md5 = it.md5, - metadata = metadata, - nodeMetadata = MetadataUtils.toList(it.metadata), - copyFromCredentialsKey = it.copyFromCredentialsKey, - copyIntoCredentialsKey = it.copyIntoCredentialsKey, - deleted = it.deleted?.format(DateTimeFormatter.ISO_DATE_TIME), - lastAccessDate = it.lastAccessDate?.format(DateTimeFormatter.ISO_DATE_TIME), - clusterNames = it.clusterNames, - archived = it.archived, - compressed = it.compressed, - ) - } - } - - fun convertToDetail(tNode: TNode?): NodeDetail? { - return convert(tNode)?.let { NodeDetail(it) } - } - - /** - * 根据有效天数，计算到期时间 - */ - fun parseExpireDate(expireDays: Long?): LocalDateTime? { - return expireDays?.takeIf { it > 0 }?.run { LocalDateTime.now().plusDays(this) } - } } } diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeCompressSupport.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeCompressSupport.kt similarity index 92% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeCompressSupport.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeCompressSupport.kt index 01f7023236..e8059d6685 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeCompressSupport.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeCompressSupport.kt @@ -1,10 +1,10 @@ -package com.tencent.bkrepo.repository.service.node.impl +package com.tencent.bkrepo.common.metadata.service.node.impl import com.tencent.bkrepo.common.metadata.dao.node.NodeDao import com.tencent.bkrepo.common.metadata.model.TNode import com.tencent.bkrepo.repository.pojo.node.service.NodeCompressedRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeUnCompressedRequest -import com.tencent.bkrepo.repository.service.node.NodeCompressOperation +import com.tencent.bkrepo.common.metadata.service.node.NodeCompressOperation import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper import org.slf4j.LoggerFactory import org.springframework.data.mongodb.core.query.Update diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeDeleteSupport.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt similarity index 92% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeDeleteSupport.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt index 318b5f0c8c..d4ee15cc0b 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeDeleteSupport.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt @@ -25,13 +25,13 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.repository.service.node.impl +package com.tencent.bkrepo.common.metadata.service.node.impl import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.common.api.util.HumanReadable import com.tencent.bkrepo.common.artifact.path.PathUtils -import com.tencent.bkrepo.common.artifact.router.RouterControllerProperties +import com.tencent.bkrepo.common.artifact.properties.RouterControllerProperties import com.tencent.bkrepo.common.service.util.SpringContextUtils.Companion.publishEvent import com.tencent.bkrepo.common.metadata.dao.node.NodeDao import com.tencent.bkrepo.common.metadata.model.TNode @@ -39,8 +39,9 @@ import com.tencent.bkrepo.repository.pojo.node.NodeDeleteResult import com.tencent.bkrepo.repository.pojo.node.NodeListOption import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest import com.tencent.bkrepo.repository.pojo.node.service.NodesDeleteRequest -import com.tencent.bkrepo.repository.service.node.NodeDeleteOperation +import com.tencent.bkrepo.common.metadata.service.node.NodeDeleteOperation import com.tencent.bkrepo.common.metadata.service.repo.QuotaService +import com.tencent.bkrepo.common.metadata.util.NodeDeleteHelper.buildCriteria import com.tencent.bkrepo.common.metadata.util.NodeEventFactory.buildDeletedEvent import com.tencent.bkrepo.common.metadata.util.NodeEventFactory.buildNodeCleanEvent import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper @@ -183,24 +184,6 @@ open class NodeDeleteSupport( return nodeDeleteResult } - private fun buildCriteria( - projectId: String, - repoName: String, - fullPath: String, - ): Criteria { - val normalizedFullPath = PathUtils.normalizeFullPath(fullPath) - val normalizedPath = PathUtils.toPath(normalizedFullPath) - val escapedPath = PathUtils.escapeRegex(normalizedPath) - val criteria = where(TNode::projectId).isEqualTo(projectId) - .and(TNode::repoName).isEqualTo(repoName) - .and(TNode::deleted).isEqualTo(null) - .orOperator( - where(TNode::fullPath).regex("^$escapedPath"), - where(TNode::fullPath).isEqualTo(normalizedFullPath) - ) - return criteria - } - private fun delete( query: Query, operator: String, diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeMoveCopySupport.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeMoveCopySupport.kt similarity index 76% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeMoveCopySupport.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeMoveCopySupport.kt index 4fc5414b27..3b693bc0aa 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeMoveCopySupport.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeMoveCopySupport.kt @@ -29,36 +29,38 @@ * SOFTWARE. */ -package com.tencent.bkrepo.repository.service.node.impl +package com.tencent.bkrepo.common.metadata.service.node.impl import com.tencent.bkrepo.common.api.exception.ErrorCodeException -import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.common.artifact.path.PathUtils.combineFullPath import com.tencent.bkrepo.common.artifact.path.PathUtils.resolveName import com.tencent.bkrepo.common.artifact.path.PathUtils.resolveParent import com.tencent.bkrepo.common.artifact.path.PathUtils.toPath -import com.tencent.bkrepo.common.artifact.pojo.RepositoryCategory -import com.tencent.bkrepo.common.service.util.SpringContextUtils.Companion.publishEvent -import com.tencent.bkrepo.common.storage.core.StorageService -import com.tencent.bkrepo.common.storage.credentials.StorageCredentials -import com.tencent.bkrepo.repository.constant.DEFAULT_STORAGE_CREDENTIALS_KEY import com.tencent.bkrepo.common.metadata.dao.node.NodeDao import com.tencent.bkrepo.common.metadata.dao.repo.RepositoryDao import com.tencent.bkrepo.common.metadata.model.TNode import com.tencent.bkrepo.common.metadata.model.TRepository -import com.tencent.bkrepo.repository.pojo.node.NodeDetail -import com.tencent.bkrepo.repository.pojo.node.NodeListOption -import com.tencent.bkrepo.repository.pojo.node.service.NodeMoveCopyRequest -import com.tencent.bkrepo.repository.service.node.NodeMoveCopyOperation +import com.tencent.bkrepo.common.metadata.service.node.NodeMoveCopyOperation import com.tencent.bkrepo.common.metadata.service.repo.QuotaService import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService +import com.tencent.bkrepo.common.metadata.util.NodeBaseServiceHelper.convertToDetail import com.tencent.bkrepo.common.metadata.util.NodeEventFactory +import com.tencent.bkrepo.common.metadata.util.NodeMoveCopyHelper +import com.tencent.bkrepo.common.metadata.util.NodeMoveCopyHelper.MoveCopyContext +import com.tencent.bkrepo.common.metadata.util.NodeMoveCopyHelper.buildDstNode +import com.tencent.bkrepo.common.metadata.util.NodeMoveCopyHelper.canIgnore +import com.tencent.bkrepo.common.metadata.util.NodeMoveCopyHelper.preCheck import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper +import com.tencent.bkrepo.common.service.util.SpringContextUtils.Companion.publishEvent +import com.tencent.bkrepo.common.storage.credentials.StorageCredentials +import com.tencent.bkrepo.repository.constant.DEFAULT_STORAGE_CREDENTIALS_KEY +import com.tencent.bkrepo.repository.pojo.node.NodeDetail +import com.tencent.bkrepo.repository.pojo.node.NodeListOption +import com.tencent.bkrepo.repository.pojo.node.service.NodeMoveCopyRequest import org.slf4j.LoggerFactory import org.springframework.data.mongodb.core.query.Query -import java.time.LocalDateTime /** * 节点移动/拷贝接口实现 @@ -67,7 +69,6 @@ open class NodeMoveCopySupport( private val nodeBaseService: NodeBaseService ) : NodeMoveCopyOperation { - private val storageService: StorageService = nodeBaseService.storageService private val nodeDao: NodeDao = nodeBaseService.nodeDao private val repositoryDao: RepositoryDao = nodeBaseService.repositoryDao private val storageCredentialService: StorageCredentialService = nodeBaseService.storageCredentialService @@ -92,7 +93,7 @@ open class NodeMoveCopySupport( with(resolveContext(request, move)) { preCheck(this) if (canIgnore(this)) { - return NodeBaseService.convertToDetail( + return convertToDetail( nodeBaseService.nodeDao.findNode( projectId = dstProjectId, repoName = dstRepoName, @@ -110,7 +111,7 @@ open class NodeMoveCopySupport( } else { publishEvent(NodeEventFactory.buildCopiedEvent(request)) } - return NodeBaseService.convertToDetail( + return convertToDetail( nodeBaseService.nodeDao.findNode( projectId = dstProjectId, repoName = dstRepoName, @@ -188,36 +189,6 @@ open class NodeMoveCopySupport( } } - open fun buildDstNode( - context: MoveCopyContext, - node: TNode, - dstPath: String, - dstName: String, - dstFullPath: String - ): TNode { - with(context) { - val dstNode = node.copy( - id = null, - projectId = dstProjectId, - repoName = dstRepoName, - path = dstPath, - name = dstName, - fullPath = dstFullPath, - size = if (node.folder) 0 else node.size, - nodeNum = if (node.folder) null else node.nodeNum, - lastModifiedBy = operator, - lastModifiedDate = LocalDateTime.now() - ) - // move操作，create信息保留 - if (move) { - dstNode.createdBy = operator - dstNode.createdDate = LocalDateTime.now() - } - - return dstNode - } - } - private fun resolveContext(request: NodeMoveCopyRequest, move: Boolean): MoveCopyContext { with(request) { val srcFullPath = PathUtils.normalizeFullPath(srcFullPath) @@ -253,50 +224,8 @@ open class NodeMoveCopySupport( } } - /** - * 预检查 - */ - private fun preCheck(context: MoveCopyContext) { - // 只允许local或者composite类型仓库操作 - val canSrcRepoMove = context.srcRepo.category.let { - it == RepositoryCategory.LOCAL || it == RepositoryCategory.COMPOSITE - } - val canDstRepoMove = context.dstRepo.category.let { - it == RepositoryCategory.LOCAL || it == RepositoryCategory.COMPOSITE - } - if (!canSrcRepoMove || !canDstRepoMove) { - throw ErrorCodeException(CommonMessageCode.METHOD_NOT_ALLOWED, "Only local repository is supported") - } - } - - /** - * 判断能否忽略执行 - */ - private fun canIgnore(context: MoveCopyContext): Boolean { - with(context) { - var canIgnore = false - if (isSameRepo()) { - if (srcNode.fullPath == dstNode?.fullPath) { - // 同路径，跳过 - canIgnore = true - } else if (dstNode?.folder == true && srcNode.path == toPath(dstNode.fullPath)) { - // src为dst目录下的子节点，跳过 - canIgnore = true - } - } - return canIgnore - } - } - open fun checkConflict(context: MoveCopyContext, node: TNode, existNode: TNode?) { - // 目录 -> 文件: 出错 - if (node.folder && existNode?.folder == false) { - throw ErrorCodeException(ArtifactMessageCode.NODE_CONFLICT, existNode.fullPath) - } - // 文件 -> 文件 & 不允许覆盖: 出错 - if (!node.folder && existNode?.folder == false && !context.overwrite) { - throw ErrorCodeException(ArtifactMessageCode.NODE_CONFLICT, existNode.fullPath) - } + NodeMoveCopyHelper.checkConflict(context, node, existNode) } /** @@ -340,10 +269,7 @@ open class NodeMoveCopySupport( srcRootNodePath: String, listOption: NodeListOption ): Query { - with(context) { - val query = NodeQueryHelper.nodeListQuery(srcNode.projectId, srcNode.repoName, srcRootNodePath, listOption) - return query - } + return NodeMoveCopyHelper.buildSubNodesQuery(context, srcRootNodePath, listOption) } /** @@ -368,24 +294,6 @@ open class NodeMoveCopySupport( return key?.let { storageCredentialService.findByKey(it) } } - data class MoveCopyContext( - val srcRepo: TRepository, - val srcCredentials: StorageCredentials?, - val srcNode: TNode, - val dstProjectId: String, - val dstRepoName: String, - val dstFullPath: String, - val dstRepo: TRepository, - val dstCredentials: StorageCredentials?, - val dstNode: TNode?, - val dstNodeFolder: Boolean?, - val overwrite: Boolean, - val operator: String, - val move: Boolean - ) { - fun isSameRepo() = srcNode.projectId == dstProjectId && srcNode.repoName == dstRepoName - } - companion object { private val logger = LoggerFactory.getLogger(NodeMoveCopySupport::class.java) } diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRenameSupport.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeRenameSupport.kt similarity index 97% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRenameSupport.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeRenameSupport.kt index 64d6f1cbb5..c3ace2c5f2 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRenameSupport.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeRenameSupport.kt @@ -29,7 +29,7 @@ * SOFTWARE. */ -package com.tencent.bkrepo.repository.service.node.impl +package com.tencent.bkrepo.common.metadata.service.node.impl import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode @@ -39,7 +39,7 @@ import com.tencent.bkrepo.common.metadata.dao.node.NodeDao import com.tencent.bkrepo.common.metadata.model.TNode import com.tencent.bkrepo.repository.pojo.node.NodeListOption import com.tencent.bkrepo.repository.pojo.node.service.NodeRenameRequest -import com.tencent.bkrepo.repository.service.node.NodeRenameOperation +import com.tencent.bkrepo.common.metadata.service.node.NodeRenameOperation import com.tencent.bkrepo.common.metadata.util.NodeEventFactory.buildRenamedEvent import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper import org.slf4j.LoggerFactory diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeRestoreSupport.kt similarity index 93% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeRestoreSupport.kt index 0408064030..ff1c73501c 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeRestoreSupport.kt @@ -29,27 +29,22 @@ * SOFTWARE. */ -package com.tencent.bkrepo.repository.service.node.impl +package com.tencent.bkrepo.common.metadata.service.node.impl import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.artifact.path.PathUtils.isRoot import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 -import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService -import com.tencent.bkrepo.common.security.util.SecurityUtils -import com.tencent.bkrepo.fs.server.constant.FS_ATTR_KEY import com.tencent.bkrepo.common.metadata.dao.node.NodeDao import com.tencent.bkrepo.common.metadata.model.TNode -import com.tencent.bkrepo.repository.pojo.node.ConflictStrategy -import com.tencent.bkrepo.repository.pojo.node.NodeDeletedPoint -import com.tencent.bkrepo.repository.pojo.node.NodeDetail -import com.tencent.bkrepo.repository.pojo.node.NodeListOption -import com.tencent.bkrepo.repository.pojo.node.NodeRestoreOption -import com.tencent.bkrepo.repository.pojo.node.NodeRestoreResult -import com.tencent.bkrepo.repository.service.node.NodeRestoreOperation -import com.tencent.bkrepo.repository.service.node.impl.NodeBaseService.Companion.convertToDetail +import com.tencent.bkrepo.common.metadata.pojo.node.ConflictStrategy +import com.tencent.bkrepo.common.metadata.pojo.node.NodeRestoreOption +import com.tencent.bkrepo.common.metadata.pojo.node.RestoreContext +import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService +import com.tencent.bkrepo.common.metadata.service.node.NodeRestoreOperation import com.tencent.bkrepo.common.metadata.util.MetadataUtils +import com.tencent.bkrepo.common.metadata.util.NodeBaseServiceHelper.convertToDetail import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper.nodeDeletedFolderQuery import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper.nodeDeletedPointListQuery @@ -58,11 +53,16 @@ import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper.nodeDeletedPointQ import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper.nodeListQuery import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper.nodeQuery import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper.nodeRestoreUpdate +import com.tencent.bkrepo.common.security.util.SecurityUtils +import com.tencent.bkrepo.fs.server.constant.FS_ATTR_KEY +import com.tencent.bkrepo.repository.pojo.node.NodeDeletedPoint +import com.tencent.bkrepo.repository.pojo.node.NodeDetail +import com.tencent.bkrepo.repository.pojo.node.NodeListOption +import com.tencent.bkrepo.repository.pojo.node.NodeRestoreResult import org.slf4j.LoggerFactory import org.springframework.dao.DuplicateKeyException import org.springframework.data.mongodb.core.FindAndModifyOptions import java.time.Instant -import java.time.LocalDateTime import java.time.ZoneId /** @@ -283,18 +283,6 @@ open class NodeRestoreSupport( } } - data class RestoreContext( - val projectId: String, - val repoName: String, - val rootFullPath: String, - val deletedTime: LocalDateTime, - val conflictStrategy: ConflictStrategy, - val operator: String, - var restoreCount: Long = 0L, - var conflictCount: Long = 0L, - var skipCount: Long = 0L - ) - companion object { private val logger = LoggerFactory.getLogger(NodeRestoreSupport::class.java) } diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeSearchServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeSearchServiceImpl.kt similarity index 94% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeSearchServiceImpl.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeSearchServiceImpl.kt index 1dfd17f8c6..2acac1af7f 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeSearchServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeSearchServiceImpl.kt @@ -25,12 +25,13 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.repository.service.node.impl +package com.tencent.bkrepo.common.metadata.service.node.impl import com.tencent.bkrepo.common.api.pojo.Page import com.tencent.bkrepo.common.api.util.HumanReadable import com.tencent.bkrepo.common.api.util.toJsonString import com.tencent.bkrepo.common.artifact.pojo.RepositoryType +import com.tencent.bkrepo.common.metadata.condition.SyncCondition import com.tencent.bkrepo.common.query.model.QueryModel import com.tencent.bkrepo.common.metadata.config.RepositoryProperties import com.tencent.bkrepo.common.metadata.dao.node.NodeDao @@ -38,12 +39,13 @@ import com.tencent.bkrepo.common.metadata.model.TNode import com.tencent.bkrepo.repository.pojo.node.NodeInfo import com.tencent.bkrepo.repository.pojo.repo.RepoListOption import com.tencent.bkrepo.repository.pojo.software.ProjectPackageOverview -import com.tencent.bkrepo.repository.search.node.NodeQueryContext -import com.tencent.bkrepo.repository.search.node.NodeQueryInterpreter -import com.tencent.bkrepo.repository.service.node.NodeSearchService +import com.tencent.bkrepo.common.metadata.search.node.NodeQueryContext +import com.tencent.bkrepo.common.metadata.search.node.NodeQueryInterpreter +import com.tencent.bkrepo.common.metadata.service.node.NodeSearchService import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.metadata.util.MetadataUtils import org.slf4j.LoggerFactory +import org.springframework.context.annotation.Conditional import org.springframework.data.mongodb.core.query.Query import org.springframework.stereotype.Service import java.time.LocalDateTime @@ -57,6 +59,7 @@ import kotlin.system.measureTimeMillis */ @Suppress("UNCHECKED_CAST") @Service +@Conditional(SyncCondition::class) class NodeSearchServiceImpl( private val nodeDao: NodeDao, private val nodeQueryInterpreter: NodeQueryInterpreter, diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeServiceImpl.kt similarity index 95% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeServiceImpl.kt index e6bd32b3cc..f4e2db348b 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeServiceImpl.kt @@ -25,23 +25,28 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.repository.service.node.impl +package com.tencent.bkrepo.common.metadata.service.node.impl import com.tencent.bkrepo.archive.api.ArchiveClient import com.tencent.bkrepo.auth.api.ServicePermissionClient import com.tencent.bkrepo.common.artifact.api.ArtifactInfo -import com.tencent.bkrepo.common.artifact.router.RouterControllerProperties -import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService -import com.tencent.bkrepo.common.service.cluster.condition.DefaultCondition -import com.tencent.bkrepo.common.storage.core.StorageService -import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier +import com.tencent.bkrepo.common.artifact.properties.RouterControllerProperties +import com.tencent.bkrepo.common.metadata.condition.SyncCondition import com.tencent.bkrepo.common.metadata.config.RepositoryProperties import com.tencent.bkrepo.common.metadata.dao.node.NodeDao import com.tencent.bkrepo.common.metadata.dao.repo.RepositoryDao +import com.tencent.bkrepo.common.metadata.pojo.node.NodeRestoreOption +import com.tencent.bkrepo.common.metadata.pojo.node.RestoreContext +import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService +import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService +import com.tencent.bkrepo.common.metadata.service.project.ProjectService +import com.tencent.bkrepo.common.metadata.service.repo.QuotaService +import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService +import com.tencent.bkrepo.common.service.cluster.condition.DefaultCondition +import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier import com.tencent.bkrepo.repository.pojo.node.NodeDeleteResult import com.tencent.bkrepo.repository.pojo.node.NodeDeletedPoint import com.tencent.bkrepo.repository.pojo.node.NodeDetail -import com.tencent.bkrepo.repository.pojo.node.NodeRestoreOption import com.tencent.bkrepo.repository.pojo.node.NodeRestoreResult import com.tencent.bkrepo.repository.pojo.node.NodeSizeInfo import com.tencent.bkrepo.repository.pojo.node.service.NodeArchiveRequest @@ -52,10 +57,6 @@ import com.tencent.bkrepo.repository.pojo.node.service.NodeMoveCopyRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeRenameRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeUnCompressedRequest import com.tencent.bkrepo.repository.pojo.node.service.NodesDeleteRequest -import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService -import com.tencent.bkrepo.common.metadata.service.project.ProjectService -import com.tencent.bkrepo.common.metadata.service.repo.QuotaService -import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService import com.tencent.bkrepo.router.api.RouterControllerClient import org.springframework.context.annotation.Conditional import org.springframework.data.mongodb.core.query.Criteria @@ -64,13 +65,12 @@ import org.springframework.transaction.annotation.Transactional import java.time.LocalDateTime @Service -@Conditional(DefaultCondition::class) +@Conditional(SyncCondition::class, DefaultCondition::class) class NodeServiceImpl( override val nodeDao: NodeDao, override val repositoryDao: RepositoryDao, override val fileReferenceService: FileReferenceService, override val storageCredentialService: StorageCredentialService, - override val storageService: StorageService, override val quotaService: QuotaService, override val repositoryProperties: RepositoryProperties, override val messageSupplier: MessageSupplier, @@ -85,7 +85,6 @@ class NodeServiceImpl( repositoryDao, fileReferenceService, storageCredentialService, - storageService, quotaService, repositoryProperties, messageSupplier, @@ -202,7 +201,7 @@ class NodeServiceImpl( return NodeRestoreSupport(this).listDeletedPoint(artifact) } - override fun restoreNode(restoreContext: NodeRestoreSupport.RestoreContext): NodeRestoreResult { + override fun restoreNode(restoreContext: RestoreContext): NodeRestoreResult { return NodeRestoreSupport(this).restoreNode(restoreContext) } diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeStatsSupport.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeStatsSupport.kt similarity index 98% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeStatsSupport.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeStatsSupport.kt index c6cda79abc..f9f8435adf 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeStatsSupport.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeStatsSupport.kt @@ -29,7 +29,7 @@ * SOFTWARE. */ -package com.tencent.bkrepo.repository.service.node.impl +package com.tencent.bkrepo.common.metadata.service.node.impl import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.api.exception.ErrorCodeException @@ -39,7 +39,7 @@ import com.tencent.bkrepo.common.metadata.dao.node.NodeDao import com.tencent.bkrepo.common.metadata.model.TNode import com.tencent.bkrepo.repository.pojo.node.NodeListOption import com.tencent.bkrepo.repository.pojo.node.NodeSizeInfo -import com.tencent.bkrepo.repository.service.node.NodeStatsOperation +import com.tencent.bkrepo.common.metadata.service.node.NodeStatsOperation import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper import org.springframework.data.mongodb.core.aggregation.Aggregation.group import org.springframework.data.mongodb.core.aggregation.Aggregation.match diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeDeleteSupport.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeDeleteSupport.kt similarity index 96% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeDeleteSupport.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeDeleteSupport.kt index a6e1ef54c0..a97d04195d 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeDeleteSupport.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeDeleteSupport.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.repository.service.node.impl.center +package com.tencent.bkrepo.common.metadata.service.node.impl.center import com.tencent.bkrepo.common.api.constant.ensureSuffix import com.tencent.bkrepo.common.api.util.HumanReadable @@ -38,8 +38,8 @@ import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties import com.tencent.bkrepo.common.metadata.model.TNode import com.tencent.bkrepo.repository.pojo.node.NodeDeleteResult import com.tencent.bkrepo.repository.pojo.node.NodeListOption -import com.tencent.bkrepo.repository.service.node.impl.NodeBaseService -import com.tencent.bkrepo.repository.service.node.impl.NodeDeleteSupport +import com.tencent.bkrepo.common.metadata.service.node.impl.NodeBaseService +import com.tencent.bkrepo.common.metadata.service.node.impl.NodeDeleteSupport import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper import org.bson.types.ObjectId import org.jboss.logging.Logger @@ -52,7 +52,7 @@ import org.springframework.data.mongodb.core.query.isEqualTo import org.springframework.data.mongodb.core.query.where import java.time.LocalDateTime -class CommitEdgeCenterNodeDeleteSupport( +class CenterNodeDeleteSupport( private val nodeBaseService: NodeBaseService, private val clusterProperties: ClusterProperties ): NodeDeleteSupport( @@ -198,6 +198,6 @@ class CommitEdgeCenterNodeDeleteSupport( } companion object { - private val logger = Logger.getLogger(CommitEdgeCenterNodeDeleteSupport::class.java) + private val logger = Logger.getLogger(CenterNodeDeleteSupport::class.java) } } diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeMoveCopySupport.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeMoveCopySupport.kt similarity index 89% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeMoveCopySupport.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeMoveCopySupport.kt index 8260f7f59c..915c6a1cde 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeMoveCopySupport.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeMoveCopySupport.kt @@ -25,24 +25,25 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.repository.service.node.impl.center +package com.tencent.bkrepo.common.metadata.service.node.impl.center import com.tencent.bkrepo.common.artifact.path.PathUtils.resolveName import com.tencent.bkrepo.common.artifact.path.PathUtils.resolveParent import com.tencent.bkrepo.common.artifact.path.PathUtils.toPath +import com.tencent.bkrepo.common.metadata.model.TNode +import com.tencent.bkrepo.common.metadata.service.node.impl.NodeBaseService +import com.tencent.bkrepo.common.metadata.service.node.impl.NodeMoveCopySupport import com.tencent.bkrepo.common.metadata.util.ClusterUtils import com.tencent.bkrepo.common.metadata.util.ClusterUtils.isEdgeRequest +import com.tencent.bkrepo.common.metadata.util.NodeMoveCopyHelper.MoveCopyContext import com.tencent.bkrepo.common.security.util.SecurityUtils -import com.tencent.bkrepo.common.metadata.model.TNode import com.tencent.bkrepo.repository.pojo.node.NodeListOption -import com.tencent.bkrepo.repository.service.node.impl.NodeBaseService -import com.tencent.bkrepo.repository.service.node.impl.NodeMoveCopySupport import org.slf4j.LoggerFactory import org.springframework.data.mongodb.core.query.Query import org.springframework.data.mongodb.core.query.isEqualTo import org.springframework.data.mongodb.core.query.where -class CommitEdgeCenterNodeMoveCopySupport( +class CenterNodeMoveCopySupport( private val nodeBaseService: NodeBaseService ) : NodeMoveCopySupport( nodeBaseService @@ -86,6 +87,6 @@ class CommitEdgeCenterNodeMoveCopySupport( } companion object { - private val logger = LoggerFactory.getLogger(CommitEdgeCenterNodeMoveCopySupport::class.java) + private val logger = LoggerFactory.getLogger(CenterNodeMoveCopySupport::class.java) } } diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeRenameSupport.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeRenameSupport.kt similarity index 91% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeRenameSupport.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeRenameSupport.kt index d7d2ee0164..4c8f818765 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeRenameSupport.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeRenameSupport.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.repository.service.node.impl.center +package com.tencent.bkrepo.common.metadata.service.node.impl.center import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.message.CommonMessageCode @@ -35,8 +35,8 @@ import com.tencent.bkrepo.common.metadata.util.ClusterUtils.isEdgeRequest import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties import com.tencent.bkrepo.common.metadata.model.TNode -import com.tencent.bkrepo.repository.service.node.impl.NodeBaseService -import com.tencent.bkrepo.repository.service.node.impl.NodeRenameSupport +import com.tencent.bkrepo.common.metadata.service.node.impl.NodeBaseService +import com.tencent.bkrepo.common.metadata.service.node.impl.NodeRenameSupport import org.slf4j.LoggerFactory import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.Query @@ -44,7 +44,7 @@ import org.springframework.data.mongodb.core.query.and import org.springframework.data.mongodb.core.query.isEqualTo import org.springframework.data.mongodb.core.query.where -class CommitEdgeCenterNodeRenameSupport( +class CenterNodeRenameSupport( nodeBaseService: NodeBaseService, private val clusterProperties: ClusterProperties ) : NodeRenameSupport( @@ -84,6 +84,6 @@ class CommitEdgeCenterNodeRenameSupport( } companion object { - private val logger = LoggerFactory.getLogger(CommitEdgeCenterNodeRenameSupport::class.java) + private val logger = LoggerFactory.getLogger(CenterNodeRenameSupport::class.java) } } diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeRestoreSupport.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeRestoreSupport.kt similarity index 89% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeRestoreSupport.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeRestoreSupport.kt index f30024d701..9b62b3e623 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeRestoreSupport.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeRestoreSupport.kt @@ -25,18 +25,19 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.repository.service.node.impl.center +package com.tencent.bkrepo.common.metadata.service.node.impl.center import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode +import com.tencent.bkrepo.common.metadata.pojo.node.ConflictStrategy +import com.tencent.bkrepo.common.metadata.pojo.node.RestoreContext import com.tencent.bkrepo.common.metadata.util.ClusterUtils import com.tencent.bkrepo.common.metadata.model.TNode -import com.tencent.bkrepo.repository.pojo.node.ConflictStrategy -import com.tencent.bkrepo.repository.service.node.impl.NodeBaseService -import com.tencent.bkrepo.repository.service.node.impl.NodeRestoreSupport +import com.tencent.bkrepo.common.metadata.service.node.impl.NodeBaseService +import com.tencent.bkrepo.common.metadata.service.node.impl.NodeRestoreSupport import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper -class CommitEdgeCenterNodeRestoreSupport( +class CenterNodeRestoreSupport( nodeBaseService: NodeBaseService ) : NodeRestoreSupport( nodeBaseService diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt similarity index 87% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeServiceImpl.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt index a3549447cf..8bb67187ca 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.repository.service.node.impl.center +package com.tencent.bkrepo.common.metadata.service.node.impl.center import com.tencent.bkrepo.archive.api.ArchiveClient import com.tencent.bkrepo.auth.api.ServicePermissionClient @@ -33,24 +33,28 @@ import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.artifact.path.PathUtils -import com.tencent.bkrepo.common.artifact.router.RouterControllerProperties +import com.tencent.bkrepo.common.artifact.properties.RouterControllerProperties +import com.tencent.bkrepo.common.metadata.condition.SyncCondition +import com.tencent.bkrepo.common.metadata.config.RepositoryProperties +import com.tencent.bkrepo.common.metadata.dao.node.NodeDao import com.tencent.bkrepo.common.metadata.dao.repo.RepositoryDao +import com.tencent.bkrepo.common.metadata.model.TMetadata +import com.tencent.bkrepo.common.metadata.model.TNode import com.tencent.bkrepo.common.metadata.model.TRepository +import com.tencent.bkrepo.common.metadata.pojo.node.RestoreContext import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService +import com.tencent.bkrepo.common.metadata.service.node.impl.NodeServiceImpl import com.tencent.bkrepo.common.metadata.service.project.ProjectService import com.tencent.bkrepo.common.metadata.service.repo.QuotaService import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService import com.tencent.bkrepo.common.metadata.util.ClusterUtils +import com.tencent.bkrepo.common.metadata.util.NodeBaseServiceHelper.convertToDetail +import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.cluster.condition.CommitEdgeCenterCondition import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties -import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier -import com.tencent.bkrepo.common.metadata.config.RepositoryProperties -import com.tencent.bkrepo.common.metadata.dao.node.NodeDao -import com.tencent.bkrepo.common.metadata.model.TMetadata -import com.tencent.bkrepo.common.metadata.model.TNode import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import com.tencent.bkrepo.repository.pojo.node.NodeDeleteResult import com.tencent.bkrepo.repository.pojo.node.NodeDetail @@ -60,9 +64,6 @@ import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeMoveCopyRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeRenameRequest import com.tencent.bkrepo.repository.pojo.node.service.NodesDeleteRequest -import com.tencent.bkrepo.repository.service.node.impl.NodeRestoreSupport -import com.tencent.bkrepo.repository.service.node.impl.NodeServiceImpl -import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper import com.tencent.bkrepo.router.api.RouterControllerClient import org.slf4j.LoggerFactory import org.springframework.context.annotation.Conditional @@ -72,13 +73,12 @@ import org.springframework.stereotype.Service import java.time.LocalDateTime @Service -@Conditional(CommitEdgeCenterCondition::class) -class CommitEdgeCenterNodeServiceImpl( +@Conditional(SyncCondition::class, CommitEdgeCenterCondition::class) +class CenterNodeServiceImpl( override val nodeDao: NodeDao, override val repositoryDao: RepositoryDao, override val fileReferenceService: FileReferenceService, override val storageCredentialService: StorageCredentialService, - override val storageService: StorageService, override val quotaService: QuotaService, override val repositoryProperties: RepositoryProperties, override val messageSupplier: MessageSupplier, @@ -94,7 +94,6 @@ class CommitEdgeCenterNodeServiceImpl( repositoryDao, fileReferenceService, storageCredentialService, - storageService, quotaService, repositoryProperties, messageSupplier, @@ -141,7 +140,7 @@ class CommitEdgeCenterNodeServiceImpl( override fun deleteByFullPathWithoutDecreaseVolume( projectId: String, repoName: String, fullPath: String, operator: String ) { - return CommitEdgeCenterNodeDeleteSupport(this, clusterProperties).deleteByFullPathWithoutDecreaseVolume( + return CenterNodeDeleteSupport(this, clusterProperties).deleteByFullPathWithoutDecreaseVolume( projectId, repoName, fullPath, @@ -201,11 +200,11 @@ class CommitEdgeCenterNodeServiceImpl( } override fun deleteNode(deleteRequest: NodeDeleteRequest): NodeDeleteResult { - return CommitEdgeCenterNodeDeleteSupport(this, clusterProperties).deleteNode(deleteRequest) + return CenterNodeDeleteSupport(this, clusterProperties).deleteNode(deleteRequest) } override fun deleteNodes(nodesDeleteRequest: NodesDeleteRequest): NodeDeleteResult { - return CommitEdgeCenterNodeDeleteSupport(this, clusterProperties).deleteNodes(nodesDeleteRequest) + return CenterNodeDeleteSupport(this, clusterProperties).deleteNodes(nodesDeleteRequest) } override fun deleteByPath( @@ -214,7 +213,7 @@ class CommitEdgeCenterNodeServiceImpl( fullPath: String, operator: String ): NodeDeleteResult { - return CommitEdgeCenterNodeDeleteSupport(this, clusterProperties).deleteByPath( + return CenterNodeDeleteSupport(this, clusterProperties).deleteByPath( projectId, repoName, fullPath, @@ -228,7 +227,7 @@ class CommitEdgeCenterNodeServiceImpl( fullPaths: List, operator: String ): NodeDeleteResult { - return CommitEdgeCenterNodeDeleteSupport(this, clusterProperties).deleteByPaths( + return CenterNodeDeleteSupport(this, clusterProperties).deleteByPaths( projectId, repoName, fullPaths, @@ -243,7 +242,7 @@ class CommitEdgeCenterNodeServiceImpl( operator: String, path: String ): NodeDeleteResult { - return CommitEdgeCenterNodeDeleteSupport(this, clusterProperties).deleteBeforeDate( + return CenterNodeDeleteSupport(this, clusterProperties).deleteBeforeDate( projectId, repoName, date, @@ -252,24 +251,24 @@ class CommitEdgeCenterNodeServiceImpl( ) } - override fun restoreNode(restoreContext: NodeRestoreSupport.RestoreContext): NodeRestoreResult { - return CommitEdgeCenterNodeRestoreSupport(this).restoreNode(restoreContext) + override fun restoreNode(restoreContext: RestoreContext): NodeRestoreResult { + return CenterNodeRestoreSupport(this).restoreNode(restoreContext) } override fun copyNode(copyRequest: NodeMoveCopyRequest): NodeDetail { - return CommitEdgeCenterNodeMoveCopySupport(this).copyNode(copyRequest) + return CenterNodeMoveCopySupport(this).copyNode(copyRequest) } override fun moveNode(moveRequest: NodeMoveCopyRequest): NodeDetail { - return CommitEdgeCenterNodeMoveCopySupport(this).moveNode(moveRequest) + return CenterNodeMoveCopySupport(this).moveNode(moveRequest) } override fun renameNode(renameRequest: NodeRenameRequest) { - return CommitEdgeCenterNodeRenameSupport(this, clusterProperties).renameNode(renameRequest) + return CenterNodeRenameSupport(this, clusterProperties).renameNode(renameRequest) } companion object { - private val logger = LoggerFactory.getLogger(CommitEdgeCenterNodeServiceImpl::class.java) + private val logger = LoggerFactory.getLogger(CenterNodeServiceImpl::class.java) private fun convert(metadataModel: MetadataModel): TMetadata { with(metadataModel) { diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeBaseService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/edge/EdgeNodeBaseService.kt similarity index 94% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeBaseService.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/edge/EdgeNodeBaseService.kt index 9c59e0a712..349437263c 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeBaseService.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/edge/EdgeNodeBaseService.kt @@ -25,10 +25,10 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.repository.service.node.impl.edge +package com.tencent.bkrepo.common.metadata.service.node.impl.edge import com.tencent.bkrepo.auth.api.ServicePermissionClient -import com.tencent.bkrepo.common.artifact.router.RouterControllerProperties +import com.tencent.bkrepo.common.artifact.properties.RouterControllerProperties import com.tencent.bkrepo.common.metadata.dao.repo.RepositoryDao import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService @@ -37,7 +37,6 @@ import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService import com.tencent.bkrepo.common.metadata.util.ClusterUtils.reportMetadataToCenter import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties import com.tencent.bkrepo.common.service.feign.FeignClientFactory -import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier import com.tencent.bkrepo.repository.api.cluster.ClusterNodeClient import com.tencent.bkrepo.common.metadata.config.RepositoryProperties @@ -47,7 +46,7 @@ import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeUpdateAccessDateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeUpdateRequest -import com.tencent.bkrepo.repository.service.node.impl.NodeBaseService +import com.tencent.bkrepo.common.metadata.service.node.impl.NodeBaseService import com.tencent.bkrepo.common.metadata.service.repo.QuotaService import com.tencent.bkrepo.router.api.RouterControllerClient @@ -56,7 +55,6 @@ abstract class EdgeNodeBaseService( override val repositoryDao: RepositoryDao, override val fileReferenceService: FileReferenceService, override val storageCredentialService: StorageCredentialService, - override val storageService: StorageService, override val quotaService: QuotaService, override val repositoryProperties: RepositoryProperties, override val messageSupplier: MessageSupplier, @@ -71,7 +69,6 @@ abstract class EdgeNodeBaseService( repositoryDao, fileReferenceService, storageCredentialService, - storageService, quotaService, repositoryProperties, messageSupplier, diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/edge/EdgeNodeServiceImpl.kt similarity index 91% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/edge/EdgeNodeServiceImpl.kt index 38e12c1f5a..d127744651 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/edge/EdgeNodeServiceImpl.kt @@ -25,30 +25,40 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.repository.service.node.impl.edge +package com.tencent.bkrepo.common.metadata.service.node.impl.edge import com.tencent.bkrepo.archive.api.ArchiveClient import com.tencent.bkrepo.auth.api.ServicePermissionClient import com.tencent.bkrepo.common.artifact.api.ArtifactInfo -import com.tencent.bkrepo.common.artifact.router.RouterControllerProperties +import com.tencent.bkrepo.common.artifact.properties.RouterControllerProperties +import com.tencent.bkrepo.common.metadata.condition.SyncCondition +import com.tencent.bkrepo.common.metadata.config.RepositoryProperties +import com.tencent.bkrepo.common.metadata.dao.node.NodeDao import com.tencent.bkrepo.common.metadata.dao.repo.RepositoryDao +import com.tencent.bkrepo.common.metadata.pojo.node.NodeRestoreOption +import com.tencent.bkrepo.common.metadata.pojo.node.NodeRestoreRequest +import com.tencent.bkrepo.common.metadata.pojo.node.RestoreContext import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService +import com.tencent.bkrepo.common.metadata.service.node.impl.NodeArchiveSupport +import com.tencent.bkrepo.common.metadata.service.node.impl.NodeCompressSupport +import com.tencent.bkrepo.common.metadata.service.node.impl.NodeDeleteSupport +import com.tencent.bkrepo.common.metadata.service.node.impl.NodeMoveCopySupport +import com.tencent.bkrepo.common.metadata.service.node.impl.NodeRenameSupport +import com.tencent.bkrepo.common.metadata.service.node.impl.NodeRestoreSupport +import com.tencent.bkrepo.common.metadata.service.node.impl.NodeStatsSupport import com.tencent.bkrepo.common.metadata.service.project.ProjectService +import com.tencent.bkrepo.common.metadata.service.repo.QuotaService import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService import com.tencent.bkrepo.common.metadata.util.ClusterUtils.ignoreException import com.tencent.bkrepo.common.metadata.util.ClusterUtils.nodeLevelNotFoundError import com.tencent.bkrepo.common.metadata.util.ClusterUtils.repoLevelNotFoundError import com.tencent.bkrepo.common.service.cluster.condition.CommitEdgeEdgeCondition import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties -import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier -import com.tencent.bkrepo.common.metadata.config.RepositoryProperties -import com.tencent.bkrepo.common.metadata.dao.node.NodeDao import com.tencent.bkrepo.repository.pojo.node.NodeDeleteResult import com.tencent.bkrepo.repository.pojo.node.NodeDeletedPoint import com.tencent.bkrepo.repository.pojo.node.NodeDetail -import com.tencent.bkrepo.repository.pojo.node.NodeRestoreOption import com.tencent.bkrepo.repository.pojo.node.NodeRestoreResult import com.tencent.bkrepo.repository.pojo.node.NodeSizeInfo import com.tencent.bkrepo.repository.pojo.node.service.NodeArchiveRequest @@ -57,17 +67,8 @@ import com.tencent.bkrepo.repository.pojo.node.service.NodeCompressedRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeMoveCopyRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeRenameRequest -import com.tencent.bkrepo.repository.pojo.node.service.NodeRestoreRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeUnCompressedRequest import com.tencent.bkrepo.repository.pojo.node.service.NodesDeleteRequest -import com.tencent.bkrepo.repository.service.node.impl.NodeArchiveSupport -import com.tencent.bkrepo.repository.service.node.impl.NodeCompressSupport -import com.tencent.bkrepo.repository.service.node.impl.NodeDeleteSupport -import com.tencent.bkrepo.repository.service.node.impl.NodeMoveCopySupport -import com.tencent.bkrepo.repository.service.node.impl.NodeRenameSupport -import com.tencent.bkrepo.repository.service.node.impl.NodeRestoreSupport -import com.tencent.bkrepo.repository.service.node.impl.NodeStatsSupport -import com.tencent.bkrepo.common.metadata.service.repo.QuotaService import com.tencent.bkrepo.router.api.RouterControllerClient import org.springframework.context.annotation.Conditional import org.springframework.data.mongodb.core.query.Criteria @@ -76,13 +77,12 @@ import org.springframework.transaction.annotation.Transactional import java.time.LocalDateTime @Service -@Conditional(CommitEdgeEdgeCondition::class) +@Conditional(SyncCondition::class, CommitEdgeEdgeCondition::class) class EdgeNodeServiceImpl( override val nodeDao: NodeDao, override val repositoryDao: RepositoryDao, override val fileReferenceService: FileReferenceService, override val storageCredentialService: StorageCredentialService, - override val storageService: StorageService, override val quotaService: QuotaService, override val repositoryProperties: RepositoryProperties, override val messageSupplier: MessageSupplier, @@ -98,7 +98,6 @@ class EdgeNodeServiceImpl( repositoryDao, fileReferenceService, storageCredentialService, - storageService, quotaService, repositoryProperties, messageSupplier, @@ -271,7 +270,7 @@ class EdgeNodeServiceImpl( return NodeRestoreSupport(this).listDeletedPoint(artifact) } - override fun restoreNode(restoreContext: NodeRestoreSupport.RestoreContext): NodeRestoreResult { + override fun restoreNode(restoreContext: RestoreContext): NodeRestoreResult { return NodeRestoreSupport(this).restoreNode(restoreContext) } diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/NodeBaseServiceHelper.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/NodeBaseServiceHelper.kt new file mode 100644 index 0000000000..52725a5e88 --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/NodeBaseServiceHelper.kt @@ -0,0 +1,112 @@ +package com.tencent.bkrepo.common.metadata.util + +import com.tencent.bkrepo.common.api.util.Preconditions +import com.tencent.bkrepo.common.artifact.path.PathUtils +import com.tencent.bkrepo.common.metadata.model.TNode +import com.tencent.bkrepo.common.query.model.Sort +import com.tencent.bkrepo.repository.pojo.node.NodeDetail +import com.tencent.bkrepo.repository.pojo.node.NodeInfo +import com.tencent.bkrepo.repository.pojo.node.NodeListOption +import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest +import org.slf4j.LoggerFactory +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter + +object NodeBaseServiceHelper { + + private val logger = LoggerFactory.getLogger(NodeBaseServiceHelper::class.java) + const val TOPIC = "bkbase_bkrepo_artifact_node_created" + + fun checkNodeListOption(option: NodeListOption) { + Preconditions.checkArgument( + option.sortProperty.none { !TNode::class.java.declaredFields.map { f -> f.name }.contains(it) }, + "sortProperty", + ) + Preconditions.checkArgument( + option.direction.none { it != Sort.Direction.DESC.name && it != Sort.Direction.ASC.name }, + "direction", + ) + } + + fun validateParameter(node: TNode): Boolean { + if (node.folder) return false + if (node.sha256.isNullOrBlank()) { + logger.warn("Failed to change file reference, node[$node] sha256 is null or blank.") + return false + } + return true + } + + fun buildTNode(request: NodeCreateRequest, allowUserAddSystemMetadata: List): TNode { + with(request) { + val normalizeFullPath = PathUtils.normalizeFullPath(fullPath) + return TNode( + projectId = projectId, + repoName = repoName, + path = PathUtils.resolveParent(normalizeFullPath), + name = PathUtils.resolveName(normalizeFullPath), + fullPath = normalizeFullPath, + folder = folder, + expireDate = if (folder) null else parseExpireDate(expires), + size = if (folder) 0 else size ?: 0, + sha256 = if (folder) null else sha256, + md5 = if (folder) null else md5, + nodeNum = null, + metadata = MetadataUtils.compatibleConvertAndCheck( + metadata, + MetadataUtils.changeSystem(nodeMetadata, allowUserAddSystemMetadata), + ), + createdBy = createdBy ?: operator, + createdDate = createdDate ?: LocalDateTime.now(), + lastModifiedBy = createdBy ?: operator, + lastModifiedDate = lastModifiedDate ?: LocalDateTime.now(), + lastAccessDate = LocalDateTime.now(), + ) + } + } + + fun convert(tNode: TNode?): NodeInfo? { + return tNode?.let { + val metadata = MetadataUtils.toMap(it.metadata) + NodeInfo( + id = it.id, + createdBy = it.createdBy, + createdDate = it.createdDate.format(DateTimeFormatter.ISO_DATE_TIME), + lastModifiedBy = it.lastModifiedBy, + lastModifiedDate = it.lastModifiedDate.format(DateTimeFormatter.ISO_DATE_TIME), + projectId = it.projectId, + repoName = it.repoName, + folder = it.folder, + path = it.path, + name = it.name, + fullPath = it.fullPath, + size = if (it.size < 0L) 0L else it.size, + nodeNum = it.nodeNum?.let { nodeNum -> + if (nodeNum < 0L) 0L else nodeNum + }, + sha256 = it.sha256, + md5 = it.md5, + metadata = metadata, + nodeMetadata = MetadataUtils.toList(it.metadata), + copyFromCredentialsKey = it.copyFromCredentialsKey, + copyIntoCredentialsKey = it.copyIntoCredentialsKey, + deleted = it.deleted?.format(DateTimeFormatter.ISO_DATE_TIME), + lastAccessDate = it.lastAccessDate?.format(DateTimeFormatter.ISO_DATE_TIME), + clusterNames = it.clusterNames, + archived = it.archived, + compressed = it.compressed, + ) + } + } + + fun convertToDetail(tNode: TNode?): NodeDetail? { + return convert(tNode)?.let { NodeDetail(it) } + } + + /** + * 根据有效天数，计算到期时间 + */ + fun parseExpireDate(expireDays: Long?): LocalDateTime? { + return expireDays?.takeIf { it > 0 }?.run { LocalDateTime.now().plusDays(this) } + } +} diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/NodeDeleteHelper.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/NodeDeleteHelper.kt new file mode 100644 index 0000000000..7230b47b59 --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/NodeDeleteHelper.kt @@ -0,0 +1,28 @@ +package com.tencent.bkrepo.common.metadata.util + +import com.tencent.bkrepo.common.artifact.path.PathUtils +import com.tencent.bkrepo.common.metadata.model.TNode +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.and +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.data.mongodb.core.query.where + +object NodeDeleteHelper { + fun buildCriteria( + projectId: String, + repoName: String, + fullPath: String, + ): Criteria { + val normalizedFullPath = PathUtils.normalizeFullPath(fullPath) + val normalizedPath = PathUtils.toPath(normalizedFullPath) + val escapedPath = PathUtils.escapeRegex(normalizedPath) + val criteria = where(TNode::projectId).isEqualTo(projectId) + .and(TNode::repoName).isEqualTo(repoName) + .and(TNode::deleted).isEqualTo(null) + .orOperator( + where(TNode::fullPath).regex("^$escapedPath"), + where(TNode::fullPath).isEqualTo(normalizedFullPath) + ) + return criteria + } +} diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/NodeMoveCopyHelper.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/NodeMoveCopyHelper.kt new file mode 100644 index 0000000000..9b8d0c4cb4 --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/NodeMoveCopyHelper.kt @@ -0,0 +1,121 @@ +package com.tencent.bkrepo.common.metadata.util + +import com.tencent.bkrepo.common.api.exception.ErrorCodeException +import com.tencent.bkrepo.common.api.message.CommonMessageCode +import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode +import com.tencent.bkrepo.common.artifact.path.PathUtils.toPath +import com.tencent.bkrepo.common.artifact.pojo.RepositoryCategory +import com.tencent.bkrepo.common.metadata.model.TNode +import com.tencent.bkrepo.common.metadata.model.TRepository +import com.tencent.bkrepo.common.storage.credentials.StorageCredentials +import com.tencent.bkrepo.repository.pojo.node.NodeListOption +import org.springframework.data.mongodb.core.query.Query +import java.time.LocalDateTime + +object NodeMoveCopyHelper { + + /** + * 预检查 + */ + fun preCheck(context: MoveCopyContext) { + // 只允许local或者composite类型仓库操作 + val canSrcRepoMove = context.srcRepo.category.let { + it == RepositoryCategory.LOCAL || it == RepositoryCategory.COMPOSITE + } + val canDstRepoMove = context.dstRepo.category.let { + it == RepositoryCategory.LOCAL || it == RepositoryCategory.COMPOSITE + } + if (!canSrcRepoMove || !canDstRepoMove) { + throw ErrorCodeException(CommonMessageCode.METHOD_NOT_ALLOWED, "Only local repository is supported") + } + } + + /** + * 判断能否忽略执行 + */ + fun canIgnore(context: MoveCopyContext): Boolean { + with(context) { + var canIgnore = false + if (isSameRepo()) { + if (srcNode.fullPath == dstNode?.fullPath) { + // 同路径，跳过 + canIgnore = true + } else if (dstNode?.folder == true && srcNode.path == toPath(dstNode.fullPath)) { + // src为dst目录下的子节点，跳过 + canIgnore = true + } + } + return canIgnore + } + } + + fun buildDstNode( + context: MoveCopyContext, + node: TNode, + dstPath: String, + dstName: String, + dstFullPath: String + ): TNode { + with(context) { + val dstNode = node.copy( + id = null, + projectId = dstProjectId, + repoName = dstRepoName, + path = dstPath, + name = dstName, + fullPath = dstFullPath, + size = if (node.folder) 0 else node.size, + nodeNum = if (node.folder) null else node.nodeNum, + lastModifiedBy = operator, + lastModifiedDate = LocalDateTime.now() + ) + // move操作，create信息保留 + if (move) { + dstNode.createdBy = operator + dstNode.createdDate = LocalDateTime.now() + } + + return dstNode + } + } + + fun checkConflict(context: MoveCopyContext, node: TNode, existNode: TNode?) { + // 目录 -> 文件: 出错 + if (node.folder && existNode?.folder == false) { + throw ErrorCodeException(ArtifactMessageCode.NODE_CONFLICT, existNode.fullPath) + } + // 文件 -> 文件 & 不允许覆盖: 出错 + if (!node.folder && existNode?.folder == false && !context.overwrite) { + throw ErrorCodeException(ArtifactMessageCode.NODE_CONFLICT, existNode.fullPath) + } + } + + fun buildSubNodesQuery( + context: MoveCopyContext, + srcRootNodePath: String, + listOption: NodeListOption + ): Query { + with(context) { + val query = NodeQueryHelper.nodeListQuery(srcNode.projectId, srcNode.repoName, srcRootNodePath, listOption) + return query + } + } + + data class MoveCopyContext( + val srcRepo: TRepository, + val srcCredentials: StorageCredentials?, + val srcNode: TNode, + val dstProjectId: String, + val dstRepoName: String, + val dstFullPath: String, + val dstRepo: TRepository, + val dstCredentials: StorageCredentials?, + val dstNode: TNode?, + val dstNodeFolder: Boolean?, + val overwrite: Boolean, + val operator: String, + val move: Boolean + ) { + fun isSameRepo() = srcNode.projectId == dstProjectId && srcNode.repoName == dstRepoName + } +} diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/NodeQueryHelper.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/NodeQueryHelper.kt index 2d034f6526..5ae980998b 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/NodeQueryHelper.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/NodeQueryHelper.kt @@ -277,7 +277,7 @@ object NodeQueryHelper { return Pair(null, emptyList()) } - private fun listPermissionPaths( + fun listPermissionPaths( userId: String, projectId: String, repoName: String, diff --git a/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/AbstractMongoReactiveDao.kt b/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/AbstractMongoReactiveDao.kt index a069919bcd..35a6ebe8f6 100644 --- a/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/AbstractMongoReactiveDao.kt +++ b/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/AbstractMongoReactiveDao.kt @@ -35,10 +35,12 @@ import org.slf4j.LoggerFactory import org.springframework.dao.DuplicateKeyException import org.springframework.data.mongodb.MongoCollectionUtils import org.springframework.data.mongodb.core.FindAndModifyOptions -import org.springframework.data.mongodb.core.ReactiveMongoOperations +import org.springframework.data.mongodb.core.ReactiveMongoTemplate +import org.springframework.data.mongodb.core.aggregation.Aggregation import org.springframework.data.mongodb.core.mapping.Document import org.springframework.data.mongodb.core.query.Query import org.springframework.data.mongodb.core.query.Update +import reactor.core.publisher.Flux import java.lang.reflect.ParameterizedType abstract class AbstractMongoReactiveDao : MongoReactiveDao { @@ -63,6 +65,18 @@ abstract class AbstractMongoReactiveDao : MongoReactiveDao { return findAll(classType) } + suspend fun stream(query: Query): Flux { + return stream(query, classType) + } + + override suspend fun stream(query: Query, clazz: Class): Flux { + if (logger.isDebugEnabled) { + logger.debug("Mongo Dao stream: [$query] [$clazz]") + } + return determineReactiveMongoOperations() + .find(query, clazz, determineCollectionName(query)) + } + override suspend fun find(query: Query, clazz: Class): List { if (logger.isDebugEnabled) { logger.debug("Mongo Dao find: [$query] [$clazz]") @@ -179,6 +193,17 @@ abstract class AbstractMongoReactiveDao : MongoReactiveDao { .findAndModify(query, update, options, clazz, determineCollectionName(query)).awaitSingle() } + override suspend fun aggregate(aggregation: Aggregation, outputType: Class): MutableList { + if (logger.isDebugEnabled) { + logger.debug("Mongo aggregate: [$aggregation]") + } + return determineReactiveMongoOperations().aggregate( + aggregation, + determineCollectionName(aggregation), + outputType + ).collectList().awaitSingle() + } + protected open fun determineCollectionName(): String { var collectionName: String? = null if (classType.isAnnotationPresent(Document::class.java)) { @@ -191,12 +216,14 @@ abstract class AbstractMongoReactiveDao : MongoReactiveDao { } else collectionName } - abstract fun determineReactiveMongoOperations(): ReactiveMongoOperations + abstract fun determineReactiveMongoOperations(): ReactiveMongoTemplate abstract fun determineCollectionName(query: Query): String abstract fun determineCollectionName(entity: E): String + abstract fun determineCollectionName(aggregation: Aggregation): String + companion object { private val logger = LoggerFactory.getLogger(AbstractMongoReactiveDao::class.java) diff --git a/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/MongoReactiveDao.kt b/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/MongoReactiveDao.kt index fd95a0c78e..0e89f74722 100644 --- a/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/MongoReactiveDao.kt +++ b/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/MongoReactiveDao.kt @@ -30,8 +30,10 @@ package com.tencent.bkrepo.common.mongo.reactive.dao import com.mongodb.client.result.DeleteResult import com.mongodb.client.result.UpdateResult import org.springframework.data.mongodb.core.FindAndModifyOptions +import org.springframework.data.mongodb.core.aggregation.Aggregation import org.springframework.data.mongodb.core.query.Query import org.springframework.data.mongodb.core.query.Update +import reactor.core.publisher.Flux /** * mongo db reactive 数据访问层接口 @@ -102,4 +104,14 @@ interface MongoReactiveDao { * 查询并更新操作 */ suspend fun findAndModify(query: Query, update: Update, options: FindAndModifyOptions, clazz: Class): T? + + /** + * 流查询 + */ + suspend fun stream(query: Query, clazz: Class): Flux + + /** + * 文档聚合操作 + */ + suspend fun aggregate(aggregation: Aggregation, outputType: Class): MutableList } diff --git a/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/ShardingMongoReactiveDao.kt b/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/ShardingMongoReactiveDao.kt index 0c8cf25a95..cd574e4646 100644 --- a/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/ShardingMongoReactiveDao.kt +++ b/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/ShardingMongoReactiveDao.kt @@ -42,7 +42,8 @@ import org.springframework.core.annotation.AnnotationUtils import org.springframework.data.domain.Page import org.springframework.data.domain.PageImpl import org.springframework.data.domain.PageRequest -import org.springframework.data.mongodb.core.ReactiveMongoOperations +import org.springframework.data.mongodb.core.ReactiveMongoTemplate +import org.springframework.data.mongodb.core.aggregation.Aggregation import org.springframework.data.mongodb.core.index.IndexDefinition import org.springframework.data.mongodb.core.query.Query import java.lang.reflect.Field @@ -52,7 +53,7 @@ abstract class ShardingMongoReactiveDao : AbstractMongoReactiveDao() { @Suppress("LateinitUsage") @Autowired - lateinit var reactiveMongoOperations: ReactiveMongoOperations + lateinit var reactiveMongoTemplate: ReactiveMongoTemplate @Value("\${sharding.count:#{null}}") private val fixedShardingCount: Int? = null @@ -129,8 +130,8 @@ abstract class ShardingMongoReactiveDao : AbstractMongoReactiveDao() { } } - override fun determineReactiveMongoOperations(): ReactiveMongoOperations { - return reactiveMongoOperations + override fun determineReactiveMongoOperations(): ReactiveMongoTemplate { + return reactiveMongoTemplate } override fun determineCollectionName(query: Query): String { @@ -147,6 +148,22 @@ abstract class ShardingMongoReactiveDao : AbstractMongoReactiveDao() { return shardingKeyToCollectionName(shardingValue) } + override fun determineCollectionName(aggregation: Aggregation): String { + var shardingValue: Any? = null + val pipeline = aggregation.toPipeline(Aggregation.DEFAULT_CONTEXT) + for (document in pipeline) { + if (document.containsKey("\$match")) { + val subDocument = document["\$match"] + require(subDocument is Document) + shardingValue = subDocument["projectId"] + break + } + } + + requireNotNull(shardingValue) { "sharding value can not be empty!" } + return shardingKeyToCollectionName(shardingValue) + } + override fun determineCollectionName(): String { if (classType.isAnnotationPresent(ShardingDocument::class.java)) { val document = classType.getAnnotation(ShardingDocument::class.java) diff --git a/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/SimpleMongoReactiveDao.kt b/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/SimpleMongoReactiveDao.kt index 6d8a2be1c7..9d711c95eb 100644 --- a/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/SimpleMongoReactiveDao.kt +++ b/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/SimpleMongoReactiveDao.kt @@ -28,7 +28,8 @@ package com.tencent.bkrepo.common.mongo.reactive.dao import org.springframework.beans.factory.annotation.Autowired -import org.springframework.data.mongodb.core.ReactiveMongoOperations +import org.springframework.data.mongodb.core.ReactiveMongoTemplate +import org.springframework.data.mongodb.core.aggregation.Aggregation import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.Query import org.springframework.data.mongodb.core.query.isEqualTo @@ -38,10 +39,10 @@ open class SimpleMongoReactiveDao : AbstractMongoReactiveDao() { // 抽象类使用构造器注入不方便 @Suppress("LateinitUsage") @Autowired - lateinit var reactiveMongoOperations: ReactiveMongoOperations + lateinit var reactiveMongoTemplate: ReactiveMongoTemplate - override fun determineReactiveMongoOperations(): ReactiveMongoOperations { - return reactiveMongoOperations + override fun determineReactiveMongoOperations(): ReactiveMongoTemplate { + return reactiveMongoTemplate } override fun determineCollectionName(query: Query): String { @@ -52,6 +53,9 @@ open class SimpleMongoReactiveDao : AbstractMongoReactiveDao() { return collectionName } + override fun determineCollectionName(aggregation: Aggregation): String { + return collectionName + } /** * 根据主键"_id"查找记录 */ diff --git a/src/backend/common/common-query/query-mongo/build.gradle.kts b/src/backend/common/common-query/query-mongo/build.gradle.kts index a564d2619c..367326b314 100644 --- a/src/backend/common/common-query/query-mongo/build.gradle.kts +++ b/src/backend/common/common-query/query-mongo/build.gradle.kts @@ -32,5 +32,5 @@ dependencies { api(project(":common:common-api")) api(project(":common:common-query:query-api")) - implementation("org.springframework.boot:spring-boot-starter-data-mongodb") + implementation("org.springframework.data:spring-data-mongodb") } diff --git a/src/backend/composer/biz-composer/src/main/kotlin/com/tencent/bkrepo/composer/artifact/repository/ComposerLocalRepository.kt b/src/backend/composer/biz-composer/src/main/kotlin/com/tencent/bkrepo/composer/artifact/repository/ComposerLocalRepository.kt index ed00f7747a..1ecc8f8b69 100644 --- a/src/backend/composer/biz-composer/src/main/kotlin/com/tencent/bkrepo/composer/artifact/repository/ComposerLocalRepository.kt +++ b/src/backend/composer/biz-composer/src/main/kotlin/com/tencent/bkrepo/composer/artifact/repository/ComposerLocalRepository.kt @@ -30,6 +30,8 @@ package com.tencent.bkrepo.composer.artifact.repository import com.tencent.bkrepo.common.api.exception.MethodNotAllowedException import com.tencent.bkrepo.common.api.util.toJsonString import com.tencent.bkrepo.common.artifact.api.ArtifactFile +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo +import com.tencent.bkrepo.common.artifact.pojo.RepositoryId import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext @@ -137,9 +139,7 @@ class ComposerLocalRepository(private val stageClient: StageClient) : LocalRepos context: ArtifactContext ): ComposerArtifact { // 通过URL查询已保存的构件的版本。 - val oldNode = with(context.artifactInfo) { - nodeClient.getNodeDetail(projectId, repoName, getArtifactFullPath()) - }.data!! + val oldNode = nodeService.getNodeDetail(context.artifactInfo)!! val oldPackageKey = oldNode.metadata["packageKey"] ?: throw ComposerArtifactMetadataException( "${oldNode.projectId} | ${oldNode.repoName}\"${oldNode.fullPath}: not found metadata.packageKey" ) @@ -162,7 +162,7 @@ class ComposerLocalRepository(private val stageClient: StageClient) : LocalRepos ) { // 查询对应的 "/p/%package%.json" 是否存在 val pArtifactUri = "/p/${composerArtifact.name}.json" - val jsonNode = nodeClient.getNodeDetail(context.projectId, context.repoName, pArtifactUri).data + val jsonNode = nodeService.getNodeDetail(ArtifactInfo(context.projectId, context.repoName, pArtifactUri)) val resultJson = if (jsonNode == null) { with(composerArtifact) { JsonUtil.addComposerVersion(String.format(COMPOSER_VERSION_INIT, name), json, name, version) @@ -206,10 +206,9 @@ class ComposerLocalRepository(private val stageClient: StageClient) : LocalRepos * [ArtifactRepeat.NONE] 无重复构件 */ private fun checkRepeatArtifact(context: ArtifactUploadContext): ArtifactRepeat { - val artifactUri = context.artifactInfo.getArtifactFullPath() val artifactSha256 = context.getArtifactSha256() return with(context.artifactInfo) { - val node = nodeClient.getNodeDetail(projectId, repoName, artifactUri).data ?: return ArtifactRepeat.NONE + val node = nodeService.getNodeDetail(context.artifactInfo) ?: return ArtifactRepeat.NONE if (node.sha256 == artifactSha256) { ArtifactRepeat.FULLPATH_SHA256 } else { @@ -310,26 +309,26 @@ class ComposerLocalRepository(private val stageClient: StageClient) : LocalRepos VersionListOption(1, versions!!.toInt(), null, null) ).data?.records ?: return for (packageVersion in pages) { - val node = nodeClient.getNodeDetail(projectId, repoName, packageVersion.contentPath!!).data ?: continue + val node = nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, packageVersion.contentPath!!)) + ?: continue removeComposerArtifact(node, packageKey, packageVersion.name, context) } } else { - with(context.artifactInfo) { - val packageVersion = packageClient.findVersionByName( - projectId = projectId, - repoName = repoName, - packageKey = packageKey, - version = version - ).data ?: return - val node = nodeClient.getNodeDetail(projectId, repoName, packageVersion.contentPath!!).data ?: return - removeComposerArtifact(node, packageKey, version, context) - } + val packageVersion = packageClient.findVersionByName( + projectId = projectId, + repoName = repoName, + packageKey = packageKey, + version = version + ).data ?: return + val node = nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, packageVersion.contentPath!!)) + ?: return + removeComposerArtifact(node, packageKey, version, context) } } private fun getPackageJson(context: ArtifactContext, name: String): String? { val jsonPath = "/p/$name.json" - val node = nodeClient.getNodeDetail(context.projectId, context.repoName, jsonPath).data ?: return null + val node = nodeService.getNodeDetail(ArtifactInfo(context.projectId, context.repoName, jsonPath)) ?: return null return nodeToJson(node) } @@ -365,7 +364,7 @@ class ComposerLocalRepository(private val stageClient: StageClient) : LocalRepos // 更新索引 deleteJsonVersion(context, PackageKeys.resolveComposer(packageKey), version) val nodeDeleteRequest = NodeDeleteRequest(projectId, repoName, node.fullPath, context.userId) - nodeClient.deleteNode(nodeDeleteRequest) + nodeService.deleteNode(nodeDeleteRequest) logger.info("Success to delete node $nodeDeleteRequest") deleteVersion(projectId, repoName, packageKey, version) logger.info("Success to delete version $projectId | $repoName : $packageKey $version") @@ -413,9 +412,7 @@ class ComposerLocalRepository(private val stageClient: StageClient) : LocalRepos */ fun getPackages(context: ArtifactQueryContext): String { val host = getHost(context) - val node = with(context.artifactInfo) { - nodeClient.getNodeDetail(projectId, repoName, getArtifactFullPath()).data - } + val node = nodeService.getNodeDetail(context.artifactInfo) return if (node == null) { val artifactFile = ByteArrayInputStream(INIT_PACKAGES.toByteArray()).use { ArtifactFileFactory.build(it) @@ -439,7 +436,7 @@ class ComposerLocalRepository(private val stageClient: StageClient) : LocalRepos } private fun nodeToJson(node: NodeDetail): String { - val repoId = ArtifactContextHolder.RepositoryId(node.projectId, node.repoName) + val repoId = RepositoryId(node.projectId, node.repoName) val storageCredentials = ArtifactContextHolder.getRepoDetail(repoId).storageCredentials val inputStream = storageManager.loadFullArtifactInputStream(node, storageCredentials) ?: throw RuntimeException("load ${node.projectId} | ${node.repoName} | ${node.fullPath} error") @@ -462,7 +459,7 @@ class ComposerLocalRepository(private val stageClient: StageClient) : LocalRepos */ private fun stream2Json(context: ArtifactContext): String? { return with(context.artifactInfo) { - val node = nodeClient.getNodeDetail(projectId, repoName, getArtifactFullPath()).data ?: return null + val node = nodeService.getNodeDetail(this) ?: return null nodeToJson(node) } } @@ -511,9 +508,9 @@ class ComposerLocalRepository(private val stageClient: StageClient) : LocalRepos ).data ?: return null val artifactPath = trueVersion.contentPath ?: return null with(context.artifactInfo) { - val jarNode = nodeClient.getNodeDetail( - projectId, repoName, artifactPath - ).data ?: return null + val jarNode = nodeService.getNodeDetail( + ArtifactInfo(projectId, repoName, artifactPath) + ) ?: return null val stageTag = stageClient.query(projectId, repoName, packageKey, version).data val packageVersion = packageClient.findVersionByName( projectId, repoName, packageKey, version diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/artifact/repository/ConanLocalRepository.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/artifact/repository/ConanLocalRepository.kt index 2ca2a34ea7..af9b94f06e 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/artifact/repository/ConanLocalRepository.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/artifact/repository/ConanLocalRepository.kt @@ -27,6 +27,7 @@ package com.tencent.bkrepo.conan.artifact.repository +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactUploadContext import com.tencent.bkrepo.common.artifact.repository.local.LocalRepository @@ -140,7 +141,7 @@ class ConanLocalRepository : LocalRepository() { with(context.artifactInfo as ConanArtifactInfo) { val fullPath = generateFullPath(this) logger.info("File $fullPath will be downloaded in repo $projectId|$repoName") - val node = nodeClient.getNodeDetail(context.projectId, context.repoName, fullPath).data + val node = nodeService.getNodeDetail(ArtifactInfo(context.projectId, context.repoName, fullPath)) node?.let { context.artifactInfo.setArtifactMappingUri(node.fullPath) downloadIntercept(context, node) diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/CommonService.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/CommonService.kt index 04be315b0e..1d88307792 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/CommonService.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/CommonService.kt @@ -31,12 +31,15 @@ import com.tencent.bkrepo.common.api.constant.CharPool import com.tencent.bkrepo.common.api.util.readJsonString import com.tencent.bkrepo.common.api.util.toJsonString import com.tencent.bkrepo.common.artifact.api.ArtifactFile +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException import com.tencent.bkrepo.common.artifact.exception.PackageNotFoundException import com.tencent.bkrepo.common.artifact.exception.RepoNotFoundException import com.tencent.bkrepo.common.artifact.manager.StorageManager import com.tencent.bkrepo.common.artifact.resolve.file.ArtifactFileFactory import com.tencent.bkrepo.common.lock.service.LockOperation +import com.tencent.bkrepo.common.metadata.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.conan.config.ConanProperties @@ -65,9 +68,8 @@ import com.tencent.bkrepo.conan.utils.ConanPathUtils.getPackageRevisionsFile import com.tencent.bkrepo.conan.utils.ConanPathUtils.getRecipeRevisionsFile import com.tencent.bkrepo.conan.utils.ConanPathUtils.joinString import com.tencent.bkrepo.conan.utils.TimeFormatUtil.convertToUtcTime -import com.tencent.bkrepo.repository.api.NodeClient -import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.pojo.node.NodeDetail +import com.tencent.bkrepo.repository.pojo.node.NodeListOption import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -80,14 +82,11 @@ class CommonService( private val properties: ConanProperties ) { @Autowired - lateinit var nodeClient: NodeClient - + lateinit var nodeService: NodeService @Autowired lateinit var storageManager: StorageManager - @Autowired - lateinit var repositoryClient: RepositoryClient - + lateinit var repositoryService: RepositoryService @Autowired lateinit var lockOperation: LockOperation @@ -249,7 +248,7 @@ class CommonService( val result = mutableMapOf() val pathList = getPaths(projectId, repoName, path, subFileset) pathList.forEach { it -> - nodeClient.getNodeDetail(projectId, repoName, it).data?.let { + nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, it))?.let { when (type) { MD5 -> result[it.name] = it.md5!! else -> result[it.name] = it.fullPath @@ -264,7 +263,7 @@ class CommonService( repoName: String, path: String, ): NodeDetail { - return nodeClient.getNodeDetail(projectId, repoName, path).data + return nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, path)) ?: throw ConanRecipeNotFoundException(ConanMessageCode.CONAN_RECIPE_NOT_FOUND, path, "$projectId|$repoName") } @@ -274,9 +273,9 @@ class CommonService( path: String, ): ConanInfo { val fullPath = "/$path" - val node = nodeClient.getNodeDetail(projectId, repoName, fullPath).data + val node = nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, fullPath)) ?: throw NodeNotFoundException(path) - val repo = repositoryClient.getRepoDetail(projectId, repoName).data + val repo = repositoryService.getRepoDetail(projectId, repoName) ?: throw RepoNotFoundException("$projectId|$repoName not found") val inputStream = storageManager.loadArtifactInputStream(node, repo.storageCredentials) // TODO 需要调整 @@ -291,10 +290,13 @@ class CommonService( subFileset: List = emptyList() ): List { val fullPath = "/$path" - nodeClient.getNodeDetail(projectId, repoName, fullPath).data + nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, fullPath)) ?: throw NodeNotFoundException(path) val subFileMap = mutableMapOf() - nodeClient.listNode(projectId, repoName, fullPath, includeFolder = false, deep = true).data!!.forEach { + nodeService.listNode( + ArtifactInfo(projectId, repoName, fullPath), + NodeListOption(includeFolder = false, deep = true) + ).forEach { subFileMap[it.name] = it.fullPath } if (subFileset.isEmpty()) return subFileMap.values.toList() @@ -381,7 +383,7 @@ class CommonService( ) if (indexJson.revisions.isEmpty()) { val revisionV1Path = joinString(revPath, DEFAULT_REVISION_V1) - nodeClient.getNodeDetail(projectId, repoName, revisionV1Path).data ?: return null + nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, revisionV1Path)) ?: return null return RevisionInfo(DEFAULT_REVISION_V1, convertToUtcTime(LocalDateTime.now())) } else { return indexJson.revisions.first() @@ -395,8 +397,9 @@ class CommonService( refStr: String ): IndexInfo { val fullPath = "/$revPath" - val indexNode = nodeClient.getNodeDetail(projectId, repoName, fullPath).data ?: return IndexInfo(refStr) - val repo = repositoryClient.getRepoDetail(projectId, repoName).data + val indexNode = nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, fullPath)) + ?: return IndexInfo(refStr) + val repo = repositoryService.getRepoDetail(projectId, repoName) ?: throw RepoNotFoundException("$projectId|$repoName not found") storageManager.loadArtifactInputStream(indexNode, repo.storageCredentials)?.use { return it.readJsonString() @@ -409,10 +412,11 @@ class CommonService( repoName: String, revPath: String, ): List { - nodeClient.getNodeDetail(projectId, repoName, revPath).data ?: return emptyList() - return nodeClient.listNode(projectId, repoName, revPath, includeFolder = true, deep = false).data!!.map { - it.name - } + nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, revPath)) ?: return emptyList() + return nodeService.listNode( + ArtifactInfo(projectId, repoName, revPath), + NodeListOption(includeFolder = true, deep = false) + ).map { it.name } } fun buildFileAndNodeCreateRequest( @@ -453,10 +457,10 @@ class CommonService( indexInfo = indexInfo, operator = SecurityUtils.getUserId() ) - val repository = repositoryClient.getRepoDetail( + val repository = repositoryService.getRepoDetail( nodeCreateRequest.projectId, nodeCreateRequest.repoName - ).data + ) ?: throw RepoNotFoundException("Repository[${nodeCreateRequest.repoName}] does not exist") storageManager.storeArtifactFile(nodeCreateRequest, artifactFile, repository.storageCredentials) } diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanDeleteServiceImpl.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanDeleteServiceImpl.kt index a7c4dd8f6e..df41c0458c 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanDeleteServiceImpl.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanDeleteServiceImpl.kt @@ -27,10 +27,12 @@ package com.tencent.bkrepo.conan.service.impl +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException import com.tencent.bkrepo.common.artifact.exception.VersionNotFoundException import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.common.artifact.util.PackageKeys +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.SpringContextUtils.Companion.publishEvent import com.tencent.bkrepo.conan.constant.ConanMessageCode @@ -56,7 +58,6 @@ import com.tencent.bkrepo.conan.utils.ConanPathUtils.joinString import com.tencent.bkrepo.conan.utils.ObjectBuildUtil import com.tencent.bkrepo.conan.utils.ObjectBuildUtil.buildBasicInfo import com.tencent.bkrepo.conan.utils.ObjectBuildUtil.toConanFileReference -import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.PackageClient import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest import com.tencent.bkrepo.repository.pojo.packages.PackageVersion @@ -69,11 +70,9 @@ import org.springframework.stereotype.Service class ConanDeleteServiceImpl : ConanDeleteService { @Autowired - lateinit var nodeClient: NodeClient - + lateinit var nodeService: NodeService @Autowired lateinit var packageClient: PackageClient - @Autowired lateinit var commonService: CommonService @@ -84,7 +83,7 @@ class ConanDeleteServiceImpl : ConanDeleteService { val rootPath = PathUtils.normalizeFullPath(buildPackagePath(conanFileReference)) logger.info("Will delete folder $rootPath in repo $projectId|$repoName") val request = NodeDeleteRequest(projectId, repoName, rootPath, SecurityUtils.getUserId()) - nodeClient.deleteNode(request) + nodeService.deleteNode(request) val refStr = ConanPathUtils.buildReferenceWithoutVersion(conanFileReference) val packageKey = PackageKeys.ofConan(refStr) packageClient.deleteVersion(projectId, repoName, packageKey, version) @@ -93,7 +92,7 @@ class ConanDeleteServiceImpl : ConanDeleteService { val rootPath = PathUtils.normalizeFullPath(buildRevisionPath(conanFileReference)) logger.info("Will delete folder $rootPath in repo $projectId|$repoName") val request = NodeDeleteRequest(projectId, repoName, rootPath, SecurityUtils.getUserId()) - nodeClient.deleteNode(request) + nodeService.deleteNode(request) publishEvent( ConanRecipeDeleteEvent( ObjectBuildUtil.buildConanRecipeDeleteRequest(this, SecurityUtils.getUserId()) @@ -110,7 +109,7 @@ class ConanDeleteServiceImpl : ConanDeleteService { val path = PathUtils.normalizeFullPath(buildPackageFolderPath(conanFileReference)) logger.info("Will delete folder $path in repo $projectId|$repoName") val request = NodeDeleteRequest(projectId, repoName, path, SecurityUtils.getUserId()) - nodeClient.deleteNode(request) + nodeService.deleteNode(request) return } val revPath = PathUtils.normalizeFullPath(getPackageRevisionsFile(conanFileReference)) @@ -122,7 +121,7 @@ class ConanDeleteServiceImpl : ConanDeleteService { val path = PathUtils.normalizeFullPath(buildPackageIdFolderPath(conanFileReference, packageId)) logger.info("Will delete folder $path in repo $projectId|$repoName") val request = NodeDeleteRequest(projectId, repoName, path, SecurityUtils.getUserId()) - nodeClient.deleteNode(request) + nodeService.deleteNode(request) } } } @@ -134,13 +133,13 @@ class ConanDeleteServiceImpl : ConanDeleteService { val rootPath = PathUtils.normalizeFullPath(buildPackageIdFolderPath(conanFileReference, packageId!!)) logger.info("Will delete folder $rootPath in repo $projectId|$repoName") val request = NodeDeleteRequest(projectId, repoName, rootPath, SecurityUtils.getUserId()) - nodeClient.deleteNode(request) + nodeService.deleteNode(request) } else { val packageReference = convertToPackageReference(conanArtifactInfo) val rootPath = PathUtils.normalizeFullPath(buildPackageRevisionFolderPath(packageReference)) logger.info("Will delete folder $rootPath in repo $projectId|$repoName") val request = NodeDeleteRequest(projectId, repoName, rootPath, SecurityUtils.getUserId()) - nodeClient.deleteNode(request) + nodeService.deleteNode(request) publishEvent( ConanPackageDeleteEvent( ObjectBuildUtil.buildConanPackageDeleteRequest(this, SecurityUtils.getUserId()) @@ -157,13 +156,13 @@ class ConanDeleteServiceImpl : ConanDeleteService { var path: String? for (file in files) { path = joinString(rootPath, file) - nodeClient.getNodeDetail(projectId, repoName, path).data + nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, path)) ?: throw ConanFileNotFoundException( ConanMessageCode.CONAN_FILE_NOT_FOUND, path, "$projectId|$repoName" ) logger.info("Will delete folder $path in repo $projectId|$repoName") val request = NodeDeleteRequest(projectId, repoName, path, SecurityUtils.getUserId()) - nodeClient.deleteNode(request) + nodeService.deleteNode(request) publishEvent( ConanRecipeDeleteEvent( ObjectBuildUtil.buildConanRecipeDeleteRequest(this, SecurityUtils.getUserId()) @@ -202,9 +201,8 @@ class ConanDeleteServiceImpl : ConanDeleteService { val versionDetail = packageClient.findVersionByName(projectId, repoName, packageKey, version).data ?: throw VersionNotFoundException(version) if (versionDetail.contentPath.isNullOrEmpty()) throw VersionNotFoundException(version) - val nodeDetail = nodeClient.getNodeDetail(projectId, repoName, versionDetail.contentPath!!).data ?: run { - throw NodeNotFoundException(versionDetail.contentPath!!) - } + val nodeDetail = nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, versionDetail.contentPath!!)) + ?: throw NodeNotFoundException(versionDetail.contentPath!!) val basicInfo = buildBasicInfo(nodeDetail, versionDetail) return PackageVersionInfo(basicInfo, versionDetail.packageMetadata) } @@ -221,7 +219,7 @@ class ConanDeleteServiceImpl : ConanDeleteService { val rootPath = PathUtils.normalizeFullPath(buildPackagePath(conanFileReference)) logger.info("Will delete folder $rootPath in repo $projectId|$repoName") val request = NodeDeleteRequest(projectId, repoName, rootPath, SecurityUtils.getUserId()) - nodeClient.deleteNode(request) + nodeService.deleteNode(request) if (deleteVersion) { packageClient.deleteVersion(projectId, repoName, packageKey, versionDetail.name) } diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/BlobService.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/BlobService.kt index c7f393cb72..e42a5ffc25 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/BlobService.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/BlobService.kt @@ -27,23 +27,25 @@ package com.tencent.bkrepo.ddc.service +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.manager.StorageManager +import com.tencent.bkrepo.common.artifact.pojo.RepositoryId import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.ddc.exception.BlobNotFoundException import com.tencent.bkrepo.ddc.model.TDdcBlob import com.tencent.bkrepo.ddc.pojo.Blob import com.tencent.bkrepo.ddc.pojo.Reference import com.tencent.bkrepo.ddc.repository.BlobRepository -import com.tencent.bkrepo.repository.api.NodeClient import org.springframework.stereotype.Service import java.time.LocalDateTime @Service class BlobService( private val blobRepository: BlobRepository, - private val nodeClient: NodeClient, + private val nodeService: NodeService, private val storageManager: StorageManager ) { fun create(blob: Blob): Blob { @@ -86,8 +88,8 @@ class BlobService( fun loadBlob(blob: Blob): ArtifactInputStream { val repo = - ArtifactContextHolder.getRepoDetail(ArtifactContextHolder.RepositoryId(blob.projectId, blob.repoName)) - val node = nodeClient.getNodeDetail(blob.projectId, blob.repoName, blob.fullPath).data + ArtifactContextHolder.getRepoDetail(RepositoryId(blob.projectId, blob.repoName)) + val node = nodeService.getNodeDetail(ArtifactInfo(blob.projectId, blob.repoName, blob.fullPath)) ?: throw BlobNotFoundException(blob.projectId, blob.repoName, blob.blobId.toString()) return storageManager.loadArtifactInputStream(node, repo.storageCredentials) ?: throw BlobNotFoundException(blob.projectId, blob.repoName, blob.blobId.toString()) diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/ReferenceService.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/ReferenceService.kt index 0d19a4b88f..bc83bc02ac 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/ReferenceService.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/ReferenceService.kt @@ -29,8 +29,11 @@ package com.tencent.bkrepo.ddc.service import com.tencent.bkrepo.common.api.exception.BadRequestException import com.tencent.bkrepo.common.api.message.CommonMessageCode +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.manager.StorageManager +import com.tencent.bkrepo.common.artifact.pojo.RepositoryId import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.ddc.config.DdcProperties import com.tencent.bkrepo.ddc.exception.ReferenceIsMissingBlobsException @@ -45,7 +48,6 @@ import com.tencent.bkrepo.ddc.repository.RefBaseRepository import com.tencent.bkrepo.ddc.repository.RefRepository import com.tencent.bkrepo.ddc.serialization.CbObject import com.tencent.bkrepo.ddc.utils.hasAttachments -import com.tencent.bkrepo.repository.api.NodeClient import org.bson.types.Binary import org.slf4j.LoggerFactory import org.springframework.stereotype.Service @@ -59,7 +61,7 @@ class ReferenceService( private val refResolver: ReferenceResolver, private val refRepository: RefRepository, private val legacyRefRepository: LegacyRefRepository, - private val nodeClient: NodeClient, + private val nodeService: NodeService, private val storageManager: StorageManager, ) { fun create(ref: Reference): Reference { @@ -135,8 +137,8 @@ class ReferenceService( val ref = Reference.from(tRef) if (ref.inlineBlob == null) { - val repo = ArtifactContextHolder.getRepoDetail(ArtifactContextHolder.RepositoryId(projectId, repoName)) - ref.inlineBlob = nodeClient.getNodeDetail(projectId, repoName, ref.fullPath()).data?.let { node -> + val repo = ArtifactContextHolder.getRepoDetail(RepositoryId(projectId, repoName)) + ref.inlineBlob = nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, ref.fullPath()))?.let { node -> storageManager.loadArtifactInputStream(node, repo.storageCredentials)?.use { it.readBytes() } } } diff --git a/src/backend/ddc/biz-ddc/src/test/kotlin/com/tencent/bkrepo/ddc/serialization/CbFieldTest.kt b/src/backend/ddc/biz-ddc/src/test/kotlin/com/tencent/bkrepo/ddc/serialization/CbFieldTest.kt index 15539c5d64..9a6a4e6c11 100644 --- a/src/backend/ddc/biz-ddc/src/test/kotlin/com/tencent/bkrepo/ddc/serialization/CbFieldTest.kt +++ b/src/backend/ddc/biz-ddc/src/test/kotlin/com/tencent/bkrepo/ddc/serialization/CbFieldTest.kt @@ -68,7 +68,9 @@ class CbFieldTest { // Test CbField(None|Type|Name) var fieldType = CbFieldType.None.value or CbFieldType.HasFieldName.value - var noneBytes = byteBufferOf(fieldType, 4, 'N'.toByte(), 'a'.toByte(), 'm'.toByte(), 'e'.toByte()) + var noneBytes = byteBufferOf( + fieldType, 4, 'N'.code.toByte(), 'a'.code.toByte(), 'm'.code.toByte(), 'e'.code.toByte() + ) noneField = CbField(noneBytes) assertEquals(noneField.getSize(), noneBytes.remaining()) @@ -100,7 +102,9 @@ class CbFieldTest { // Test CbField(None|Name) fieldType = CbFieldType.None.value or CbFieldType.HasFieldName.value - noneBytes = byteBufferOf(fieldType, 4, 'N'.toByte(), 'a'.toByte(), 'm'.toByte(), 'e'.toByte()) + noneBytes = byteBufferOf( + fieldType, 4, 'N'.code.toByte(), 'a'.code.toByte(), 'm'.code.toByte(), 'e'.code.toByte() + ) var b = noneBytes.asReadOnlyBuffer() b.position(1) @@ -176,9 +180,9 @@ class CbFieldTest { val intType: Byte = (CbFieldType.HasFieldName.value or CbFieldType.IntegerPositive.value) var payload = byteBufferOf( 12, - intType, 1, 'A'.toByte(), 1, - intType, 1, 'B'.toByte(), 2, - intType, 1, 'C'.toByte(), 3 + intType, 1, 'A'.code.toByte(), 1, + intType, 1, 'B'.code.toByte(), 2, + intType, 1, 'C'.code.toByte(), 3 ) var field = CbField(payload.asReadOnlyBuffer(), CbFieldType.Object) testField(CbFieldType.Object, field, CbObject(payload, CbFieldType.Object)) @@ -197,9 +201,9 @@ class CbFieldTest { // Test CbField(UniformObject, NotEmpty) payload = byteBufferOf( 10, intType, - 1, 'A'.toByte(), 1, - 1, 'B'.toByte(), 2, - 1, 'C'.toByte(), 3 + 1, 'A'.code.toByte(), 1, + 1, 'B'.code.toByte(), 2, + 1, 'C'.code.toByte(), 3 ) field = CbField(payload.asReadOnlyBuffer(), CbFieldType.UniformObject) testField(CbFieldType.UniformObject, field, CbObject(payload, CbFieldType.UniformObject)) @@ -216,11 +220,11 @@ class CbFieldTest { // Equals val namedPayload = byteBufferOf( - 1, 'O'.toByte(), + 1, 'O'.code.toByte(), 10, intType, - 1, 'A'.toByte(), 1, - 1, 'B'.toByte(), 2, - 1, 'C'.toByte(), 3 + 1, 'A'.code.toByte(), 1, + 1, 'B'.code.toByte(), 2, + 1, 'C'.code.toByte(), 3 ) val namedField = CbField(namedPayload, CbFieldType.UniformObject.value or CbFieldType.HasFieldName.value) assertTrue(field.asObject() == namedField.asObject()) @@ -244,10 +248,10 @@ class CbFieldTest { val objectType = (CbFieldType.Object.value or CbFieldType.HasFieldName.value) val buffer = byteBufferOf( objectType, - 3, 'K'.toByte(), 'e'.toByte(), 'y'.toByte(), + 3, 'K'.code.toByte(), 'e'.code.toByte(), 'y'.code.toByte(), 4, (CbFieldType.HasFieldName.value or CbFieldType.IntegerPositive.value), - 1, 'F'.toByte(), + 1, 'F'.code.toByte(), 8 ) var o = CbObject(buffer) @@ -304,7 +308,7 @@ class CbFieldTest { assertTrue(array == field.asArray()) // Equals - val namedPayload = byteBufferOf(1, 'A'.toByte(), 5, 3, intType, 1, 2, 3) + val namedPayload = byteBufferOf(1, 'A'.code.toByte(), 5, 3, intType, 1, 2, 3) val namedField = CbField(namedPayload, CbFieldType.UniformArray.value or CbFieldType.HasFieldName.value) assertTrue(field.asArray() == namedField.asArray()) assertTrue(field == field.asArray().asField()) @@ -334,7 +338,7 @@ class CbFieldTest { val arrayType: Byte = (CbFieldType.Array.value or CbFieldType.HasFieldName.value) val buffer = byteBufferOf( arrayType, - 3, 'K'.toByte(), 'e'.toByte(), 'y'.toByte(), + 3, 'K'.code.toByte(), 'e'.code.toByte(), 'y'.code.toByte(), 3, 1, CbFieldType.IntegerPositive.value, 8 ) array = CbArray(buffer) @@ -388,7 +392,8 @@ class CbFieldTest { // Test CbField(String, Value) run { - val payload = byteBufferOf(3, 'A'.toByte(), 'B'.toByte(), 'C'.toByte()) // Size: 3, Data: ABC + // Size: 3, Data: ABC + val payload = byteBufferOf(3, 'A'.code.toByte(), 'B'.code.toByte(), 'C'.code.toByte()) testField(CbFieldType.String, payload, "ABC") } diff --git a/src/backend/ddc/biz-ddc/src/test/kotlin/com/tencent/bkrepo/ddc/serialization/CbWriterTest.kt b/src/backend/ddc/biz-ddc/src/test/kotlin/com/tencent/bkrepo/ddc/serialization/CbWriterTest.kt index ad88114776..3b388123cc 100644 --- a/src/backend/ddc/biz-ddc/src/test/kotlin/com/tencent/bkrepo/ddc/serialization/CbWriterTest.kt +++ b/src/backend/ddc/biz-ddc/src/test/kotlin/com/tencent/bkrepo/ddc/serialization/CbWriterTest.kt @@ -52,17 +52,17 @@ class CbWriterTest { assertEquals(data[2], CbFieldType.IntegerPositive.value or CbFieldType.HasFieldName.value) assertEquals(data[3], 1.toByte()) - assertEquals(data[4], 'a'.toByte()) + assertEquals(data[4], 'a'.code.toByte()) assertEquals(data[5], 1.toByte()) assertEquals(data[6], CbFieldType.IntegerPositive.value or CbFieldType.HasFieldName.value) assertEquals(data[7], 1.toByte()) - assertEquals(data[8], 'b'.toByte()) + assertEquals(data[8], 'b'.code.toByte()) assertEquals(data[9], 2.toByte()) assertEquals(data[10], CbFieldType.IntegerPositive.value or CbFieldType.HasFieldName.value) assertEquals(data[11], 1.toByte()) - assertEquals(data[12], 'c'.toByte()) + assertEquals(data[12], 'c'.code.toByte()) assertEquals(data[13], 3.toByte()) } @@ -125,7 +125,7 @@ class CbWriterTest { assertEquals((CbFieldType.Array.value or CbFieldType.HasFieldName.value), data[2]) assertEquals(1, data[3]) // Name length - assertEquals('a'.toByte(), data[4]) // Name + assertEquals('a'.code.toByte(), data[4]) // Name assertEquals(3, data[5]) // Length assertEquals(1, data[6]) // Item count @@ -149,7 +149,7 @@ class CbWriterTest { assertEquals((CbFieldType.Binary.value or CbFieldType.HasFieldName.value), data[2]) assertEquals(1, data[3]) // Name length - assertEquals('a'.toByte(), data[4]) // Name + assertEquals('a'.code.toByte(), data[4]) // Name assertEquals(4, data[5]) // Length assertEquals(1, data[6]) diff --git a/src/backend/fs/boot-fs-server/build.gradle.kts b/src/backend/fs/boot-fs-server/build.gradle.kts index 0a72ea11ee..62873c8ebd 100644 --- a/src/backend/fs/boot-fs-server/build.gradle.kts +++ b/src/backend/fs/boot-fs-server/build.gradle.kts @@ -39,7 +39,7 @@ dependencies { implementation("com.playtika.reactivefeign:feign-reactor-spring-cloud-starter") implementation("io.micrometer:micrometer-registry-influx") implementation("io.micrometer:micrometer-registry-prometheus") - implementation("com.google.guava:guava") + implementation("com.github.ben-manes.caffeine:caffeine:2.9.3") testImplementation("org.mockito.kotlin:mockito-kotlin") diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/RepositoryCache.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/RepositoryCache.kt index 0486c6c899..f423d4ef51 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/RepositoryCache.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/RepositoryCache.kt @@ -29,10 +29,9 @@ package com.tencent.bkrepo.fs.server import com.google.common.cache.CacheBuilder import com.tencent.bkrepo.common.artifact.exception.RepoNotFoundException -import com.tencent.bkrepo.common.metadata.client.RRepositoryClient +import com.tencent.bkrepo.common.metadata.service.repo.RRepositoryService import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail import java.util.concurrent.TimeUnit -import kotlinx.coroutines.reactor.awaitSingle /** * 仓库缓存 @@ -40,15 +39,15 @@ import kotlinx.coroutines.reactor.awaitSingle * 因为仓库需要通过请求获取，并且一般情况下没有变化，所以这里进行缓存 * */ class RepositoryCache( - rRepositoryClient: RRepositoryClient + repositoryService: RRepositoryService ) { init { - Companion.rRepositoryClient = rRepositoryClient + Companion.repositoryService = repositoryService } companion object { - private lateinit var rRepositoryClient: RRepositoryClient + private lateinit var repositoryService: RRepositoryService private val repositoryDetailCache = CacheBuilder.newBuilder() .maximumSize(1000) .expireAfterWrite(60, TimeUnit.SECONDS) @@ -63,7 +62,7 @@ class RepositoryCache( private suspend fun queryRepoDetail(repositoryId: RepositoryId): RepositoryDetail { with(repositoryId) { - return rRepositoryClient.getRepoDetail(projectId, repoName).awaitSingle().data + return repositoryService.getRepoDetail(projectId, repoName) ?: throw RepoNotFoundException(repositoryId.toString()) } } diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/config/BeanConfiguration.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/config/BeanConfiguration.kt index ca7ff55f2b..6cfe45b77a 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/config/BeanConfiguration.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/config/BeanConfiguration.kt @@ -43,7 +43,6 @@ import com.tencent.bkrepo.fs.server.handler.LoginHandler import com.tencent.bkrepo.fs.server.handler.NodeOperationsHandler import com.tencent.bkrepo.fs.server.handler.service.FsNodeHandler import com.tencent.bkrepo.fs.server.metrics.ServerMetrics -import com.tencent.bkrepo.fs.server.service.BlockNodeServiceImpl import com.tencent.bkrepo.fs.server.service.ClientService import com.tencent.bkrepo.fs.server.service.FileNodeService import com.tencent.bkrepo.fs.server.service.FileOperationService @@ -70,7 +69,6 @@ val beans = beans { bean() bean() bean() - bean() bean() bean() bean() diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/FileOperationsHandler.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/FileOperationsHandler.kt index d34bad13b8..1916ee0876 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/FileOperationsHandler.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/FileOperationsHandler.kt @@ -29,12 +29,12 @@ package com.tencent.bkrepo.fs.server.handler import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.message.CommonMessageCode +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.exception.ArtifactNotFoundException import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException import com.tencent.bkrepo.common.artifact.pojo.RepositoryCategory import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.stream.FileArtifactInputStream -import com.tencent.bkrepo.common.metadata.client.RRepositoryClient import com.tencent.bkrepo.fs.server.bodyToArtifactFile import com.tencent.bkrepo.fs.server.context.ReactiveArtifactContextHolder import com.tencent.bkrepo.fs.server.io.RegionInputStreamResource @@ -44,9 +44,9 @@ import com.tencent.bkrepo.fs.server.request.NodeRequest import com.tencent.bkrepo.fs.server.request.StreamRequest import com.tencent.bkrepo.fs.server.resolveRange import com.tencent.bkrepo.fs.server.service.FileOperationService +import com.tencent.bkrepo.fs.server.service.node.RNodeService import com.tencent.bkrepo.fs.server.utils.ReactiveResponseBuilder import com.tencent.bkrepo.fs.server.utils.ReactiveSecurityUtils -import kotlinx.coroutines.reactor.awaitSingle import kotlinx.coroutines.reactor.awaitSingleOrNull import org.slf4j.LoggerFactory import org.springframework.core.io.buffer.DataBufferUtils @@ -68,8 +68,8 @@ import java.net.URI * 处理文件操作请求 * */ class FileOperationsHandler( - private val rRepositoryClient: RRepositoryClient, - private val fileOperationService: FileOperationService + private val fileOperationService: FileOperationService, + private val nodeService: RNodeService ) { /** @@ -82,7 +82,7 @@ class FileOperationsHandler( if (category == RepositoryCategory.REMOTE.name && repoType == RepositoryType.GENERIC) { return temporaryRedirect(URI.create("/generic${request.uri().path}")).buildAndAwait() } - val node = rRepositoryClient.getNodeDetail(projectId, repoName, fullPath).awaitSingle().data + val node = nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, fullPath)) if (node?.folder == true || node == null) { throw NodeNotFoundException(this.toString()) } diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/NodeOperationsHandler.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/NodeOperationsHandler.kt index e94bbd50d2..c278ad681c 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/NodeOperationsHandler.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/NodeOperationsHandler.kt @@ -27,14 +27,15 @@ package com.tencent.bkrepo.fs.server.handler -import com.github.benmanes.caffeine.cache.Caffeine import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.message.CommonMessageCode +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.metadata.constant.FAKE_MD5 import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 -import com.tencent.bkrepo.common.storage.core.overlay.OverlayRangeUtils -import com.tencent.bkrepo.common.metadata.client.RRepositoryClient +import com.tencent.bkrepo.common.metadata.service.fs.FsService import com.tencent.bkrepo.common.metadata.service.metadata.RMetadataService +import com.tencent.bkrepo.common.metadata.service.repo.RRepositoryService +import com.tencent.bkrepo.common.storage.core.overlay.OverlayRangeUtils import com.tencent.bkrepo.fs.server.constant.FS_ATTR_KEY import com.tencent.bkrepo.fs.server.context.ReactiveArtifactContextHolder import com.tencent.bkrepo.fs.server.model.NodeAttribute @@ -49,6 +50,7 @@ import com.tencent.bkrepo.fs.server.request.SetLengthRequest import com.tencent.bkrepo.fs.server.resolveRange import com.tencent.bkrepo.fs.server.response.StatResponse import com.tencent.bkrepo.fs.server.service.FileNodeService +import com.tencent.bkrepo.fs.server.service.node.RNodeService import com.tencent.bkrepo.fs.server.toNode import com.tencent.bkrepo.fs.server.utils.ReactiveResponseBuilder import com.tencent.bkrepo.fs.server.utils.ReactiveSecurityUtils @@ -61,14 +63,12 @@ import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeLinkRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeRenameRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeSetLengthRequest -import kotlinx.coroutines.reactor.awaitSingle import org.slf4j.LoggerFactory import org.springframework.http.HttpHeaders import org.springframework.web.reactive.function.server.ServerRequest import org.springframework.web.reactive.function.server.ServerResponse import org.springframework.web.reactive.function.server.buildAndAwait import reactivefeign.client.ReadTimeoutException -import java.time.Duration /** * 节点操作相关的处理器 @@ -76,18 +76,18 @@ import java.time.Duration * 处理节点操作的请求 * */ class NodeOperationsHandler( - private val rRepositoryClient: RRepositoryClient, private val fileNodeService: FileNodeService, + private val fsService: FsService, + private val nodeService: RNodeService, + private val repositoryService: RRepositoryService, private val metadataService: RMetadataService ) { suspend fun getNode(request: ServerRequest): ServerResponse { with(NodeRequest(request)) { - val nodeDetail = rRepositoryClient.getNodeDetail( - projectId = projectId, - repoName = repoName, - fullPath = fullPath - ).awaitSingle().data ?: return ServerResponse.notFound().buildAndAwait() + val nodeDetail = nodeService.getNodeDetail( + ArtifactInfo(projectId, repoName, fullPath) + ) ?: return ServerResponse.notFound().buildAndAwait() return ReactiveResponseBuilder.success(nodeDetail.nodeInfo.toNode()) } } @@ -95,13 +95,10 @@ class NodeOperationsHandler( suspend fun listNodes(request: ServerRequest): ServerResponse { val pageRequest = NodePageRequest(request) with(pageRequest) { - val nodes = rRepositoryClient.listNodePage( - path = fullPath, - projectId = projectId, - repoName = repoName, + val nodes = nodeService.listNodePage( + artifact = ArtifactInfo(projectId, repoName, fullPath), option = listOption - ).awaitSingle().data?.records?.map { it.toNode() }?.toList() - ?: return ServerResponse.notFound().buildAndAwait() + ).records.map { it.toNode() }.toList() return ReactiveResponseBuilder.success(nodes) } } @@ -117,7 +114,7 @@ class NodeOperationsHandler( fullPath = fullPath, operator = ReactiveSecurityUtils.getUser() ) - rRepositoryClient.deleteNode(nodeDeleteRequest).awaitSingle().data + nodeService.deleteNode(nodeDeleteRequest) fileNodeService.deleteNodeBlocks(projectId, repoName, fullPath) return ReactiveResponseBuilder.success() } @@ -163,25 +160,20 @@ class NodeOperationsHandler( suspend fun getStat(request: ServerRequest): ServerResponse { with(NodeRequest(request)) { - val cacheKey = "$projectId-$repoName-$fullPath" - var res = statCache.getIfPresent(cacheKey) - if (res == null) { - val cap = ReactiveArtifactContextHolder.getRepoDetail().quota - val nodeStat = try { - rRepositoryClient.statRepo(projectId, repoName).awaitSingle().data - } catch (e: ReadTimeoutException) { - logger.warn("get repo[$projectId/$repoName] stat timeout") - NodeSizeInfo(0, 0, UNKNOWN) - } - - res = StatResponse( - subNodeCount = nodeStat?.subNodeCount ?: UNKNOWN, - size = nodeStat?.size ?: UNKNOWN, - capacity = cap ?: UNKNOWN - ) - statCache.put(cacheKey, res) + val cap = ReactiveArtifactContextHolder.getRepoDetail().quota + val nodeStat = try { + repositoryService.statRepo(projectId, repoName) + } catch (e: ReadTimeoutException) { + logger.warn("get repo[$projectId/$repoName] stat timeout") + NodeSizeInfo(0, 0, UNKNOWN) } + val res = StatResponse( + subNodeCount = nodeStat.subNodeCount, + size = nodeStat.size, + capacity = cap ?: UNKNOWN + ) + return ReactiveResponseBuilder.success(res) } } @@ -192,18 +184,14 @@ class NodeOperationsHandler( suspend fun move(request: ServerRequest): ServerResponse { with(MoveRequest(request)) { if (overwrite) { - rRepositoryClient.getNodeDetail( - projectId = projectId, - repoName = repoName, - fullPath = dst - ).awaitSingle().data?.let { + nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, dst))?.let { val nodeDeleteRequest = NodeDeleteRequest( projectId = projectId, repoName = repoName, fullPath = dst, operator = ReactiveSecurityUtils.getUser() ) - rRepositoryClient.deleteNode(nodeDeleteRequest).awaitSingle() + nodeService.deleteNode(nodeDeleteRequest) } } val moveRequest = NodeRenameRequest( @@ -213,7 +201,7 @@ class NodeOperationsHandler( newFullPath = dst, operator = ReactiveSecurityUtils.getUser() ) - rRepositoryClient.renameNode(moveRequest).awaitSingle() + nodeService.renameNode(moveRequest) fileNodeService.renameNodeBlocks(projectId, repoName, fullPath, dst) return ReactiveResponseBuilder.success() } @@ -221,11 +209,9 @@ class NodeOperationsHandler( suspend fun info(request: ServerRequest): ServerResponse { with(NodeRequest(request)) { - val nodeDetail = rRepositoryClient.getNodeDetail( - projectId = projectId, - repoName = repoName, - fullPath = fullPath - ).awaitSingle().data ?: return ServerResponse.notFound().buildAndAwait() + val nodeDetail = nodeService.getNodeDetail( + ArtifactInfo(projectId, repoName, fullPath) + ) ?: return ServerResponse.notFound().buildAndAwait() val range = try { request.resolveRange(nodeDetail.size) } catch (e: IllegalArgumentException) { @@ -266,7 +252,7 @@ class NodeOperationsHandler( operator = ReactiveSecurityUtils.getUser() ) } - val node = rRepositoryClient.link(nodeLinkRequest).awaitSingle().data!! + val node = nodeService.link(nodeLinkRequest) return ReactiveResponseBuilder.success(node.nodeInfo.toNode()) } @@ -297,7 +283,7 @@ class NodeOperationsHandler( nodeMetadata = listOf(fsAttr), operator = user ) - return rRepositoryClient.createFsNode(nodeCreateRequest).awaitSingle().data!! + return fsService.createNode(nodeCreateRequest) } } @@ -311,14 +297,11 @@ class NodeOperationsHandler( newLength = length, operator = user ) - rRepositoryClient.setLength(nodeSetLengthRequest).awaitSingle() + fsService.setLength(nodeSetLengthRequest) return ReactiveResponseBuilder.success() } } - private val statCache = Caffeine.newBuilder() - .maximumSize(1000).expireAfterWrite(Duration.ofDays(1)).build() - companion object { private const val UNKNOWN = -1L private val logger = LoggerFactory.getLogger(NodeOperationsHandler::class.java) diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/service/FsNodeHandler.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/service/FsNodeHandler.kt index 29a4571a18..1afcb5073b 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/service/FsNodeHandler.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/service/FsNodeHandler.kt @@ -27,28 +27,26 @@ package com.tencent.bkrepo.fs.server.handler.service +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.stream.Range -import com.tencent.bkrepo.common.metadata.client.RRepositoryClient import com.tencent.bkrepo.fs.server.request.service.ListBlocksRequest import com.tencent.bkrepo.fs.server.service.FileNodeService +import com.tencent.bkrepo.fs.server.service.node.RNodeService import com.tencent.bkrepo.fs.server.utils.ReactiveResponseBuilder -import kotlinx.coroutines.reactor.awaitSingle import org.springframework.web.reactive.function.server.ServerRequest import org.springframework.web.reactive.function.server.ServerResponse import org.springframework.web.reactive.function.server.buildAndAwait class FsNodeHandler( - private val rRepositoryClient: RRepositoryClient, - private val fileNodeService: FileNodeService + private val fileNodeService: FileNodeService, + private val nodeService: RNodeService ) { suspend fun listBlocks(request: ServerRequest): ServerResponse { with(ListBlocksRequest(request)) { - val nodeDetail = rRepositoryClient.getNodeDetail( - projectId = projectId, - repoName = repoName, - fullPath = path - ).awaitSingle().data ?: return ServerResponse.notFound().buildAndAwait() + val nodeDetail = nodeService.getNodeDetail( + ArtifactInfo(projectId, repoName, fullPath) + ) ?: return ServerResponse.notFound().buildAndAwait() val range = Range(startPos, endPos, nodeDetail.size) return ReactiveResponseBuilder.success(fileNodeService.info(nodeDetail, range)) } diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/listener/NodeModifyListener.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/listener/NodeModifyListener.kt index 3b895d943c..9571b1706d 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/listener/NodeModifyListener.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/listener/NodeModifyListener.kt @@ -1,10 +1,10 @@ package com.tencent.bkrepo.fs.server.listener +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.event.base.ArtifactEvent import com.tencent.bkrepo.common.artifact.event.base.EventType -import com.tencent.bkrepo.common.metadata.client.RRepositoryClient import com.tencent.bkrepo.fs.server.service.FileNodeService -import kotlinx.coroutines.reactor.awaitSingle +import com.tencent.bkrepo.fs.server.service.node.RNodeService import kotlinx.coroutines.runBlocking import org.slf4j.LoggerFactory import org.springframework.messaging.Message @@ -13,8 +13,8 @@ import java.util.concurrent.Executors @Component class NodeModifyListener( - private val rRepositoryClient: RRepositoryClient, - private val fileNodeService: FileNodeService + private val fileNodeService: FileNodeService, + private val nodeService: RNodeService ) { fun accept(message: Message) { @@ -30,7 +30,7 @@ class NodeModifyListener( private fun consumer(event: ArtifactEvent) { runBlocking { with(event) { - val node = rRepositoryClient.getNodeDetail(projectId, repoName, resourceKey).awaitSingle().data + val node = nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, resourceKey)) if (node?.folder != true) { fileNodeService.deleteNodeBlocks(projectId, repoName, resourceKey) } diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/BlockNodeServiceImpl.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/BlockNodeServiceImpl.kt deleted file mode 100644 index 89d8297892..0000000000 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/BlockNodeServiceImpl.kt +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. - * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. - * - * A copy of the MIT License is included in this file. - * - * - * Terms of the MIT License: - * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT - * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN - * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package com.tencent.bkrepo.fs.server.service - -import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException -import com.tencent.bkrepo.common.metadata.dao.blocknode.RBlockNodeDao -import com.tencent.bkrepo.common.metadata.service.blocknode.impl.RAbstractBlockNodeService -import com.tencent.bkrepo.common.metadata.service.file.RFileReferenceService -import com.tencent.bkrepo.common.metadata.client.RRepositoryClient -import com.tencent.bkrepo.repository.pojo.node.NodeDetail -import kotlinx.coroutines.reactor.awaitSingle - -/** - * 文件块服务 - * */ -class BlockNodeServiceImpl( - rBlockNodeDao: RBlockNodeDao, - fileReferenceService: RFileReferenceService, - private val rRepositoryClient: RRepositoryClient, -) : RAbstractBlockNodeService(rBlockNodeDao, fileReferenceService) { - - override suspend fun getNodeDetail(projectId: String, repoName: String, fullPath: String): NodeDetail { - return rRepositoryClient.getNodeDetail(projectId, repoName, fullPath).awaitSingle().data - ?: throw NodeNotFoundException(fullPath) - } -} diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/FileOperationService.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/FileOperationService.kt index 087918687b..b590d7fd3f 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/FileOperationService.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/FileOperationService.kt @@ -32,7 +32,7 @@ import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.metadata.constant.FAKE_MD5 import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.metadata.model.TBlockNode -import com.tencent.bkrepo.common.metadata.client.RRepositoryClient +import com.tencent.bkrepo.common.metadata.service.fs.FsService import com.tencent.bkrepo.common.metadata.service.metadata.RMetadataService import com.tencent.bkrepo.fs.server.config.properties.StreamProperties import com.tencent.bkrepo.fs.server.constant.FS_ATTR_KEY @@ -41,6 +41,7 @@ import com.tencent.bkrepo.fs.server.model.NodeAttribute import com.tencent.bkrepo.fs.server.request.BlockRequest import com.tencent.bkrepo.fs.server.request.FlushRequest import com.tencent.bkrepo.fs.server.request.StreamRequest +import com.tencent.bkrepo.fs.server.service.node.RNodeService import com.tencent.bkrepo.fs.server.storage.CoArtifactFile import com.tencent.bkrepo.fs.server.storage.CoArtifactFileFactory import com.tencent.bkrepo.fs.server.storage.CoStorageManager @@ -51,7 +52,6 @@ import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeSetLengthRequest import kotlinx.coroutines.flow.collect import kotlinx.coroutines.flow.onCompletion -import kotlinx.coroutines.reactor.awaitSingle import org.springframework.core.io.buffer.DataBuffer import org.springframework.core.io.buffer.DataBufferUtils import org.springframework.web.reactive.function.server.bodyToFlow @@ -59,11 +59,12 @@ import java.time.LocalDateTime import java.util.concurrent.atomic.AtomicLong class FileOperationService( - private val rRepositoryClient: RRepositoryClient, private val storageManager: CoStorageManager, private val fileNodeService: FileNodeService, private val streamProperties: StreamProperties, private val metadataService: RMetadataService, + private val fsService: FsService, + private val nodeService: RNodeService, ) { suspend fun read(nodeDetail: NodeDetail, range: Range): ArtifactInputStream? { @@ -93,7 +94,7 @@ class FileOperationService( } suspend fun stream(streamRequest: StreamRequest, user: String): NodeDetail { - val nodeDetail = rRepositoryClient.createNode(buildNodeCreateRequest(streamRequest, user)).awaitSingle().data!! + val nodeDetail = nodeService.createNode(buildNodeCreateRequest(streamRequest, user)) var reactiveArtifactFile = CoArtifactFileFactory.buildArtifactFile() val offset = AtomicLong(0) streamRequest.request.bodyToFlow().onCompletion { @@ -173,7 +174,7 @@ class FileOperationService( newLength = length, operator = user ) - rRepositoryClient.setLength(nodeSetLengthRequest).awaitSingle() + fsService.setLength(nodeSetLengthRequest) } } diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeBaseOperation.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeBaseOperation.kt new file mode 100644 index 0000000000..641b5ff875 --- /dev/null +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeBaseOperation.kt @@ -0,0 +1,77 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.fs.server.service.node + +import com.tencent.bkrepo.common.api.pojo.Page +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo +import com.tencent.bkrepo.repository.pojo.node.NodeDetail +import com.tencent.bkrepo.repository.pojo.node.NodeInfo +import com.tencent.bkrepo.repository.pojo.node.NodeListOption +import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest +import com.tencent.bkrepo.repository.pojo.node.service.NodeLinkRequest +import com.tencent.bkrepo.repository.pojo.node.service.NodeUpdateAccessDateRequest + +/** + * 节点CRUD基本操作接口 + */ +interface RNodeBaseOperation { + + /** + * 查询节点详情 + */ + suspend fun getNodeDetail(artifact: ArtifactInfo, repoType: String? = null): NodeDetail? + + /** + * 分页查询节点 + */ + suspend fun listNodePage(artifact: ArtifactInfo, option: NodeListOption): Page + + /** + * 判断节点是否存在 + */ + suspend fun checkExist(artifact: ArtifactInfo): Boolean + + /** + * 创建节点，返回节点详情 + */ + suspend fun createNode(createRequest: NodeCreateRequest): NodeDetail + + /** + * 创建链接节点，返回节点详情 + */ + suspend fun link(request: NodeLinkRequest): NodeDetail + + /** + * 更新节点访问时间 + */ + suspend fun updateNodeAccessDate(updateAccessDateRequest: NodeUpdateAccessDateRequest) +} diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeBaseService.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeBaseService.kt new file mode 100644 index 0000000000..e0ef7dd6f2 --- /dev/null +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeBaseService.kt @@ -0,0 +1,413 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.fs.server.service.node + +import com.tencent.bkrepo.common.api.exception.BadRequestException +import com.tencent.bkrepo.common.api.exception.ErrorCodeException +import com.tencent.bkrepo.common.api.pojo.Page +import com.tencent.bkrepo.common.api.util.Preconditions +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo +import com.tencent.bkrepo.common.artifact.constant.METADATA_KEY_LINK_FULL_PATH +import com.tencent.bkrepo.common.artifact.constant.METADATA_KEY_LINK_PROJECT +import com.tencent.bkrepo.common.artifact.constant.METADATA_KEY_LINK_REPO +import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode +import com.tencent.bkrepo.common.artifact.path.PathUtils +import com.tencent.bkrepo.common.artifact.pojo.RepositoryType +import com.tencent.bkrepo.common.metadata.client.RAuthClient +import com.tencent.bkrepo.common.metadata.config.RepositoryProperties +import com.tencent.bkrepo.common.metadata.constant.FAKE_MD5 +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 +import com.tencent.bkrepo.common.metadata.dao.node.RNodeDao +import com.tencent.bkrepo.common.metadata.dao.repo.RRepositoryDao +import com.tencent.bkrepo.common.metadata.model.TNode +import com.tencent.bkrepo.common.metadata.model.TRepository +import com.tencent.bkrepo.common.metadata.service.blocknode.RBlockNodeService +import com.tencent.bkrepo.common.metadata.service.file.RFileReferenceService +import com.tencent.bkrepo.common.metadata.service.project.RProjectService +import com.tencent.bkrepo.common.metadata.service.repo.RQuotaService +import com.tencent.bkrepo.common.metadata.service.repo.RStorageCredentialService +import com.tencent.bkrepo.common.metadata.util.NodeBaseServiceHelper +import com.tencent.bkrepo.common.metadata.util.NodeBaseServiceHelper.checkNodeListOption +import com.tencent.bkrepo.common.metadata.util.NodeBaseServiceHelper.convert +import com.tencent.bkrepo.common.metadata.util.NodeBaseServiceHelper.convertToDetail +import com.tencent.bkrepo.common.metadata.util.NodeBaseServiceHelper.validateParameter +import com.tencent.bkrepo.common.metadata.util.NodeEventFactory.buildCreatedEvent +import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper +import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper.listPermissionPaths +import com.tencent.bkrepo.common.mongo.util.Pages +import com.tencent.bkrepo.common.query.enums.OperationType +import com.tencent.bkrepo.common.service.util.SpringContextUtils.Companion.publishEvent +import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier +import com.tencent.bkrepo.fs.server.utils.ReactiveSecurityUtils +import com.tencent.bkrepo.repository.constant.SYSTEM_USER +import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel +import com.tencent.bkrepo.repository.pojo.node.NodeDetail +import com.tencent.bkrepo.repository.pojo.node.NodeInfo +import com.tencent.bkrepo.repository.pojo.node.NodeListOption +import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest +import com.tencent.bkrepo.repository.pojo.node.service.NodeLinkRequest +import com.tencent.bkrepo.repository.pojo.node.service.NodeUpdateAccessDateRequest +import kotlinx.coroutines.reactor.awaitSingle +import org.slf4j.LoggerFactory +import org.springframework.dao.DuplicateKeyException +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.Update +import org.springframework.data.mongodb.core.query.and +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.data.mongodb.core.query.where +import org.springframework.transaction.annotation.Transactional +import java.time.LocalDateTime + +/** + * 节点基础服务，实现了CRUD基本操作 + */ +abstract class RNodeBaseService( + open val nodeDao: RNodeDao, + open val repositoryDao: RRepositoryDao, + open val fileReferenceService: RFileReferenceService, + open val storageCredentialService: RStorageCredentialService, + open val quotaService: RQuotaService, + open val repositoryProperties: RepositoryProperties, + open val messageSupplier: MessageSupplier, + open val authClient: RAuthClient, + open val blockNodeService: RBlockNodeService, + open val projectService: RProjectService, +) : RNodeService { + + override suspend fun getNodeDetail(artifact: ArtifactInfo, repoType: String?): NodeDetail? { + with(artifact) { + val node = nodeDao.findNode(projectId, repoName, getArtifactFullPath()) + return convertToDetail(node) + } + } + + override suspend fun listNodePage(artifact: ArtifactInfo, option: NodeListOption): Page { + checkNodeListOption(option) + with(artifact) { + val userId = ReactiveSecurityUtils.getUser() + val (hasPermissionPaths, noPermissionPaths) = getPermissionPaths(userId, projectId, repoName) + option.hasPermissionPath = hasPermissionPaths + option.noPermissionPath = noPermissionPaths + val pageNumber = option.pageNumber + val pageSize = option.pageSize + Preconditions.checkArgument(pageNumber >= 0, "pageNumber") + Preconditions.checkArgument(pageSize >= 0 && pageSize <= repositoryProperties.listCountLimit, "pageSize") + val query = NodeQueryHelper.nodeListQuery(projectId, repoName, getArtifactFullPath(), option) + val totalRecords = getTotalNodeNum(artifact, query) + val pageRequest = Pages.ofRequest(pageNumber, pageSize) + val records = nodeDao.find(query.with(pageRequest)).map { convert(it)!! } + return Pages.ofResponse(pageRequest, totalRecords, records) + } + } + + override suspend fun checkExist(artifact: ArtifactInfo): Boolean { + return nodeDao.exists(artifact.projectId, artifact.repoName, artifact.getArtifactFullPath()) + } + + @Transactional(rollbackFor = [Throwable::class]) + override suspend fun createNode(createRequest: NodeCreateRequest): NodeDetail { + with(createRequest) { + val fullPath = PathUtils.normalizeFullPath(fullPath) + Preconditions.checkArgument(!PathUtils.isRoot(fullPath), this::fullPath.name) + Preconditions.checkArgument(folder || !sha256.isNullOrBlank(), this::sha256.name) + Preconditions.checkArgument(folder || !md5.isNullOrBlank(), this::md5.name) + // 仓库是否存在 + val repo = checkRepo(projectId, repoName) + // 路径唯一性校验 + checkConflictAndQuota(createRequest, fullPath) + // 判断父目录是否存在，不存在先创建 + mkdirs(projectId, repoName, PathUtils.resolveParent(fullPath), operator) + // 创建节点 + val node = buildTNode(this) + doCreate(node) + afterCreate(repo, node) + logger.info("Create node[/$projectId/$repoName$fullPath], sha256[$sha256] success.") + return convertToDetail(node)!! + } + } + + @Transactional(rollbackFor = [Throwable::class]) + override suspend fun link(request: NodeLinkRequest): NodeDetail { + with(request) { + val targetArtifact = "/$targetProjectId/$targetRepoName/$targetFullPath" + if (checkTargetExist) { + val targetNode = nodeDao.findNode(targetProjectId, targetRepoName, targetFullPath) + ?: throw ErrorCodeException(ArtifactMessageCode.NODE_NOT_FOUND, targetArtifact) + + // 不支持链接到目录 + if (targetNode.folder) { + throw BadRequestException(ArtifactMessageCode.NODE_LINK_FOLDER_UNSUPPORTED, targetArtifact) + } + } + + val metadata = listOf( + MetadataModel(key = METADATA_KEY_LINK_PROJECT, targetProjectId, system = true), + MetadataModel(key = METADATA_KEY_LINK_REPO, targetRepoName, system = true), + MetadataModel(key = METADATA_KEY_LINK_FULL_PATH, targetFullPath, system = true), + ) + val createRequest = NodeCreateRequest( + projectId = projectId, + repoName = repoName, + fullPath = fullPath, + folder = false, + overwrite = overwrite, + sha256 = FAKE_SHA256, + md5 = FAKE_MD5, + nodeMetadata = nodeMetadata?.let { it + metadata } ?: metadata, + operator = operator, + ) + // 创建链接节点 + return createNode(createRequest) + } + } + + open fun buildTNode(request: NodeCreateRequest): TNode { + return NodeBaseServiceHelper.buildTNode(request, repositoryProperties.allowUserAddSystemMetadata) + } + + private suspend fun getTotalNodeNum(artifact: ArtifactInfo, query: Query): Long { + // 避免当目录下节点过多去进行count产生慢查询，使用目录对应的子节点个数进行判断 + // 只有节点个数小于配置的大小时，才去实时获取对应总节点个数 + val subNodeNum = getSubNodeNum(artifact) + val limit = if (repositoryProperties.listCountLimit > repositoryProperties.subNodeLimit) { + repositoryProperties.listCountLimit + } else { + repositoryProperties.subNodeLimit + } + return if (subNodeNum <= limit) { + nodeDao.count(query) + } else { + // *2主要是因为subnodeNum不包含目录 + subNodeNum * 2 + } + } + + /** + * 获取该节点下的子节点（不包含目录）个数 + */ + private suspend fun getSubNodeNum(artifact: ArtifactInfo): Long { + with(artifact) { + val fullPath = artifact.getArtifactFullPath() + if (PathUtils.isRoot(fullPath)) { + return try { + projectService.getProjectMetricsInfo(artifact.projectId)?.repoMetrics?.firstOrNull { + it.repoName == artifact.repoName + }?.num ?: -1 + } catch (e: Exception) { + -1 + } + } else { + val node = nodeDao.findNode(projectId, repoName, fullPath) ?: return 0 + if (!node.folder) return 0 + if (node.nodeNum == null) return -1 + return node.nodeNum!! + } + } + } + + private fun afterCreate(repo: TRepository, node: TNode) { + if (isGenericRepo(repo)) { + publishEvent(buildCreatedEvent(node)) + } + } + + /** + * 判断仓库是否为generic类型仓库 + */ + private fun isGenericRepo(repo: TRepository): Boolean { + return repo.type == RepositoryType.GENERIC + } + + /** + * 校验仓库是否存在 + */ + open suspend fun checkRepo(projectId: String, repoName: String): TRepository { + return repositoryDao.findByNameAndType(projectId, repoName) + ?: throw ErrorCodeException(ArtifactMessageCode.REPOSITORY_NOT_FOUND, repoName) + } + + override suspend fun updateNodeAccessDate(updateAccessDateRequest: NodeUpdateAccessDateRequest) { + with(updateAccessDateRequest) { + val fullPath = PathUtils.normalizeFullPath(fullPath) + val node = nodeDao.findNode(projectId, repoName, fullPath) + ?: throw ErrorCodeException(ArtifactMessageCode.NODE_NOT_FOUND, fullPath) + + val criteria = where(TNode::projectId).isEqualTo(projectId) + .and(TNode::repoName).isEqualTo(repoName) + .and(TNode::deleted).isEqualTo(null) + .and(TNode::fullPath).regex("^${PathUtils.escapeRegex(node.fullPath)}") + .and(TNode::folder).isEqualTo(false) + val query = Query(criteria).withHint(TNode.FULL_PATH_IDX) + val update = Update().set(TNode::lastAccessDate.name, accessDate) + nodeDao.updateMulti(query, update) + logger.info("Update node access time [$this] success.") + } + } + + open suspend fun doCreate(node: TNode, repository: TRepository? = null): TNode { + try { + nodeDao.insert(node) + if (!node.folder) { + // 软链接node或fs-server创建的node的sha256为FAKE_SHA256不会关联实际文件，无需增加引用数 + if (node.sha256 != FAKE_SHA256) { + incrementFileReference(node, repository) + } + quotaService.increaseUsedVolume(node.projectId, node.repoName, node.size) + } + } catch (exception: DuplicateKeyException) { + logger.warn("Insert node[$node] error: [${exception.message}]") + } + + return node + } + + /** + * 递归创建目录 + */ + suspend fun mkdirs(projectId: String, repoName: String, path: String, createdBy: String): List { + val nodes = mutableListOf() + // 格式化 + val fullPath = PathUtils.toFullPath(path) + val creatingNode = nodeDao.findNode(projectId, repoName, fullPath) + if (creatingNode != null && !creatingNode.folder) { + throw ErrorCodeException(ArtifactMessageCode.NODE_CONFLICT, fullPath) + } + if (creatingNode == null) { + val parentPath = PathUtils.resolveParent(fullPath) + val name = PathUtils.resolveName(fullPath) + val creates = mkdirs(projectId, repoName, parentPath, createdBy) + val node = TNode( + folder = true, + path = parentPath, + name = name, + fullPath = PathUtils.combineFullPath(parentPath, name), + size = 0, + expireDate = null, + metadata = mutableListOf(), + projectId = projectId, + repoName = repoName, + createdBy = createdBy, + createdDate = LocalDateTime.now(), + lastModifiedBy = createdBy, + lastModifiedDate = LocalDateTime.now(), + ) + doCreate(node) + nodes.addAll(creates) + nodes.add(node) + } + return nodes + } + + open suspend fun checkConflictAndQuota(createRequest: NodeCreateRequest, fullPath: String) { + with(createRequest) { + val existNode = nodeDao.findNode(projectId, repoName, fullPath) + if (existNode != null) { + if (!overwrite) { + throw ErrorCodeException(ArtifactMessageCode.NODE_EXISTED, fullPath) + } else if (existNode.folder || this.folder) { + throw ErrorCodeException(ArtifactMessageCode.NODE_CONFLICT, fullPath) + } else { + val changeSize = this.size?.minus(existNode.size) ?: -existNode.size + quotaService.checkRepoQuota(projectId, repoName, changeSize) + deleteByFullPathWithoutDecreaseVolume(projectId, repoName, fullPath, operator) + quotaService.decreaseUsedVolume(projectId, repoName, existNode.size) + } + } else { + quotaService.checkRepoQuota(projectId, repoName, this.size ?: 0) + } + } + } + + private suspend fun incrementFileReference(node: TNode, repository: TRepository?): Boolean { + if (!validateParameter(node)) return false + return try { + val credentialsKey = findCredentialsKey(node, repository) + fileReferenceService.increment(node.sha256!!, credentialsKey) + } catch (exception: IllegalArgumentException) { + logger.error("Failed to increment reference of node [$node], repository not found.") + false + } + } + + private suspend fun findCredentialsKey(node: TNode, repository: TRepository?): String? { + if (repository != null) { + return repository.credentialsKey + } + val tRepository = repositoryDao.findByNameAndType(node.projectId, node.repoName) + require(tRepository != null) + return tRepository.credentialsKey + } + + /** + * 获取用户无权限路径列表 + */ + private suspend fun getPermissionPaths( + userId: String, + projectId: String, + repoName: String + ): Pair?, List> { + if (userId == SYSTEM_USER) { + return Pair(null, emptyList()) + } + return authClient.listPermissionPaths(userId, projectId, repoName) + } + + /** + * 查询有权限与无权限的路径 + * + * @param userId 用户 + * @param projectId 项目 + * @param repoName 仓库 + * + * @return first为有权限的路径，为空时表示所有路径均无权限，为null时表示未配置，second为无权限路径 + */ + private suspend fun RAuthClient.listPermissionPaths( + userId: String, + projectId: String, + repoName: String, + ): Pair?, List> { + val result = listPermissionPath(userId, projectId, repoName).awaitSingle().data!! + if (result.status) { + require(result.path.isNotEmpty()) + require(result.path.all { it.key == OperationType.IN } || result.path.all { it.key == OperationType.NIN }) + val opType = result.path.entries.first().key + return listPermissionPaths( + userId, + projectId, + repoName, + opType, + result.path.values.flatten() + ) + } + return Pair(null, emptyList()) + } + + companion object { + private val logger = LoggerFactory.getLogger(RNodeBaseService::class.java) + } +} diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeDeleteOperation.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeDeleteOperation.kt new file mode 100644 index 0000000000..a961d9be7f --- /dev/null +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeDeleteOperation.kt @@ -0,0 +1,63 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.fs.server.service.node + +import com.tencent.bkrepo.repository.pojo.node.NodeDeleteResult +import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest + +/** + * 节点删除接口 + */ +interface RNodeDeleteOperation { + + /** + * 删除指定节点, 逻辑删除 + */ + suspend fun deleteNode(deleteRequest: NodeDeleteRequest): NodeDeleteResult + + /** + * 根据fullpath删除对应节点 + * 不会进行已删除节点数据返回 + * 不会进行容量清理，需要自行进行容量清理 + */ + suspend fun deleteByFullPathWithoutDecreaseVolume( + projectId: String, + repoName: String, + fullPath: String, + operator: String + ) + + /** + * 根据全路径删除文件或者目录 + */ + suspend fun deleteByPath(projectId: String, repoName: String, fullPath: String, operator: String): NodeDeleteResult +} diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeDeleteSupport.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeDeleteSupport.kt new file mode 100644 index 0000000000..3e3a38b323 --- /dev/null +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeDeleteSupport.kt @@ -0,0 +1,143 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.fs.server.service.node + +import com.tencent.bkrepo.common.api.exception.ErrorCodeException +import com.tencent.bkrepo.common.api.message.CommonMessageCode +import com.tencent.bkrepo.common.api.util.HumanReadable +import com.tencent.bkrepo.common.artifact.path.PathUtils +import com.tencent.bkrepo.common.metadata.model.TNode +import com.tencent.bkrepo.common.metadata.util.NodeEventFactory.buildDeletedEvent +import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper +import com.tencent.bkrepo.common.service.util.SpringContextUtils.Companion.publishEvent +import com.tencent.bkrepo.common.metadata.util.NodeDeleteHelper.buildCriteria +import com.tencent.bkrepo.repository.pojo.node.NodeDeleteResult +import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest +import org.slf4j.LoggerFactory +import org.springframework.dao.DuplicateKeyException +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.and +import org.springframework.data.mongodb.core.query.isEqualTo +import java.time.LocalDateTime + +/** + * 节点删除接口实现 + */ +open class RNodeDeleteSupport( + private val nodeBaseService: RNodeBaseService +) : RNodeDeleteOperation { + + private val nodeDao = nodeBaseService.nodeDao + private val quotaService = nodeBaseService.quotaService + + override suspend fun deleteNode(deleteRequest: NodeDeleteRequest): NodeDeleteResult { + with(deleteRequest) { + // 不允许直接删除根目录 + if (PathUtils.isRoot(fullPath)) { + throw ErrorCodeException(CommonMessageCode.METHOD_NOT_ALLOWED, "Can't delete root node.") + } + return deleteByPath(projectId, repoName, fullPath, operator) + } + } + + override suspend fun deleteByFullPathWithoutDecreaseVolume( + projectId: String, + repoName: String, + fullPath: String, + operator: String + ) { + val criteria = buildCriteria(projectId, repoName, fullPath) + val query = Query(criteria) + delete(query, operator, criteria, projectId, repoName, listOf(fullPath), false) + } + + override suspend fun deleteByPath( + projectId: String, + repoName: String, + fullPath: String, + operator: String + ): NodeDeleteResult { + val criteria = buildCriteria(projectId, repoName, fullPath) + val query = Query(criteria) + return delete(query, operator, criteria, projectId, repoName, listOf(fullPath)) + } + + private suspend fun delete( + query: Query, + operator: String, + criteria: Criteria, + projectId: String, + repoName: String, + fullPaths: List? = null, + decreaseVolume: Boolean = true + ): NodeDeleteResult { + var deletedNum = 0L + var deletedSize = 0L + val deleteTime = LocalDateTime.now() + val resourceKey = if (fullPaths == null) { + "/$projectId/$repoName" + } else if (fullPaths.size == 1) { + "/$projectId/$repoName${fullPaths[0]}" + } else { + "/$projectId/$repoName$fullPaths" + } + try { + val updateResult = nodeDao.updateMulti(query, NodeQueryHelper.nodeDeleteUpdate(operator, deleteTime)) + deletedNum = updateResult.modifiedCount + if (deletedNum == 0L) { + return NodeDeleteResult(deletedNum, deletedSize, deleteTime) + } + if (decreaseVolume) { + var deletedCriteria = criteria.and(TNode::deleted).isEqualTo(deleteTime) + fullPaths?.let { + // 节点删除接口返回的数据排除目录 + deletedCriteria = deletedCriteria.and(TNode::folder).isEqualTo(false) + deletedNum = nodeDao.count(Query(deletedCriteria)) + } + deletedSize = nodeBaseService.aggregateComputeSize(deletedCriteria) + quotaService.decreaseUsedVolume(projectId, repoName, deletedSize) + } + fullPaths?.forEach { + publishEvent(buildDeletedEvent(projectId, repoName, it, operator)) + } + } catch (exception: DuplicateKeyException) { + logger.warn("Delete node[$resourceKey] by [$operator] error: [${exception.message}]") + } + logger.info( + "Delete node[$resourceKey] by [$operator] success." + + "$deletedNum nodes have been deleted. The size is ${HumanReadable.size(deletedSize)}" + ) + return NodeDeleteResult(deletedNum, deletedSize, deleteTime) + } + + + companion object { + private val logger = LoggerFactory.getLogger(RNodeDeleteSupport::class.java) + } +} diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeMoveCopyOperation.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeMoveCopyOperation.kt new file mode 100644 index 0000000000..01492ad1bc --- /dev/null +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeMoveCopyOperation.kt @@ -0,0 +1,63 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.fs.server.service.node + +import com.tencent.bkrepo.repository.pojo.node.NodeDetail +import com.tencent.bkrepo.repository.pojo.node.service.NodeMoveCopyRequest + +/** + * 节点移动/拷贝接口 + */ +interface RNodeMoveCopyOperation { + + /** + * 移动文件或者文件夹 + * 采用fast-failed模式，移动过程中出现错误则抛异常，剩下的文件不会再移动 + * 行为类似linux mv命令 + * mv 文件名 文件名 将源文件名改为目标文件名 + * mv 文件名 目录名 将文件移动到目标目录 + * mv 目录名 目录名 目标目录已存在，将源目录（目录本身及子文件）移动到目标目录；目标目录不存在则改名 + * mv 目录名 文件名 出错 + */ + suspend fun moveNode(moveRequest: NodeMoveCopyRequest): NodeDetail + + /** + * 拷贝文件或者文件夹 + * 采用fast-failed模式，拷贝过程中出现错误则抛异常，剩下的文件不会再拷贝 + * 行为类似linux cp命令 + * cp 文件名 文件名 将源文件拷贝到目标文件 + * cp 文件名 目录名 将文件移动到目标目录下 + * cp 目录名 目录名 目标目录已存在，将源目录（目录本身及子文件）拷贝到目标目录；目标目录不存在则将源目录下文件拷贝到目标目录 + * cp 目录名 文件名 出错 + */ + suspend fun copyNode(copyRequest: NodeMoveCopyRequest): NodeDetail +} diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeMoveCopySupport.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeMoveCopySupport.kt new file mode 100644 index 0000000000..b7be3b5952 --- /dev/null +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeMoveCopySupport.kt @@ -0,0 +1,295 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.fs.server.service.node + +import com.tencent.bkrepo.common.api.exception.ErrorCodeException +import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode +import com.tencent.bkrepo.common.artifact.path.PathUtils +import com.tencent.bkrepo.common.artifact.path.PathUtils.combineFullPath +import com.tencent.bkrepo.common.artifact.path.PathUtils.resolveName +import com.tencent.bkrepo.common.artifact.path.PathUtils.resolveParent +import com.tencent.bkrepo.common.artifact.path.PathUtils.toPath +import com.tencent.bkrepo.common.metadata.model.TNode +import com.tencent.bkrepo.common.metadata.model.TRepository +import com.tencent.bkrepo.common.metadata.util.NodeBaseServiceHelper.convertToDetail +import com.tencent.bkrepo.common.metadata.util.NodeEventFactory +import com.tencent.bkrepo.common.metadata.util.NodeMoveCopyHelper +import com.tencent.bkrepo.common.metadata.util.NodeMoveCopyHelper.MoveCopyContext +import com.tencent.bkrepo.common.metadata.util.NodeMoveCopyHelper.buildDstNode +import com.tencent.bkrepo.common.metadata.util.NodeMoveCopyHelper.canIgnore +import com.tencent.bkrepo.common.metadata.util.NodeMoveCopyHelper.preCheck +import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper +import com.tencent.bkrepo.common.service.util.SpringContextUtils.Companion.publishEvent +import com.tencent.bkrepo.common.storage.credentials.StorageCredentials +import com.tencent.bkrepo.repository.constant.DEFAULT_STORAGE_CREDENTIALS_KEY +import com.tencent.bkrepo.repository.pojo.node.NodeDetail +import com.tencent.bkrepo.repository.pojo.node.NodeListOption +import com.tencent.bkrepo.repository.pojo.node.service.NodeMoveCopyRequest +import org.slf4j.LoggerFactory +import org.springframework.data.mongodb.core.query.Query + +/** + * 节点移动/拷贝接口实现 + */ +open class RNodeMoveCopySupport( + private val nodeBaseService: RNodeBaseService +) : RNodeMoveCopyOperation { + + private val nodeDao = nodeBaseService.nodeDao + private val repositoryDao = nodeBaseService.repositoryDao + private val storageCredentialService = nodeBaseService.storageCredentialService + private val quotaService = nodeBaseService.quotaService + + override suspend fun moveNode(moveRequest: NodeMoveCopyRequest): NodeDetail { + val dstNode = moveCopy(moveRequest, true) + logger.info("Move node success: [$moveRequest]") + return dstNode + } + + override suspend fun copyNode(copyRequest: NodeMoveCopyRequest): NodeDetail { + val dstNode = moveCopy(copyRequest, false) + logger.info("Copy node success: [$copyRequest]") + return dstNode + } + + /** + * 处理节点操作请求 + */ + private suspend fun moveCopy(request: NodeMoveCopyRequest, move: Boolean): NodeDetail { + with(resolveContext(request, move)) { + preCheck(this) + if (canIgnore(this)) { + return convertToDetail( + nodeBaseService.nodeDao.findNode( + projectId = dstProjectId, + repoName = dstRepoName, + fullPath = dstFullPath + ) + )!! + } + if (srcNode.folder) { + moveCopyFolder(this) + } else { + moveCopyFile(this) + } + if (move) { + publishEvent(NodeEventFactory.buildMovedEvent(request)) + } else { + publishEvent(NodeEventFactory.buildCopiedEvent(request)) + } + return convertToDetail( + nodeBaseService.nodeDao.findNode( + projectId = dstProjectId, + repoName = dstRepoName, + fullPath = dstFullPath + ) + )!! + } + } + + /** + * 移动/复制节点 + */ + protected suspend fun doMoveCopy( + context: MoveCopyContext, + node: TNode, + dstPath: String, + dstName: String + ) { + with(context) { + val dstFullPath = combineFullPath(dstPath, dstName) + // 冲突检查 + val existNode = nodeDao.findNode(dstProjectId, dstRepoName, dstFullPath) + // 目录 -> 目录: 跳过 + if (node.folder && existNode?.folder == true) return + checkConflict(context, node, existNode) + // copy目标节点 + val dstNode = buildDstNode(this, node, dstPath, dstName, dstFullPath) + // 仓库配额检查 + checkQuota(context, node, existNode) + + // 文件 & 跨存储node + if (!node.folder && srcCredentials != dstCredentials) { + // 默认存储为null,所以需要使用一个默认key，以区分该节点是拷贝节点 + dstNode.copyFromCredentialsKey = srcNode.copyFromCredentialsKey + ?: srcCredentials?.key + ?: DEFAULT_STORAGE_CREDENTIALS_KEY + dstNode.copyIntoCredentialsKey = dstCredentials?.key ?: DEFAULT_STORAGE_CREDENTIALS_KEY + } + // 创建dst节点 + nodeBaseService.doCreate(dstNode, dstRepo) + // move操作，创建dst节点后，还需要删除src节点 + // 因为分表所以不能直接更新src节点，必须创建新的并删除旧的 + if (move) { + val query = NodeQueryHelper.nodeQuery(node.projectId, node.repoName, node.fullPath) + val update = NodeQueryHelper.nodeDeleteUpdate(operator) + if (!node.folder) { + quotaService.decreaseUsedVolume(node.projectId, node.repoName, node.size) + } + nodeDao.updateFirst(query, update) + } + } + } + + private suspend fun checkQuota(context: MoveCopyContext, node: TNode, existNode: TNode?) { + // 目录不占仓库容量，不需要检查 + if (node.folder) return + + with(context) { + // 文件 -> 文件，目标文件不存在 + if (existNode == null) { + // 同仓库的移动操作不需要检查仓库已使用容量 + if (!(isSameRepo() && move)) { + quotaService.checkRepoQuota(dstProjectId, dstRepoName, node.size) + } + } + + // 文件 -> 文件 & 允许覆盖: 删除old + if (existNode?.folder == false && overwrite) { + quotaService.checkRepoQuota(existNode.projectId, existNode.repoName, node.size - existNode.size) + nodeBaseService.deleteByFullPathWithoutDecreaseVolume( + existNode.projectId, existNode.repoName, existNode.fullPath, operator + ) + quotaService.decreaseUsedVolume(existNode.projectId, existNode.repoName, existNode.size) + } + } + } + + private suspend fun resolveContext(request: NodeMoveCopyRequest, move: Boolean): MoveCopyContext { + with(request) { + val srcFullPath = PathUtils.normalizeFullPath(srcFullPath) + val dstProjectId = request.destProjectId ?: srcProjectId + val dstRepoName = request.destRepoName ?: srcRepoName + val dstFullPath = PathUtils.normalizeFullPath(request.destFullPath) + val isSameRepo = srcProjectId == destProjectId && srcRepoName == destRepoName + // 查询repository + val srcRepo = findRepository(srcProjectId, srcRepoName) + val dstRepo = if (!isSameRepo) findRepository(dstProjectId, dstRepoName) else srcRepo + // 查询storageCredentials + val srcCredentials = findCredential(srcRepo.credentialsKey) + val dstCredentials = if (!isSameRepo) findCredential(dstRepo.credentialsKey) else srcCredentials + val srcNode = nodeDao.findNode(srcProjectId, srcRepoName, srcFullPath) + ?: throw ErrorCodeException(ArtifactMessageCode.NODE_NOT_FOUND, srcFullPath) + val dstNode = nodeDao.findNode(dstProjectId, dstRepoName, dstFullPath) + + return MoveCopyContext( + srcRepo = srcRepo, + srcCredentials = srcCredentials, + srcNode = srcNode, + dstProjectId = dstProjectId, + dstRepoName = dstRepoName, + dstFullPath = dstFullPath, + dstRepo = dstRepo, + dstCredentials = dstCredentials, + dstNode = dstNode, + dstNodeFolder = destNodeFolder, + overwrite = overwrite, + operator = request.operator, + move = move + ) + } + } + + open fun checkConflict(context: MoveCopyContext, node: TNode, existNode: TNode?) { + NodeMoveCopyHelper.checkConflict(context, node, existNode) + } + + /** + * 移动/复制目录 + */ + private suspend fun moveCopyFolder(context: MoveCopyContext) { + with(context) { + // 目录 -> 文件: error + if (dstNode?.folder == false) { + throw ErrorCodeException(ArtifactMessageCode.NODE_CONFLICT, dstFullPath) + } + val dstRootNodePath = if (dstNode == null) { + // 目录 -> 不存在的目录 + val path = resolveParent(dstFullPath) + val name = resolveName(dstFullPath) + // 创建dst父目录 + nodeBaseService.mkdirs(dstProjectId, dstRepoName, path, operator) + // 操作节点 + doMoveCopy(this, srcNode, path, name) + PathUtils.combinePath(path, name) + } else { + // 目录 -> 存在的目录 + val path = toPath(dstNode!!.fullPath) + val name = srcNode.name + // 操作节点 + doMoveCopy(this, srcNode, path, name) + PathUtils.combinePath(path, name) + } + val srcRootNodePath = toPath(srcNode.fullPath) + val listOption = NodeListOption(includeFolder = true, includeMetadata = true, deep = true, sort = false) + val query = buildSubNodesQuery(this, srcRootNodePath, listOption) + // 目录下的节点 -> 创建好的目录 + nodeDao.stream(query).toIterable().forEach { + doMoveCopy(this, it, it.path.replaceFirst(srcRootNodePath, dstRootNodePath), it.name) + } + } + } + + open fun buildSubNodesQuery( + context: MoveCopyContext, + srcRootNodePath: String, + listOption: NodeListOption + ): Query { + return NodeMoveCopyHelper.buildSubNodesQuery(context, srcRootNodePath, listOption) + } + + /** + * 移动/复制文件 + */ + open suspend fun moveCopyFile(context: MoveCopyContext) { + with(context) { + val dstPath = if (dstNode?.folder == true) toPath(dstNode!!.fullPath) else resolveParent(dstFullPath) + val dstName = if (dstNode?.folder == true) srcNode.name else resolveName(dstFullPath) + // 创建dst父目录 + nodeBaseService.mkdirs(dstProjectId, dstRepoName, dstPath, operator) + doMoveCopy(context, srcNode, dstPath, dstName) + } + } + + private suspend fun findRepository(projectId: String, repoName: String): TRepository { + return repositoryDao.findByNameAndType(projectId, repoName) + ?: throw ErrorCodeException(ArtifactMessageCode.REPOSITORY_NOT_FOUND, repoName) + } + + private suspend fun findCredential(key: String?): StorageCredentials? { + return key?.let { storageCredentialService.findByKey(it) } + } + + companion object { + private val logger = LoggerFactory.getLogger(RNodeMoveCopySupport::class.java) + } +} diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeRenameOperation.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeRenameOperation.kt new file mode 100644 index 0000000000..c9f22900c4 --- /dev/null +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeRenameOperation.kt @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.fs.server.service.node + +import com.tencent.bkrepo.repository.pojo.node.service.NodeRenameRequest + +/** + * 节点重命名接口 + */ +interface RNodeRenameOperation { + + /** + * 重命名文件或者文件夹 + * 重命名过程中出现错误则抛异常，剩下的文件不会再移动 + * 遇到同名文件或者文件夹直接抛异常 + */ + suspend fun renameNode(renameRequest: NodeRenameRequest) +} diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeRenameSupport.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeRenameSupport.kt new file mode 100644 index 0000000000..8e365a0810 --- /dev/null +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeRenameSupport.kt @@ -0,0 +1,111 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.fs.server.service.node + +import com.tencent.bkrepo.common.api.exception.ErrorCodeException +import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode +import com.tencent.bkrepo.common.artifact.path.PathUtils +import com.tencent.bkrepo.common.metadata.model.TNode +import com.tencent.bkrepo.common.metadata.util.NodeEventFactory.buildRenamedEvent +import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper +import com.tencent.bkrepo.common.service.util.SpringContextUtils.Companion.publishEvent +import com.tencent.bkrepo.repository.pojo.node.NodeListOption +import com.tencent.bkrepo.repository.pojo.node.service.NodeRenameRequest +import org.slf4j.LoggerFactory + +/** + * 节点重命名接口实现 + */ +open class RNodeRenameSupport( + private val nodeBaseService: RNodeBaseService +) : RNodeRenameOperation { + + protected val nodeDao = nodeBaseService.nodeDao + + override suspend fun renameNode(renameRequest: NodeRenameRequest) { + with(renameRequest) { + val fullPath = PathUtils.normalizeFullPath(fullPath) + val newFullPath = PathUtils.normalizeFullPath(newFullPath) + val node = nodeDao.findNode(projectId, repoName, fullPath) + ?: throw ErrorCodeException(ArtifactMessageCode.NODE_NOT_FOUND, fullPath) + checkNodeCluster(node) + doRename(node, newFullPath, operator) + publishEvent(buildRenamedEvent(renameRequest)) + logger.info("Rename node [$this] success.") + } + } + + open fun checkNodeCluster(node: TNode) { + return + } + + /** + * 将节点重命名为指定名称 + */ + private suspend fun doRename(node: TNode, newFullPath: String, operator: String) { + val projectId = node.projectId + val repoName = node.repoName + val newPath = PathUtils.resolveParent(newFullPath) + val newName = PathUtils.resolveName(newFullPath) + + // 检查新路径是否被占用 + if (nodeDao.exists(projectId, repoName, newFullPath)) { + logger.warn("Rename node [${node.fullPath}] failed: $newFullPath is exist.") + throw ErrorCodeException(ArtifactMessageCode.NODE_EXISTED, newFullPath) + } + + // 如果为文件夹，查询子节点并修改 + if (node.folder) { + nodeBaseService.mkdirs(projectId, repoName, newFullPath, operator) + val newParentPath = PathUtils.toPath(newFullPath) + val listOption = NodeListOption( + includeFolder = true, + includeMetadata = false, + deep = false, + sort = false + ) + val query = NodeQueryHelper.nodeListQuery(projectId, repoName, node.fullPath, listOption) + nodeDao.find(query).forEach { doRename(it, newParentPath + it.name, operator) } + // 删除自己 + nodeDao.remove(NodeQueryHelper.nodeQuery(projectId, repoName, node.fullPath)) + } else { + // 修改自己 + val selfQuery = NodeQueryHelper.nodeQuery(projectId, repoName, node.fullPath) + val selfUpdate = NodeQueryHelper.nodePathUpdate(newPath, newName, operator) + nodeDao.updateFirst(selfQuery, selfUpdate) + } + } + + companion object { + private val logger = LoggerFactory.getLogger(RNodeRenameSupport::class.java) + } +} diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/NodeModel.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeService.kt similarity index 57% rename from src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/NodeModel.kt rename to src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeService.kt index e385c1caee..8629d93866 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/NodeModel.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeService.kt @@ -29,32 +29,14 @@ * SOFTWARE. */ -package com.tencent.bkrepo.opdata.model +package com.tencent.bkrepo.fs.server.service.node -import com.tencent.bkrepo.common.artifact.path.PathUtils -import com.tencent.bkrepo.common.service.log.LoggerHolder -import com.tencent.bkrepo.opdata.pojo.RepoMetrics -import com.tencent.bkrepo.repository.api.NodeClient -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.stereotype.Service - -@Service -class NodeModel @Autowired constructor( - private val nodeClient: NodeClient -) { - - fun getNodeSize(projectId: String, repoName: String): RepoMetrics { - try { - val result = nodeClient.computeSize(projectId, repoName, PathUtils.ROOT).data - ?: return RepoMetrics(repoName = repoName, size = 0L, num = 0L) - return RepoMetrics(repoName = repoName, size = result.size, num = result.subNodeCount) - } catch (ignored: Exception) { - logger.warn("get node size exception [$projectId, $repoName ,$ignored]") - return RepoMetrics(repoName = repoName, size = 0L, num = 0L) - } - } - - companion object { - private val logger = LoggerHolder.jobLogger - } -} +/** + * 节点服务接口 + */ +interface RNodeService : + RNodeBaseOperation, + RNodeStatsOperation, + RNodeDeleteOperation, + RNodeMoveCopyOperation, + RNodeRenameOperation diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeServiceImpl.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeServiceImpl.kt new file mode 100644 index 0000000000..984d6c7a51 --- /dev/null +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeServiceImpl.kt @@ -0,0 +1,124 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.fs.server.service.node + +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo +import com.tencent.bkrepo.common.metadata.client.RAuthClient +import com.tencent.bkrepo.common.metadata.config.RepositoryProperties +import com.tencent.bkrepo.common.metadata.dao.node.RNodeDao +import com.tencent.bkrepo.common.metadata.dao.repo.RRepositoryDao +import com.tencent.bkrepo.common.metadata.service.blocknode.RBlockNodeService +import com.tencent.bkrepo.common.metadata.service.file.RFileReferenceService +import com.tencent.bkrepo.common.metadata.service.project.RProjectService +import com.tencent.bkrepo.common.metadata.service.repo.RQuotaService +import com.tencent.bkrepo.common.metadata.service.repo.RStorageCredentialService +import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier +import com.tencent.bkrepo.repository.pojo.node.NodeDeleteResult +import com.tencent.bkrepo.repository.pojo.node.NodeDetail +import com.tencent.bkrepo.repository.pojo.node.NodeSizeInfo +import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest +import com.tencent.bkrepo.repository.pojo.node.service.NodeMoveCopyRequest +import com.tencent.bkrepo.repository.pojo.node.service.NodeRenameRequest +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.stereotype.Service +import org.springframework.transaction.annotation.Transactional + +@Service +class RNodeServiceImpl( + override val nodeDao: RNodeDao, + override val repositoryDao: RRepositoryDao, + override val fileReferenceService: RFileReferenceService, + override val storageCredentialService: RStorageCredentialService, + override val quotaService: RQuotaService, + override val repositoryProperties: RepositoryProperties, + override val messageSupplier: MessageSupplier, + override val authClient: RAuthClient, + override val blockNodeService: RBlockNodeService, + override val projectService: RProjectService, +) : RNodeBaseService( + nodeDao, + repositoryDao, + fileReferenceService, + storageCredentialService, + quotaService, + repositoryProperties, + messageSupplier, + authClient, + blockNodeService, + projectService +) { + + override suspend fun computeSize( + artifact: ArtifactInfo, estimated: Boolean + ): NodeSizeInfo { + return RNodeStatsSupport(this).computeSize(artifact, estimated) + } + + override suspend fun aggregateComputeSize(criteria: Criteria): Long { + return RNodeStatsSupport(this).aggregateComputeSize(criteria) + } + + @Transactional(rollbackFor = [Throwable::class]) + override suspend fun deleteNode(deleteRequest: NodeDeleteRequest): NodeDeleteResult { + return RNodeDeleteSupport(this).deleteNode(deleteRequest) + } + + override suspend fun deleteByFullPathWithoutDecreaseVolume( + projectId: String, repoName: String, fullPath: String, operator: String + ) { + return RNodeDeleteSupport(this).deleteByFullPathWithoutDecreaseVolume( + projectId, repoName, fullPath, operator + ) + } + + @Transactional(rollbackFor = [Throwable::class]) + override suspend fun deleteByPath( + projectId: String, + repoName: String, + fullPath: String, + operator: String, + ): NodeDeleteResult { + return RNodeDeleteSupport(this).deleteByPath(projectId, repoName, fullPath, operator) + } + + + @Transactional(rollbackFor = [Throwable::class]) + override suspend fun moveNode(moveRequest: NodeMoveCopyRequest): NodeDetail { + return RNodeMoveCopySupport(this).moveNode(moveRequest) + } + + @Transactional(rollbackFor = [Throwable::class]) + override suspend fun copyNode(copyRequest: NodeMoveCopyRequest): NodeDetail { + return RNodeMoveCopySupport(this).copyNode(copyRequest) + } + + @Transactional(rollbackFor = [Throwable::class]) + override suspend fun renameNode(renameRequest: NodeRenameRequest) { + RNodeRenameSupport(this).renameNode(renameRequest) + } +} diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeStatsOperation.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeStatsOperation.kt new file mode 100644 index 0000000000..b67410047d --- /dev/null +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeStatsOperation.kt @@ -0,0 +1,53 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.fs.server.service.node + +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo +import com.tencent.bkrepo.repository.pojo.node.NodeSizeInfo +import org.springframework.data.mongodb.core.query.Criteria + +/** + * 节点重命名接口 + */ +interface RNodeStatsOperation { + + /** + * 计算文件或者文件夹大小 + */ + suspend fun computeSize(artifact: ArtifactInfo, estimated: Boolean = false): NodeSizeInfo + + /** + * 聚合查询节点大小 + */ + suspend fun aggregateComputeSize(criteria: Criteria): Long + +} diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeStatsSupport.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeStatsSupport.kt new file mode 100644 index 0000000000..7f7a8cd0a5 --- /dev/null +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/node/RNodeStatsSupport.kt @@ -0,0 +1,156 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.fs.server.service.node + +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.api.exception.ErrorCodeException +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo +import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode +import com.tencent.bkrepo.common.metadata.model.TNode +import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper +import com.tencent.bkrepo.repository.pojo.node.NodeListOption +import com.tencent.bkrepo.repository.pojo.node.NodeSizeInfo +import org.springframework.data.mongodb.core.aggregation.Aggregation.group +import org.springframework.data.mongodb.core.aggregation.Aggregation.match +import org.springframework.data.mongodb.core.aggregation.Aggregation.newAggregation +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query + +/** + * 节点统计接口 + */ +open class RNodeStatsSupport( + nodeBaseService: RNodeBaseService +) : RNodeStatsOperation { + + private val nodeDao = nodeBaseService.nodeDao + + override suspend fun computeSize( + artifact: ArtifactInfo, + estimated: Boolean + ): NodeSizeInfo { + val node = getNodeInfo(artifact) + // 节点为文件直接返回 + if (!node.folder) { + return NodeSizeInfo(subNodeCount = 0, subNodeWithoutFolderCount = 0, size = node.size) + } + if (estimated) { + return computeEstimatedSize(node) + } + val listOption = NodeListOption(includeFolder = true, deep = true) + val criteria = NodeQueryHelper.nodeListCriteria( + artifact.projectId, + artifact.repoName, + artifact.getArtifactFullPath(), + listOption) + val listOptionWithOutFolder = NodeListOption(includeFolder = false, deep = true) + val criteriaWithOutFolder = NodeQueryHelper.nodeListCriteria( + artifact.projectId, + artifact.repoName, + artifact.getArtifactFullPath(), + listOptionWithOutFolder + ) + val nodeSizeInfo = accurateSizeCalculate(criteria, criteriaWithOutFolder) + nodeDao.setSizeAndNodeNumOfFolder( + artifact.projectId, + artifact.repoName, + artifact.getArtifactFullPath(), + size = nodeSizeInfo.size, + nodeNum = nodeSizeInfo.subNodeWithoutFolderCount) + return nodeSizeInfo + } + + private suspend fun getNodeInfo(artifact: ArtifactInfo): TNode { + val projectId = artifact.projectId + val repoName = artifact.repoName + val fullPath = artifact.getArtifactFullPath() + val node = nodeDao.findNode(projectId, repoName, fullPath) + ?: throw ErrorCodeException(ArtifactMessageCode.NODE_NOT_FOUND, fullPath) + return node + } + + /** + * 计算大小 + */ + private suspend fun accurateSizeCalculate(criteria: Criteria, criteriaWithOutFolder: Criteria):NodeSizeInfo { + val count = nodeDao.count(Query(criteria)) + val countWithOutFolder = nodeDao.count(Query(criteriaWithOutFolder)) + val size = aggregateComputeSize(criteriaWithOutFolder) + return NodeSizeInfo(subNodeCount = count, subNodeWithoutFolderCount = countWithOutFolder, size = size) + } + + /** + * 计算目录大小信息的估计值 + * + */ + private suspend fun computeEstimatedSize(node: TNode): NodeSizeInfo { + val countCriteria = NodeQueryHelper.nodeListCriteria( + projectId = node.projectId, + repoName = node.repoName, + path = node.fullPath, + option = NodeListOption(includeFolder = true, deep = true) + ) + val count = nodeDao.count(Query(countCriteria)) + if (node.fullPath != StringPool.ROOT) { + return NodeSizeInfo(count, node.nodeNum ?: 0, node.size) + } + val criteria = NodeQueryHelper.nodeListCriteria( + projectId = node.projectId, + repoName = node.repoName, + path = node.fullPath, + option = NodeListOption(includeFolder = true, deep = false) + ) + + val aggregation = newAggregation( + match(criteria), + group().sum(TNode::size.name).`as`(NodeSizeInfo::size.name) + .sum(TNode::nodeNum.name).`as`(NodeSizeInfo::subNodeCount.name) + ) + val data = nodeDao.aggregate(aggregation, HashMap::class.java).firstOrNull() + return NodeSizeInfo( + subNodeCount = count, + subNodeWithoutFolderCount = data?.get(NodeSizeInfo::subNodeCount.name) as? Long ?: 0, + size = data?.get(NodeSizeInfo::size.name) as? Long ?: 0 + ) + } + + + + override suspend fun aggregateComputeSize(criteria: Criteria): Long { + val aggregation = newAggregation( + match(criteria), + group().sum(TNode::size.name).`as`(NodeSizeInfo::size.name) + ) + val data = nodeDao.aggregate(aggregation, HashMap::class.java).firstOrNull() + return data?.get(NodeSizeInfo::size.name) as? Long ?: 0 + } +} diff --git a/src/backend/fs/boot-fs-server/src/test/kotlin/com/tencent/com/bkrepo/fs/service/BlockNodeServiceTest.kt b/src/backend/fs/boot-fs-server/src/test/kotlin/com/tencent/com/bkrepo/fs/service/BlockNodeServiceTest.kt index 6381ffa0a4..80dd569b63 100644 --- a/src/backend/fs/boot-fs-server/src/test/kotlin/com/tencent/com/bkrepo/fs/service/BlockNodeServiceTest.kt +++ b/src/backend/fs/boot-fs-server/src/test/kotlin/com/tencent/com/bkrepo/fs/service/BlockNodeServiceTest.kt @@ -1,16 +1,13 @@ package com.tencent.com.bkrepo.fs.service import com.tencent.bkrepo.common.api.constant.StringPool -import com.tencent.bkrepo.common.api.message.CommonMessageCode -import com.tencent.bkrepo.common.api.pojo.Response import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.metadata.dao.blocknode.RBlockNodeDao +import com.tencent.bkrepo.common.metadata.dao.node.RNodeDao import com.tencent.bkrepo.common.metadata.model.TBlockNode +import com.tencent.bkrepo.common.metadata.model.TNode import com.tencent.bkrepo.common.metadata.service.blocknode.RBlockNodeService import com.tencent.bkrepo.common.storage.credentials.FileSystemCredentials -import com.tencent.bkrepo.common.metadata.client.RRepositoryClient -import com.tencent.bkrepo.repository.pojo.node.NodeDetail -import com.tencent.bkrepo.repository.pojo.node.NodeInfo import com.tencent.com.bkrepo.fs.UT_PROJECT_ID import com.tencent.com.bkrepo.fs.UT_REPO_NAME import com.tencent.com.bkrepo.fs.UT_USER @@ -31,7 +28,6 @@ import org.springframework.data.mongodb.core.query.Query import org.springframework.data.mongodb.core.query.isEqualTo import org.springframework.data.mongodb.core.query.where import org.springframework.test.context.TestPropertySource -import reactor.core.publisher.Mono import java.time.LocalDateTime @DataMongoTest @@ -42,7 +38,7 @@ import java.time.LocalDateTime class BlockNodeServiceTest { @MockBean - lateinit var rRepositoryClient: RRepositoryClient + lateinit var nodeDao: RNodeDao @Autowired lateinit var blockNodeService: RBlockNodeService @@ -126,8 +122,8 @@ class BlockNodeServiceTest { @Test fun testMoveNode() { runBlocking { - val createdDate = LocalDateTime.now().minusSeconds(1).toString() - val nodeInfo = NodeInfo( + val createdDate = LocalDateTime.now().minusSeconds(1) + val node = TNode( createdBy = UT_USER, createdDate = createdDate, lastModifiedBy = UT_USER, @@ -140,8 +136,8 @@ class BlockNodeServiceTest { projectId = UT_PROJECT_ID, repoName = UT_REPO_NAME ) - Mockito.`when`(rRepositoryClient.getNodeDetail(any(), any(), any())) - .thenReturn(Mono.just(successResponse(NodeDetail(nodeInfo)))) + Mockito.`when`(nodeDao.findNode(any(), any(), any())) + .thenReturn(node) createBlockNode(startPos = 10) createBlockNode(startPos = 20) val fullPath = "/file" @@ -152,7 +148,7 @@ class BlockNodeServiceTest { UT_PROJECT_ID, UT_REPO_NAME, newFullPath, - createdDate + createdDate.toString() ) Assertions.assertEquals(2, blocks.size) val blocks1 = blockNodeService.listBlocks( @@ -160,7 +156,7 @@ class BlockNodeServiceTest { UT_PROJECT_ID, UT_REPO_NAME, fullPath, - createdDate + createdDate.toString() ) Assertions.assertEquals(0, blocks1.size) } @@ -183,6 +179,4 @@ class BlockNodeServiceTest { ) return blockNodeService.createBlock(blockNode, storageCredentials) } - - private fun successResponse(data: T) = Response(CommonMessageCode.SUCCESS.getCode(), null, data, null) } diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt index 96ea86911a..f0a7d3c4f0 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt @@ -125,6 +125,7 @@ import org.springframework.stereotype.Component import org.springframework.util.unit.DataSize import java.net.URLDecoder import java.util.Base64 +import java.util.Locale import java.util.UUID import java.util.concurrent.TimeUnit import javax.servlet.http.HttpServletRequest @@ -253,7 +254,7 @@ class GenericLocalRepository( val uploadType = HeaderUtils.getHeader(HEADER_UPLOAD_TYPE) if (!overwrite && !isBlockUpload(uploadId, sequence) && !isChunkedUpload(uploadType)) { with(context.artifactInfo) { - nodeClient.getNodeDetail(projectId, repoName, getArtifactFullPath()).data?.let { + nodeService.getNodeDetail(this)?.let { throw ErrorCodeException(ArtifactMessageCode.NODE_EXISTED, getArtifactName()) } } @@ -266,7 +267,7 @@ class GenericLocalRepository( return } with(context.artifactInfo) { - val existNode = nodeClient.getNodeDetail(projectId, repoName, getArtifactFullPath()).data + val existNode = nodeService.getNodeDetail(this) val metadata = resolveMetadata(context.request) val mPipelineId = metadata.find { it.key.equals(METADATA_SUB_PIPELINE_ID, true) }?.value?.toString() ?: metadata.find { it.key.equals(METADATA_PIPELINE_ID, true) }?.value?.toString() @@ -396,11 +397,11 @@ class GenericLocalRepository( existNode?.metadata?.keys?.forEach { key -> if (CIPermissionManager.PIPELINE_METADATA.any { it.equals(key, true) }) { val mProjectId = existNode.metadata[METADATA_PROJECT_ID] - ?: existNode.metadata[METADATA_PROJECT_ID.toLowerCase()] + ?: existNode.metadata[METADATA_PROJECT_ID.lowercase(Locale.getDefault())] val mPipelineId = existNode.metadata[METADATA_PIPELINE_ID] - ?: existNode.metadata[METADATA_PIPELINE_ID.toLowerCase()] + ?: existNode.metadata[METADATA_PIPELINE_ID.lowercase(Locale.getDefault())] val mBuildId = existNode.metadata[METADATA_BUILD_ID] - ?: existNode.metadata[METADATA_BUILD_ID.toLowerCase()] + ?: existNode.metadata[METADATA_BUILD_ID.lowercase(Locale.getDefault())] return ciPermissionManager.throwOrLogError( messageCode = GenericMessageCode.CUSTOM_ARTIFACT_OVERWRITE_NOT_ALLOWED, existNode.fullPath, "$mProjectId/$mPipelineId/$mBuildId" @@ -499,8 +500,9 @@ class GenericLocalRepository( includeMetadata = true, deep = true ) - val records = nodeClient.listNodePage(folder.projectId, folder.repoName, folder.fullPath, option).data - ?.records.takeUnless { it.isNullOrEmpty() }?.map { NodeDetail(it) } ?: break + val records = + nodeService.listNodePage(ArtifactInfo(folder.projectId, folder.repoName, folder.fullPath), option) + .records.takeUnless { it.isEmpty() }?.map { NodeDetail(it) } ?: break records.filterNot { it.folder }.forEach { downloadIntercept(context, it) } totalSize += records.sumOf { it.size } checkFileTotalSize(totalSize) @@ -528,8 +530,8 @@ class GenericLocalRepository( includeMetadata = true, deep = true ) - val records = nodeClient.listNodePage(projectId, repoName, prefix, option).data?.records - if (records.isNullOrEmpty()) { + val records = nodeService.listNodePage(ArtifactInfo(projectId, repoName, prefix), option).records + if (records.isEmpty()) { break } nodeDetailList.addAll( @@ -593,15 +595,15 @@ class GenericLocalRepository( override fun remove(context: ArtifactRemoveContext) { with(context.artifactInfo) { - val node = nodeClient.getNodeDetail(projectId, repoName, getArtifactFullPath()).data + val node = nodeService.getNodeDetail(this) ?: throw NodeNotFoundException(this.getArtifactFullPath()) if (node.folder) { - if (nodeClient.countFileNode(projectId, repoName, getArtifactFullPath()).data!! > 0) { + if (nodeService.countFileNode(this) > 0) { throw ErrorCodeException(ArtifactMessageCode.FOLDER_CONTAINS_FILE) } } val nodeDeleteRequest = NodeDeleteRequest(projectId, repoName, getArtifactFullPath(), context.userId) - nodeClient.deleteNode(nodeDeleteRequest) + nodeService.deleteNode(nodeDeleteRequest) } } @@ -619,11 +621,7 @@ class GenericLocalRepository( override fun query(context: ArtifactQueryContext): Any? { val artifactInfo = context.artifactInfo - return nodeClient.getNodeDetail( - artifactInfo.projectId, - artifactInfo.repoName, - artifactInfo.getArtifactFullPath() - ).data + return nodeService.getNodeDetail(artifactInfo) } override fun search(context: ArtifactSearchContext): List { @@ -648,7 +646,7 @@ class GenericLocalRepository( } else { // 强制替换为请求的projectId与repoName避免越权 val newRule = replaceProjectIdAndRepo(queryModel.rule, context.projectId, context.repoName) - nodeClient.queryWithoutCount(queryModel.copy(rule = newRule)).data!!.records.onEach { node -> + nodeSearchService.searchWithoutCount(queryModel.copy(rule = newRule)).records.onEach { node -> (node as MutableMap)[RepositoryInfo::category.name] = RepositoryCategory.LOCAL.name } } @@ -876,7 +874,7 @@ class GenericLocalRepository( md5 = fileInfo.md5, size = fileInfo.size ) - nodeClient.createNode(nodeRequest) + nodeService.createNode(nodeRequest) return property } } diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/remote/AsyncRemoteArtifactCacheWriter.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/remote/AsyncRemoteArtifactCacheWriter.kt index 04444cc7fa..81c6fe45a7 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/remote/AsyncRemoteArtifactCacheWriter.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/remote/AsyncRemoteArtifactCacheWriter.kt @@ -29,14 +29,15 @@ package com.tencent.bkrepo.generic.artifact.remote import com.tencent.bkrepo.common.api.constant.HttpHeaders import com.tencent.bkrepo.common.artifact.api.ArtifactFile +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.manager.StorageManager import com.tencent.bkrepo.common.artifact.pojo.configuration.remote.RemoteConfiguration import com.tencent.bkrepo.common.artifact.resolve.file.ArtifactFileFactory import com.tencent.bkrepo.common.metadata.service.metadata.MetadataService +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.generic.artifact.findRemoteMetadata import com.tencent.bkrepo.generic.artifact.updateParentMetadata -import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import okhttp3.Request @@ -50,7 +51,7 @@ import org.springframework.stereotype.Component @Component class AsyncRemoteArtifactCacheWriter( private val storageManager: StorageManager, - private val nodeClient: NodeClient, + private val nodeService: NodeService, private val httpClientBuilderFactory: AsyncCacheHttpClientBuilderFactory, private val executor: ThreadPoolTaskExecutor, private val cacheLocks: RemoteArtifactCacheLocks, @@ -79,7 +80,7 @@ class AsyncRemoteArtifactCacheWriter( val repoName = cacheTask.repoName val fullPath = cacheTask.fullPath try { - if (nodeClient.getNodeDetail(projectId, repoName, fullPath).data != null) { + if (nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, fullPath)) != null) { logger.info("artifact[$projectId/$repoName$fullPath] was cached, skip cache") } else if (cacheLocks.tryLock(projectId, repoName, fullPath)) { try { diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/CompressedFileService.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/CompressedFileService.kt index 19628f4d54..5277d51934 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/CompressedFileService.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/CompressedFileService.kt @@ -40,9 +40,9 @@ import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResourceWriter import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.generic.artifact.GenericArtifactInfo import com.tencent.bkrepo.generic.pojo.CompressedFileInfo -import com.tencent.bkrepo.repository.api.NodeClient import org.apache.commons.compress.archivers.ArchiveEntry import org.apache.commons.compress.archivers.ArchiveException import org.apache.commons.compress.archivers.ArchiveInputStream @@ -55,7 +55,7 @@ import java.io.InputStream @Service class CompressedFileService( - private val nodeClient: NodeClient, + private val nodeService: NodeService, private val storageManager: StorageManager, private val artifactResourceWriter: ArtifactResourceWriter ) : ArtifactService() { @@ -113,7 +113,7 @@ class CompressedFileService( if (!Regex(COMPRESSED_FILE_TYPE_PATTERN).matches(fileExtension)) { throw ErrorCodeException(ArtifactMessageCode.ARTIFACT_TYPE_UNSUPPORTED, fileExtension) } - val node = nodeClient.getNodeDetail(projectId, repoName, getArtifactFullPath()).data + val node = nodeService.getNodeDetail(artifactInfo) ?: throw NodeNotFoundException(getArtifactFullPath()) if (node.size > COMPRESSED_FILE_SIZE_LIMIT) { throw ErrorCodeException(ArtifactMessageCode.ARTIFACT_SIZE_TOO_LARGE, COMPRESSED_FILE_SIZE_LIMIT_DESC) diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/DeltaSyncService.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/DeltaSyncService.kt index b87e681dcb..d1adfa4c30 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/DeltaSyncService.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/DeltaSyncService.kt @@ -27,6 +27,8 @@ import com.tencent.bkrepo.common.artifact.stream.FileArtifactInputStream import com.tencent.bkrepo.common.bksync.BlockChannel import com.tencent.bkrepo.common.bksync.FileBlockChannel import com.tencent.bkrepo.common.bksync.transfer.http.BkSyncMetrics +import com.tencent.bkrepo.common.metadata.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.redis.RedisOperation import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.otel.util.AsyncUtils.trace @@ -46,8 +48,6 @@ import com.tencent.bkrepo.generic.enum.GenericAction import com.tencent.bkrepo.generic.model.TSignFile import com.tencent.bkrepo.generic.pojo.bkbase.QueryRequest import com.tencent.bkrepo.generic.pojo.bkbase.QueryResponse -import com.tencent.bkrepo.repository.api.NodeClient -import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest @@ -80,9 +80,9 @@ import java.util.concurrent.TimeUnit class DeltaSyncService( genericProperties: GenericProperties, val storageManager: StorageManager, - val nodeClient: NodeClient, + val nodeService: NodeService, val signFileDao: SignFileDao, - val repositoryClient: RepositoryClient, + val repositoryService: RepositoryService, private val redisOperation: RedisOperation, ) : ArtifactService() { @@ -95,7 +95,7 @@ class DeltaSyncService( val signFileProjectId = deltaProperties.projectId val signFileRepoName = deltaProperties.repoName val signRepo: RepositoryDetail by lazy { - repositoryClient.getRepoDetail(signFileProjectId, signFileRepoName).data + repositoryService.getRepoDetail(signFileProjectId, signFileRepoName) ?: throw ErrorCodeException(ArtifactMessageCode.REPOSITORY_NOT_FOUND, signFileRepoName) } private val httpClient = HttpClientBuilderFactory.create().build() @@ -127,17 +127,19 @@ class DeltaSyncService( * */ fun patch(oldFilePath: String, deltaFile: ArtifactFile): SseEmitter { with(ArtifactContext()) { - val node = nodeClient.getNodeDetail( - projectId = projectId, - repoName = repoName, - fullPath = UrlEncoded.decodeString(oldFilePath, 0, oldFilePath.length, Charsets.UTF_8), - ).data + val node = nodeService.getNodeDetail( + ArtifactInfo( + projectId, + repoName, + UrlEncoded.decodeString(oldFilePath, 0, oldFilePath.length, Charsets.UTF_8) + ), + ) if (node == null || node.folder) { throw NodeNotFoundException(artifactInfo.getArtifactFullPath()) } val overwrite = HeaderUtils.getBooleanHeader(HEADER_OVERWRITE) if (!overwrite) { - nodeClient.getNodeDetail(projectId, repoName, artifactInfo.getArtifactFullPath()).data?.let { + nodeService.getNodeDetail(artifactInfo)?.let { throw ErrorCodeException( ArtifactMessageCode.NODE_EXISTED, artifactInfo.getArtifactName(), @@ -328,7 +330,7 @@ class DeltaSyncService( * */ private fun getMd5FromNode(context: ArtifactContext): String { with(context) { - val node = nodeClient.getNodeDetail(projectId, repoName, artifactInfo.getArtifactFullPath()).data + val node = nodeService.getNodeDetail(artifactInfo) if (node == null || node.folder) { throw NodeNotFoundException(artifactInfo.getArtifactFullPath()) } diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/DownloadService.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/DownloadService.kt index 113cb89c77..032dd8b13c 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/DownloadService.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/DownloadService.kt @@ -45,16 +45,17 @@ import com.tencent.bkrepo.common.artifact.repository.context.ArtifactQueryContex import com.tencent.bkrepo.common.artifact.repository.core.ArtifactService import com.tencent.bkrepo.common.artifact.view.ViewModelService import com.tencent.bkrepo.common.generic.configuration.AutoIndexRepositorySettings +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.query.model.QueryModel import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.generic.artifact.GenericArtifactInfo import com.tencent.bkrepo.generic.artifact.context.GenericArtifactSearchContext import com.tencent.bkrepo.generic.constant.GenericMessageCode -import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.pojo.list.HeaderItem import com.tencent.bkrepo.repository.pojo.list.RowItem import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.NodeInfo +import com.tencent.bkrepo.repository.pojo.node.NodeListOption import com.tencent.bkrepo.repository.pojo.node.NodeListViewItem import com.tencent.bkrepo.repository.pojo.repo.RepositoryInfo import org.springframework.beans.factory.annotation.Value @@ -65,7 +66,7 @@ import org.springframework.stereotype.Service */ @Service class DownloadService( - private val nodeClient: NodeClient, + private val nodeService: NodeService, private val viewModelService: ViewModelService, ) : ArtifactService() { @@ -121,11 +122,8 @@ class DownloadService( if (fromApp) DownloadInterceptorType.MOBILE else DownloadInterceptorType.WEB ) val context = ArtifactDownloadContext() - val nodeDetail = nodeClient.getNodeDetail( - projectId = artifactInfo.projectId, - repoName = artifactInfo.repoName, - fullPath = artifactInfo.getArtifactFullPath() - ).data ?: throw NodeNotFoundException(artifactInfo.getArtifactFullPath()) + val nodeDetail = nodeService.getNodeDetail(artifactInfo) + ?: throw NodeNotFoundException(artifactInfo.getArtifactFullPath()) context.getInterceptors().forEach { it.intercept(nodeDetail.projectId, nodeDetail) } return true } @@ -152,13 +150,7 @@ class DownloadService( private fun renderListView(node: NodeDetail, artifactInfo: GenericArtifactInfo) { viewModelService.trailingSlash(applicationName) - val nodeList = nodeClient.listNode( - projectId = artifactInfo.projectId, - repoName = artifactInfo.repoName, - path = artifactInfo.getArtifactFullPath(), - includeFolder = true, - deep = false - ).data + val nodeList = nodeService.listNode(artifactInfo, NodeListOption(includeFolder = true, deep = false)) val currentPath = viewModelService.computeCurrentPath(node) val headerList = listOf( HeaderItem("Name"), @@ -167,10 +159,10 @@ class DownloadService( HeaderItem("Size"), HeaderItem("Sha256") ) - val itemList = nodeList?.map { NodeListViewItem.from(it) }?.sorted() - val rowList = itemList?.map { + val itemList = nodeList.map { NodeListViewItem.from(it) }.sorted() + val rowList = itemList.map { RowItem(listOf(it.name, it.createdBy, it.lastModified, it.size, it.sha256)) - } ?: listOf() + } viewModelService.render(currentPath, headerList, rowList) } } diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/OperateService.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/OperateService.kt index 2b6c442791..b54b0a5433 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/OperateService.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/OperateService.kt @@ -31,9 +31,11 @@ package com.tencent.bkrepo.generic.service +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.generic.pojo.FileInfo -import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.pojo.node.NodeInfo +import com.tencent.bkrepo.repository.pojo.node.NodeListOption import org.springframework.stereotype.Service /** @@ -41,7 +43,7 @@ import org.springframework.stereotype.Service */ @Service class OperateService( - private val nodeClient: NodeClient + private val nodeService: NodeService, ) { fun listFile( @@ -53,10 +55,11 @@ class OperateService( deep: Boolean, includeMetadata: Boolean, ): List { - return nodeClient - .listNode(projectId, repoName, path, includeFolder, deep, includeMetadata) - .data - .orEmpty() + return nodeService + .listNode( + ArtifactInfo(projectId, repoName, path), + NodeListOption(includeFolder = includeFolder, deep = deep, includeMetadata = includeMetadata) + ) .map { toFileInfo(it) } } diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/ProxyService.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/ProxyService.kt index d327c40509..e04c804811 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/ProxyService.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/ProxyService.kt @@ -36,6 +36,7 @@ import com.tencent.bkrepo.common.artifact.exception.ArtifactNotFoundException import com.tencent.bkrepo.common.artifact.exception.ArtifactResponseException import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException import com.tencent.bkrepo.common.artifact.manager.StorageManager +import com.tencent.bkrepo.common.artifact.pojo.RepositoryId import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder import com.tencent.bkrepo.common.artifact.repository.core.ArtifactService import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource @@ -77,7 +78,7 @@ class ProxyService( fun download(projectId: String, name: String) { permissionManager.checkProjectPermission(PermissionAction.MANAGE, projectId) val repo = ArtifactContextHolder.getRepoDetail( - ArtifactContextHolder.RepositoryId( + RepositoryId( genericProperties.proxy.projectId, genericProperties.proxy.repoName ) diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/TemporaryAccessService.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/TemporaryAccessService.kt index a315bb6d5a..3ec533da62 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/TemporaryAccessService.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/TemporaryAccessService.kt @@ -36,11 +36,11 @@ import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.auth.pojo.token.TemporaryTokenCreateRequest import com.tencent.bkrepo.auth.pojo.token.TemporaryTokenInfo import com.tencent.bkrepo.auth.pojo.token.TokenType -import com.tencent.bkrepo.common.api.constant.HttpStatus -import com.tencent.bkrepo.common.api.constant.USER_KEY import com.tencent.bkrepo.common.api.constant.ANONYMOUS_USER import com.tencent.bkrepo.common.api.constant.AUTH_HEADER_UID +import com.tencent.bkrepo.common.api.constant.HttpStatus import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.api.constant.USER_KEY import com.tencent.bkrepo.common.api.exception.BadRequestException import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.message.CommonMessageCode @@ -60,6 +60,7 @@ import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactUploadContext +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.security.manager.PermissionManager import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.HeaderUtils @@ -80,7 +81,6 @@ import com.tencent.bkrepo.generic.pojo.TemporaryAccessToken import com.tencent.bkrepo.generic.pojo.TemporaryAccessUrl import com.tencent.bkrepo.generic.pojo.TemporaryUrlCreateRequest import com.tencent.bkrepo.generic.util.ChunkedRequestUtil.uploadResponse -import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.devops.plugin.api.PluginManager import com.tencent.devops.plugin.api.applyExtension import org.slf4j.LoggerFactory @@ -95,7 +95,7 @@ import java.time.format.DateTimeFormatter @Service class TemporaryAccessService( private val temporaryTokenClient: ServiceTemporaryTokenClient, - private val repositoryClient: RepositoryClient, + private val repositoryService: RepositoryService, private val genericProperties: GenericProperties, private val pluginManager: PluginManager, private val deltaSyncService: DeltaSyncService, @@ -108,7 +108,7 @@ class TemporaryAccessService( */ fun upload(artifactInfo: GenericArtifactInfo, file: ArtifactFile) { with(artifactInfo) { - val repo = repositoryClient.getRepoDetail(projectId, repoName).data + val repo = repositoryService.getRepoDetail(projectId, repoName) ?: throw ErrorCodeException(ArtifactMessageCode.REPOSITORY_NOT_FOUND, repoName) val context = ArtifactUploadContext(repo, file) ArtifactContextHolder.getRepository(repo.category).upload(context) @@ -120,7 +120,7 @@ class TemporaryAccessService( */ fun download(artifactInfo: GenericArtifactInfo) { with(artifactInfo) { - val repo = repositoryClient.getRepoDetail(projectId, repoName).data + val repo = repositoryService.getRepoDetail(projectId, repoName) ?: throw ErrorCodeException(ArtifactMessageCode.REPOSITORY_NOT_FOUND, repoName) HttpContextHolder.getRequest().setAttribute(REPO_KEY, repo) val context = ArtifactDownloadContext(repo) @@ -133,7 +133,7 @@ class TemporaryAccessService( checkAlphaApkDownloadUser(userId, artifactInfo, shareBy) with(artifactInfo) { val downloadUser = if (userId == ANONYMOUS_USER) shareBy else userId - val repo = repositoryClient.getRepoDetail(projectId, repoName).data + val repo = repositoryService.getRepoDetail(projectId, repoName) ?: throw ErrorCodeException(ArtifactMessageCode.REPOSITORY_NOT_FOUND, repoName) val context = ArtifactDownloadContext(repo = repo, userId = downloadUser) context.shareUserId = shareBy @@ -261,7 +261,7 @@ class TemporaryAccessService( * */ fun sign(artifactInfo: GenericArtifactInfo, md5: String?) { with(artifactInfo) { - val repo = repositoryClient.getRepoDetail(projectId, repoName).data + val repo = repositoryService.getRepoDetail(projectId, repoName) ?: throw ErrorCodeException(ArtifactMessageCode.REPOSITORY_NOT_FOUND, repoName) val request = HttpContextHolder.getRequest() request.setAttribute(REPO_KEY, repo) @@ -274,7 +274,7 @@ class TemporaryAccessService( * */ fun patch(artifactInfo: GenericArtifactInfo, oldFilePath: String, deltaFile: ArtifactFile): SseEmitter { with(artifactInfo) { - val repo = repositoryClient.getRepoDetail(projectId, repoName).data + val repo = repositoryService.getRepoDetail(projectId, repoName) ?: throw ErrorCodeException(ArtifactMessageCode.REPOSITORY_NOT_FOUND, repoName) val request = HttpContextHolder.getRequest() request.setAttribute(REPO_KEY, repo) @@ -292,7 +292,7 @@ class TemporaryAccessService( fun getUuidForChunkedUpload(artifactInfo: GenericChunkedArtifactInfo, artifactFile: ArtifactFile) { with(artifactInfo) { - val result = repositoryClient.getRepoDetail(projectId, repoName).data + val result = repositoryService.getRepoDetail(projectId, repoName) ?: throw RepoNotFoundException(repoName) val responseProperty = if (uuid.isNullOrEmpty()) { val uuidCreated = storageService.createAppendId(result.storageCredentials) @@ -332,7 +332,7 @@ class TemporaryAccessService( fun reportChunkedMetrics(metrics: ChunkArtifactTransferMetrics) { if (metrics.success) { - val repo = repositoryClient.getRepoDetail(metrics.projectId, metrics.repoName).data ?: return + val repo = repositoryService.getRepoDetail(metrics.projectId, metrics.repoName) ?: return metrics.storage = repo.storageCredentials?.key ?: DEFAULT_STORAGE_KEY SpringContextUtils.publishEvent(ChunkArtifactTransferEvent(metrics)) } diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt index 4e3dec9c4f..801869ba46 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt @@ -42,6 +42,8 @@ import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHold import com.tencent.bkrepo.common.artifact.repository.context.ArtifactRemoveContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactUploadContext import com.tencent.bkrepo.common.artifact.repository.core.ArtifactService +import com.tencent.bkrepo.common.metadata.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.HeaderUtils.getBooleanHeader import com.tencent.bkrepo.common.service.util.HeaderUtils.getLongHeader @@ -57,8 +59,6 @@ import com.tencent.bkrepo.generic.constant.HEADER_EXPIRES import com.tencent.bkrepo.generic.constant.HEADER_OVERWRITE import com.tencent.bkrepo.generic.pojo.BlockInfo import com.tencent.bkrepo.generic.pojo.UploadTransactionInfo -import com.tencent.bkrepo.repository.api.NodeClient -import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import org.slf4j.LoggerFactory import org.springframework.stereotype.Service @@ -68,9 +68,9 @@ import org.springframework.stereotype.Service */ @Service class UploadService( - private val nodeClient: NodeClient, + private val nodeService: NodeService, private val storageService: StorageService, - private val repositoryClient: RepositoryClient, + private val repositoryService: RepositoryService, ) : ArtifactService() { fun upload(artifactInfo: GenericArtifactInfo, file: ArtifactFile) { @@ -90,7 +90,7 @@ class UploadService( val overwrite = getBooleanHeader(HEADER_OVERWRITE) Preconditions.checkArgument(expires >= 0, "expires") // 判断文件是否存在 - if (!overwrite && nodeClient.checkExist(projectId, repoName, getArtifactFullPath()).data == true) { + if (!overwrite && nodeService.checkExist(this)) { logger.warn( "User[${SecurityUtils.getPrincipal()}] start block upload [$artifactInfo] failed: " + "artifact already exists." @@ -148,7 +148,7 @@ class UploadService( } // 保存节点 val repository = ArtifactContextHolder.getRepository(RepositoryCategory.LOCAL) as GenericLocalRepository - nodeClient.createNode( + nodeService.createNode( NodeCreateRequest( projectId = artifactInfo.projectId, repoName = artifactInfo.repoName, @@ -184,7 +184,7 @@ class UploadService( private fun getStorageCredentials(artifactInfo: GenericArtifactInfo): StorageCredentials? { with(artifactInfo) { - val repoDetail = repositoryClient.getRepoDetail(projectId, repoName).data + val repoDetail = repositoryService.getRepoDetail(projectId, repoName) ?: throw ErrorCodeException(ArtifactMessageCode.REPOSITORY_NOT_FOUND, repoName) return repoDetail.storageCredentials } diff --git a/src/backend/git/biz-git/src/main/kotlin/com/tencent/bkrepo/git/artifact/GitRepoInterceptor.kt b/src/backend/git/biz-git/src/main/kotlin/com/tencent/bkrepo/git/artifact/GitRepoInterceptor.kt index 98cdc008d7..112cea4626 100644 --- a/src/backend/git/biz-git/src/main/kotlin/com/tencent/bkrepo/git/artifact/GitRepoInterceptor.kt +++ b/src/backend/git/biz-git/src/main/kotlin/com/tencent/bkrepo/git/artifact/GitRepoInterceptor.kt @@ -6,29 +6,29 @@ import com.tencent.bkrepo.common.artifact.constant.REPO_NAME import com.tencent.bkrepo.common.artifact.pojo.RepositoryCategory import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.pojo.configuration.remote.RemoteConfiguration +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.git.config.GitProperties +import com.tencent.bkrepo.git.constant.DOT_GIT import com.tencent.bkrepo.git.constant.GitMessageCode import com.tencent.bkrepo.git.constant.HubType import com.tencent.bkrepo.git.constant.PARAMETER_HUBTYPE import com.tencent.bkrepo.git.constant.PARAMETER_OWNER -import com.tencent.bkrepo.git.constant.DOT_GIT import com.tencent.bkrepo.git.constant.PATH_SYNC import com.tencent.bkrepo.git.constant.X_DEVOPS_BUILD_ID import com.tencent.bkrepo.git.constant.X_DEVOPS_PIPELINE_ID -import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.pojo.repo.RepoCreateRequest import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired import org.springframework.web.servlet.HandlerInterceptor import org.springframework.web.servlet.HandlerMapping -import java.lang.IllegalArgumentException +import java.util.Locale import javax.servlet.http.HttpServletRequest import javax.servlet.http.HttpServletResponse -import org.springframework.beans.factory.annotation.Autowired class GitRepoInterceptor : HandlerInterceptor { @Autowired - lateinit var repositoryClient: RepositoryClient + lateinit var repositoryService: RepositoryService @Autowired lateinit var properties: GitProperties @@ -38,7 +38,7 @@ class GitRepoInterceptor : HandlerInterceptor { val type = request.getParameter(PARAMETER_HUBTYPE) type ?: return true try { - val domain = properties.getDomain(HubType.valueOf(type.toUpperCase())) ?: let { + val domain = properties.getDomain(HubType.valueOf(type.uppercase(Locale.getDefault()))) ?: let { throw ErrorCodeException(GitMessageCode.GIT_HUB_TYPE_NOT_SUPPORT, type) } val owner = request.getParameter(PARAMETER_OWNER) @@ -58,7 +58,7 @@ class GitRepoInterceptor : HandlerInterceptor { "$X_DEVOPS_PIPELINE_ID:${request.getHeader(X_DEVOPS_PIPELINE_ID)} " + "sync request $uri" ) - repositoryClient.getRepoDetail(projectId, realRepoName).data ?: let { + repositoryService.getRepoDetail(projectId, realRepoName) ?: let { val req = RepoCreateRequest( projectId = projectId, name = realRepoName, @@ -70,7 +70,7 @@ class GitRepoInterceptor : HandlerInterceptor { url = uri ) ) - repositoryClient.createRepo(req) + repositoryService.createRepo(req) logger.info("create projectId $projectId repo $realRepoName") } return true diff --git a/src/backend/git/biz-git/src/main/kotlin/com/tencent/bkrepo/git/artifact/repository/GitLocalRepository.kt b/src/backend/git/biz-git/src/main/kotlin/com/tencent/bkrepo/git/artifact/repository/GitLocalRepository.kt index 69ee0643be..4346c4bf6b 100644 --- a/src/backend/git/biz-git/src/main/kotlin/com/tencent/bkrepo/git/artifact/repository/GitLocalRepository.kt +++ b/src/backend/git/biz-git/src/main/kotlin/com/tencent/bkrepo/git/artifact/repository/GitLocalRepository.kt @@ -55,7 +55,7 @@ class GitLocalRepository : LocalRepository() { override fun onDownload(context: ArtifactDownloadContext): ArtifactResource? { with(context) { - val node = nodeClient.getNodeDetail(projectId, repoName, artifactInfo.getArtifactFullPath()).data + val node = nodeService.getNodeDetail(artifactInfo) val responseName = artifactInfo.getResponseName() storageManager.loadArtifactInputStream(node, storageCredentials)?.let { return ArtifactResource(it, responseName, node, ArtifactChannel.PROXY, useDisposition) diff --git a/src/backend/git/biz-git/src/main/kotlin/com/tencent/bkrepo/git/service/CodeRepositoryDataService.kt b/src/backend/git/biz-git/src/main/kotlin/com/tencent/bkrepo/git/service/CodeRepositoryDataService.kt index db71ec0e35..c015394a21 100644 --- a/src/backend/git/biz-git/src/main/kotlin/com/tencent/bkrepo/git/service/CodeRepositoryDataService.kt +++ b/src/backend/git/biz-git/src/main/kotlin/com/tencent/bkrepo/git/service/CodeRepositoryDataService.kt @@ -2,10 +2,12 @@ package com.tencent.bkrepo.git.service import com.tencent.bkrepo.common.api.util.readJsonString import com.tencent.bkrepo.common.api.util.toJsonString +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException import com.tencent.bkrepo.common.artifact.manager.StorageManager import com.tencent.bkrepo.common.artifact.resolve.file.ArtifactFileFactory import com.tencent.bkrepo.common.artifact.stream.FileArtifactInputStream +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.git.artifact.GitPackFileArtifactInfo import com.tencent.bkrepo.git.constant.BLANK import com.tencent.bkrepo.git.context.DfsDataReadersHolder @@ -21,11 +23,8 @@ import com.tencent.bkrepo.git.internal.storage.DfsReadableChannel import com.tencent.bkrepo.git.internal.storage.RepositoryDataService import com.tencent.bkrepo.git.model.TDfsPackDescription import com.tencent.bkrepo.git.repository.DfsPackDescriptionRepository -import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest -import java.io.ByteArrayOutputStream -import java.time.LocalDateTime import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase import org.eclipse.jgit.internal.storage.dfs.DfsOutputStream import org.eclipse.jgit.internal.storage.dfs.DfsPackDescription @@ -36,11 +35,13 @@ import org.springframework.data.mongodb.core.query.Query import org.springframework.data.mongodb.core.query.isEqualTo import org.springframework.data.mongodb.core.query.where import org.springframework.stereotype.Service +import java.io.ByteArrayOutputStream +import java.time.LocalDateTime @Service class CodeRepositoryDataService( val storageManager: StorageManager, - val nodeClient: NodeClient, + val nodeService: NodeService, val dfsPackDescriptionRepository: DfsPackDescriptionRepository ) : RepositoryDataService { @@ -126,10 +127,9 @@ class CodeRepositoryDataService( packArtifactInfo: GitPackFileArtifactInfo ): DfsDataReader { with(repository) { - val node = nodeClient.getNodeDetail( - projectId, repoName, - packArtifactInfo.getArtifactFullPath() - ).data ?: throw NodeNotFoundException(packArtifactInfo.getArtifactFullPath()) + val node = nodeService.getNodeDetail( + ArtifactInfo(projectId, repoName, packArtifactInfo.getArtifactFullPath()) + ) ?: throw NodeNotFoundException(packArtifactInfo.getArtifactFullPath()) val artifactInputStream = storageManager.loadArtifactInputStream(node, storageCredentials) ?: throw IllegalStateException("Stream load failed.") artifactInputStream.use { @@ -213,7 +213,7 @@ class CodeRepositoryDataService( fullPath = packArtifactInfo.getArtifactFullPath(), operator = userId ) - nodeClient.deleteNode(deleteRequest) + nodeService.deleteNode(deleteRequest) } } } diff --git a/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/artifact/repository/HelmLocalRepository.kt b/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/artifact/repository/HelmLocalRepository.kt index c0de152d46..48f5ec290b 100644 --- a/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/artifact/repository/HelmLocalRepository.kt +++ b/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/artifact/repository/HelmLocalRepository.kt @@ -31,6 +31,7 @@ package com.tencent.bkrepo.helm.artifact.repository +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactQueryContext @@ -118,7 +119,7 @@ class HelmLocalRepository( val repoName = repositoryDetail.name val fullPath = getStringAttribute(FULL_PATH).orEmpty() helmOperationService.checkNodePermission(fullPath) - val isExist = nodeClient.checkExist(projectId, repoName, fullPath).data!! + val isExist = nodeService.checkExist(ArtifactInfo(projectId, repoName, fullPath)) val isOverwrite = isOverwrite(fullPath, isForce) putAttribute(OVERWRITE, isOverwrite) if (isExist && !isOverwrite) { @@ -213,7 +214,7 @@ class HelmLocalRepository( val projectId = repositoryDetail.projectId val repoName = repositoryDetail.name val fullPath = context.getStringAttribute(FULL_PATH)!! - val node = nodeClient.getNodeDetail(projectId, repoName, fullPath).data + val node = nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, fullPath)) if (node == null || node.folder) return null return storageManager.loadFullArtifactInputStream(node, context.storageCredentials) } diff --git a/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/artifact/repository/HelmRemoteRepository.kt b/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/artifact/repository/HelmRemoteRepository.kt index ebeb005106..369cbfd9e7 100644 --- a/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/artifact/repository/HelmRemoteRepository.kt +++ b/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/artifact/repository/HelmRemoteRepository.kt @@ -34,6 +34,7 @@ package com.tencent.bkrepo.helm.artifact.repository import com.tencent.bkrepo.common.api.util.readYamlString import com.tencent.bkrepo.common.api.util.toYamlString import com.tencent.bkrepo.common.artifact.api.ArtifactFile +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.constant.SOURCE_TYPE import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext @@ -137,7 +138,7 @@ class HelmRemoteRepository( "in repo ${context.artifactInfo.getRepoIdentify()}" ) val sha256 = artifactFile.getFileSha256() - nodeClient.getNodeDetail(projectId, repoName, fullPath).data?.let { + nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, fullPath))?.let { if (it.sha256.equals(sha256)) { logger.info("artifact [$fullPath] hits the cache.") return artifactFile.getInputStream().artifactStream(Range.full(artifactFile.getSize())) diff --git a/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/AbstractChartService.kt b/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/AbstractChartService.kt index a256de8e02..49a2c811f3 100644 --- a/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/AbstractChartService.kt +++ b/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/AbstractChartService.kt @@ -59,7 +59,10 @@ import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream import com.tencent.bkrepo.common.artifact.util.PackageKeys import com.tencent.bkrepo.common.lock.service.LockOperation import com.tencent.bkrepo.common.metadata.service.metadata.MetadataService +import com.tencent.bkrepo.common.metadata.service.node.NodeSearchService +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.metadata.service.repo.ProxyChannelService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.query.enums.OperationType import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.exception.RemoteErrorCodeException @@ -97,10 +100,8 @@ import com.tencent.bkrepo.helm.utils.HelmUtils import com.tencent.bkrepo.helm.utils.ObjectBuilderUtil import com.tencent.bkrepo.helm.utils.RemoteDownloadUtil import com.tencent.bkrepo.helm.utils.TimeFormatUtil -import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.PackageClient import com.tencent.bkrepo.repository.api.PackageMetadataClient -import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.NodeInfo import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest @@ -121,8 +122,12 @@ import java.util.concurrent.ThreadPoolExecutor // LateinitUsage: 抽象类中使用构造器注入会造成不便 @Suppress("LateinitUsage") open class AbstractChartService : ArtifactService() { + + @Autowired + lateinit var nodeService: NodeService + @Autowired - lateinit var nodeClient: NodeClient + lateinit var nodeSearchService: NodeSearchService @Autowired lateinit var metadataService: MetadataService @@ -131,7 +136,7 @@ open class AbstractChartService : ArtifactService() { lateinit var packageMetadataClient: PackageMetadataClient @Autowired - lateinit var repositoryClient: RepositoryClient + lateinit var repositoryService: RepositoryService @Autowired lateinit var packageClient: PackageClient @@ -179,7 +184,7 @@ open class AbstractChartService : ArtifactService() { */ fun getOriginalIndexYaml(projectId: String, repoName: String): HelmIndexYamlMetadata { val nodeDetail = getOriginalIndexNode(projectId, repoName) - val repository = repositoryClient.getRepoDetail(projectId, repoName, RepositoryType.HELM.name).data + val repository = repositoryService.getRepoDetail(projectId, repoName, RepositoryType.HELM.name) ?: throw RepoNotFoundException("Repository[$repoName] does not exist") val inputStream = storageManager.loadArtifactInputStream(nodeDetail, repository.storageCredentials) ?: throw HelmFileNotFoundException( @@ -190,7 +195,7 @@ open class AbstractChartService : ArtifactService() { private fun getOriginalIndexNode(projectId: String, repoName: String): NodeDetail? { val fullPath = HelmUtils.getIndexCacheYamlFullPath() - return nodeClient.getNodeDetail(projectId, repoName, fullPath).data + return nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, fullPath)) } /** @@ -207,11 +212,11 @@ open class AbstractChartService : ArtifactService() { * upload index.yaml file */ fun uploadIndexYamlMetadata(artifactFile: ArtifactFile, nodeCreateRequest: NodeCreateRequest) { - val repository = repositoryClient.getRepoDetail( + val repository = repositoryService.getRepoDetail( nodeCreateRequest.projectId, nodeCreateRequest.repoName, RepositoryType.HELM.name - ).data + ) ?: throw RepoNotFoundException("Repository[${nodeCreateRequest.repoName}] does not exist") storageManager.storeArtifactFile(nodeCreateRequest, artifactFile, repository.storageCredentials) } @@ -221,7 +226,7 @@ open class AbstractChartService : ArtifactService() { */ fun getRepositoryInfo(artifactInfo: ArtifactInfo): RepositoryDetail { with(artifactInfo) { - val result = repositoryClient.getRepoDetail(projectId, repoName, REPO_TYPE).data ?: run { + val result = repositoryService.getRepoDetail(projectId, repoName, REPO_TYPE) ?: run { logger.warn("check repository [$repoName] in projectId [$projectId] failed!") throw HelmRepoNotFoundException(HelmMessageCode.HELM_REPO_NOT_FOUND, "$projectId|$repoName") } @@ -231,9 +236,9 @@ open class AbstractChartService : ArtifactService() { fun getChartYaml(projectId: String, repoName: String, fullPath: String): HelmChartMetadata { - val repository = repositoryClient.getRepoDetail(projectId, repoName, RepositoryType.HELM.name).data + val repository = repositoryService.getRepoDetail(projectId, repoName, RepositoryType.HELM.name) ?: throw RepoNotFoundException("Repository[$repoName] does not exist") - val nodeDetail = nodeClient.getNodeDetail(projectId, repoName, fullPath).data + val nodeDetail = nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, fullPath)) val inputStream = storageManager.loadArtifactInputStream(nodeDetail, repository.storageCredentials) ?: throw HelmFileNotFoundException( HelmMessageCode.HELM_FILE_NOT_FOUND, fullPath, "$projectId|$repoName" @@ -249,7 +254,7 @@ open class AbstractChartService : ArtifactService() { */ fun checkRepositoryExistAndCategory(artifactInfo: ArtifactInfo) { with(artifactInfo) { - val repo = repositoryClient.getRepoDetail(projectId, repoName, REPO_TYPE).data ?: run { + val repo = repositoryService.getRepoDetail(projectId, repoName, REPO_TYPE) ?: run { logger.warn("check repository [$repoName] in projectId [$projectId] failed!") throw HelmRepoNotFoundException(HelmMessageCode.HELM_REPO_NOT_FOUND, "$projectId|$repoName") } @@ -307,10 +312,7 @@ open class AbstractChartService : ArtifactService() { if (exist) { lastModifyTime?.let { queryModelBuilder.rule(true, NODE_CREATE_DATE, it, OperationType.AFTER) } } - val result = nodeClient.queryWithoutCount(queryModelBuilder.build()).data ?: run { - logger.warn("don't find node list in repository: [$projectId/$repoName].") - return emptyList() - } + val result = nodeSearchService.searchWithoutCount(queryModelBuilder.build()) return result.records } } @@ -327,8 +329,8 @@ open class AbstractChartService : ArtifactService() { .projectId(artifactInfo.projectId) .repoName(artifactInfo.repoName) .fullPath(TGZ_SUFFIX, OperationType.SUFFIX) - val result = nodeClient.queryWithoutCount(queryModelBuilder.build()).data - if (result == null || result.records.isEmpty()) break + val result = nodeSearchService.searchWithoutCount(queryModelBuilder.build()) + if (result.records.isEmpty()) break result.records.forEach { try { ChartParserUtil.addIndexEntries(indexYamlMetadata, createChartMetadata(it, artifactInfo)) @@ -368,7 +370,7 @@ open class AbstractChartService : ArtifactService() { * check node exists */ fun exist(projectId: String, repoName: String, fullPath: String): Boolean { - return nodeClient.checkExist(projectId, repoName, fullPath).data ?: false + return nodeService.checkExist(ArtifactInfo(projectId, repoName, fullPath)) } /** @@ -637,7 +639,7 @@ open class AbstractChartService : ArtifactService() { * 检查该仓库是否remote仓库或者composite仓库 */ fun checkRepo(projectId: String, repoName: String): RepositoryDetail? { - val repoDetail = repositoryClient.getRepoDetail(projectId, repoName, REPO_TYPE).data ?: run { + val repoDetail = repositoryService.getRepoDetail(projectId, repoName, REPO_TYPE) ?: run { throw HelmRepoNotFoundException(HelmMessageCode.HELM_REPO_NOT_FOUND, "$projectId|$repoName") } if (RepositoryCategory.LOCAL == repoDetail.category) { diff --git a/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/ChartRepositoryServiceImpl.kt b/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/ChartRepositoryServiceImpl.kt index 8a333b99c2..dd3d102725 100644 --- a/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/ChartRepositoryServiceImpl.kt +++ b/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/ChartRepositoryServiceImpl.kt @@ -145,8 +145,8 @@ class ChartRepositoryServiceImpl( select = mutableListOf(PROJECT_ID, REPO_NAME, NODE_FULL_PATH, NODE_METADATA), rule = rule ) - val nodeList: List>? = nodeClient.queryWithoutCount(queryModel).data?.records - if (nodeList.isNullOrEmpty()) HttpStatus.NOT_FOUND else HttpStatus.OK + val nodeList: List> = nodeSearchService.searchWithoutCount(queryModel).records + if (nodeList.isEmpty()) HttpStatus.NOT_FOUND else HttpStatus.OK } else { HttpStatus.NOT_FOUND } @@ -163,7 +163,7 @@ class ChartRepositoryServiceImpl( with(artifactInfo) { val name = PackageKeys.resolveHelm(packageKey) val fullPath = String.format("/%s-%s.tgz", name, version) - val nodeDetail = nodeClient.getNodeDetail(projectId, repoName, fullPath).data ?: run { + val nodeDetail = nodeService.getNodeDetail(artifactInfo) ?: run { logger.warn("node [$fullPath] don't found.") throw HelmFileNotFoundException(HelmMessageCode.HELM_FILE_NOT_FOUND, fullPath, "$projectId|$repoName") } diff --git a/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/FixToolServiceImpl.kt b/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/FixToolServiceImpl.kt index a719010da7..5166d7397b 100644 --- a/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/FixToolServiceImpl.kt +++ b/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/FixToolServiceImpl.kt @@ -71,7 +71,6 @@ import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.NodeInfo import com.tencent.bkrepo.repository.pojo.packages.request.PopulatedPackageVersion import com.tencent.bkrepo.repository.pojo.repo.RepoUpdateRequest -import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail import org.slf4j.LoggerFactory import org.springframework.stereotype.Service import java.time.Duration @@ -87,10 +86,7 @@ class FixToolServiceImpl : FixToolService, AbstractChartService() { val successRepoName: MutableList = mutableListOf() val failedRepoName: MutableList = mutableListOf() logger.info("starting repair package created date for historical data") - val repositoryList = repositoryClient.pageByType(0, 1000, "HELM").data?.records ?: run { - logger.warn("no helm repository found, return.") - emptyList() - } + val repositoryList = repositoryService.listRepoPageByType("HELM", 0, 1000).records val helmLocalRepositoryList = repositoryList.filter { it.category == RepositoryCategory.LOCAL }.toList() logger.info( "find [${helmLocalRepositoryList.size}] HELM local " + @@ -172,7 +168,7 @@ class FixToolServiceImpl : FixToolService, AbstractChartService() { private fun helmIndexYamlMetadata(projectId: String, repoName: String): HelmIndexYamlMetadata { val nodeDetail = - nodeClient.getNodeDetail(projectId, repoName, HelmUtils.getIndexCacheYamlFullPath()).data ?: run { + nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, HelmUtils.getIndexCacheYamlFullPath())) ?: run { logger.error("query index-cache.yaml file failed in repo [$projectId/$repoName]") throw HelmFileNotFoundException( HelmMessageCode.HELM_FILE_NOT_FOUND, "index.yaml", "$projectId|$repoName" @@ -191,10 +187,7 @@ class FixToolServiceImpl : FixToolService, AbstractChartService() { val packageManagerList = mutableListOf() // 查找所有仓库 logger.info("starting add package manager function to historical data") - val repositoryList = repositoryClient.pageByType(0, 1000, "HELM").data?.records ?: run { - logger.warn("no helm repository found, return.") - return emptyList() - } + val repositoryList = repositoryService.listRepoPageByType("HELM", 0, 1000).records val helmLocalRepositoryList = repositoryList.filter { it.category == RepositoryCategory.LOCAL }.toList() logger.info( "find [${helmLocalRepositoryList.size}] HELM local " + @@ -218,12 +211,13 @@ class FixToolServiceImpl : FixToolService, AbstractChartService() { try { // 查询索引文件 val nodeDetail = - nodeClient.getNodeDetail(projectId, repoName, HelmUtils.getIndexCacheYamlFullPath()).data ?: run { - logger.error("query index-cache.yaml file failed in repo [$projectId/$repoName]") - throw HelmFileNotFoundException( - HelmMessageCode.HELM_FILE_NOT_FOUND, "index.yaml", "$projectId|$repoName" - ) - } + nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, HelmUtils.getIndexCacheYamlFullPath())) + ?: run { + logger.error("query index-cache.yaml file failed in repo [$projectId/$repoName]") + throw HelmFileNotFoundException( + HelmMessageCode.HELM_FILE_NOT_FOUND, "index.yaml", "$projectId|$repoName" + ) + } // sleep 0.1s Thread.sleep(100) val inputStream = storageManager.loadFullArtifactInputStream(nodeDetail, null) ?: run { @@ -369,7 +363,7 @@ class FixToolServiceImpl : FixToolService, AbstractChartService() { select = mutableListOf(), rule = Rule.NestedRule(ruleList, Rule.NestedRule.RelationType.AND) ) - return nodeClient.queryWithoutCount(queryModel).data!! + return nodeSearchService.searchWithoutCount(queryModel) } private fun resolveNode(record: Map): NodeInfo { @@ -450,7 +444,7 @@ class FixToolServiceImpl : FixToolService, AbstractChartService() { operator = userId, configuration = repositoryDetail.configuration ) - repositoryClient.updateRepo(request) + repositoryService.updateRepo(request) } companion object { diff --git a/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/HelmOperationService.kt b/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/HelmOperationService.kt index cbe200911a..e8e0839da9 100644 --- a/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/HelmOperationService.kt +++ b/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/HelmOperationService.kt @@ -28,6 +28,7 @@ package com.tencent.bkrepo.helm.service.impl import com.tencent.bkrepo.auth.pojo.enums.PermissionAction +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.exception.VersionNotFoundException import com.tencent.bkrepo.common.artifact.pojo.configuration.RepositoryConfiguration import com.tencent.bkrepo.common.artifact.pojo.configuration.composite.CompositeConfiguration @@ -100,7 +101,7 @@ class HelmOperationService( return } // 删除index文件 - nodeClient.deleteNode( + nodeService.deleteNode( NodeDeleteRequest( projectId = projectId, repoName = repoName, @@ -127,10 +128,10 @@ class HelmOperationService( val provPath = HelmUtils.getProvFileFullPath(packageName, version) if (chartPath.isNotBlank()) { val request = NodeDeleteRequest(projectId, repoName, chartPath, userId) - nodeClient.deleteNode(request) + nodeService.deleteNode(request) } if (provPath.isNotBlank()) { - nodeClient.deleteNode(NodeDeleteRequest(projectId, repoName, provPath, userId)) + nodeService.deleteNode(NodeDeleteRequest(projectId, repoName, provPath, userId)) } } @@ -142,7 +143,7 @@ class HelmOperationService( val version = packageClient.findPackageByKey(projectId, repoName, packageName).data?.latest try { val chartPath = HelmUtils.getChartFileFullPath(getArtifactName(), version!!) - val map = nodeClient.getNodeDetail(projectId, repoName, chartPath).data?.metadata + val map = nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, chartPath))?.metadata val chartInfo = map?.let { it1 -> HelmMetadataUtils.convertToObject(it1) } chartInfo?.appVersion?.let { val packageUpdateRequest = ObjectBuilderUtil.buildPackageUpdateRequest( diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/ArchivedNodeCompleteJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/ArchivedNodeCompleteJob.kt index 7a1ef4e9c8..db4d2722a9 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/ArchivedNodeCompleteJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/ArchivedNodeCompleteJob.kt @@ -30,6 +30,7 @@ package com.tencent.bkrepo.job.batch.task.archive import com.tencent.bkrepo.archive.ArchiveStatus import com.tencent.bkrepo.archive.api.ArchiveClient import com.tencent.bkrepo.archive.request.ArchiveFileRequest +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.job.batch.base.MongoDbBatchJob @@ -37,16 +38,15 @@ import com.tencent.bkrepo.job.batch.context.NodeContext import com.tencent.bkrepo.job.batch.utils.NodeCommonUtils import com.tencent.bkrepo.job.batch.utils.RepositoryCommonUtils import com.tencent.bkrepo.job.config.properties.ArchivedNodeCompleteJobProperties -import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.constant.SYSTEM_USER import com.tencent.bkrepo.repository.pojo.node.service.NodeArchiveRequest -import java.time.Duration import org.slf4j.LoggerFactory import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.Query import org.springframework.data.mongodb.core.query.isEqualTo import org.springframework.stereotype.Component +import java.time.Duration import kotlin.reflect.KClass /** @@ -62,7 +62,7 @@ import kotlin.reflect.KClass class ArchivedNodeCompleteJob( val properties: ArchivedNodeCompleteJobProperties, private val archiveClient: ArchiveClient, - private val nodeClient: NodeClient, + private val nodeService: NodeService, private val storageService: StorageService, ) : MongoDbBatchJob(properties) { @@ -125,7 +125,7 @@ class ArchivedNodeCompleteJob( fullPath = fullPath, operator = SYSTEM_USER, ) - nodeClient.archiveNode(nodeArchiveRequest) + nodeService.archiveNode(nodeArchiveRequest) // 删除原存储 storageService.delete(sha256, storageCredentials) } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/ArchivedNodeRestoreJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/ArchivedNodeRestoreJob.kt index a2afa14168..c20dd5e6f7 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/ArchivedNodeRestoreJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/ArchivedNodeRestoreJob.kt @@ -30,19 +30,19 @@ package com.tencent.bkrepo.job.batch.task.archive import com.tencent.bkrepo.archive.ArchiveStatus import com.tencent.bkrepo.archive.api.ArchiveClient import com.tencent.bkrepo.archive.request.ArchiveFileRequest +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.job.batch.base.MongoDbBatchJob import com.tencent.bkrepo.job.batch.context.NodeContext import com.tencent.bkrepo.job.batch.utils.NodeCommonUtils import com.tencent.bkrepo.job.config.properties.ArchivedNodeRestoreJobProperties -import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.pojo.node.service.NodeArchiveRequest -import java.time.Duration import org.slf4j.LoggerFactory import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.Query import org.springframework.data.mongodb.core.query.isEqualTo import org.springframework.stereotype.Component +import java.time.Duration import kotlin.reflect.KClass /** @@ -57,7 +57,7 @@ import kotlin.reflect.KClass class ArchivedNodeRestoreJob( private val properties: ArchivedNodeRestoreJobProperties, private val archiveClient: ArchiveClient, - private val nodeClient: NodeClient, + private val nodeService: NodeService ) : MongoDbBatchJob(properties) { override fun createJobContext(): NodeContext { @@ -87,7 +87,7 @@ class ArchivedNodeRestoreJob( fullPath = fullPath, operator = lastModifiedBy, ) - nodeClient.restoreNode(request) + nodeService.restoreNode(request) context.count.incrementAndGet() context.size.addAndGet(it.size) logger.info("Success to restore node $projectId/$repoName/$fullPath.") diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/NodeCompressedJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/NodeCompressedJob.kt index f1f39076f1..aff99ce2bc 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/NodeCompressedJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/NodeCompressedJob.kt @@ -30,21 +30,21 @@ package com.tencent.bkrepo.job.batch.task.archive import com.tencent.bkrepo.archive.CompressStatus import com.tencent.bkrepo.archive.api.ArchiveClient import com.tencent.bkrepo.archive.request.CompleteCompressRequest +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.job.batch.base.MongoDbBatchJob import com.tencent.bkrepo.job.batch.context.NodeContext import com.tencent.bkrepo.job.batch.utils.NodeCommonUtils import com.tencent.bkrepo.job.batch.utils.RepositoryCommonUtils import com.tencent.bkrepo.job.config.properties.NodeCompressedJobProperties -import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.pojo.node.service.NodeCompressedRequest import org.slf4j.LoggerFactory -import java.time.Duration import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.Query import org.springframework.data.mongodb.core.query.isEqualTo import org.springframework.stereotype.Component +import java.time.Duration import kotlin.reflect.KClass /** @@ -60,11 +60,10 @@ import kotlin.reflect.KClass @EnableConfigurationProperties(NodeCompressedJobProperties::class) class NodeCompressedJob( properties: NodeCompressedJobProperties, - val nodeClient: NodeClient, + val nodeService: NodeService, val archiveClient: ArchiveClient, val storageService: StorageService, -) : - MongoDbBatchJob(properties) { +) : MongoDbBatchJob(properties) { override fun createJobContext(): NodeContext { return NodeContext() } @@ -96,7 +95,7 @@ class NodeCompressedJob( fullPath = it.fullPath, operator = lastModifiedBy, ) - nodeClient.compressedNode(compressedRequest) + nodeService.compressedNode(compressedRequest) } storageService.delete(sha256, storageCredentials) val request = CompleteCompressRequest(sha256, storageCredentialsKey, lastModifiedBy) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/NodeUncompressedJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/NodeUncompressedJob.kt index 90fedb200f..49d9be8275 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/NodeUncompressedJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/NodeUncompressedJob.kt @@ -30,21 +30,21 @@ package com.tencent.bkrepo.job.batch.task.archive import com.tencent.bkrepo.archive.CompressStatus import com.tencent.bkrepo.archive.api.ArchiveClient import com.tencent.bkrepo.archive.request.DeleteCompressRequest +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.job.batch.base.MongoDbBatchJob import com.tencent.bkrepo.job.batch.context.NodeContext import com.tencent.bkrepo.job.batch.utils.NodeCommonUtils import com.tencent.bkrepo.job.batch.utils.RepositoryCommonUtils import com.tencent.bkrepo.job.config.properties.NodeUncompressedJobProperties -import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.pojo.node.service.NodeUnCompressedRequest import org.slf4j.LoggerFactory -import java.time.Duration import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.Query import org.springframework.data.mongodb.core.query.isEqualTo import org.springframework.stereotype.Component +import java.time.Duration import kotlin.reflect.KClass /** @@ -59,7 +59,7 @@ import kotlin.reflect.KClass @EnableConfigurationProperties(NodeUncompressedJobProperties::class) class NodeUncompressedJob( properties: NodeUncompressedJobProperties, - val nodeClient: NodeClient, + val nodeService: NodeService, val archiveClient: ArchiveClient, val storageService: StorageService, ) : @@ -94,7 +94,7 @@ class NodeUncompressedJob( fullPath = it.fullPath, operator = lastModifiedBy, ) - nodeClient.uncompressedNode(compressedRequest) + nodeService.uncompressedNode(compressedRequest) } val request = DeleteCompressRequest(sha256, storageCredentialsKey, lastModifiedBy) archiveClient.deleteCompress(request) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/ArtifactCleanupJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/ArtifactCleanupJob.kt index 37caad9ea3..c40c9b7abc 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/ArtifactCleanupJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/ArtifactCleanupJob.kt @@ -33,6 +33,7 @@ import com.tencent.bkrepo.common.api.util.readJsonString import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.pojo.configuration.RepositoryConfiguration +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.security.constant.MS_AUTH_HEADER_SECURITY_TOKEN import com.tencent.bkrepo.common.security.service.ServiceAuthManager import com.tencent.bkrepo.common.service.log.LoggerHolder @@ -43,9 +44,7 @@ import com.tencent.bkrepo.job.batch.base.DefaultContextMongoDbJob import com.tencent.bkrepo.job.batch.base.JobContext import com.tencent.bkrepo.job.config.properties.ArtifactCleanupJobProperties import com.tencent.bkrepo.job.exception.JobExecuteException -import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.constant.SYSTEM_USER -import com.tencent.bkrepo.repository.pojo.node.service.NodeCleanRequest import org.springframework.beans.factory.annotation.Value import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.cloud.client.discovery.DiscoveryClient @@ -71,7 +70,7 @@ import kotlin.reflect.KClass @EnableConfigurationProperties(ArtifactCleanupJobProperties::class) class ArtifactCleanupJob( private val properties: ArtifactCleanupJobProperties, - private val nodeClient: NodeClient, + private val nodeService: NodeService, private val discoveryClient: DiscoveryClient, private val serviceAuthManager: ServiceAuthManager ) : DefaultContextMongoDbJob(properties) { @@ -186,14 +185,12 @@ class ArtifactCleanupJob( } folders.forEach { try { - nodeClient.cleanNodes( - (NodeCleanRequest( - projectId = projectId, - repoName = repoName, - path = PathUtils.toPath(it), - date = cleanupDate, - operator = SYSTEM_USER - )) + nodeService.deleteBeforeDate( + projectId = projectId, + repoName = repoName, + path = PathUtils.toPath(it), + date = cleanupDate, + operator = SYSTEM_USER ) } catch (e: Exception) { logger.warn( diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/PipelineArtifactCleanupJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/PipelineArtifactCleanupJob.kt index c571e9ed2a..5c8274e6d3 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/PipelineArtifactCleanupJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/PipelineArtifactCleanupJob.kt @@ -30,14 +30,13 @@ package com.tencent.bkrepo.job.batch.task.clean import com.tencent.bkrepo.common.artifact.constant.PIPELINE import com.tencent.bkrepo.common.artifact.constant.REPORT import com.tencent.bkrepo.common.artifact.path.PathUtils +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.job.SHARDING_COUNT import com.tencent.bkrepo.job.batch.base.DefaultContextMongoDbJob import com.tencent.bkrepo.job.batch.base.JobContext import com.tencent.bkrepo.job.batch.utils.TimeUtils import com.tencent.bkrepo.job.config.properties.PipelineArtifactCleanupJobProperties -import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.constant.SYSTEM_USER -import com.tencent.bkrepo.repository.pojo.node.service.NodeCleanRequest import org.slf4j.LoggerFactory import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.data.domain.Sort @@ -57,7 +56,7 @@ import kotlin.reflect.KClass @EnableConfigurationProperties(PipelineArtifactCleanupJobProperties::class) class PipelineArtifactCleanupJob( private val properties: PipelineArtifactCleanupJobProperties, - private val nodeClient: NodeClient, + private val nodeService: NodeService ) : DefaultContextMongoDbJob(properties) { override fun collectionNames(): List { return (0 until SHARDING_COUNT) @@ -123,15 +122,15 @@ class PipelineArtifactCleanupJob( */ private fun deleteBeforeBuild(buildNode: Node) { try { - val result = nodeClient.cleanNodes((NodeCleanRequest( + val result = nodeService.deleteBeforeDate( projectId = buildNode.projectId, repoName = buildNode.repoName, path = buildNode.path, date = buildNode.createdDate, operator = SYSTEM_USER - ))).data + ) logger.info( - "delete ${result?.deletedNumber} node " + + "delete ${result.deletedNumber} node " + "in [${buildNode.projectId}/${buildNode.repoName}${buildNode.path}]" ) } catch (e: NullPointerException) { diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/SignFileCleanupJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/SignFileCleanupJob.kt index f75a89a306..e26f3f0cad 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/SignFileCleanupJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/SignFileCleanupJob.kt @@ -27,12 +27,12 @@ package com.tencent.bkrepo.job.batch.task.clean +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.mongo.constant.ID import com.tencent.bkrepo.job.CREATED_DATE import com.tencent.bkrepo.job.batch.base.DefaultContextMongoDbJob import com.tencent.bkrepo.job.batch.base.JobContext import com.tencent.bkrepo.job.config.properties.SignFileCleanupJobProperties -import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.constant.SYSTEM_USER import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest import org.springframework.boot.context.properties.EnableConfigurationProperties @@ -47,7 +47,7 @@ import kotlin.reflect.KClass @Component @EnableConfigurationProperties(SignFileCleanupJobProperties::class) class SignFileCleanupJob( - private val nodeClient: NodeClient, + private val nodeService: NodeService, val properties: SignFileCleanupJobProperties ) : DefaultContextMongoDbJob(properties) { @@ -75,7 +75,7 @@ class SignFileCleanupJob( fullPath = fullPath, operator = SYSTEM_USER ) - nodeClient.deleteNode(deleteReq) + nodeService.deleteNode(deleteReq) mongoTemplate.remove(Query(Criteria(ID).isEqualTo(id)), collectionName) } } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/ddc/DdcBlobCleanupJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/ddc/DdcBlobCleanupJob.kt index cd1114a144..ccc4726460 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/ddc/DdcBlobCleanupJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/ddc/DdcBlobCleanupJob.kt @@ -27,10 +27,10 @@ package com.tencent.bkrepo.job.batch.task.ddc +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.mongo.constant.ID import com.tencent.bkrepo.job.batch.base.DefaultContextMongoDbJob import com.tencent.bkrepo.job.batch.base.JobContext -import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.constant.SYSTEM_USER import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest import org.springframework.boot.context.properties.EnableConfigurationProperties @@ -46,7 +46,7 @@ import java.time.LocalDateTime @EnableConfigurationProperties(DdcBlobCleanupJobProperties::class) class DdcBlobCleanupJob( private val properties: DdcBlobCleanupJobProperties, - private val nodeClient: NodeClient, + private val nodeService: NodeService ) : DefaultContextMongoDbJob(properties) { override fun collectionNames() = listOf(COLLECTION_NAME) @@ -77,7 +77,7 @@ class DdcBlobCleanupJob( override fun entityClass() = Blob::class override fun run(row: Blob, collectionName: String, context: JobContext) { - nodeClient.deleteNode( + nodeService.deleteNode( NodeDeleteRequest( projectId = row.projectId, repoName = row.repoName, diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/ddc/ExpiredDdcRefCleanupJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/ddc/ExpiredDdcRefCleanupJob.kt index 808072043c..ae5610352e 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/ddc/ExpiredDdcRefCleanupJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/ddc/ExpiredDdcRefCleanupJob.kt @@ -27,10 +27,10 @@ package com.tencent.bkrepo.job.batch.task.ddc +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.mongo.constant.ID import com.tencent.bkrepo.job.batch.base.DefaultContextMongoDbJob import com.tencent.bkrepo.job.batch.base.JobContext -import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.constant.SYSTEM_USER import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest import org.bson.types.Binary @@ -49,7 +49,7 @@ import kotlin.reflect.KClass @EnableConfigurationProperties(ExpiredDdcRefCleanupJobProperties::class) class ExpiredDdcRefCleanupJob( private val properties: ExpiredDdcRefCleanupJobProperties, - private val nodeClient: NodeClient, + private val nodeService: NodeService ) : DefaultContextMongoDbJob(properties) { override fun collectionNames(): List = listOf(COLLECTION_NAME, COLLECTION_NAME_LEGACY) @@ -80,7 +80,7 @@ class ExpiredDdcRefCleanupJob( mongoTemplate.remove(Query(Criteria.where(ID).isEqualTo(row.id)), collectionName) if (row.inlineBlob == null && collectionName == COLLECTION_NAME) { // inlineBlob为null时表示inlineBlob不存在数据库中而是单独存放于后端存储中，需要一并清理 - nodeClient.deleteNode( + nodeService.deleteNode( NodeDeleteRequest(row.projectId, row.repoName, "/${row.bucket}/${row.key}", SYSTEM_USER) ) } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/other/ExpiredNodeMarkupJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/other/ExpiredNodeMarkupJob.kt index 555eef1cec..7892a1d7a8 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/other/ExpiredNodeMarkupJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/other/ExpiredNodeMarkupJob.kt @@ -27,12 +27,12 @@ package com.tencent.bkrepo.job.batch.task.other +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.job.SHARDING_COUNT import com.tencent.bkrepo.job.batch.base.DefaultContextMongoDbJob import com.tencent.bkrepo.job.batch.base.JobContext import com.tencent.bkrepo.job.batch.utils.TimeUtils import com.tencent.bkrepo.job.config.properties.ExpiredNodeMarkupJobProperties -import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.constant.SYSTEM_USER import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest import org.slf4j.LoggerFactory @@ -53,7 +53,7 @@ import kotlin.reflect.KClass @EnableConfigurationProperties(ExpiredNodeMarkupJobProperties::class) class ExpiredNodeMarkupJob( properties: ExpiredNodeMarkupJobProperties, - private val nodeClient: NodeClient + private val nodeService: NodeService ) : DefaultContextMongoDbJob(properties) { data class Node( @@ -97,7 +97,7 @@ class ExpiredNodeMarkupJob( override fun run(row: Node, collectionName: String, context: JobContext) { try { - nodeClient.deleteNode(NodeDeleteRequest(row.projectId, row.repoName, row.fullPath, SYSTEM_USER)) + nodeService.deleteNode(NodeDeleteRequest(row.projectId, row.repoName, row.fullPath, SYSTEM_USER)) } catch (e: Exception) { logger.warn("delete expired node[$row] failed: $e") } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/utils/RepositoryCommonUtils.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/utils/RepositoryCommonUtils.kt index 91840b1cba..f0e660ff36 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/utils/RepositoryCommonUtils.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/utils/RepositoryCommonUtils.kt @@ -4,9 +4,9 @@ import com.google.common.cache.Cache import com.google.common.cache.CacheBuilder import com.tencent.bkrepo.common.artifact.exception.RepoNotFoundException import com.tencent.bkrepo.common.artifact.pojo.RepositoryType +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService import com.tencent.bkrepo.common.storage.credentials.StorageCredentials -import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail import org.springframework.stereotype.Component import java.util.concurrent.TimeUnit @@ -14,17 +14,17 @@ import java.util.concurrent.TimeUnit @Component class RepositoryCommonUtils( storageCredentialService: StorageCredentialService, - repositoryClient: RepositoryClient + repositoryService: RepositoryService ) { init { Companion.storageCredentialService = storageCredentialService - Companion.repositoryClient = repositoryClient + Companion.repositoryService = repositoryService } companion object { private lateinit var storageCredentialService: StorageCredentialService - private lateinit var repositoryClient: RepositoryClient + private lateinit var repositoryService: RepositoryService private val repositoryCache = CacheBuilder.newBuilder() .maximumSize(10000) .expireAfterWrite(5, TimeUnit.MINUTES) @@ -48,7 +48,7 @@ class RepositoryCommonUtils( ): RepositoryDetail { val repositoryId = RepositoryId(projectId, repoName, type) return repositoryCache.getOrPut(repositoryId) { - repositoryClient.getRepoDetail(projectId, repoName, type.name).data + repositoryService.getRepoDetail(projectId, repoName, type.name) ?: throw RepoNotFoundException("$projectId/$repoName") } } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/MigrateRepoStorageService.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/MigrateRepoStorageService.kt index 9cfb0604e3..bc1639d56a 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/MigrateRepoStorageService.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/MigrateRepoStorageService.kt @@ -32,6 +32,7 @@ import com.google.common.base.CaseFormat.UPPER_CAMEL import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.common.api.pojo.Page +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.mongo.util.Pages import com.tencent.bkrepo.common.service.actuator.ActuatorConfiguration.Companion.SERVICE_INSTANCE_ID import com.tencent.bkrepo.common.storage.config.StorageProperties @@ -59,7 +60,6 @@ import com.tencent.bkrepo.job.migrate.pojo.MigrateRepoStorageTaskState.MIGRATING import com.tencent.bkrepo.job.migrate.pojo.MigrateRepoStorageTaskState.PENDING import com.tencent.bkrepo.job.migrate.pojo.MigrationContext import com.tencent.bkrepo.job.migrate.utils.ExecutingTaskRecorder -import com.tencent.bkrepo.repository.api.RepositoryClient import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Value import org.springframework.data.domain.PageRequest @@ -76,7 +76,7 @@ class MigrateRepoStorageService( private val migrateRepoStorageTaskDao: MigrateRepoStorageTaskDao, private val executors: Map, private val executingTaskRecorder: ExecutingTaskRecorder, - private val repositoryClient: RepositoryClient, + private val repositoryService: RepositoryService, ) { @Value(SERVICE_INSTANCE_ID) protected lateinit var instanceId: String @@ -92,7 +92,7 @@ class MigrateRepoStorageService( throw ErrorCodeException(CommonMessageCode.RESOURCE_EXISTED, "$projectId/$repoName") } val now = LocalDateTime.now() - val repo = repositoryClient.getRepoDetail(projectId, repoName).data!! + val repo = repositoryService.getRepoDetail(projectId, repoName)!! if (repo.storageCredentials?.key == dstCredentialsKey) { throw ErrorCodeException(CommonMessageCode.PARAMETER_INVALID, "src key cant be same as dst key") } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/FinishExecutor.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/FinishExecutor.kt index 951f4c54d0..80307fb737 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/FinishExecutor.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/FinishExecutor.kt @@ -28,6 +28,7 @@ package com.tencent.bkrepo.job.migrate.executor import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.job.migrate.config.MigrateRepoStorageProperties import com.tencent.bkrepo.job.migrate.dao.MigrateFailedNodeDao @@ -36,7 +37,6 @@ import com.tencent.bkrepo.job.migrate.pojo.MigrateRepoStorageTaskState.FINISHING import com.tencent.bkrepo.job.migrate.pojo.MigrateRepoStorageTaskState.MIGRATE_FAILED_NODE_FINISHED import com.tencent.bkrepo.job.migrate.pojo.MigrationContext import com.tencent.bkrepo.job.migrate.utils.ExecutingTaskRecorder -import com.tencent.bkrepo.repository.api.RepositoryClient import org.slf4j.LoggerFactory import org.springframework.stereotype.Component @@ -48,7 +48,7 @@ class FinishExecutor( migrateFailedNodeDao: MigrateFailedNodeDao, storageService: StorageService, executingTaskRecorder: ExecutingTaskRecorder, - private val repositoryClient: RepositoryClient, + private val repositoryService: RepositoryService, ) : BaseTaskExecutor( properties, migrateRepoStorageTaskDao, @@ -64,7 +64,7 @@ class FinishExecutor( val newContext = checkExecutable(context, MIGRATE_FAILED_NODE_FINISHED.name, FINISHING.name) ?: return null with(newContext.task) { logger.info("migrate finished, task[${newContext.task}]") - repositoryClient.unsetOldStorageCredentialsKey(projectId, repoName) + repositoryService.unsetOldStorageCredentialsKey(projectId, repoName) migrateRepoStorageTaskDao.removeById(id!!) logger.info("clean migrate task[${projectId}/${repoName}] success") return context diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/MigrateExecutor.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/MigrateExecutor.kt index 3a161986b4..8ccd6311f7 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/MigrateExecutor.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/MigrateExecutor.kt @@ -28,6 +28,7 @@ package com.tencent.bkrepo.job.migrate.executor import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.job.migrate.config.MigrateRepoStorageProperties import com.tencent.bkrepo.job.migrate.dao.MigrateFailedNodeDao @@ -40,7 +41,6 @@ import com.tencent.bkrepo.job.migrate.utils.MigrateRepoStorageUtils.buildThreadP import com.tencent.bkrepo.job.migrate.utils.MigratedTaskNumberPriorityQueue import com.tencent.bkrepo.job.migrate.utils.NodeIterator import com.tencent.bkrepo.job.migrate.utils.TransferDataExecutor -import com.tencent.bkrepo.repository.api.RepositoryClient import org.slf4j.LoggerFactory import org.springframework.data.mongodb.core.MongoTemplate import org.springframework.stereotype.Component @@ -58,7 +58,7 @@ class MigrateExecutor( executingTaskRecorder: ExecutingTaskRecorder, private val migrateFailedHandler: MigrateFailedHandler, private val transferDataExecutor: TransferDataExecutor, - private val repositoryClient: RepositoryClient, + private val repositoryService: RepositoryService, private val mongoTemplate: MongoTemplate ) : BaseTaskExecutor( properties, @@ -134,14 +134,14 @@ class MigrateExecutor( override fun prepare(context: MigrationContext): MigrationContext { val task = context.task - val repo = repositoryClient.getRepoDetail(task.projectId, task.repoName).data!! + val repo = repositoryService.getRepoDetail(task.projectId, task.repoName)!! // 任务首次执行才更新仓库配置，从上次中断点继续执行时不需要重复更新 return if (repo.storageCredentials?.key != task.dstStorageKey) { val startDate = LocalDateTime.now() val newTask = migrateRepoStorageTaskDao.updateStartDate(task.id!!, startDate)!! logger.info("update migrate task of [${task.projectId}/${task.repoName}] startDate[$startDate]") // 修改repository配置，保证之后上传的文件直接保存到新存储实例中，文件下载时，当前实例找不到的情况下会去默认存储找 - repositoryClient.updateStorageCredentialsKey(task.projectId, task.repoName, task.dstStorageKey) + repositoryService.updateStorageCredentialsKey(task.projectId, task.repoName, task.dstStorageKey) logger.info("update repo[${task.projectId}/${task.repoName}] dstStorageKey[${task.dstStorageKey}]") context.copy(task = newTask.toDto()) } else { diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/SeparationTaskServiceImpl.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/SeparationTaskServiceImpl.kt index cdae04128a..1a2cb4cf93 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/SeparationTaskServiceImpl.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/SeparationTaskServiceImpl.kt @@ -32,6 +32,7 @@ import com.tencent.bkrepo.common.api.exception.NotFoundException import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.common.api.pojo.Page import com.tencent.bkrepo.common.artifact.pojo.RepositoryType +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.mongo.util.Pages import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.job.RESTORE @@ -47,7 +48,6 @@ import com.tencent.bkrepo.job.separation.pojo.task.SeparationTask.Companion.toDt import com.tencent.bkrepo.job.separation.pojo.task.SeparationTaskRequest import com.tencent.bkrepo.job.separation.pojo.task.SeparationTaskState import com.tencent.bkrepo.job.separation.service.SeparationTaskService -import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail import org.slf4j.LoggerFactory import org.springframework.data.domain.PageRequest @@ -64,7 +64,7 @@ import java.time.format.DateTimeParseException @Component class SeparationTaskServiceImpl( private val dataSeparationConfig: DataSeparationConfig, - private val repositoryClient: RepositoryClient, + private val repositoryService: RepositoryService, private val separationTaskDao: SeparationTaskDao, private val separationFailedRecordDao: SeparationFailedRecordDao, private val mongoTemplate: MongoTemplate, @@ -183,7 +183,7 @@ class SeparationTaskServiceImpl( } private fun getRepoInfo(projectId: String, repoName: String): RepositoryDetail { - val repo = repositoryClient.getRepoDetail(projectId, repoName).data + val repo = repositoryService.getRepoDetail(projectId, repoName) ?: run { logger.warn("Repo [$projectId|$repoName] not exist.") throw NotFoundException( diff --git a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/FileReferenceCleanupJobTest.kt b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/FileReferenceCleanupJobTest.kt index 293c7e6a41..a220209de8 100644 --- a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/FileReferenceCleanupJobTest.kt +++ b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/FileReferenceCleanupJobTest.kt @@ -29,22 +29,25 @@ package com.tencent.bkrepo.job.batch import com.tencent.bkrepo.archive.api.ArchiveClient import com.tencent.bkrepo.archive.constant.DEFAULT_KEY +import com.tencent.bkrepo.auth.api.ServiceBkiamV3ResourceClient +import com.tencent.bkrepo.auth.api.ServicePermissionClient import com.tencent.bkrepo.common.artifact.pojo.RepositoryCategory import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.pojo.configuration.local.LocalConfiguration import com.tencent.bkrepo.common.metadata.service.log.OperateLogService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService -import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.storage.credentials.InnerCosCredentials +import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier import com.tencent.bkrepo.job.SHARDING_COUNT import com.tencent.bkrepo.job.batch.task.clean.FileReferenceCleanupJob import com.tencent.bkrepo.job.batch.utils.NodeCommonUtils import com.tencent.bkrepo.job.batch.utils.RepositoryCommonUtils import com.tencent.bkrepo.job.config.properties.FileReferenceCleanupJobProperties import com.tencent.bkrepo.job.repository.JobSnapshotRepository -import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail +import com.tencent.bkrepo.router.api.RouterControllerClient import org.bson.Document import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Assertions @@ -80,7 +83,19 @@ class FileReferenceCleanupJobTest : JobBaseTest() { lateinit var archiveClient: ArchiveClient @MockBean - lateinit var repositoryClient: RepositoryClient + lateinit var repositoryService: RepositoryService + + @MockBean + private lateinit var messageSupplier: MessageSupplier + + @MockBean + private lateinit var servicePermissionClient: ServicePermissionClient + + @MockBean + private lateinit var routerControllerClient: RouterControllerClient + + @MockBean + private lateinit var serviceBkiamV3ResourceClient: ServiceBkiamV3ResourceClient @MockBean lateinit var jobSnapshotRepository: JobSnapshotRepository @@ -113,26 +128,24 @@ class FileReferenceCleanupJobTest : JobBaseTest() { Mockito.`when`(storageCredentialService.findByKey(anyString())).thenReturn( credentials ) - Mockito.`when`(repositoryClient.getRepoDetail(anyString(), anyString(), anyString())).thenReturn( - ResponseBuilder.success( - RepositoryDetail( - projectId = "ut-project", - name = "ut-repo", - storageCredentials = InnerCosCredentials(key = "0"), - type = RepositoryType.NONE, - category = RepositoryCategory.LOCAL, - public = false, - description = "", - configuration = LocalConfiguration(), - createdBy = "", - createdDate = "", - lastModifiedBy = "", - lastModifiedDate = "", - oldCredentialsKey = null, - quota = 0, - used = 0, - ), - ), + Mockito.`when`(repositoryService.getRepoDetail(anyString(), anyString(), anyString())).thenReturn( + RepositoryDetail( + projectId = "ut-project", + name = "ut-repo", + storageCredentials = InnerCosCredentials(key = "0"), + type = RepositoryType.NONE, + category = RepositoryCategory.LOCAL, + public = false, + description = "", + configuration = LocalConfiguration(), + createdBy = "", + createdDate = "", + lastModifiedBy = "", + lastModifiedDate = "", + oldCredentialsKey = null, + quota = 0, + used = 0, + ) ) fileReferenceCleanupJobProperties.expectedNodes = 50_000 } diff --git a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/JobBaseTest.kt b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/JobBaseTest.kt index 149d6be26b..c158dbbabd 100644 --- a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/JobBaseTest.kt +++ b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/JobBaseTest.kt @@ -27,11 +27,14 @@ package com.tencent.bkrepo.job.batch +import com.tencent.bkrepo.common.artifact.event.base.ArtifactEvent +import com.tencent.bkrepo.common.artifact.properties.RouterControllerProperties import com.tencent.bkrepo.common.job.JobAutoConfiguration import com.tencent.bkrepo.common.metadata.properties.ProjectUsageStatisticsProperties import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties import com.tencent.bkrepo.common.service.util.SpringContextUtils import com.tencent.bkrepo.common.storage.StorageAutoConfiguration +import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier import com.tencent.bkrepo.job.batch.file.ExpireFileResolverConfig import com.tencent.bkrepo.job.config.JobConfig import io.mockk.every @@ -59,6 +62,7 @@ import org.springframework.test.context.TestPropertySource ClusterProperties::class, RedisAutoConfiguration::class, StorageAutoConfiguration::class, + RouterControllerProperties::class, ProjectUsageStatisticsProperties::class, ) @TestPropertySource( @@ -87,5 +91,8 @@ class JobBaseTest { every { SpringContextUtils.getBean() } returns tracer every { tracer.currentSpan() } returns null every { SpringContextUtils.publishEvent(any()) } returns Unit + + val messageSupplier = mockk() + every { messageSupplier.delegateToSupplier(any(), any(), any(), any(), any())}.returns(Unit) } } diff --git a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/task/clean/DeletedNodeCleanupJobTest.kt b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/task/clean/DeletedNodeCleanupJobTest.kt index f6b8c7fd2e..b495870f66 100644 --- a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/task/clean/DeletedNodeCleanupJobTest.kt +++ b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/task/clean/DeletedNodeCleanupJobTest.kt @@ -1,11 +1,15 @@ package com.tencent.bkrepo.job.batch.task.clean +import com.tencent.bkrepo.archive.api.ArchiveClient +import com.tencent.bkrepo.auth.api.ServiceBkiamV3ResourceClient +import com.tencent.bkrepo.auth.api.ServicePermissionClient import com.tencent.bkrepo.common.artifact.hash.sha256 import com.tencent.bkrepo.common.metadata.service.log.OperateLogService import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService import com.tencent.bkrepo.common.mongo.constant.ID import com.tencent.bkrepo.common.mongo.dao.util.sharding.HashShardingUtils import com.tencent.bkrepo.common.storage.credentials.InnerCosCredentials +import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier import com.tencent.bkrepo.job.SHARDING_COUNT import com.tencent.bkrepo.job.UT_PROJECT_ID import com.tencent.bkrepo.job.UT_REPO_NAME @@ -14,6 +18,7 @@ import com.tencent.bkrepo.job.batch.JobBaseTest import com.tencent.bkrepo.job.batch.context.DeletedNodeCleanupJobContext import com.tencent.bkrepo.job.migrate.MigrateRepoStorageService import com.tencent.bkrepo.job.separation.service.SeparationTaskService +import com.tencent.bkrepo.router.api.RouterControllerClient import org.bson.types.ObjectId import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNotNull @@ -42,6 +47,21 @@ class DeletedNodeCleanupJobTest @Autowired constructor( private val mongoTemplate: MongoTemplate, ) : JobBaseTest() { + @MockBean + private lateinit var messageSupplier: MessageSupplier + + @MockBean + private lateinit var servicePermissionClient: ServicePermissionClient + + @MockBean + private lateinit var routerControllerClient: RouterControllerClient + + @MockBean + private lateinit var serviceBkiamV3ResourceClient: ServiceBkiamV3ResourceClient + + @MockBean + private lateinit var archiveClient: ArchiveClient + @MockBean private lateinit var migrateRepoStorageService: MigrateRepoStorageService diff --git a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/task/storage/StorageReconcileJobTest.kt b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/task/storage/StorageReconcileJobTest.kt index a3ac9252cc..84e4a05b8d 100644 --- a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/task/storage/StorageReconcileJobTest.kt +++ b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/task/storage/StorageReconcileJobTest.kt @@ -1,5 +1,8 @@ package com.tencent.bkrepo.job.batch.task.storage +import com.tencent.bkrepo.archive.api.ArchiveClient +import com.tencent.bkrepo.auth.api.ServiceBkiamV3ResourceClient +import com.tencent.bkrepo.auth.api.ServicePermissionClient import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.api.FileSystemArtifactFile import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService @@ -9,10 +12,12 @@ import com.tencent.bkrepo.common.storage.config.StorageProperties import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.storage.credentials.FileSystemCredentials import com.tencent.bkrepo.common.storage.util.toPath +import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier import com.tencent.bkrepo.job.batch.JobBaseTest import com.tencent.bkrepo.job.batch.utils.NodeCommonUtils import com.tencent.bkrepo.job.migrate.MigrateRepoStorageService import com.tencent.bkrepo.repository.api.RepositoryClient +import com.tencent.bkrepo.router.api.RouterControllerClient import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.BeforeEach @@ -35,9 +40,24 @@ class StorageReconcileJobTest @Autowired constructor( private val storageProperties: StorageProperties, ) : JobBaseTest() { + @MockBean + private lateinit var routerControllerClient: RouterControllerClient + + @MockBean + lateinit var servicePermissionClient: ServicePermissionClient + + @MockBean + lateinit var serviceBkiamV3ResourceClient: ServiceBkiamV3ResourceClient + @MockBean lateinit var migrateRepoStorageService: MigrateRepoStorageService + @MockBean + lateinit var messageSupplier: MessageSupplier + + @MockBean + lateinit var archiveClient: ArchiveClient + @MockBean lateinit var storageCredentialService: StorageCredentialService @@ -52,6 +72,7 @@ class StorageReconcileJobTest @Autowired constructor( @Autowired lateinit var nodeCommonUtils: NodeCommonUtils + private val cred = storageProperties.defaultStorageCredentials() as FileSystemCredentials init { diff --git a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/utils/NodeCommonUtilsTest.kt b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/utils/NodeCommonUtilsTest.kt index a564f5813a..ca514899eb 100644 --- a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/utils/NodeCommonUtilsTest.kt +++ b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/utils/NodeCommonUtilsTest.kt @@ -27,8 +27,12 @@ package com.tencent.bkrepo.job.batch.utils +import com.tencent.bkrepo.archive.api.ArchiveClient +import com.tencent.bkrepo.auth.api.ServiceBkiamV3ResourceClient +import com.tencent.bkrepo.auth.api.ServicePermissionClient import com.tencent.bkrepo.common.metadata.service.log.OperateLogService import com.tencent.bkrepo.common.mongo.dao.util.sharding.HashShardingUtils +import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier import com.tencent.bkrepo.job.SHARDING_COUNT import com.tencent.bkrepo.job.UT_PROJECT_ID import com.tencent.bkrepo.job.UT_REPO_NAME @@ -36,6 +40,7 @@ import com.tencent.bkrepo.job.UT_SHA256 import com.tencent.bkrepo.job.batch.JobBaseTest import com.tencent.bkrepo.job.migrate.MigrateRepoStorageService import com.tencent.bkrepo.job.separation.service.SeparationTaskService +import com.tencent.bkrepo.router.api.RouterControllerClient import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.DisplayName import org.junit.jupiter.api.Test @@ -57,6 +62,16 @@ import java.time.LocalDateTime class NodeCommonUtilsTest @Autowired constructor( private val mongoTemplate: MongoTemplate, ) : JobBaseTest() { + @MockBean + private lateinit var routerControllerClient: RouterControllerClient + @MockBean + lateinit var servicePermissionClient: ServicePermissionClient + @MockBean + lateinit var serviceBkiamV3ResourceClient: ServiceBkiamV3ResourceClient + @MockBean + lateinit var messageSupplier: MessageSupplier + @MockBean + lateinit var archiveClient: ArchiveClient @MockBean lateinit var migrateRepoStorageService: MigrateRepoStorageService @MockBean diff --git a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/MigrateRepoStorageServiceTest.kt b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/MigrateRepoStorageServiceTest.kt index 48bf74b73d..9670caeabf 100644 --- a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/MigrateRepoStorageServiceTest.kt +++ b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/MigrateRepoStorageServiceTest.kt @@ -28,7 +28,7 @@ package com.tencent.bkrepo.job.migrate import com.tencent.bkrepo.common.api.exception.ErrorCodeException -import com.tencent.bkrepo.common.api.pojo.Response +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService import com.tencent.bkrepo.common.mongo.constant.ID import com.tencent.bkrepo.common.service.actuator.ActuatorConfiguration @@ -51,7 +51,6 @@ import com.tencent.bkrepo.job.migrate.pojo.MigrationContext import com.tencent.bkrepo.job.migrate.utils.ExecutingTaskRecorder import com.tencent.bkrepo.job.migrate.utils.MigrateTestUtils.buildRepo import com.tencent.bkrepo.job.migrate.utils.MigrateTestUtils.buildTask -import com.tencent.bkrepo.repository.api.RepositoryClient import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertNotNull @@ -100,7 +99,7 @@ class MigrateRepoStorageServiceTest @Autowired constructor( private lateinit var instanceId: String @MockBean - private lateinit var repositoryClient: RepositoryClient + private lateinit var repositoryService: RepositoryService @MockBean private lateinit var storageCredentialService: StorageCredentialService @@ -220,8 +219,8 @@ class MigrateRepoStorageServiceTest @Autowired constructor( ) private fun initMock() { - whenever(repositoryClient.getRepoDetail(anyString(), anyString(), anyOrNull())).thenReturn( - Response(0, "", buildRepo()), + whenever(repositoryService.getRepoDetail(anyString(), anyString(), anyOrNull())).thenReturn( + buildRepo() ) whenever(storageCredentialService.findByKey(anyString())) .thenReturn(FileSystemCredentials()) diff --git a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/executor/ExecutorBaseTest.kt b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/executor/ExecutorBaseTest.kt index 59a2c79882..644b268104 100644 --- a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/executor/ExecutorBaseTest.kt +++ b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/executor/ExecutorBaseTest.kt @@ -27,8 +27,8 @@ package com.tencent.bkrepo.job.migrate.executor -import com.tencent.bkrepo.common.api.pojo.Response import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService import com.tencent.bkrepo.common.mongo.constant.ID import com.tencent.bkrepo.common.storage.config.StorageProperties @@ -48,7 +48,6 @@ import com.tencent.bkrepo.job.migrate.pojo.MigrateRepoStorageTask import com.tencent.bkrepo.job.migrate.pojo.MigrationContext import com.tencent.bkrepo.job.migrate.utils.ExecutingTaskRecorder import com.tencent.bkrepo.job.migrate.utils.MigrateTestUtils -import com.tencent.bkrepo.repository.api.RepositoryClient import org.junit.jupiter.api.TestInstance import org.mockito.ArgumentMatchers.anyString import org.mockito.kotlin.any @@ -73,7 +72,6 @@ import java.time.LocalDateTime @Import( TaskExecutionAutoConfiguration::class, MigrateRepoStorageProperties::class, - RepositoryClient::class, RepositoryCommonUtils::class, StorageProperties::class, ) @@ -102,7 +100,7 @@ open class ExecutorBaseTest { protected lateinit var fileReferenceService: FileReferenceService @MockBean - protected lateinit var repositoryClient: RepositoryClient + protected lateinit var repositoryService: RepositoryService @MockBean protected lateinit var storageCredentialService: StorageCredentialService @@ -115,11 +113,10 @@ open class ExecutorBaseTest { whenever(fileReferenceService.decrement(any(), anyOrNull())).thenReturn(true) whenever(fileReferenceService.count(anyString(), anyOrNull())).thenReturn(0) - whenever(repositoryClient.getRepoDetail(anyString(), anyString(), anyOrNull())) - .thenReturn(Response(0, "", MigrateTestUtils.buildRepo())) - whenever(repositoryClient.updateStorageCredentialsKey(anyString(), anyString(), anyString())) - .thenReturn(Response(0)) - whenever(repositoryClient.unsetOldStorageCredentialsKey(anyString(), anyString())).thenReturn(Response(0)) + whenever(repositoryService.getRepoDetail(anyString(), anyString(), anyOrNull())) + .thenReturn(MigrateTestUtils.buildRepo()) + whenever(repositoryService.updateStorageCredentialsKey(anyString(), anyString(), anyString())).then { } + whenever(repositoryService.unsetOldStorageCredentialsKey(anyString(), anyString())).then { } whenever(storageCredentialService.findByKey(anyString())) .thenReturn(FileSystemCredentials()) whenever(storageService.copy(anyString(), anyOrNull(), anyOrNull())).then { diff --git a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/strategy/FileNotFoundAutoFixStrategyTest.kt b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/strategy/FileNotFoundAutoFixStrategyTest.kt index cd81e54b13..77e4c9130c 100644 --- a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/strategy/FileNotFoundAutoFixStrategyTest.kt +++ b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/strategy/FileNotFoundAutoFixStrategyTest.kt @@ -1,10 +1,10 @@ package com.tencent.bkrepo.job.migrate.strategy import com.sun.xml.internal.messaging.saaj.util.ByteInputStream -import com.tencent.bkrepo.common.api.pojo.Response import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.artifact.stream.artifactStream import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService import com.tencent.bkrepo.common.storage.config.StorageProperties import com.tencent.bkrepo.common.storage.core.StorageService @@ -17,7 +17,6 @@ import com.tencent.bkrepo.job.migrate.utils.MigrateTestUtils import com.tencent.bkrepo.job.migrate.utils.MigrateTestUtils.createNode import com.tencent.bkrepo.job.migrate.utils.MigrateTestUtils.insertFailedNode import com.tencent.bkrepo.job.migrate.utils.MigrateTestUtils.removeNodes -import com.tencent.bkrepo.repository.api.RepositoryClient import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue @@ -67,7 +66,7 @@ class FileNotFoundAutoFixStrategyTest @Autowired constructor( private lateinit var fileReferenceService: FileReferenceService @MockBean - private lateinit var repositoryClient: RepositoryClient + private lateinit var repositoryService: RepositoryService @MockBean private lateinit var storageService: StorageService @@ -80,8 +79,8 @@ class FileNotFoundAutoFixStrategyTest @Autowired constructor( .thenReturn(FileSystemCredentials()) whenever(fileReferenceService.increment(anyString(), anyOrNull(), any())) .thenReturn(true) - whenever(repositoryClient.getRepoDetail(anyString(), anyString(), anyOrNull())) - .thenReturn(Response(0, "", MigrateTestUtils.buildRepo())) + whenever(repositoryService.getRepoDetail(anyString(), anyString(), anyOrNull())) + .thenReturn(MigrateTestUtils.buildRepo()) whenever(storageService.exist(anyString(), anyOrNull())).thenReturn(false) migrateFailedNodeDao.remove(Query()) archiveMigrateFailedNodeDao.remove(Query()) diff --git a/src/backend/lfs/biz-lfs/src/main/kotlin/com/tencent/bkrepo/lfs/service/ObjectService.kt b/src/backend/lfs/biz-lfs/src/main/kotlin/com/tencent/bkrepo/lfs/service/ObjectService.kt index ef9af25b09..121dc72d91 100644 --- a/src/backend/lfs/biz-lfs/src/main/kotlin/com/tencent/bkrepo/lfs/service/ObjectService.kt +++ b/src/backend/lfs/biz-lfs/src/main/kotlin/com/tencent/bkrepo/lfs/service/ObjectService.kt @@ -39,6 +39,7 @@ import com.tencent.bkrepo.common.api.util.Preconditions import com.tencent.bkrepo.common.api.util.readJsonString import com.tencent.bkrepo.common.api.util.toJsonString import com.tencent.bkrepo.common.artifact.api.ArtifactFile +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.exception.RepoNotFoundException import com.tencent.bkrepo.common.artifact.pojo.RepositoryCategory import com.tencent.bkrepo.common.artifact.pojo.RepositoryType @@ -48,6 +49,8 @@ import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHold import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactUploadContext import com.tencent.bkrepo.common.artifact.repository.core.ArtifactService +import com.tencent.bkrepo.common.metadata.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.security.manager.PermissionManager import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.HeaderUtils @@ -67,8 +70,6 @@ import com.tencent.bkrepo.lfs.pojo.BatchRequest import com.tencent.bkrepo.lfs.pojo.BatchResponse import com.tencent.bkrepo.lfs.pojo.LfsObject import com.tencent.bkrepo.lfs.utils.OidUtils -import com.tencent.bkrepo.repository.api.NodeClient -import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail import okhttp3.Headers.Companion.toHeaders import okhttp3.MediaType.Companion.toMediaType @@ -77,9 +78,9 @@ import org.springframework.stereotype.Service @Service class ObjectService( - private val nodeClient: NodeClient, + private val nodeService: NodeService, private val temporaryTokenClient: ServiceTemporaryTokenClient, - private val repositoryClient: RepositoryClient, + private val repositoryService: RepositoryService, private val permissionManager: PermissionManager, private val lfsProperties: LfsProperties ) : ArtifactService() { @@ -116,7 +117,7 @@ class ObjectService( getArtifactFullPath() ) } - nodeClient.getNodeDetail(projectId, repoName, getArtifactFullPath()).data ?: return + nodeService.getNodeDetail(this) ?: return } val context = ArtifactUploadContext(file) repository.upload(context) @@ -277,13 +278,13 @@ class ObjectService( private fun checkObjectExist(repo: RepositoryDetail, oid: String): Boolean { val realPath = OidUtils.convertToFullPath(oid) - val node = nodeClient.getNodeDetail(repo.projectId, repo.name, realPath).data + val node = nodeService.getNodeDetail(ArtifactInfo(repo.projectId, repo.name, realPath)) return node != null } private fun getRepoDetail(projectId: String, repoName: String): RepositoryDetail { if (!lfsProperties.enabledGitProxy) { - return repositoryClient.getRepoDetail(projectId, repoName).data + return repositoryService.getRepoDetail(projectId, repoName) ?: throw RepoNotFoundException("$projectId/$repoName") } val repo = if (!repoName.startsWith(LFS_REPO_PREFIX)) { @@ -291,8 +292,8 @@ class ObjectService( } else { repoName } - return repositoryClient.getRepoDetail(projectId, repo, RepositoryType.LFS.name).data - ?: repositoryClient.getRepoDetail(projectId, repoName.removeSuffix(".git"), RepositoryType.GIT.name).data + return repositoryService.getRepoDetail(projectId, repo, RepositoryType.LFS.name) + ?: repositoryService.getRepoDetail(projectId, repoName.removeSuffix(".git"), RepositoryType.GIT.name) ?: throw RepoNotFoundException("$projectId/$repoName") } diff --git a/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/artifact/repository/MavenLocalRepository.kt b/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/artifact/repository/MavenLocalRepository.kt index 00583ee35e..170c2a0203 100644 --- a/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/artifact/repository/MavenLocalRepository.kt +++ b/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/artifact/repository/MavenLocalRepository.kt @@ -265,11 +265,7 @@ class MavenLocalRepository( if (noOverwrite) { // -SNAPSHOT/** 路径下的metadata.xml 文件不做判断 if (!path.endsWith(MAVEN_METADATA_FILE_NAME)) { - val node = nodeClient.getNodeDetail( - context.artifactInfo.projectId, - context.artifactInfo.repoName, - path - ).data + val node = nodeService.getNodeDetail(context.artifactInfo) if (node != null && checksumType(path) == null) { val message = "The File $path already existed in the ${context.artifactInfo.getRepoIdentify()}, " + "please check your overwrite configuration." @@ -311,9 +307,8 @@ class MavenLocalRepository( return } val node = - nodeClient.getNodeDetail(projectId, repoName, artifactFilePath).data ?: throw NotFoundException( - ArtifactMessageCode.NODE_NOT_FOUND, artifactFilePath - ) + nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, artifactFilePath)) + ?: throw NotFoundException(ArtifactMessageCode.NODE_NOT_FOUND, artifactFilePath) val serverDigest = node.metadata[hashType.ext].toString() val clientDigest = MavenUtil.extractDigest(getArtifactFile().getInputStream()) if (clientDigest != serverDigest) { @@ -456,7 +451,7 @@ class MavenLocalRepository( val mimeType = fullPath.fileMimeType() if (mimeType != null) { val node = - nodeClient.getNodeDetail(projectId, repoName, fullPath).data ?: return + nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, fullPath)) ?: return val uri = "$mavenDomain/$projectId/$repoName/${node.fullPath}" val mavenArtifactResponse = MavenArtifactResponse( projectId = node.projectId, @@ -566,11 +561,7 @@ class MavenLocalRepository( "Will go to update checkSum files for $fullPath " + "in repo ${context.artifactInfo.getRepoIdentify()}" ) - val node = nodeClient.getNodeDetail( - projectId = context.projectId, - repoName = context.repoName, - fullPath = fullPath - ).data ?: return + val node = nodeService.getNodeDetail(ArtifactInfo(context.projectId, context.repoName, fullPath)) ?: return val typeArray = if (hashType == null) { HashType.values() } else { @@ -962,7 +953,7 @@ class MavenLocalRepository( else -> { val fullPath = context.artifactInfo.getArtifactFullPath() - val nodeInfo = nodeClient.getNodeDetail(context.projectId, context.repoName, fullPath).data + val nodeInfo = nodeService.getNodeDetail(context.artifactInfo) ?: throw MavenArtifactNotFoundException( MavenMessageCode.MAVEN_ARTIFACT_NOT_FOUND, fullPath, context.artifactInfo.getRepoIdentify() ) @@ -998,7 +989,7 @@ class MavenLocalRepository( fullPath = url, operator = userId ) - nodeClient.deleteNode(request) + nodeService.deleteNode(request) } else { packageClient.findVersionByName(projectId, repoName, packageName, version).data?.let { removeVersion(artifactInfo, it, userId) @@ -1020,7 +1011,7 @@ class MavenLocalRepository( private fun folderRemoveHandler(context: ArtifactRemoveContext, node: NodeDetail) { logger.info("Will try to delete folder ${node.fullPath} in repo ${context.artifactInfo.getRepoIdentify()}") val option = NodeListOption(pageNumber = 0, pageSize = 10, includeFolder = true, sort = true) - val nodes = nodeClient.listNodePage(context.projectId, context.repoName, node.fullPath, option).data!! + val nodes = nodeService.listNodePage(ArtifactInfo(context.projectId, context.repoName, node.fullPath), option) if (nodes.records.isEmpty()) { // 如果目录下没有任何节点，则删除当前目录并返回 val request = NodeDeleteRequest( @@ -1029,7 +1020,7 @@ class MavenLocalRepository( fullPath = node.fullPath, operator = context.userId ) - nodeClient.deleteNode(request) + nodeService.deleteNode(request) return } // 如果当前目录下级节点包含目录，当前目录可能是artifactId所在目录 @@ -1133,7 +1124,7 @@ class MavenLocalRepository( fullPath = artifactPath, operator = userId ) - nodeClient.deleteNode(request) + nodeService.deleteNode(request) } } @@ -1144,7 +1135,7 @@ class MavenLocalRepository( with(artifactInfo) { val fullPath = artifactInfo.getArtifactFullPath() logger.info("Will prepare to delete file $fullPath in repo ${artifactInfo.getRepoIdentify()} ") - nodeClient.getNodeDetail(projectId, repoName, fullPath).data?.let { + nodeService.getNodeDetail(artifactInfo)?.let { if (checksumType(it.fullPath) == null) { deleteArtifactCheckSums( projectId = projectId, @@ -1161,7 +1152,7 @@ class MavenLocalRepository( fullPath = fullPath, operator = userId ) - nodeClient.deleteNode(request) + nodeService.deleteNode(request) } ?: throw MavenArtifactNotFoundException( MavenMessageCode.MAVEN_ARTIFACT_NOT_FOUND, fullPath, "$projectId|$repoName" ) @@ -1180,14 +1171,14 @@ class MavenLocalRepository( ) { for (hashType in typeArray) { val fullPath = "${node.fullPath}.${hashType.ext}" - nodeClient.getNodeDetail(projectId, repoName, fullPath).data?.let { + nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, fullPath))?.let { val request = NodeDeleteRequest( projectId = projectId, repoName = repoName, fullPath = fullPath, operator = userId ) - nodeClient.deleteNode(request) + nodeService.deleteNode(request) } } } @@ -1204,13 +1195,13 @@ class MavenLocalRepository( // reference https://maven.apache.org/guides/getting-started/#what-is-a-snapshot-version // 查找 `/groupId/artifactId/maven-metadata.xml` with(artifactInfo) { - val node = nodeClient.getNodeDetail(projectId, repoName, artifactInfo.getArtifactFullPath()).data ?: return + val node = nodeService.getNodeDetail(this) ?: return storageManager.loadFullArtifactInputStream(node, storageCredentials).use { artifactInputStream -> // 更新 `/groupId/artifactId/maven-metadata.xml` val mavenMetadata = MetadataXpp3Reader().read(artifactInputStream) mavenMetadata.versioning.versions.remove(version) if (mavenMetadata.versioning.versions.size == 0) { - nodeClient.deleteNode(NodeDeleteRequest(projectId, repoName, node.fullPath, userId)) + nodeService.deleteNode(NodeDeleteRequest(projectId, repoName, node.fullPath, userId)) deleteArtifactCheckSums( projectId = projectId, repoName = repoName, @@ -1261,9 +1252,9 @@ class MavenLocalRepository( version ).data ?: return null with(context.artifactInfo) { - val jarNode = nodeClient.getNodeDetail( - projectId, repoName, trueVersion.contentPath!! - ).data ?: return null + val jarNode = nodeService.getNodeDetail( + ArtifactInfo(projectId, repoName, trueVersion.contentPath!!) + ) ?: return null val type = jarNode.nodeMetadata.find { it.key == METADATA_KEY_PACKAGING }?.value as String? val stageTag = stageClient.query(projectId, repoName, packageKey, version).data val packageVersion = packageClient.findVersionByName( @@ -1295,7 +1286,7 @@ class MavenLocalRepository( ): PackageDownloadRecord? { with(context) { val fullPath = artifactInfo.getArtifactFullPath() - val node = nodeClient.getNodeDetail(projectId, repoName, fullPath).data + val node = nodeService.getNodeDetail(artifactInfo) return if (node != null && node.metadata[METADATA_KEY_PACKAGING] != null) { val mavenGAVC = fullPath.mavenGAVC() val version = mavenGAVC.version diff --git a/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/service/MavenExtService.kt b/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/service/MavenExtService.kt index b468e5952f..1a4b979e5b 100644 --- a/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/service/MavenExtService.kt +++ b/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/service/MavenExtService.kt @@ -3,19 +3,19 @@ package com.tencent.bkrepo.maven.service import com.tencent.bkrepo.common.api.exception.ParameterInvalidException import com.tencent.bkrepo.common.api.pojo.Page import com.tencent.bkrepo.common.api.pojo.Response +import com.tencent.bkrepo.common.metadata.service.node.NodeSearchService import com.tencent.bkrepo.common.query.model.PageLimit import com.tencent.bkrepo.common.query.model.QueryModel import com.tencent.bkrepo.common.query.model.Rule import com.tencent.bkrepo.common.query.model.Sort import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.maven.pojo.response.MavenGAVCResponse -import com.tencent.bkrepo.repository.api.NodeClient import org.springframework.beans.factory.annotation.Value import org.springframework.stereotype.Service @Service class MavenExtService( - private val nodeClient: NodeClient + private val nodeSearchService: NodeSearchService ) { @Value("\${maven.domain:http://127.0.0.1:25803}") @@ -92,6 +92,6 @@ class MavenExtService( select = listOf("projectId", "repoName", "fullPath"), rule = rule ) - return nodeClient.search(queryModel) + return ResponseBuilder.success(nodeSearchService.search(queryModel)) } } diff --git a/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/service/impl/MavenServiceImpl.kt b/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/service/impl/MavenServiceImpl.kt index 2dad37edd3..519e22c123 100644 --- a/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/service/impl/MavenServiceImpl.kt +++ b/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/service/impl/MavenServiceImpl.kt @@ -38,6 +38,7 @@ import com.tencent.bkrepo.common.artifact.repository.context.ArtifactRemoveConte import com.tencent.bkrepo.common.artifact.repository.context.ArtifactUploadContext import com.tencent.bkrepo.common.artifact.repository.core.ArtifactService import com.tencent.bkrepo.common.artifact.view.ViewModelService +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.security.permission.Permission import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.maven.artifact.MavenArtifactInfo @@ -45,10 +46,10 @@ import com.tencent.bkrepo.maven.artifact.MavenDeleteArtifactInfo import com.tencent.bkrepo.maven.enum.MavenMessageCode import com.tencent.bkrepo.maven.exception.MavenBadRequestException import com.tencent.bkrepo.maven.service.MavenService -import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.pojo.list.HeaderItem import com.tencent.bkrepo.repository.pojo.list.RowItem import com.tencent.bkrepo.repository.pojo.node.NodeDetail +import com.tencent.bkrepo.repository.pojo.node.NodeListOption import com.tencent.bkrepo.repository.pojo.node.NodeListViewItem import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -58,7 +59,7 @@ import java.util.regex.PatternSyntaxException @Service class MavenServiceImpl( - private val nodeClient: NodeClient, + private val nodeService: NodeService, private val viewModelService: ViewModelService ) : ArtifactService(), MavenService { @@ -88,7 +89,7 @@ class MavenServiceImpl( override fun dependency(mavenArtifactInfo: MavenArtifactInfo) { // 为了兼容jfrog，当查询到目录时，会展示当前目录下所有子项，而不是直接报错 with(mavenArtifactInfo) { - val node = nodeClient.getNodeDetail(projectId, repoName, getArtifactFullPath()).data + val node = nodeService.getNodeDetail(mavenArtifactInfo) val download = HttpContextHolder.getRequest().getParameter(PARAM_DOWNLOAD)?.toBoolean() ?: false if (node != null) { if (node.folder && !download) { @@ -114,13 +115,9 @@ class MavenServiceImpl( with(artifactInfo) { viewModelService.trailingSlash(applicationName) // listNodePage 接口没办法满足当前情况 - val nodeList = nodeClient.listNode( - projectId = projectId, - repoName = repoName, - path = getArtifactFullPath(), - includeFolder = true, - deep = false - ).data + val nodeList = nodeService.listNode( + this, NodeListOption(includeFolder = true, deep = false) + ) val currentPath = viewModelService.computeCurrentPath(node) val headerList = listOf( HeaderItem("Name"), @@ -129,10 +126,10 @@ class MavenServiceImpl( HeaderItem("Size"), HeaderItem("Sha256") ) - val itemList = nodeList?.map { NodeListViewItem.from(it) }?.sorted() - val rowList = itemList?.map { + val itemList = nodeList.map { NodeListViewItem.from(it) }.sorted() + val rowList = itemList.map { RowItem(listOf(it.name, it.createdBy, it.lastModified, it.size, it.sha256)) - } ?: listOf() + } viewModelService.render(currentPath, headerList, rowList) } } diff --git a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/artifact/repository/MediaLocalRepository.kt b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/artifact/repository/MediaLocalRepository.kt index 7addbad8fb..857793b1c2 100644 --- a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/artifact/repository/MediaLocalRepository.kt +++ b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/artifact/repository/MediaLocalRepository.kt @@ -10,7 +10,7 @@ class MediaLocalRepository : LocalRepository() { override fun remove(context: ArtifactRemoveContext) { with(context.artifactInfo) { val nodeDeleteRequest = NodeDeleteRequest(projectId, repoName, getArtifactFullPath(), context.userId) - nodeClient.deleteNode(nodeDeleteRequest) + nodeService.deleteNode(nodeDeleteRequest) } } } diff --git a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/service/StreamService.kt b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/service/StreamService.kt index 5fe23ae735..1833726fe9 100644 --- a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/service/StreamService.kt +++ b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/service/StreamService.kt @@ -4,25 +4,26 @@ import com.tencent.bkrepo.auth.pojo.token.TemporaryTokenCreateRequest import com.tencent.bkrepo.auth.pojo.token.TokenType import com.tencent.bkrepo.common.artifact.manager.StorageManager import com.tencent.bkrepo.common.artifact.pojo.RepositoryCategory +import com.tencent.bkrepo.common.artifact.pojo.RepositoryId import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext import com.tencent.bkrepo.common.artifact.repository.core.ArtifactService import com.tencent.bkrepo.common.artifact.resolve.file.ArtifactFileFactory +import com.tencent.bkrepo.common.metadata.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.storage.config.StorageProperties import com.tencent.bkrepo.media.STREAM_PATH import com.tencent.bkrepo.media.artifact.MediaArtifactInfo import com.tencent.bkrepo.media.config.MediaProperties -import com.tencent.bkrepo.media.stream.MediaArtifactFileConsumer import com.tencent.bkrepo.media.stream.ArtifactFileRecordingListener import com.tencent.bkrepo.media.stream.ClientStream +import com.tencent.bkrepo.media.stream.MediaArtifactFileConsumer import com.tencent.bkrepo.media.stream.MediaType import com.tencent.bkrepo.media.stream.RemuxRecordingListener import com.tencent.bkrepo.media.stream.StreamManger import com.tencent.bkrepo.media.stream.StreamMode import com.tencent.bkrepo.media.stream.TranscodeConfig -import com.tencent.bkrepo.repository.api.NodeClient -import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import com.tencent.bkrepo.repository.pojo.repo.RepoCreateRequest import org.slf4j.LoggerFactory @@ -32,8 +33,8 @@ import org.springframework.stereotype.Service @Service class StreamService( private val mediaProperties: MediaProperties, - private val repositoryClient: RepositoryClient, - private val nodeClient: NodeClient, + private val repositoryService: RepositoryService, + private val nodeService: NodeService, private val tokenService: TokenService, private val scheduler: ThreadPoolTaskScheduler, private val storageManager: StorageManager, @@ -51,7 +52,7 @@ class StreamService( * 2. 创建streams目录 * 3. 创建streams目录写权限url =》 推流地址/{projectId}/{pushId}/streams * */ - repositoryClient.getRepoDetail(projectId, repoName).data ?: let { + repositoryService.getRepoDetail(projectId, repoName) ?: let { val createRepoRequest = RepoCreateRequest( projectId = projectId, name = repoName, @@ -60,7 +61,7 @@ class StreamService( public = false, display = display ) - repositoryClient.createRepo(createRepoRequest) + repositoryService.createRepo(createRepoRequest) val nodeCreateRequest = NodeCreateRequest( projectId = projectId, repoName = repoName, @@ -68,7 +69,7 @@ class StreamService( folder = true, overwrite = false, ) - nodeClient.createNode(nodeCreateRequest) + nodeService.createNode(nodeCreateRequest) logger.info("Create media repository for [$repoName].") } val temporaryTokenRequest = TemporaryTokenCreateRequest( @@ -100,7 +101,7 @@ class StreamService( remux: Boolean = false, saveType: MediaType = MediaType.RAW, ): ClientStream { - val repoId = ArtifactContextHolder.RepositoryId(projectId, repoName) + val repoId = RepositoryId(projectId, repoName) val repo = ArtifactContextHolder.getRepoDetail(repoId) val credentials = repo.storageCredentials ?: storageProperties.defaultStorageCredentials() val fileConsumer = MediaArtifactFileConsumer( diff --git a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/ArtifactFileRecordingListener.kt b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/ArtifactFileRecordingListener.kt index 3ec2fe39d9..5946c3e2f2 100644 --- a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/ArtifactFileRecordingListener.kt +++ b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/ArtifactFileRecordingListener.kt @@ -2,6 +2,7 @@ package com.tencent.bkrepo.media.stream import com.tencent.bkrepo.common.artifact.resolve.file.chunk.ChunkedArtifactFile import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import java.util.Locale /** * 保存流为制品构件 @@ -16,7 +17,7 @@ class ArtifactFileRecordingListener( private lateinit var name: String override fun init(name: String) { - this.name = "$name.${type.name.toLowerCase()}" + this.name = "$name.${type.name.lowercase(Locale.getDefault())}" } override fun handler(packet: StreamPacket) { diff --git a/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/artifact/repository/NpmLocalRepository.kt b/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/artifact/repository/NpmLocalRepository.kt index 1485ab9b7d..cd2d0de219 100644 --- a/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/artifact/repository/NpmLocalRepository.kt +++ b/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/artifact/repository/NpmLocalRepository.kt @@ -32,6 +32,7 @@ import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.util.JsonUtils import com.tencent.bkrepo.common.api.util.UrlFormatter import com.tencent.bkrepo.common.artifact.api.ArtifactFile +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.hash.sha1 import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext @@ -134,7 +135,7 @@ class NpmLocalRepository( val projectId = repositoryDetail.projectId val repoName = repositoryDetail.name val fullPath = context.getStringAttribute(NPM_FILE_FULL_PATH)!! - val node = nodeClient.getNodeDetail(projectId, repoName, fullPath).data + val node = nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, fullPath)) if (node == null || node.folder) return null return storageManager.loadFullArtifactInputStream(node, context.storageCredentials) .also { @@ -170,10 +171,7 @@ class NpmLocalRepository( .metadata("version", searchRequest.text, OperationType.MATCH) .metadata("keywords", searchRequest.text, OperationType.MATCH) .build() - val data = nodeClient.queryWithoutCount(queryModel).data ?: run { - logger.warn("failed to find npm package in repo [${context.projectId}/${context.repoName}]") - return emptyList() - } + val data = nodeSearchService.searchWithoutCount(queryModel) return transferRecords(data.records) } @@ -207,7 +205,7 @@ class NpmLocalRepository( val fullPath = context.getAttribute>(NPM_FILE_FULL_PATH) val userId = context.userId fullPath?.forEach { - nodeClient.deleteNode(NodeDeleteRequest(projectId, repoName, it.toString(), userId)) + nodeService.deleteNode(NodeDeleteRequest(projectId, repoName, it.toString(), userId)) logger.info("delete artifact $it success in repo [${context.artifactInfo.getRepoIdentify()}].") } } @@ -364,7 +362,7 @@ class NpmLocalRepository( } private fun ArtifactMigrateContext.npmPackageMetaData(fullPath: String): NpmPackageMetaData? { - val node = nodeClient.getNodeDetail(projectId, repoName, fullPath).data + val node = nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, fullPath)) return node?.let { val inputStream = storageManager.loadFullArtifactInputStream(it, storageCredentials) JsonUtils.objectMapper.readValue(inputStream, NpmPackageMetaData::class.java) @@ -513,7 +511,7 @@ class NpmLocalRepository( val artifactFile = inputStream.use { ArtifactFileFactory.build(it) } val fullPath = NpmUtils.getVersionPackageMetadataPath(name, version) context.putAttribute(NPM_FILE_FULL_PATH, fullPath) - if (nodeClient.checkExist(projectId, repoName, fullPath).data!!) { + if (nodeService.checkExist(ArtifactInfo(projectId, repoName, fullPath))) { logger.info( "package [$name] with version metadata [$name-$version.json] " + "is already exists in repository [$projectId/$repoName], skip migration." @@ -540,7 +538,7 @@ class NpmLocalRepository( val fullPath = NpmUtils.getTgzPath(name, version) context.putAttribute(NPM_FILE_FULL_PATH, fullPath) // hit cache continue - if (nodeClient.checkExist(projectId, repoName, fullPath).data!!) { + if (nodeService.checkExist(ArtifactInfo(projectId, repoName, fullPath))) { logger.info( "package [$name] with tgz file [$fullPath] is " + "already exists in repository [$projectId/$repoName], skip migration." @@ -595,9 +593,9 @@ class NpmLocalRepository( private fun deleteVersionMetadata(context: ArtifactMigrateContext, name: String, version: String) { val fullPath = NpmUtils.getVersionPackageMetadataPath(name, version) with(context) { - if (nodeClient.checkExist(projectId, repoName, fullPath).data!!) { + if (nodeService.checkExist(ArtifactInfo(projectId, repoName, fullPath))) { val nodeDeleteRequest = NodeDeleteRequest(projectId, repoName, fullPath, userId) - nodeClient.deleteNode(nodeDeleteRequest) + nodeService.deleteNode(nodeDeleteRequest) logger.info( "migrate package [$name] with version [$version] failed, " + "delete package version metadata [$fullPath] success." diff --git a/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/artifact/repository/NpmRemoteRepository.kt b/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/artifact/repository/NpmRemoteRepository.kt index c3291ab668..cb5f5a20fc 100644 --- a/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/artifact/repository/NpmRemoteRepository.kt +++ b/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/artifact/repository/NpmRemoteRepository.kt @@ -34,6 +34,7 @@ package com.tencent.bkrepo.npm.artifact.repository import com.tencent.bkrepo.common.api.util.JsonUtils import com.tencent.bkrepo.common.api.util.UrlFormatter import com.tencent.bkrepo.common.artifact.api.ArtifactFile +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactMigrateContext @@ -77,7 +78,7 @@ class NpmRemoteRepository( with(context) { val packageInfo = NpmUtils.parseNameAndVersionFromFullPath(artifactInfo.getArtifactFullPath()) val versionMetadataFullPath = NpmUtils.getVersionPackageMetadataPath(packageInfo.first, packageInfo.second) - if (nodeClient.checkExist(projectId, repoName, versionMetadataFullPath).data!!) { + if (nodeService.checkExist(ArtifactInfo(projectId, repoName, versionMetadataFullPath))) { logger.info( "version metadata [$versionMetadataFullPath] is already exits " + "in repo [$projectId/$repoName]" @@ -133,7 +134,7 @@ class NpmRemoteRepository( val artifactFile = createTempFile(body) val sha256 = artifactFile.getFileSha256() with(context) { - nodeClient.getNodeDetail(projectId, repoName, fullPath).data?.let { + nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, fullPath))?.let { if (it.sha256.equals(sha256)) { logger.info("artifact [$fullPath] is hit the cache.") return artifactFile.getInputStream() diff --git a/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/artifact/repository/NpmVirtualRepository.kt b/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/artifact/repository/NpmVirtualRepository.kt index 6671a87692..7a8b130c0e 100644 --- a/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/artifact/repository/NpmVirtualRepository.kt +++ b/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/artifact/repository/NpmVirtualRepository.kt @@ -57,7 +57,7 @@ class NpmVirtualRepository : VirtualRepository() { } traversedList.add(repoIdentify) try { - val subRepoInfo = repositoryClient.getRepoDetail(repoIdentify.projectId, repoIdentify.name).data!! + val subRepoInfo = repositoryService.getRepoDetail(repoIdentify.projectId, repoIdentify.name)!! val repository = ArtifactContextHolder.getRepository(subRepoInfo.category) as AbstractArtifactRepository val subContext = context.copy(repositoryDetail = subRepoInfo) as ArtifactSearchContext repository.search(subContext).let { map -> diff --git a/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/service/impl/AbstractNpmService.kt b/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/service/impl/AbstractNpmService.kt index 2b4156fd68..57d1cb9a78 100644 --- a/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/service/impl/AbstractNpmService.kt +++ b/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/service/impl/AbstractNpmService.kt @@ -34,6 +34,9 @@ package com.tencent.bkrepo.npm.service.impl import com.tencent.bkrepo.common.api.util.JsonUtils import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder import com.tencent.bkrepo.common.artifact.repository.context.ArtifactQueryContext +import com.tencent.bkrepo.common.metadata.service.node.NodeSearchService +import com.tencent.bkrepo.common.metadata.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.service.util.HeaderUtils import com.tencent.bkrepo.npm.artifact.NpmArtifactInfo import com.tencent.bkrepo.npm.constants.NPM_FILE_FULL_PATH @@ -44,9 +47,7 @@ import com.tencent.bkrepo.npm.model.metadata.NpmPackageMetaData import com.tencent.bkrepo.npm.model.metadata.NpmVersionMetadata import com.tencent.bkrepo.npm.properties.NpmProperties import com.tencent.bkrepo.npm.utils.NpmUtils -import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.PackageClient -import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -58,10 +59,13 @@ import java.io.InputStream open class AbstractNpmService { @Autowired - lateinit var nodeClient: NodeClient + lateinit var nodeService: NodeService @Autowired - lateinit var repositoryClient: RepositoryClient + lateinit var nodeSearchService: NodeSearchService + + @Autowired + lateinit var repositoryService: RepositoryService @Autowired lateinit var packageClient: PackageClient @@ -73,7 +77,7 @@ open class AbstractNpmService { * 查询仓库是否存在 */ fun checkRepositoryExist(projectId: String, repoName: String): RepositoryDetail { - return repositoryClient.getRepoDetail(projectId, repoName, "NPM").data ?: run { + return repositoryService.getRepoDetail(projectId, repoName, "NPM") ?: run { logger.error("check repository [$repoName] in projectId [$projectId] failed!") throw NpmRepoNotFoundException("repository [$repoName] in projectId [$projectId] not existed.") } diff --git a/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/service/impl/NpmFixToolServiceImpl.kt b/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/service/impl/NpmFixToolServiceImpl.kt index e452ea4f37..1752090dbc 100644 --- a/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/service/impl/NpmFixToolServiceImpl.kt +++ b/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/service/impl/NpmFixToolServiceImpl.kt @@ -223,10 +223,7 @@ class NpmFixToolServiceImpl( val packageManagerList = mutableListOf() // 查找所有仓库 logger.info("starting add package manager function to historical data.") - val repositoryList = repositoryClient.pageByType(0, 1000, "NPM").data?.records ?: run { - logger.warn("no npm repository found, return.") - return emptyList() - } + val repositoryList = repositoryService.listRepoPageByType("NPM", 0, 1000).records val npmLocalRepositoryList = repositoryList.filter { it.category == RepositoryCategory.LOCAL }.toList() logger.info( "find [${npmLocalRepositoryList.size}] NPM local" + @@ -333,7 +330,7 @@ class NpmFixToolServiceImpl( select = mutableListOf(), rule = Rule.NestedRule(ruleList, Rule.NestedRule.RelationType.AND) ) - val queryResult = nodeClient.queryWithoutCount(queryModel).data!! + val queryResult = nodeSearchService.searchWithoutCount(queryModel) return queryResult.records.associateBy( { resolverVersion(packageName, it["fullPath"] as String) }, { resolveNode(it) } @@ -363,7 +360,7 @@ class NpmFixToolServiceImpl( select = mutableListOf(), rule = Rule.NestedRule(ruleList, Rule.NestedRule.RelationType.AND) ) - return nodeClient.queryWithoutCount(queryModel).data!! + return nodeSearchService.searchWithoutCount(queryModel) } private fun resolveNode(record: Map): NodeInfo { diff --git a/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/service/impl/NpmWebServiceImpl.kt b/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/service/impl/NpmWebServiceImpl.kt index 078e83f815..ba50f1a530 100644 --- a/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/service/impl/NpmWebServiceImpl.kt +++ b/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/service/impl/NpmWebServiceImpl.kt @@ -33,6 +33,7 @@ package com.tencent.bkrepo.npm.service.impl import com.tencent.bkrepo.common.api.util.JsonUtils import com.tencent.bkrepo.common.api.util.UrlFormatter +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder import com.tencent.bkrepo.common.artifact.repository.context.ArtifactUploadContext import com.tencent.bkrepo.common.artifact.resolve.file.ArtifactFileFactory @@ -84,7 +85,7 @@ class NpmWebServiceImpl : NpmWebService, AbstractNpmService() { val fullPath = NpmUtils.getTgzPath(name, version, pathWithDash) with(artifactInfo) { checkRepositoryExist(projectId, repoName) - val nodeDetail = nodeClient.getNodeDetail(projectId, repoName, fullPath).data ?: run { + val nodeDetail = nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, fullPath)) ?: run { logger.warn("node [$fullPath] don't found.") throw NpmArtifactNotFoundException("node [$fullPath] don't found.") } diff --git a/src/backend/nuget/biz-nuget/src/main/kotlin/com/tencent/bkrepo/nuget/artifact/repository/NugetLocalRepository.kt b/src/backend/nuget/biz-nuget/src/main/kotlin/com/tencent/bkrepo/nuget/artifact/repository/NugetLocalRepository.kt index 1e3c2cbc75..8cbf1ac035 100644 --- a/src/backend/nuget/biz-nuget/src/main/kotlin/com/tencent/bkrepo/nuget/artifact/repository/NugetLocalRepository.kt +++ b/src/backend/nuget/biz-nuget/src/main/kotlin/com/tencent/bkrepo/nuget/artifact/repository/NugetLocalRepository.kt @@ -35,6 +35,7 @@ import com.fasterxml.jackson.core.JsonProcessingException import com.tencent.bkrepo.common.api.constant.HttpStatus import com.tencent.bkrepo.common.api.constant.MediaTypes import com.tencent.bkrepo.common.api.exception.ErrorCodeException +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.exception.PackageNotFoundException import com.tencent.bkrepo.common.artifact.exception.VersionNotFoundException import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext @@ -256,7 +257,7 @@ class NugetLocalRepository( val nugetPath = version.contentPath.orEmpty() if (nugetPath.isNotBlank()) { val request = NodeDeleteRequest(projectId, repoName, nugetPath, userId) - nodeClient.deleteNode(request) + nodeService.deleteNode(request) } } } @@ -265,13 +266,13 @@ class NugetLocalRepository( with(context) { val nugetArtifactInfo = artifactInfo as NugetDownloadArtifactInfo val nuspecFullPath = NugetUtils.getNuspecFullPath(nugetArtifactInfo.packageName, nugetArtifactInfo.version) - val nuspecNode = nodeClient.getNodeDetail(projectId, repoName, nuspecFullPath).data + val nuspecNode = nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, nuspecFullPath)) val inputStream = if (nuspecNode != null) { storageManager.loadArtifactInputStream(nuspecNode, storageCredentials) } else { val nupkgFullPath = NugetUtils.getNupkgFullPath(nugetArtifactInfo.packageName, nugetArtifactInfo.version) - val nupkgNode = nodeClient.getNodeDetail(projectId, repoName, nupkgFullPath).data + val nupkgNode = nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, nupkgFullPath)) storageManager.loadArtifactInputStream(nupkgNode, storageCredentials)?.use { it.resolverNuspecMetadata().byteInputStream() } diff --git a/src/backend/nuget/biz-nuget/src/main/kotlin/com/tencent/bkrepo/nuget/artifact/repository/NugetVirtualRepository.kt b/src/backend/nuget/biz-nuget/src/main/kotlin/com/tencent/bkrepo/nuget/artifact/repository/NugetVirtualRepository.kt index ec762b66ec..449a515b81 100644 --- a/src/backend/nuget/biz-nuget/src/main/kotlin/com/tencent/bkrepo/nuget/artifact/repository/NugetVirtualRepository.kt +++ b/src/backend/nuget/biz-nuget/src/main/kotlin/com/tencent/bkrepo/nuget/artifact/repository/NugetVirtualRepository.kt @@ -141,7 +141,7 @@ class NugetVirtualRepository( val repoList = virtualConfiguration.repositoryList // 分隔出本地仓库和远程仓库 val repoCategoryMap = repoList.map { - repositoryClient.getRepoDetail(it.projectId, it.name).data!! + repositoryService.getRepoDetail(it.projectId, it.name)!! }.groupBy { it.category } // 分别查询出本地仓库和远程仓库下面的数组，然后在进行整合 val allRemoteReposPageItems = repoCategoryMap[RepositoryCategory.REMOTE].orEmpty().stream().map { diff --git a/src/backend/nuget/biz-nuget/src/main/kotlin/com/tencent/bkrepo/nuget/service/impl/NugetWebServiceImpl.kt b/src/backend/nuget/biz-nuget/src/main/kotlin/com/tencent/bkrepo/nuget/service/impl/NugetWebServiceImpl.kt index 6a32b2c5c5..01d8a66a6d 100644 --- a/src/backend/nuget/biz-nuget/src/main/kotlin/com/tencent/bkrepo/nuget/service/impl/NugetWebServiceImpl.kt +++ b/src/backend/nuget/biz-nuget/src/main/kotlin/com/tencent/bkrepo/nuget/service/impl/NugetWebServiceImpl.kt @@ -2,10 +2,12 @@ package com.tencent.bkrepo.nuget.service.impl import com.tencent.bkrepo.common.api.exception.NotFoundException import com.tencent.bkrepo.common.api.util.UrlFormatter +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.manager.PackageManager import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.artifact.repository.context.ArtifactRemoveContext import com.tencent.bkrepo.common.artifact.repository.core.ArtifactService +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.nuget.artifact.NugetArtifactInfo import com.tencent.bkrepo.nuget.constant.NugetProperties import com.tencent.bkrepo.nuget.pojo.artifact.NugetDeleteArtifactInfo @@ -13,7 +15,6 @@ import com.tencent.bkrepo.nuget.pojo.domain.NugetDomainInfo import com.tencent.bkrepo.nuget.pojo.user.BasicInfo import com.tencent.bkrepo.nuget.pojo.user.PackageVersionInfo import com.tencent.bkrepo.nuget.service.NugetWebService -import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.packages.PackageVersion import org.slf4j.LoggerFactory @@ -23,7 +24,7 @@ import org.springframework.stereotype.Service class NugetWebServiceImpl( private val nugetProperties: NugetProperties, private val packageManager: PackageManager, - private val nodeClient: NodeClient + private val nodeService: NodeService ) : NugetWebService, ArtifactService() { override fun deletePackage(userId: String, artifactInfo: NugetDeleteArtifactInfo) { @@ -42,7 +43,7 @@ class NugetWebServiceImpl( return with(artifactInfo) { val packageVersion = packageManager.findVersionByName(projectId, repoName, packageKey, version) val fullPath = packageVersion.contentPath.orEmpty() - val nodeDetail = nodeClient.getNodeDetail(projectId, repoName, fullPath).data ?: run { + val nodeDetail = nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, fullPath)) ?: run { logger.warn("node [$fullPath] don't found.") throw NotFoundException(ArtifactMessageCode.NODE_NOT_FOUND, fullPath) } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/auth/OciLoginAuthHandler.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/auth/OciLoginAuthHandler.kt index 391f31f88e..7ef36b0a33 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/auth/OciLoginAuthHandler.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/auth/OciLoginAuthHandler.kt @@ -32,6 +32,7 @@ package com.tencent.bkrepo.oci.artifact.auth import com.tencent.bkrepo.common.api.constant.ANONYMOUS_USER +import com.tencent.bkrepo.common.artifact.pojo.RepositoryId import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder import com.tencent.bkrepo.common.security.exception.AuthenticationException import com.tencent.bkrepo.common.security.http.basic.BasicAuthHandler @@ -94,7 +95,7 @@ class OciLoginAuthHandler( val scope = params?.get("scope")?.first ?: throw authenticationException val scopeValues = scope.split(":") val values = scopeValues[1].split("/") - val repositoryId = ArtifactContextHolder.RepositoryId(values[0], values[1]) + val repositoryId = RepositoryId(values[0], values[1]) val repo = ArtifactContextHolder.getRepoDetail(repositoryId) // 针对仓库类型的为public的，允许下载。 if (repo.public) { diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt index cb8212f64c..0c9f526a42 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt @@ -33,6 +33,7 @@ package com.tencent.bkrepo.oci.artifact.repository import com.tencent.bkrepo.common.api.constant.HttpStatus import com.tencent.bkrepo.common.api.constant.MediaTypes +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactQueryContext @@ -80,6 +81,7 @@ import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest import com.tencent.bkrepo.repository.pojo.packages.PackageVersion import org.slf4j.LoggerFactory import org.springframework.stereotype.Component +import java.util.Locale @Component class OciRegistryLocalRepository( @@ -102,7 +104,7 @@ class OciRegistryLocalRepository( val projectId = repositoryDetail.projectId val repoName = repositoryDetail.name val fullPath = context.artifactInfo.getArtifactFullPath() - val isExist = nodeClient.checkExist(projectId, repoName, fullPath).data!! + val isExist = nodeService.checkExist(ArtifactInfo(projectId, repoName, fullPath)) logger.info( "The file $fullPath that will be uploaded to server is exist: $isExist " + "in repo ${artifactInfo.getRepoIdentify()}, and the flag of force overwrite is $isForce" @@ -252,9 +254,7 @@ class OciRegistryLocalRepository( } catch (e: StorageErrorException) { // 计算sha256和转存文件导致时间较长，会出现请求超时，然后发起重试，导致并发操作该临时文件，文件可能已经被删除 if (storageService.exist(sha256, context.repositoryDetail.storageCredentials)) { - val nodeDetail = nodeClient.getNodeDetail( - artifactInfo.projectId, artifactInfo.repoName, artifactInfo.getArtifactFullPath() - ).data + val nodeDetail = nodeService.getNodeDetail(artifactInfo) if (nodeDetail == null || nodeDetail.sha256 != sha256) { throw e } else { @@ -413,13 +413,13 @@ class OciRegistryLocalRepository( ?: throw OciFileNotFoundException( OciMessageCode.OCI_FILE_NOT_FOUND, getArtifactFullPath(), getRepoIdentify() ) - nodeClient.getNodeDetail(projectId, repoName, fullPath).data + nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, fullPath)) ?: throw OciFileNotFoundException( OciMessageCode.OCI_FILE_NOT_FOUND, getArtifactFullPath(), getRepoIdentify() ) logger.info("Ready to delete $fullPath in repo ${getRepoIdentify()}") val request = NodeDeleteRequest(projectId, repoName, fullPath, context.userId) - nodeClient.deleteNode(request) + nodeService.deleteNode(request) OciResponseUtils.buildDeleteResponse(context.response) } } @@ -495,7 +495,8 @@ class OciRegistryLocalRepository( with(context.artifactInfo as OciTagArtifactInfo) { val n = context.getAttribute(N) val last = context.getAttribute(LAST_TAG) - val packageKey = PackageKeys.ofName(context.repositoryDetail.type.name.toLowerCase(), packageName) + val packageKey = + PackageKeys.ofName(context.repositoryDetail.type.name.lowercase(Locale.getDefault()), packageName) val versionList = packageClient.listAllVersion( projectId, repoName, diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt index 703fbc565d..b0fdf7d6ad 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt @@ -46,6 +46,7 @@ import com.tencent.bkrepo.common.api.util.JsonUtils import com.tencent.bkrepo.common.api.util.toJsonString import com.tencent.bkrepo.common.api.util.UrlFormatter import com.tencent.bkrepo.common.artifact.api.ArtifactFile +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException import com.tencent.bkrepo.common.artifact.pojo.configuration.remote.RemoteConfiguration import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContext @@ -455,7 +456,7 @@ class OciRegistryRemoteRepository( override fun findCacheNodeDetail(context: ArtifactDownloadContext): NodeDetail? { with(context) { val fullPath = ociOperationService.getNodeFullPath(context.artifactInfo as OciArtifactInfo) ?: return null - return nodeClient.getNodeDetail(projectId, repoName, fullPath).data + return nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, fullPath)) } } @@ -518,7 +519,8 @@ class OciRegistryRemoteRepository( val fullPath = ociOperationService.getNodeFullPath(ociArtifactInfo) // 针对manifest文件获取会通过tag或manifest获取，避免重复创建 fullPath?.let { - val node = nodeClient.getNodeDetail(ociArtifactInfo.projectId, ociArtifactInfo.repoName, fullPath).data + val node = + nodeService.getNodeDetail(ArtifactInfo(ociArtifactInfo.projectId, ociArtifactInfo.repoName, fullPath)) if (node != null && artifactFile.getFileSha256() == node.sha256) return node } val url = context.getStringAttribute(PROXY_URL) @@ -531,11 +533,7 @@ class OciRegistryRemoteRepository( // 针对manifest文件需要更新metadata if (context.artifactInfo is OciManifestArtifactInfo) { updateManifestAndBlob(context, nodeDetail!!) - nodeDetail = nodeClient.getNodeDetail( - projectId = context.artifactInfo.projectId, - repoName = context.artifactInfo.repoName, - fullPath = context.artifactInfo.getArtifactFullPath() - ).data + nodeDetail = nodeService.getNodeDetail(context.artifactInfo) } return nodeDetail } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/base/EventExecutor.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/base/EventExecutor.kt index c8bcdf504e..4ec536b844 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/base/EventExecutor.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/base/EventExecutor.kt @@ -32,12 +32,12 @@ import com.tencent.bkrepo.common.artifact.event.base.EventType import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException import com.tencent.bkrepo.common.artifact.exception.RepoNotFoundException import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactChannel +import com.tencent.bkrepo.common.metadata.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.oci.listener.pool.EventHandlerThreadPoolExecutor import com.tencent.bkrepo.oci.pojo.artifact.OciManifestArtifactInfo import com.tencent.bkrepo.oci.pojo.digest.OciDigest import com.tencent.bkrepo.oci.service.OciOperationService -import com.tencent.bkrepo.repository.api.NodeClient -import com.tencent.bkrepo.repository.api.RepositoryClient import org.slf4j.Logger import org.slf4j.LoggerFactory import org.springframework.stereotype.Component @@ -46,8 +46,8 @@ import java.util.concurrent.ThreadPoolExecutor @Component class EventExecutor( - open val nodeClient: NodeClient, - open val repositoryClient: RepositoryClient, + open val nodeService: NodeService, + open val repositoryService: RepositoryService, open val ociOperationService: OciOperationService ) { private val threadPoolExecutor: ThreadPoolExecutor = EventHandlerThreadPoolExecutor.instance @@ -87,12 +87,12 @@ class EventExecutor( val ociArtifactInfo = OciManifestArtifactInfo( projectId, repoName, packageName, "", version, false ) - val nodeInfo = nodeClient.getNodeDetail(projectId, repoName, ociArtifactInfo.getArtifactFullPath()).data + val nodeInfo = nodeService.getNodeDetail(ociArtifactInfo) ?: throw NodeNotFoundException( "${ociArtifactInfo.getArtifactFullPath()} not found in repo in $projectId|$repoName" ) val ociDigest = OciDigest.fromSha256(sha256) - val repositoryDetail = repositoryClient.getRepoDetail(projectId, repoName).data + val repositoryDetail = repositoryService.getRepoDetail(projectId, repoName) ?: throw RepoNotFoundException("$projectId|$repoName") ociOperationService.updateOciInfo( ociArtifactInfo = ociArtifactInfo, diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt index 3c61e06e8e..1a0ca90e21 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt @@ -41,6 +41,8 @@ import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHold import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactRemoveContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactUploadContext +import com.tencent.bkrepo.common.metadata.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.security.manager.PermissionManager import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.common.storage.core.StorageService @@ -57,8 +59,6 @@ import com.tencent.bkrepo.oci.service.OciOperationService import com.tencent.bkrepo.oci.util.ObjectBuildUtils import com.tencent.bkrepo.oci.util.OciLocationUtils import com.tencent.bkrepo.oci.util.OciResponseUtils -import com.tencent.bkrepo.repository.api.NodeClient -import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import org.slf4j.LoggerFactory import org.springframework.stereotype.Service @@ -66,8 +66,8 @@ import org.springframework.stereotype.Service @Service class OciBlobServiceImpl( private val storage: StorageService, - private val repoClient: RepositoryClient, - private val nodeClient: NodeClient, + private val repositoryService: RepositoryService, + private val nodeService: NodeService, private val ociOperationService: OciOperationService, private val permissionManager: PermissionManager, ) : OciBlobService { @@ -144,9 +144,9 @@ class OciBlobServiceImpl( return } // 当mount仓库和当前仓库不在一个存储实例时，直接上传 - val mountRepo = repoClient.getRepoDetail(mountProjectId, mountRepoName).data + val mountRepo = repositoryService.getRepoDetail(mountProjectId, mountRepoName) ?: throw RepoNotFoundException("$mountProjectId|$mountRepoName") - val currentRepo = repoClient.getRepoDetail(projectId, repoName).data + val currentRepo = repositoryService.getRepoDetail(projectId, repoName) ?: throw RepoNotFoundException("$projectId|$repoName") if (mountRepo.storageCredentials?.key != currentRepo.storageCredentials?.key) { buildSessionIdLocationForUpload(this, domain) @@ -166,7 +166,7 @@ class OciBlobServiceImpl( md5 = nodeProperty.md5!!, metadata = metadata ) - nodeClient.createNode(nodeCreateRequest) + nodeService.createNode(nodeCreateRequest) val blobLocation = OciLocationUtils.blobLocation(ociDigest, this) val responseProperty = ResponseProperty( location = blobLocation, @@ -208,7 +208,7 @@ class OciBlobServiceImpl( fun startAppend(artifactInfo: OciBlobArtifactInfo): String { with(artifactInfo) { // check repository - val result = repoClient.getRepoDetail(projectId, repoName, REPO_TYPE).data ?: run { + val result = repositoryService.getRepoDetail(projectId, repoName, REPO_TYPE) ?: run { ArtifactContextHolder.queryRepoDetailFormExtraRepoType(projectId, repoName) } logger.debug("Start to append file in ${getRepoIdentify()}") diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index 97cff576d7..50cf06a4f1 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -33,6 +33,7 @@ import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.api.util.StreamUtils.readText import com.tencent.bkrepo.common.api.util.UrlFormatter import com.tencent.bkrepo.common.artifact.api.ArtifactFile +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.constant.SOURCE_TYPE import com.tencent.bkrepo.common.artifact.exception.RepoNotFoundException import com.tencent.bkrepo.common.artifact.exception.VersionNotFoundException @@ -46,6 +47,9 @@ import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactChannel import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream import com.tencent.bkrepo.common.artifact.util.PackageKeys import com.tencent.bkrepo.common.metadata.service.metadata.MetadataService +import com.tencent.bkrepo.common.metadata.service.node.NodeSearchService +import com.tencent.bkrepo.common.metadata.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService import com.tencent.bkrepo.common.query.enums.OperationType import com.tencent.bkrepo.common.security.util.SecurityUtils @@ -95,13 +99,12 @@ import com.tencent.bkrepo.oci.util.OciLocationUtils import com.tencent.bkrepo.oci.util.OciLocationUtils.buildBlobsFolderPath import com.tencent.bkrepo.oci.util.OciResponseUtils import com.tencent.bkrepo.oci.util.OciUtils -import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.PackageClient import com.tencent.bkrepo.repository.api.PackageMetadataClient -import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.constant.SYSTEM_USER import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import com.tencent.bkrepo.repository.pojo.node.NodeDetail +import com.tencent.bkrepo.repository.pojo.node.NodeListOption import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest import com.tencent.bkrepo.repository.pojo.node.service.NodesDeleteRequest @@ -124,16 +127,18 @@ import java.nio.charset.Charset import java.time.Instant import java.time.LocalDateTime import java.time.ZoneId +import java.util.Locale import javax.servlet.http.HttpServletRequest @Service class OciOperationServiceImpl( - private val nodeClient: NodeClient, + private val nodeService: NodeService, + private val nodeSearchService: NodeSearchService, private val metadataService: MetadataService, private val packageMetadataClient: PackageMetadataClient, private val packageClient: PackageClient, private val storageManager: StorageManager, - private val repositoryClient: RepositoryClient, + private val repositoryService: RepositoryService, private val ociProperties: OciProperties, private val ociReplicationRecordDao: OciReplicationRecordDao, private val storageCredentialService: StorageCredentialService, @@ -168,7 +173,7 @@ class OciOperationServiceImpl( with(artifactInfo) { // 可能存在支持多种type val repoDetail = getRepositoryInfo(artifactInfo) - val packageKey = PackageKeys.ofName(repoDetail.type.name.toLowerCase(), packageName) + val packageKey = PackageKeys.ofName(repoDetail.type.name.lowercase(Locale.getDefault()), packageName) if (version.isNotBlank()) { packageClient.findVersionByName( projectId = projectId, @@ -255,7 +260,7 @@ class OciOperationServiceImpl( fullPath = fullPath, operator = userId ) - nodeClient.deleteNode(request) + nodeService.deleteNode(request) } /** @@ -263,7 +268,7 @@ class OciOperationServiceImpl( */ private fun getRepositoryInfo(artifactInfo: OciArtifactInfo): RepositoryDetail { with(artifactInfo) { - val result = repositoryClient.getRepoDetail(projectId, repoName, REPO_TYPE).data ?: run { + val result = repositoryService.getRepoDetail(projectId, repoName, REPO_TYPE) ?: run { ArtifactContextHolder.queryRepoDetailFormExtraRepoType(projectId, repoName) } return result @@ -279,7 +284,7 @@ class OciOperationServiceImpl( with(artifactInfo) { logger.info("Try to get detail of the [$packageKey/$version] in repo ${artifactInfo.getRepoIdentify()}") val repoDetail = getRepositoryInfo(artifactInfo) - val name = PackageKeys.resolveName(repoDetail.type.name.toLowerCase(), packageKey) + val name = PackageKeys.resolveName(repoDetail.type.name.lowercase(Locale.getDefault()), packageKey) checkVersionExist( projectId = projectId, repoName = repoName, @@ -332,9 +337,9 @@ class OciOperationServiceImpl( ) } val fullPath = ociArtifactInfo.getArtifactFullPath() - val nodeDetail = nodeClient.getNodeDetail(projectId, repoName, fullPath).data ?: run { + val nodeDetail = nodeService.getNodeDetail(ociArtifactInfo) ?: run { val oldDockerFullPath = getDockerNode(ociArtifactInfo) ?: return@run null - nodeClient.getNodeDetail(projectId, repoName, oldDockerFullPath).data ?: run { + nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, oldDockerFullPath)) ?: run { logger.warn("node [$fullPath] don't found.") null } @@ -402,7 +407,7 @@ class OciOperationServiceImpl( md5 = fileInfo.md5, sha256 = fileInfo.sha256 ) - nodeClient.createNode(newNodeRequest).data + nodeService.createNode(newNodeRequest) } else { storageManager.storeArtifactFile(request, artifactFile, storageCredentials) } @@ -475,8 +480,8 @@ class OciOperationServiceImpl( manifestPath: String, ): Boolean { with(ociArtifactInfo) { - val repositoryDetail = repositoryClient.getRepoDetail(projectId, repoName).data ?: return false - val nodeDetail = nodeClient.getNodeDetail(projectId, repoName, manifestPath).data ?: return false + val repositoryDetail = repositoryService.getRepoDetail(projectId, repoName) ?: return false + val nodeDetail = nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, manifestPath)) ?: return false val manifest = loadManifest(nodeDetail, repositoryDetail.storageCredentials) ?: return false return syncBlobInfo( ociArtifactInfo = ociArtifactInfo, @@ -658,8 +663,7 @@ class OciOperationServiceImpl( // 并发情况下，版本目录下可能存在着非该版本的blob // 覆盖上传时会先删除原有目录，并发情况下可能导致blobs节点不存在 val nodeProperty = getNodeByDigest(projectId, repoName, descriptor.digest) ?: run { - nodeClient.getDeletedNodeDetailBySha256( - projectId, repoName, descriptor.sha256).data?.let { + nodeService.getDeletedNodeDetailBySha256(projectId, repoName, descriptor.sha256)?.let { NodeProperty(StringPool.EMPTY, it.md5, it.size) } ?: return false } @@ -675,7 +679,7 @@ class OciOperationServiceImpl( md5 = nodeProperty.md5 ?: StringPool.UNKNOWN, userId = userId ) - nodeClient.createNode(nodeCreateRequest) + nodeService.createNode(nodeCreateRequest) } val metadataMap = metadataService.listMetadata(projectId, repoName, fullPath) if (metadataMap[BLOB_PATH_VERSION_KEY] != null) { @@ -701,8 +705,8 @@ class OciOperationServiceImpl( with(ociArtifactInfo) { logger.info("Will create package info for [$packageName/$version in repo ${getRepoIdentify()} ") // 针对支持多仓库类型，如docker和oci - val repoType = repositoryClient.getRepoDetail(projectId, repoName).data!!.type.name - val packageKey = PackageKeys.ofName(repoType.toLowerCase(), packageName) + val repoType = repositoryService.getRepoDetail(projectId, repoName)!!.type.name + val packageKey = PackageKeys.ofName(repoType.lowercase(Locale.getDefault()), packageName) val metadata = mutableMapOf(MANIFEST_DIGEST to manifestDigest.toString()) .apply { sourceType?.let { this[SOURCE_TYPE] = sourceType } @@ -792,8 +796,8 @@ class OciOperationServiceImpl( this.path(path, OperationType.PREFIX) } } - val result = nodeClient.queryWithoutCount(queryModel.build()).data - if (result == null || result.records.isEmpty()) { + val result = nodeSearchService.searchWithoutCount(queryModel.build()) + if (result.records.isEmpty()) { logger.warn( "Could not find $digestStr " + "in repo $projectId|$repoName" @@ -937,7 +941,7 @@ class OciOperationServiceImpl( ): OciTagResult { val artifactInfo = OciArtifactInfo(projectId, repoName, StringPool.EMPTY, StringPool.EMPTY) val repoDetail = getRepositoryInfo(artifactInfo) - val packageKey = PackageKeys.ofName(repoDetail.type.name.toLowerCase(), packageName) + val packageKey = PackageKeys.ofName(repoDetail.type.name.lowercase(Locale.getDefault()), packageName) val result = packageClient.listVersionPage( projectId, repoName, @@ -961,7 +965,7 @@ class OciOperationServiceImpl( } override fun getPackagesFromThirdPartyRepo(projectId: String, repoName: String) { - val repositoryDetail = repositoryClient.getRepoDetail(projectId, repoName).data + val repositoryDetail = repositoryService.getRepoDetail(projectId, repoName) ?: throw RepoNotFoundException("$projectId|$repoName") buildImagePackagePullContext(projectId, repoName, repositoryDetail.configuration).forEach { pluginManager.applyExtension { @@ -973,13 +977,13 @@ class OciOperationServiceImpl( override fun deleteBlobsFolderAfterRefreshed( projectId: String, repoName: String, pName: String, userId: String ) { - repositoryClient.getRepoInfo(projectId, repoName).data + repositoryService.getRepoInfo(projectId, repoName) ?: throw RepoNotFoundException("$projectId|$repoName") val blobsFolderPath = buildBlobsFolderPath(pName) - val fullPaths = nodeClient.listNode( - projectId, repoName, blobsFolderPath, includeFolder = false, deep = false - ).data?.map { it.fullPath } - if (fullPaths.isNullOrEmpty()) return + val fullPaths = nodeService.listNode( + ArtifactInfo(projectId, repoName, blobsFolderPath), NodeListOption(includeFolder = false, deep = false) + ).map { it.fullPath } + if (fullPaths.isEmpty()) return logger.info("Blobs of package $pName in folder $blobsFolderPath will be deleted in $projectId|$repoName") val request = NodesDeleteRequest( projectId = projectId, @@ -987,7 +991,7 @@ class OciOperationServiceImpl( fullPaths = fullPaths, operator = userId ) - nodeClient.deleteNodes(request) + nodeService.deleteNodes(request) } /** @@ -1001,9 +1005,9 @@ class OciOperationServiceImpl( pVersion: String, userId: String ): Boolean { - val repoInfo = repositoryClient.getRepoInfo(projectId, repoName).data + val repoInfo = repositoryService.getRepoInfo(projectId, repoName) ?: throw RepoNotFoundException("$projectId|$repoName") - val packageName = PackageKeys.resolveName(repoInfo.type.name.toLowerCase(), pName) + val packageName = PackageKeys.resolveName(repoInfo.type.name.lowercase(Locale.getDefault()), pName) val manifestPath = OciLocationUtils.buildManifestPath(packageName, pVersion) logger.info("Manifest $manifestPath will be refreshed") val ociArtifactInfo = OciManifestArtifactInfo( @@ -1014,13 +1018,13 @@ class OciOperationServiceImpl( reference = pVersion, isValidDigest = false ) - val manifestNode = nodeClient.getNodeDetail( - repoInfo.projectId, repoInfo.name, manifestPath - ).data ?: run { + val manifestNode = nodeService.getNodeDetail( + ArtifactInfo(repoInfo.projectId, repoInfo.name, manifestPath) + ) ?: run { val oldDockerFullPath = getDockerNode(ociArtifactInfo) ?: return false - nodeClient.getNodeDetail( - repoInfo.projectId, repoInfo.name, oldDockerFullPath - ).data ?: return false + nodeService.getNodeDetail( + ArtifactInfo(repoInfo.projectId, repoInfo.name, oldDockerFullPath) + ) ?: return false } val refreshedMetadat = manifestNode.nodeMetadata.firstOrNull { it.key == BLOB_PATH_REFRESHED_KEY} if (refreshedMetadat != null) { diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/service/RepoService.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/service/RepoService.kt index 418a739dd6..b1102205d8 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/service/RepoService.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/service/RepoService.kt @@ -29,8 +29,8 @@ package com.tencent.bkrepo.opdata.service import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.metadata.service.project.ProjectService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.opdata.pojo.CleanupRules -import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.constant.SYSTEM_USER import com.tencent.bkrepo.repository.pojo.project.ProjectInfo import com.tencent.bkrepo.repository.pojo.project.ProjectRangeQueryRequest @@ -41,7 +41,7 @@ import org.springframework.stereotype.Service @Service class RepoService( private val projectService: ProjectService, - private val repoClient: RepositoryClient, + private val repositoryService: RepositoryService, ) { fun batchUpdateCleanupStrategy(rule: CleanupRules) { @@ -122,12 +122,12 @@ class RepoService( if (repoNames.isNullOrEmpty() || cleanupValue.isEmpty() || cleanupType.isEmpty()) return repoNames.forEach { if (specialRepoRules.containsKey("$projectId/$it")) return - val repoInfo = repoClient.getRepoInfo(projectId, it).data ?: return + val repoInfo = repositoryService.getRepoInfo(projectId, it) ?: return val configuration = repoInfo.configuration if (configuration.settings["cleanupStrategy"] != null && !forceRefresh) return var useDefault = true if (!relatedRepo.isNullOrEmpty()) { - val relatedConfig = repoClient.getRepoInfo(projectId, relatedRepo).data?.configuration + val relatedConfig = repositoryService.getRepoInfo(projectId, relatedRepo)?.configuration ?.getSetting>("cleanupStrategy") val relatedCleanupStrategy = toCleanupStrategy(relatedConfig) if (relatedCleanupStrategy != null) { @@ -150,7 +150,7 @@ class RepoService( configuration = configuration, operator = SYSTEM_USER ) - repoClient.updateRepo(request) + repositoryService.updateRepo(request) } } diff --git a/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/artifact/repository/PypiLocalRepository.kt b/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/artifact/repository/PypiLocalRepository.kt index a1c975bef2..e4edffa7e4 100644 --- a/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/artifact/repository/PypiLocalRepository.kt +++ b/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/artifact/repository/PypiLocalRepository.kt @@ -29,6 +29,7 @@ package com.tencent.bkrepo.pypi.artifact.repository import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.artifact.api.ArtifactFile +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.path.PathUtils.ROOT import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactQueryContext @@ -79,6 +80,7 @@ import com.tencent.bkrepo.repository.constant.SHA256 import com.tencent.bkrepo.repository.pojo.download.PackageDownloadRecord import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import com.tencent.bkrepo.repository.pojo.node.NodeInfo +import com.tencent.bkrepo.repository.pojo.node.NodeListOption import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest import com.tencent.bkrepo.repository.pojo.packages.PackageType @@ -216,7 +218,7 @@ class PypiLocalRepository( select = mutableListOf("projectId", "repoName", "fullPath", "metadata"), rule = rule ) - val nodeList: List>? = nodeClient.queryWithoutCount(queryModel).data?.records + val nodeList: List>? = nodeSearchService.searchWithoutCount(queryModel).records if (nodeList != null) { return XmlUtil.nodeLis2Values(nodeList) } @@ -233,7 +235,7 @@ class PypiLocalRepository( val contentPath = HttpContextHolder.getRequest().getParameter("contentPath") if (version.isNullOrBlank()) { // 删除包 - nodeClient.deleteNode( + nodeService.deleteNode( NodeDeleteRequest( context.projectId, context.repoName, @@ -249,7 +251,7 @@ class PypiLocalRepository( ) } else { // 删除版本 - nodeClient.deleteNode( + nodeService.deleteNode( NodeDeleteRequest( context.projectId, context.repoName, @@ -293,9 +295,9 @@ class PypiLocalRepository( ).data ?: return null val artifactPath = trueVersion.contentPaths?.firstOrNull() ?: trueVersion.contentPath ?: return null with(context.artifactInfo) { - val jarNode = nodeClient.getNodeDetail( - projectId, repoName, artifactPath - ).data ?: return null + val jarNode = nodeService.getNodeDetail( + ArtifactInfo(projectId, repoName, artifactPath) + ) ?: return null val stageTag = stageClient.query(projectId, repoName, packageKey, version).data val packageVersion = packageClient.findVersionByName( projectId, repoName, packageKey, version @@ -323,21 +325,18 @@ class PypiLocalRepository( logger.info("Get simple html, artifactInfo: $this") // 请求不带包名，返回包名列表. if (packageName == null) { - val nodeList = nodeClient.listNode(projectId, repoName, ROOT, includeFolder = true).data - ?.filter { it.folder }?.takeIf { it.isNotEmpty() } - ?: throw PypiSimpleNotFoundException(StringPool.SLASH) + val nodeList = nodeService.listNode( + ArtifactInfo(projectId, repoName, ROOT), + NodeListOption(includeFolder = true) + ).filter { it.folder }.takeIf { it.isNotEmpty() } ?: throw PypiSimpleNotFoundException(StringPool.SLASH) // 过滤掉'根节点', return buildPypiPageContent(PACKAGE_INDEX_TITLE, buildPackageListContent(nodeList)) } // 请求中带包名，返回对应包的文件列表。 - val nodes = nodeClient.listNode( - projectId = projectId, - repoName = repoName, - path = "/$packageName", - includeFolder = false, - deep = true, - includeMetadata = true - ).data + val nodes = nodeService.listNode( + ArtifactInfo(projectId, repoName, "/$packageName"), + NodeListOption(includeFolder = false, deep = true, includeMetadata = true) + ) if (!nodes.isNullOrEmpty()) { return buildPypiPageContent( String.format(VERSION_INDEX_TITLE, packageName), @@ -361,7 +360,7 @@ class PypiLocalRepository( .path("^/${packageName!!.replace("-", NON_ALPHANUMERIC_SEQ_REGEX)}/", OperationType.REGEX_I) .excludeFolder() .build() - val records = nodeClient.queryWithoutCount(queryModel).data!!.records + val records = nodeSearchService.searchWithoutCount(queryModel).records nodeList.addAll(records) pageNumber++ } while (records.size == PAGE_SIZE) diff --git a/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/artifact/repository/PypiRemoteRepository.kt b/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/artifact/repository/PypiRemoteRepository.kt index 47e09261eb..de7adb4864 100644 --- a/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/artifact/repository/PypiRemoteRepository.kt +++ b/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/artifact/repository/PypiRemoteRepository.kt @@ -34,6 +34,7 @@ package com.tencent.bkrepo.pypi.artifact.repository import com.tencent.bkrepo.common.api.exception.BadRequestException import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.common.artifact.api.ArtifactFile +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactQueryContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactSearchContext @@ -53,7 +54,7 @@ import kotlinx.coroutines.launch import okhttp3.MediaType.Companion.toMediaTypeOrNull import okhttp3.OkHttpClient import okhttp3.Request -import okhttp3.RequestBody +import okhttp3.RequestBody.Companion.toRequestBody import org.slf4j.Logger import org.slf4j.LoggerFactory import org.springframework.stereotype.Component @@ -110,10 +111,10 @@ class PypiRemoteRepository : RemoteRepository() { val projectId = repositoryDetail.projectId val repoName = repositoryDetail.name val fullPath = REMOTE_HTML_CACHE_FULL_PATH - var node = nodeClient.getNodeDetail(projectId, repoName, fullPath).data + var node = nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, fullPath)) loop@for (i in 1..3) { cacheRemoteRepoList(context) - node = nodeClient.getNodeDetail(projectId, repoName, fullPath).data + node = nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, fullPath)) if (node != null) break@loop } if (node == null) return "Can not cache remote html" @@ -169,7 +170,7 @@ class PypiRemoteRepository : RemoteRepository() { val xmlString = context.request.reader.readXml() val remoteConfiguration = context.getRemoteConfiguration() val okHttpClient: OkHttpClient = createHttpClient(remoteConfiguration) - val body = RequestBody.create("text/xml".toMediaTypeOrNull(), xmlString) + val body = xmlString.toRequestBody("text/xml".toMediaTypeOrNull()) val build: Request = Request.Builder().url("${remoteConfiguration.url}$XML_RPC_URI") .addHeader("Connection", "keep-alive") .post(body) diff --git a/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/artifact/repository/PypiVirtualRepository.kt b/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/artifact/repository/PypiVirtualRepository.kt index 082ec8f1f0..0b939fb81c 100644 --- a/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/artifact/repository/PypiVirtualRepository.kt +++ b/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/artifact/repository/PypiVirtualRepository.kt @@ -56,7 +56,7 @@ class PypiVirtualRepository : VirtualRepository() { continue } traversedList.add(repoIdentify) - val subRepoInfo = repositoryClient.getRepoDetail(repoIdentify.projectId, repoIdentify.name).data!! + val subRepoInfo = repositoryService.getRepoDetail(repoIdentify.projectId, repoIdentify.name)!! val repository = ArtifactContextHolder.getRepository(subRepoInfo.category) stringBuilder.append(repository.query(context)) } @@ -73,7 +73,7 @@ class PypiVirtualRepository : VirtualRepository() { continue } traversedList.add(repoIdentify) - val subRepoInfo = repositoryClient.getRepoDetail(repoIdentify.projectId, repoIdentify.name).data!! + val subRepoInfo = repositoryService.getRepoDetail(repoIdentify.projectId, repoIdentify.name)!! val repository = ArtifactContextHolder.getRepository(subRepoInfo.category) val subContext = context.copy(subRepoInfo) as ArtifactSearchContext val subValueList = repository.search(subContext) diff --git a/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/service/PypiWebService.kt b/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/service/PypiWebService.kt index a67cd40a3b..69bf6ec216 100644 --- a/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/service/PypiWebService.kt +++ b/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/service/PypiWebService.kt @@ -34,6 +34,7 @@ package com.tencent.bkrepo.pypi.service import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.common.api.pojo.Page +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.common.artifact.repository.context.ArtifactQueryContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactRemoveContext @@ -41,9 +42,9 @@ import com.tencent.bkrepo.common.artifact.repository.core.ArtifactService import com.tencent.bkrepo.common.artifact.util.PackageKeys import com.tencent.bkrepo.common.artifact.util.version.SemVersion import com.tencent.bkrepo.common.artifact.util.version.SemVersionParser +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.security.permission.Permission import com.tencent.bkrepo.pypi.artifact.PypiArtifactInfo -import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.PackageClient import com.tencent.bkrepo.repository.pojo.node.NodeInfo import com.tencent.bkrepo.repository.pojo.node.NodeListOption @@ -53,7 +54,7 @@ import java.time.LocalDateTime @Service class PypiWebService( - private val nodeClient: NodeClient, + private val nodeService: NodeService, private val packageClient: PackageClient ) : ArtifactService() { @@ -82,12 +83,14 @@ class PypiWebService( pageNumber: Int, pageSize: Int ): Page { - val data = nodeClient.listNodePage( - projectId = pypiArtifactInfo.projectId, - repoName = pypiArtifactInfo.repoName, - path = PathUtils.normalizePath(PackageKeys.resolvePypi(packageKey)), + val data = nodeService.listNodePage( + ArtifactInfo( + pypiArtifactInfo.projectId, + pypiArtifactInfo.repoName, + PathUtils.normalizePath(PackageKeys.resolvePypi(packageKey)) + ), option = NodeListOption(pageNumber, pageSize, includeFolder = false, deep = true) - ).data!! + ) val packageVersionList = data.records.map { val version = parseSemVersion(it.path).toString() val packageVersion = packageClient.findVersionByName(it.projectId, it.repoName, packageKey, version).data diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/controller/service/ArtifactReplicaController.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/controller/service/ArtifactReplicaController.kt index 80c9c7ef17..6d78797872 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/controller/service/ArtifactReplicaController.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/controller/service/ArtifactReplicaController.kt @@ -30,8 +30,11 @@ package com.tencent.bkrepo.replication.controller.service import com.tencent.bkrepo.auth.api.ServiceUserClient import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.common.api.pojo.Response +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.metadata.service.metadata.MetadataService +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.metadata.service.project.ProjectService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.security.exception.PermissionException import com.tencent.bkrepo.common.security.manager.PermissionManager import com.tencent.bkrepo.common.security.permission.Principal @@ -42,9 +45,7 @@ import com.tencent.bkrepo.replication.constant.DEFAULT_VERSION import com.tencent.bkrepo.replication.pojo.request.CheckPermissionRequest import com.tencent.bkrepo.replication.pojo.request.NodeExistCheckRequest import com.tencent.bkrepo.replication.pojo.request.PackageVersionExistCheckRequest -import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.PackageClient -import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.pojo.metadata.MetadataDeleteRequest import com.tencent.bkrepo.repository.pojo.metadata.MetadataSaveRequest import com.tencent.bkrepo.repository.pojo.node.NodeDeleteResult @@ -71,8 +72,8 @@ import org.springframework.web.bind.annotation.RestController @RestController class ArtifactReplicaController( private val projectService: ProjectService, - private val repositoryClient: RepositoryClient, - private val nodeClient: NodeClient, + private val repositoryService: RepositoryService, + private val nodeService: NodeService, private val packageClient: PackageClient, private val metadataService: MetadataService, private val userResource: ServiceUserClient, @@ -92,56 +93,60 @@ class ArtifactReplicaController( repoName: String, fullPath: String ): Response { - return nodeClient.checkExist(projectId, repoName, fullPath) + return ResponseBuilder.success(nodeService.checkExist(ArtifactInfo(projectId, repoName, fullPath))) } override fun checkNodeExistList( request: NodeExistCheckRequest ): Response> { - return nodeClient.listExistFullPath( + return ResponseBuilder.success(nodeService.listExistFullPath( request.projectId, request.repoName, request.fullPathList - ) + )) } override fun replicaNodeCreateRequest(request: NodeCreateRequest): Response { - return nodeClient.createNode(request) + return ResponseBuilder.success(nodeService.createNode(request)) } override fun replicaNodeRenameRequest(request: NodeRenameRequest): Response { - return nodeClient.renameNode(request) + nodeService.renameNode(request) + return ResponseBuilder.success() } override fun replicaNodeUpdateRequest(request: NodeUpdateRequest): Response { - return nodeClient.updateNode(request) + nodeService.updateNode(request) + return ResponseBuilder.success() } override fun replicaNodeCopyRequest(request: NodeMoveCopyRequest): Response { - nodeClient.copyNode(request) + nodeService.copyNode(request) return ResponseBuilder.success() } override fun replicaNodeMoveRequest(request: NodeMoveCopyRequest): Response { - nodeClient.moveNode(request) + nodeService.moveNode(request) return ResponseBuilder.success() } override fun replicaNodeDeleteRequest(request: NodeDeleteRequest): Response { - return nodeClient.deleteNode(request) + return ResponseBuilder.success(nodeService.deleteNode(request)) } override fun replicaRepoCreateRequest(request: RepoCreateRequest): Response { - return repositoryClient.getRepoDetail(request.projectId, request.name).data?.let { ResponseBuilder.success(it) } - ?: repositoryClient.createRepo(request) + return repositoryService.getRepoDetail(request.projectId, request.name)?.let { ResponseBuilder.success(it) } + ?: ResponseBuilder.success(repositoryService.createRepo(request)) } override fun replicaRepoUpdateRequest(request: RepoUpdateRequest): Response { - return repositoryClient.updateRepo(request) + repositoryService.updateRepo(request) + return ResponseBuilder.success() } override fun replicaRepoDeleteRequest(request: RepoDeleteRequest): Response { - return repositoryClient.deleteRepo(request) + repositoryService.deleteRepo(request) + return ResponseBuilder.success() } override fun checkRepoPermission(request: CheckPermissionRequest): Response { diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/manager/LocalDataManager.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/manager/LocalDataManager.kt index 254e7beaa8..4fcbb8036c 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/manager/LocalDataManager.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/manager/LocalDataManager.kt @@ -27,6 +27,7 @@ package com.tencent.bkrepo.replication.manager +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.constant.PROJECT_ID import com.tencent.bkrepo.common.artifact.constant.REPO_NAME import com.tencent.bkrepo.common.artifact.exception.ArtifactNotFoundException @@ -37,8 +38,11 @@ import com.tencent.bkrepo.common.artifact.exception.RepoNotFoundException import com.tencent.bkrepo.common.artifact.exception.VersionNotFoundException import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.common.artifact.stream.Range -import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService +import com.tencent.bkrepo.common.metadata.service.node.NodeSearchService +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.metadata.service.project.ProjectService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService +import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService import com.tencent.bkrepo.common.mongo.dao.util.Pages import com.tencent.bkrepo.common.mongo.dao.util.sharding.HashShardingUtils import com.tencent.bkrepo.common.storage.config.StorageProperties @@ -48,13 +52,12 @@ import com.tencent.bkrepo.common.storage.pojo.FileInfo import com.tencent.bkrepo.replication.constant.MD5 import com.tencent.bkrepo.replication.constant.NODE_FULL_PATH import com.tencent.bkrepo.replication.constant.SIZE -import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.PackageClient -import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.constant.SHARDING_COUNT import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.NodeInfo +import com.tencent.bkrepo.repository.pojo.node.NodeListOption import com.tencent.bkrepo.repository.pojo.packages.PackageListOption import com.tencent.bkrepo.repository.pojo.packages.PackageSummary import com.tencent.bkrepo.repository.pojo.packages.PackageVersion @@ -78,8 +81,9 @@ import java.time.format.DateTimeFormatter @Component class LocalDataManager( private val projectService: ProjectService, - private val repositoryClient: RepositoryClient, - private val nodeClient: NodeClient, + private val repositoryService: RepositoryService, + private val nodeService: NodeService, + private val nodeSearchService: NodeSearchService, private val packageClient: PackageClient, private val storageService: StorageService, private val storageCredentialService: StorageCredentialService, @@ -147,7 +151,7 @@ class LocalDataManager( * 仓库不存在抛异常 */ fun findRepoByName(projectId: String, repoName: String, type: String? = null): RepositoryDetail { - return repositoryClient.getRepoDetail(projectId, repoName, type).data + return repositoryService.getRepoDetail(projectId, repoName, type) ?: throw RepoNotFoundException(repoName) } @@ -155,7 +159,7 @@ class LocalDataManager( * 判断仓库是否存在 */ fun existRepo(projectId: String, repoName: String, type: String? = null): Boolean { - return repositoryClient.getRepoDetail(projectId, repoName, type).data != null + return repositoryService.getRepoDetail(projectId, repoName, type) != null } /** @@ -209,14 +213,14 @@ class LocalDataManager( fun findDeletedNodeDetail( projectId: String, repoName: String, fullPath: String ): NodeDetail? { - return nodeClient.getDeletedNodeDetail(projectId, repoName, fullPath).data?.firstOrNull() + return nodeService.getDeletedNodeDetail(ArtifactInfo(projectId, repoName, fullPath)).firstOrNull() } /** * 查找节点 */ fun findNode(projectId: String, repoName: String, fullPath: String): NodeDetail? { - return nodeClient.getNodeDetail(projectId, repoName, fullPath).data + return nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, fullPath)) } /** @@ -234,8 +238,8 @@ class LocalDataManager( .sha256(sha256) .page(1, 1) .sortByAsc(NODE_FULL_PATH) - val result = nodeClient.queryWithoutCount(queryModel.build()).data - if (result == null || result.records.isEmpty()) { + val result = nodeSearchService.searchWithoutCount(queryModel.build()) + if (result.records.isEmpty()) { throw NodeNotFoundException(sha256) } return FileInfo( @@ -264,13 +268,10 @@ class LocalDataManager( * 查询目录下的文件列表 */ fun listNode(projectId: String, repoName: String, fullPath: String): List { - val nodes = nodeClient.listNode( - projectId = projectId, - repoName = repoName, - path = fullPath, - includeFolder = true, - deep = false - ).data + val nodes = nodeService.listNode( + ArtifactInfo(projectId, repoName, fullPath), + NodeListOption(includeFolder = true, deep = false) + ) if (nodes.isNullOrEmpty()) { throw NodeNotFoundException("$projectId/$repoName") } diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/internal/type/DockerPackageNodeMapper.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/internal/type/DockerPackageNodeMapper.kt index 2542bf9d68..b0c17779a9 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/internal/type/DockerPackageNodeMapper.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/internal/type/DockerPackageNodeMapper.kt @@ -27,9 +27,12 @@ package com.tencent.bkrepo.replication.replica.repository.internal.type +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.exception.ArtifactNotFoundException import com.tencent.bkrepo.common.artifact.manager.StorageManager import com.tencent.bkrepo.common.artifact.pojo.RepositoryType +import com.tencent.bkrepo.common.metadata.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.replication.constant.BLOB_PATH_REFRESHED_KEY import com.tencent.bkrepo.replication.constant.DOCKER_LAYER_FULL_PATH import com.tencent.bkrepo.replication.constant.DOCKER_MANIFEST_JSON_FULL_PATH @@ -37,8 +40,6 @@ import com.tencent.bkrepo.replication.constant.OCI_LAYER_FULL_PATH import com.tencent.bkrepo.replication.constant.OCI_LAYER_FULL_PATH_V1 import com.tencent.bkrepo.replication.constant.OCI_MANIFEST_JSON_FULL_PATH import com.tencent.bkrepo.replication.util.ManifestParser -import com.tencent.bkrepo.repository.api.NodeClient -import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.packages.PackageSummary import com.tencent.bkrepo.repository.pojo.packages.PackageVersion @@ -49,9 +50,9 @@ import org.springframework.stereotype.Component */ @Component class DockerPackageNodeMapper( - private val nodeClient: NodeClient, + private val nodeService: NodeService, private val storageManager: StorageManager, - private val repositoryClient: RepositoryClient + private val repositoryService: RepositoryService ) : PackageNodeMapper { override fun type() = RepositoryType.DOCKER @@ -69,13 +70,13 @@ class DockerPackageNodeMapper( val name = packageSummary.name val version = packageVersion.name var isOci = false - val repository = repositoryClient.getRepoDetail(projectId, repoName, type.name).data!! + val repository = repositoryService.getRepoDetail(projectId, repoName, type.name)!! var manifestFullPath = DOCKER_MANIFEST_JSON_FULL_PATH.format(name, version) - val nodeDetail = nodeClient.getNodeDetail(projectId, repoName, manifestFullPath).data ?: run { + val nodeDetail = nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, manifestFullPath)) ?: run { // 针对使用oci替换了docker仓库，需要进行数据兼容 isOci = true manifestFullPath = OCI_MANIFEST_JSON_FULL_PATH.format(name, version) - nodeClient.getNodeDetail(projectId, repoName, manifestFullPath).data!! + nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, manifestFullPath))!! } if (nodeDetail.sha256.isNullOrEmpty()) throw ArtifactNotFoundException(manifestFullPath) val inputStream = storageManager.loadFullArtifactInputStream(nodeDetail, repository.storageCredentials)!! diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/internal/type/HelmPackageNodeMapper.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/internal/type/HelmPackageNodeMapper.kt index e6c9088b25..9c5e76591a 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/internal/type/HelmPackageNodeMapper.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/internal/type/HelmPackageNodeMapper.kt @@ -27,17 +27,16 @@ package com.tencent.bkrepo.replication.replica.repository.internal.type -import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException import com.tencent.bkrepo.common.artifact.pojo.RepositoryType -import com.tencent.bkrepo.repository.api.NodeClient +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.repository.pojo.packages.PackageSummary import com.tencent.bkrepo.repository.pojo.packages.PackageVersion import org.springframework.stereotype.Component @Component class HelmPackageNodeMapper( - private val nodeClient: NodeClient - ): PackageNodeMapper { + private val nodeService: NodeService +): PackageNodeMapper { override fun type() = RepositoryType.HELM override fun extraType(): RepositoryType? { @@ -55,9 +54,9 @@ class HelmPackageNodeMapper( CHART_PACKAGE_FILE_PATH.format(name, version), PROVENANCE_FILE_PATH.format(name, version) ) - return nodeClient.listExistFullPath( + return nodeService.listExistFullPath( packageSummary.projectId, packageSummary.repoName, chartNodeList - ).data ?: throw NodeNotFoundException(chartNodeList.toString()) + ) } companion object { diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/internal/type/MavenPackageNodeMapper.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/internal/type/MavenPackageNodeMapper.kt index 1e9195be76..5b4169ba6d 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/internal/type/MavenPackageNodeMapper.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/internal/type/MavenPackageNodeMapper.kt @@ -27,8 +27,10 @@ package com.tencent.bkrepo.replication.replica.repository.internal.type +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.pojo.RepositoryType -import com.tencent.bkrepo.repository.api.NodeClient +import com.tencent.bkrepo.common.metadata.service.node.NodeService +import com.tencent.bkrepo.repository.pojo.node.NodeListOption import com.tencent.bkrepo.repository.pojo.packages.PackageSummary import com.tencent.bkrepo.repository.pojo.packages.PackageVersion import org.slf4j.LoggerFactory @@ -36,7 +38,7 @@ import org.springframework.stereotype.Component @Component class MavenPackageNodeMapper( - private val nodeClient: NodeClient + private val nodeService: NodeService ) : PackageNodeMapper { override fun type() = RepositoryType.MAVEN @@ -53,7 +55,7 @@ class MavenPackageNodeMapper( val artifactPath = packageVersion.contentPath require(artifactPath != null) { "artifactPath for $key is null in [$projectId/$repoName]" } val path = artifactPath.substringBeforeLast('/') - val listNodePage = nodeClient.listNode(projectId, repoName, path).data!! + val listNodePage = nodeService.listNode(ArtifactInfo(projectId, repoName, path), NodeListOption()) val fullPathList = listNodePage.map { it.fullPath } if (logger.isDebugEnabled) { logger.debug( diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/edge/EdgePullReplicaExecutor.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/edge/EdgePullReplicaExecutor.kt index b563e5e17e..bc073d1518 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/edge/EdgePullReplicaExecutor.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/edge/EdgePullReplicaExecutor.kt @@ -40,6 +40,7 @@ import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.artifact.resolve.file.ArtifactFileFactory import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.metadata.service.project.ProjectService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.service.cluster.condition.CommitEdgeEdgeCondition import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties import com.tencent.bkrepo.common.service.feign.FeignClientFactory @@ -57,7 +58,6 @@ import com.tencent.bkrepo.replication.pojo.task.setting.ErrorStrategy import com.tencent.bkrepo.replication.replica.base.interceptor.SignInterceptor import com.tencent.bkrepo.replication.service.ReplicaRecordService import com.tencent.bkrepo.replication.util.OkHttpClientPool -import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.api.cluster.ClusterNodeClient import com.tencent.bkrepo.repository.api.cluster.ClusterRepositoryClient import com.tencent.bkrepo.repository.pojo.node.NodeDetail @@ -78,7 +78,7 @@ import java.time.LocalDateTime class EdgePullReplicaExecutor( private val clusterProperties: ClusterProperties, private val projectService: ProjectService, - private val repositoryClient: RepositoryClient, + private val repositoryService: RepositoryService, private val storageManager: StorageManager, private val replicaRecordService: ReplicaRecordService ) { @@ -143,7 +143,7 @@ class EdgePullReplicaExecutor( fullPath = fullPath ).data ?: throw NodeNotFoundException(fullPath) - val localRepo = repositoryClient.getRepoDetail(localProjectId, localRepoName).data + val localRepo = repositoryService.getRepoDetail(localProjectId, localRepoName) ?: createProjectAndRepo(centerRepo, localProjectId, localRepoName) // 手动重试时，需要把仓库信息添加到request attribute中 HttpContextHolder.getRequestOrNull()?.setAttribute(REPO_KEY, localRepo) @@ -207,7 +207,7 @@ class EdgePullReplicaExecutor( description = centerRepo.description, operator = centerRepo.createdBy ) - return repositoryClient.createRepo(repoCreateRequest).data!! + return repositoryService.createRepo(repoCreateRequest) } private fun buildNodeCreateRequest( diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/edge/EdgeReplicaTaskJob.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/edge/EdgeReplicaTaskJob.kt index abcbbb1cd9..66e85f4e5b 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/edge/EdgeReplicaTaskJob.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/edge/EdgeReplicaTaskJob.kt @@ -30,12 +30,14 @@ package com.tencent.bkrepo.replication.replica.type.edge import com.tencent.bkrepo.common.api.constant.HttpStatus import com.tencent.bkrepo.common.api.pojo.Response import com.tencent.bkrepo.common.api.util.readJsonString +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException import com.tencent.bkrepo.common.artifact.exception.PackageNotFoundException import com.tencent.bkrepo.common.artifact.exception.VersionNotFoundException import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 -import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.service.cluster.condition.CommitEdgeEdgeCondition +import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties import com.tencent.bkrepo.common.service.feign.FeignClientFactory import com.tencent.bkrepo.common.service.otel.util.AsyncUtils.trace import com.tencent.bkrepo.common.service.util.UrlUtils @@ -47,7 +49,6 @@ import com.tencent.bkrepo.replication.replica.base.interceptor.SignInterceptor import com.tencent.bkrepo.replication.replica.context.ReplicaContext import com.tencent.bkrepo.replication.replica.executor.ManualThreadPoolExecutor import com.tencent.bkrepo.replication.util.OkHttpClientPool -import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.PackageClient import okhttp3.Request import org.slf4j.LoggerFactory @@ -63,7 +64,7 @@ import javax.annotation.PostConstruct class EdgeReplicaTaskJob( private val clusterProperties: ClusterProperties, private val replicationProperties: ReplicationProperties, - private val nodeClient: NodeClient, + private val nodeService: NodeService, private val packageClient: PackageClient ) { @@ -150,7 +151,7 @@ class EdgeReplicaTaskJob( replicationProperties = replicationProperties ) try { - val nodeInfo = nodeClient.getNodeDetail(projectId, repoName, fullPath!!).data?.nodeInfo + val nodeInfo = nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, fullPath!!))?.nodeInfo ?: throw NodeNotFoundException(fullPath!!) if (nodeInfo.sha256 == FAKE_SHA256) { logger.warn("Node $fullPath in repo ${nodeInfo.projectId}|${nodeInfo.repoName} is link node.") diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/impl/ReplicaExtServiceImpl.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/impl/ReplicaExtServiceImpl.kt index 57545bde09..c2fa75020d 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/impl/ReplicaExtServiceImpl.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/impl/ReplicaExtServiceImpl.kt @@ -34,11 +34,14 @@ import com.tencent.bkrepo.common.api.pojo.Page import com.tencent.bkrepo.common.api.pojo.Response import com.tencent.bkrepo.common.api.util.readJsonString import com.tencent.bkrepo.common.api.util.toJsonString +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.constant.PROJECT_ID import com.tencent.bkrepo.common.artifact.constant.REPO_NAME import com.tencent.bkrepo.common.artifact.exception.RepoNotFoundException import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.common.artifact.pojo.RepositoryType +import com.tencent.bkrepo.common.metadata.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.query.enums.OperationType import com.tencent.bkrepo.common.query.model.PageLimit import com.tencent.bkrepo.common.query.model.QueryModel @@ -56,14 +59,12 @@ import com.tencent.bkrepo.replication.pojo.task.objects.PackageConstraint import com.tencent.bkrepo.replication.pojo.task.objects.PathConstraint import com.tencent.bkrepo.replication.pojo.task.setting.ConflictStrategy import com.tencent.bkrepo.replication.pojo.task.setting.ReplicaSetting -import com.tencent.bkrepo.replication.util.OkHttpClientPool import com.tencent.bkrepo.replication.replica.context.ReplicaContext.Companion.READ_TIMEOUT import com.tencent.bkrepo.replication.replica.context.ReplicaContext.Companion.WRITE_TIMEOUT import com.tencent.bkrepo.replication.service.RemoteNodeService import com.tencent.bkrepo.replication.service.ReplicaExtService -import com.tencent.bkrepo.repository.api.NodeClient +import com.tencent.bkrepo.replication.util.OkHttpClientPool import com.tencent.bkrepo.repository.api.PackageClient -import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.pojo.node.NodeInfo import com.tencent.bkrepo.repository.pojo.node.NodeListOption import com.tencent.bkrepo.repository.pojo.packages.PackageListOption @@ -80,9 +81,9 @@ import java.time.Duration @Service class ReplicaExtServiceImpl( - private val repositoryClient: RepositoryClient, + private val repositoryService: RepositoryService, private val packageClient: PackageClient, - private val nodeClient: NodeClient, + private val nodeService: NodeService, private val remoteNodeService: RemoteNodeService, private val replicationProperties: ReplicationProperties, ) : ReplicaExtService { @@ -275,7 +276,7 @@ class ReplicaExtServiceImpl( repoName: String, ): RepositoryDetail { return if (host.isNullOrEmpty()) { - repositoryClient.getRepoDetail(projectId, repoName, null).data + repositoryService.getRepoDetail(projectId, repoName, null) } else { queryRepoDetailFromHost( host = host, @@ -302,12 +303,10 @@ class ReplicaExtServiceImpl( pageNumber = pageNumber, pageSize = pageSize ) - nodeClient.listNodePage( - projectId = projectId, - repoName = repoName, - path = PathUtils.UNIX_SEPARATOR.toString(), + nodeService.listNodePage( + ArtifactInfo(projectId, repoName, PathUtils.UNIX_SEPARATOR.toString()), option = option - ).data?.records?.map { it.fullPath } + ).records.map { it.fullPath } } else { listNodesFromRemote( host = host, diff --git a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/FsNodeClient.kt b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/FsNodeClient.kt deleted file mode 100644 index 648e281011..0000000000 --- a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/FsNodeClient.kt +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. - * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. - * - * A copy of the MIT License is included in this file. - * - * - * Terms of the MIT License: - * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT - * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN - * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package com.tencent.bkrepo.repository.api - -import com.tencent.bkrepo.common.api.constant.REPOSITORY_SERVICE_NAME -import com.tencent.bkrepo.common.api.pojo.Response -import com.tencent.bkrepo.repository.pojo.node.NodeDetail -import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest -import com.tencent.bkrepo.repository.pojo.node.service.NodeSetLengthRequest -import org.springframework.cloud.openfeign.FeignClient -import org.springframework.web.bind.annotation.PostMapping -import org.springframework.web.bind.annotation.PutMapping -import org.springframework.web.bind.annotation.RequestBody -import org.springframework.web.bind.annotation.RequestMapping - -@FeignClient(REPOSITORY_SERVICE_NAME, contextId = "FsNodeClient", primary = false) -@RequestMapping("/service/node/fs") -interface FsNodeClient { - - @PutMapping("/length") - fun setLength( - @RequestBody nodeSetLengthRequest: NodeSetLengthRequest - ): Response - - @PostMapping("/create") - fun createNode(@RequestBody createRequest: NodeCreateRequest): Response -} diff --git a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/NodeClient.kt b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/NodeClient.kt index 9ca048a974..00254e5ae0 100644 --- a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/NodeClient.kt +++ b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/NodeClient.kt @@ -49,7 +49,7 @@ import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeLinkRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeMoveCopyRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeRenameRequest -import com.tencent.bkrepo.repository.pojo.node.service.NodeRestoreRequest +import com.tencent.bkrepo.common.metadata.pojo.node.NodeRestoreRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeUnCompressedRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeUpdateAccessDateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeUpdateRequest @@ -75,6 +75,7 @@ import org.springframework.web.bind.annotation.RequestParam @Primary @FeignClient(REPOSITORY_SERVICE_NAME, contextId = "NodeClient", primary = false) @RequestMapping("/service/node") +@Deprecated("replace with NodeService") interface NodeClient { @ApiOperation("根据路径查看节点详情") diff --git a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/RepositoryClient.kt b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/RepositoryClient.kt index 8d1579d023..02f72a1809 100644 --- a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/RepositoryClient.kt +++ b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/RepositoryClient.kt @@ -60,6 +60,7 @@ import org.springframework.web.bind.annotation.RequestParam @Api("仓库服务接口") @FeignClient(REPOSITORY_SERVICE_NAME, contextId = "RepositoryClient", primary = false) @RequestMapping("/service/repo") +@Deprecated("replace with RepositoryService") interface RepositoryClient { @ApiOperation("查询仓库信息") diff --git a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/cluster/ClusterNodeClient.kt b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/cluster/ClusterNodeClient.kt index 5be801e8ca..545f171dfe 100644 --- a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/cluster/ClusterNodeClient.kt +++ b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/cluster/ClusterNodeClient.kt @@ -36,7 +36,7 @@ import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeMoveCopyRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeRenameRequest -import com.tencent.bkrepo.repository.pojo.node.service.NodeRestoreRequest +import com.tencent.bkrepo.common.metadata.pojo.node.NodeRestoreRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeUpdateAccessDateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeUpdateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodesDeleteRequest diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/config/RepositoryConsumerConfig.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/config/RepositoryConsumerConfig.kt index e86704d91f..239f534c78 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/config/RepositoryConsumerConfig.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/config/RepositoryConsumerConfig.kt @@ -32,7 +32,7 @@ package com.tencent.bkrepo.repository.config import com.tencent.bkrepo.common.artifact.event.base.ArtifactEvent -import com.tencent.bkrepo.repository.listener.NodeUpdateAccessDateEventListener +import com.tencent.bkrepo.common.metadata.listener.NodeUpdateAccessDateEventListener import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import org.springframework.messaging.Message diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/cluster/ClusterFsNodeController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/cluster/ClusterFsNodeController.kt deleted file mode 100644 index bc16fcc265..0000000000 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/cluster/ClusterFsNodeController.kt +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. - * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. - * - * A copy of the MIT License is included in this file. - * - * - * Terms of the MIT License: - * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT - * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN - * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package com.tencent.bkrepo.repository.controller.cluster - -import com.tencent.bkrepo.common.api.pojo.Response -import com.tencent.bkrepo.common.service.util.ResponseBuilder -import com.tencent.bkrepo.repository.api.cluster.ClusterFsNodeClient -import com.tencent.bkrepo.repository.pojo.node.NodeDetail -import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest -import com.tencent.bkrepo.repository.pojo.node.service.NodeSetLengthRequest -import com.tencent.bkrepo.repository.service.fs.FsService -import org.springframework.web.bind.annotation.RestController - -@RestController -class ClusterFsNodeController( - private val fsService: FsService -) : ClusterFsNodeClient { - override fun setLength(nodeSetLengthRequest: NodeSetLengthRequest): Response { - fsService.setLength(nodeSetLengthRequest) - return ResponseBuilder.success() - } - - override fun createNode(createRequest: NodeCreateRequest): Response { - return ResponseBuilder.success(fsService.createNode(createRequest)) - } -} diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/cluster/ClusterNodeController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/cluster/ClusterNodeController.kt index 854b47bb48..91cf25648e 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/cluster/ClusterNodeController.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/cluster/ClusterNodeController.kt @@ -41,11 +41,11 @@ import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeMoveCopyRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeRenameRequest -import com.tencent.bkrepo.repository.pojo.node.service.NodeRestoreRequest +import com.tencent.bkrepo.common.metadata.pojo.node.NodeRestoreRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeUpdateAccessDateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeUpdateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodesDeleteRequest -import com.tencent.bkrepo.repository.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.node.NodeService import org.springframework.web.bind.annotation.RestController import java.time.LocalDateTime diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/proxy/ProxyNodeController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/proxy/ProxyNodeController.kt index 9fdbbe1e13..b0a9661ab0 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/proxy/ProxyNodeController.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/proxy/ProxyNodeController.kt @@ -37,7 +37,7 @@ import com.tencent.bkrepo.repository.pojo.node.NodeInfo import com.tencent.bkrepo.repository.pojo.node.NodeListOption import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeUpdateAccessDateRequest -import com.tencent.bkrepo.repository.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.node.NodeService import org.springframework.web.bind.annotation.RestController @RestController diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/FsNodeController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/FsNodeController.kt deleted file mode 100644 index ebaa187a41..0000000000 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/FsNodeController.kt +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. - * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. - * - * A copy of the MIT License is included in this file. - * - * - * Terms of the MIT License: - * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT - * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN - * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package com.tencent.bkrepo.repository.controller.service - -import com.tencent.bkrepo.common.api.pojo.Response -import com.tencent.bkrepo.common.service.util.ResponseBuilder -import com.tencent.bkrepo.repository.api.FsNodeClient -import com.tencent.bkrepo.repository.pojo.node.NodeDetail -import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest -import com.tencent.bkrepo.repository.pojo.node.service.NodeSetLengthRequest -import com.tencent.bkrepo.repository.service.fs.FsService -import org.springframework.web.bind.annotation.RestController - -@RestController -class FsNodeController( - private val fsService: FsService -) : FsNodeClient { - - override fun setLength(nodeSetLengthRequest: NodeSetLengthRequest): Response { - fsService.setLength(nodeSetLengthRequest) - return ResponseBuilder.success() - } - - override fun createNode(createRequest: NodeCreateRequest): Response { - return ResponseBuilder.success(fsService.createNode(createRequest)) - } -} diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/NodeController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/NodeController.kt index 6a57e8819d..8b898d0a46 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/NodeController.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/NodeController.kt @@ -51,13 +51,13 @@ import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeLinkRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeMoveCopyRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeRenameRequest -import com.tencent.bkrepo.repository.pojo.node.service.NodeRestoreRequest +import com.tencent.bkrepo.common.metadata.pojo.node.NodeRestoreRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeUnCompressedRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeUpdateAccessDateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeUpdateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodesDeleteRequest -import com.tencent.bkrepo.repository.service.node.NodeSearchService -import com.tencent.bkrepo.repository.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.node.NodeSearchService +import com.tencent.bkrepo.common.metadata.service.node.NodeService import org.springframework.context.annotation.Primary import org.springframework.web.bind.annotation.RestController diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserNodeController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserNodeController.kt index 37af85a95d..6cabfe9e36 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserNodeController.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserNodeController.kt @@ -53,7 +53,7 @@ import com.tencent.bkrepo.repository.pojo.node.NodeDeletedPoint import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.NodeInfo import com.tencent.bkrepo.repository.pojo.node.NodeListOption -import com.tencent.bkrepo.repository.pojo.node.NodeRestoreOption +import com.tencent.bkrepo.common.metadata.pojo.node.NodeRestoreOption import com.tencent.bkrepo.repository.pojo.node.NodeRestoreResult import com.tencent.bkrepo.repository.pojo.node.NodeSizeInfo import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest @@ -69,8 +69,8 @@ import com.tencent.bkrepo.repository.pojo.node.user.UserNodeMoveCopyRequest import com.tencent.bkrepo.repository.pojo.node.user.UserNodeRenameRequest import com.tencent.bkrepo.repository.pojo.node.user.UserNodeUpdateRequest import com.tencent.bkrepo.repository.pojo.software.ProjectPackageOverview -import com.tencent.bkrepo.repository.service.node.NodeSearchService -import com.tencent.bkrepo.repository.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.node.NodeSearchService +import com.tencent.bkrepo.common.metadata.service.node.NodeService import io.swagger.annotations.Api import io.swagger.annotations.ApiOperation import io.swagger.annotations.ApiParam @@ -216,18 +216,25 @@ class UserNodeController( @RequestAttribute userId: String, @RequestBody request: UserNodeLinkRequest, ): Response { - val linkReq = NodeLinkRequest( - projectId = request.projectId, - repoName = request.repoName, - fullPath = request.fullPath, - targetProjectId = request.targetProjectId, - targetRepoName = request.targetRepoName, - targetFullPath = request.targetFullPath, - overwrite = request.overwrite, - nodeMetadata = request.nodeMetadata, - operator = userId, - ) - return ResponseBuilder.success(nodeService.link(linkReq)) + with(request) { + val linkReq = NodeLinkRequest( + projectId = projectId, + repoName = repoName, + fullPath = fullPath, + targetProjectId = targetProjectId, + targetRepoName = targetRepoName, + targetFullPath = targetFullPath, + overwrite = overwrite, + nodeMetadata = nodeMetadata, + operator = userId, + ) + // 校验源仓库与目标节点权限 + permissionManager.checkRepoPermission(PermissionAction.WRITE, projectId, repoName, userId = userId) + permissionManager.checkNodePermission( + PermissionAction.READ, targetProjectId, targetRepoName, targetFullPath, userId = userId + ) + return ResponseBuilder.success(nodeService.link(linkReq)) + } } @ApiOperation("更新节点") diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserRepositoryController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserRepositoryController.kt index d5797be0e0..a3a6c87556 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserRepositoryController.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserRepositoryController.kt @@ -31,6 +31,7 @@ import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.common.api.pojo.Page import com.tencent.bkrepo.common.api.pojo.Response +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.metadata.service.repo.QuotaService import com.tencent.bkrepo.common.security.manager.PermissionManager import com.tencent.bkrepo.common.security.permission.Permission @@ -45,7 +46,6 @@ import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail import com.tencent.bkrepo.repository.pojo.repo.RepositoryInfo import com.tencent.bkrepo.repository.pojo.repo.UserRepoCreateRequest import com.tencent.bkrepo.repository.pojo.repo.UserRepoUpdateRequest -import com.tencent.bkrepo.repository.service.node.NodeService import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import io.swagger.annotations.Api import io.swagger.annotations.ApiOperation diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/job/RepoUsedVolumeSynJob.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/job/RepoUsedVolumeSynJob.kt index 85688ff31c..0dd2ee8639 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/job/RepoUsedVolumeSynJob.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/job/RepoUsedVolumeSynJob.kt @@ -38,7 +38,7 @@ import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.common.service.log.LoggerHolder import com.tencent.bkrepo.common.metadata.model.TRepository -import com.tencent.bkrepo.repository.service.node.impl.NodeBaseService +import com.tencent.bkrepo.common.metadata.service.node.impl.NodeBaseService import org.springframework.context.event.EventListener import org.springframework.data.mongodb.core.query.Query import org.springframework.data.mongodb.core.query.and diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/packages/PackageQueryContext.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/packages/PackageQueryContext.kt index fc9667209d..b9ced552cb 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/packages/PackageQueryContext.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/packages/PackageQueryContext.kt @@ -33,7 +33,7 @@ package com.tencent.bkrepo.repository.search.packages import com.tencent.bkrepo.common.query.builder.MongoQueryInterpreter import com.tencent.bkrepo.common.query.model.QueryModel -import com.tencent.bkrepo.repository.search.common.CommonQueryContext +import com.tencent.bkrepo.common.metadata.search.common.CommonQueryContext import org.springframework.data.mongodb.core.query.Query class PackageQueryContext( diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/packages/PackageSearchInterpreter.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/packages/PackageSearchInterpreter.kt index 0901f55380..5fb10a24ec 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/packages/PackageSearchInterpreter.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/packages/PackageSearchInterpreter.kt @@ -34,13 +34,13 @@ package com.tencent.bkrepo.repository.search.packages import com.tencent.bkrepo.common.query.interceptor.QueryContext import com.tencent.bkrepo.common.query.model.QueryModel import com.tencent.bkrepo.common.security.manager.PermissionManager -import com.tencent.bkrepo.repository.search.common.CommonQueryInterpreter -import com.tencent.bkrepo.repository.search.common.LocalDatetimeRuleInterceptor -import com.tencent.bkrepo.repository.search.common.MetadataRuleInterceptor -import com.tencent.bkrepo.repository.search.common.ModelValidateInterceptor -import com.tencent.bkrepo.repository.search.common.RepoNameRuleInterceptor -import com.tencent.bkrepo.repository.search.common.RepoTypeRuleInterceptor -import com.tencent.bkrepo.repository.search.common.SelectFieldInterceptor +import com.tencent.bkrepo.common.metadata.search.common.CommonQueryInterpreter +import com.tencent.bkrepo.common.metadata.search.common.LocalDatetimeRuleInterceptor +import com.tencent.bkrepo.common.metadata.search.common.MetadataRuleInterceptor +import com.tencent.bkrepo.common.metadata.search.common.ModelValidateInterceptor +import com.tencent.bkrepo.common.metadata.search.common.RepoNameRuleInterceptor +import com.tencent.bkrepo.common.metadata.search.common.RepoTypeRuleInterceptor +import com.tencent.bkrepo.common.metadata.search.common.SelectFieldInterceptor import org.springframework.data.mongodb.core.query.Query import org.springframework.stereotype.Component import javax.annotation.PostConstruct diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/software/interceptor/SoftwareRepoNameRuleInterceptor.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/software/interceptor/SoftwareRepoNameRuleInterceptor.kt index b290eff200..3e240e1813 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/software/interceptor/SoftwareRepoNameRuleInterceptor.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/software/interceptor/SoftwareRepoNameRuleInterceptor.kt @@ -36,7 +36,7 @@ import com.tencent.bkrepo.common.query.interceptor.QueryContext import com.tencent.bkrepo.common.query.interceptor.QueryRuleInterceptor import com.tencent.bkrepo.common.query.model.Rule import com.tencent.bkrepo.repository.pojo.node.NodeInfo -import com.tencent.bkrepo.repository.search.common.CommonQueryContext +import com.tencent.bkrepo.common.metadata.search.common.CommonQueryContext import org.springframework.data.mongodb.core.query.Criteria import org.springframework.stereotype.Component diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/software/interceptor/SoftwareRepoTypeRuleInterceptor.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/software/interceptor/SoftwareRepoTypeRuleInterceptor.kt index b0f08c5ab4..34103613a5 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/software/interceptor/SoftwareRepoTypeRuleInterceptor.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/software/interceptor/SoftwareRepoTypeRuleInterceptor.kt @@ -36,7 +36,7 @@ import com.tencent.bkrepo.common.query.interceptor.QueryContext import com.tencent.bkrepo.common.query.interceptor.QueryRuleInterceptor import com.tencent.bkrepo.common.query.model.Rule import com.tencent.bkrepo.repository.model.TPackage -import com.tencent.bkrepo.repository.search.common.CommonQueryContext +import com.tencent.bkrepo.common.metadata.search.common.CommonQueryContext import org.springframework.data.mongodb.core.query.Criteria import org.springframework.stereotype.Component diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/software/packages/SoftwarePackageSearchInterpreter.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/software/packages/SoftwarePackageSearchInterpreter.kt index 69168cf60b..37c27208be 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/software/packages/SoftwarePackageSearchInterpreter.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/search/software/packages/SoftwarePackageSearchInterpreter.kt @@ -34,8 +34,8 @@ package com.tencent.bkrepo.repository.search.software.packages import com.tencent.bkrepo.common.query.builder.MongoQueryInterpreter import com.tencent.bkrepo.common.query.interceptor.QueryContext import com.tencent.bkrepo.common.query.model.QueryModel -import com.tencent.bkrepo.repository.search.common.MetadataRuleInterceptor -import com.tencent.bkrepo.repository.search.common.SelectFieldInterceptor +import com.tencent.bkrepo.common.metadata.search.common.MetadataRuleInterceptor +import com.tencent.bkrepo.common.metadata.search.common.SelectFieldInterceptor import com.tencent.bkrepo.repository.search.packages.PackageQueryContext import com.tencent.bkrepo.repository.search.software.interceptor.SoftwareModelValidateInterceptor import com.tencent.bkrepo.repository.search.software.interceptor.SoftwareRepoNameRuleInterceptor diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/blocknode/BlockNodeServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/blocknode/BlockNodeServiceImpl.kt deleted file mode 100644 index c349c7cc98..0000000000 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/blocknode/BlockNodeServiceImpl.kt +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. - * - * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. - * - * A copy of the MIT License is included in this file. - * - * - * Terms of the MIT License: - * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT - * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN - * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package com.tencent.bkrepo.repository.service.blocknode - -import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException -import com.tencent.bkrepo.common.metadata.dao.blocknode.BlockNodeDao -import com.tencent.bkrepo.common.metadata.service.blocknode.impl.AbstractBlockNodeService -import com.tencent.bkrepo.common.metadata.dao.node.NodeDao -import com.tencent.bkrepo.repository.pojo.node.NodeDetail -import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService -import com.tencent.bkrepo.repository.service.node.impl.NodeBaseService -import org.springframework.context.annotation.Primary -import org.springframework.stereotype.Service - -@Service -@Primary -class BlockNodeServiceImpl( - blockNodeDao: BlockNodeDao, - fileReferenceService: FileReferenceService, - private val nodeDao: NodeDao -) : AbstractBlockNodeService(blockNodeDao, fileReferenceService) { - - override fun getNodeDetail(projectId: String, repoName: String, fullPath: String): NodeDetail { - val node = nodeDao.findNode(projectId, repoName, fullPath) ?: throw NodeNotFoundException(fullPath) - return NodeBaseService.convertToDetail(node)!! - } -} diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/file/impl/ListViewServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/file/impl/ListViewServiceImpl.kt index 1bb1ead114..c28833b606 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/file/impl/ListViewServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/file/impl/ListViewServiceImpl.kt @@ -41,7 +41,7 @@ import com.tencent.bkrepo.repository.pojo.node.NodeListViewItem import com.tencent.bkrepo.repository.pojo.project.ProjectListViewItem import com.tencent.bkrepo.repository.pojo.repo.RepoListViewItem import com.tencent.bkrepo.repository.service.file.ListViewService -import com.tencent.bkrepo.repository.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.metadata.service.project.ProjectService import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import org.springframework.beans.factory.annotation.Value diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/file/impl/ShareServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/file/impl/ShareServiceImpl.kt index 73fb57b435..55342317d6 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/file/impl/ShareServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/file/impl/ShareServiceImpl.kt @@ -49,7 +49,7 @@ import com.tencent.bkrepo.repository.model.TShareRecord import com.tencent.bkrepo.repository.pojo.share.ShareRecordCreateRequest import com.tencent.bkrepo.repository.pojo.share.ShareRecordInfo import com.tencent.bkrepo.repository.service.file.ShareService -import com.tencent.bkrepo.repository.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import org.slf4j.LoggerFactory import org.springframework.context.annotation.Conditional diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/file/impl/center/CommitEdgeCenterShareServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/file/impl/center/CommitEdgeCenterShareServiceImpl.kt index 84d57e96bf..df7b8c5df5 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/file/impl/center/CommitEdgeCenterShareServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/file/impl/center/CommitEdgeCenterShareServiceImpl.kt @@ -32,7 +32,7 @@ import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.metadata.util.ClusterUtils.isEdgeRequest import com.tencent.bkrepo.common.service.cluster.condition.CommitEdgeCenterCondition import com.tencent.bkrepo.repository.service.file.impl.ShareServiceImpl -import com.tencent.bkrepo.repository.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import org.springframework.context.annotation.Conditional import org.springframework.data.mongodb.core.MongoTemplate diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/file/impl/edge/EdgeShareServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/file/impl/edge/EdgeShareServiceImpl.kt index 1fddb62084..ca75e198d8 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/file/impl/edge/EdgeShareServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/file/impl/edge/EdgeShareServiceImpl.kt @@ -38,7 +38,7 @@ import com.tencent.bkrepo.repository.pojo.share.ClusterShareTokenCheckRequest import com.tencent.bkrepo.repository.pojo.share.ShareRecordCreateRequest import com.tencent.bkrepo.repository.pojo.share.ShareRecordInfo import com.tencent.bkrepo.repository.service.file.impl.ShareServiceImpl -import com.tencent.bkrepo.repository.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import org.springframework.context.annotation.Conditional import org.springframework.data.mongodb.core.MongoTemplate diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/fs/impl/center/CommitEdgeCenterFsServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/fs/impl/center/CommitEdgeCenterFsServiceImpl.kt deleted file mode 100644 index 1962650ca7..0000000000 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/fs/impl/center/CommitEdgeCenterFsServiceImpl.kt +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. - * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. - * - * A copy of the MIT License is included in this file. - * - * - * Terms of the MIT License: - * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT - * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN - * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package com.tencent.bkrepo.repository.service.fs.impl.center - -import com.tencent.bkrepo.common.security.util.SecurityUtils -import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties -import com.tencent.bkrepo.common.service.cluster.condition.CommitEdgeCenterCondition -import com.tencent.bkrepo.common.metadata.dao.node.NodeDao -import com.tencent.bkrepo.common.metadata.model.TNode -import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest -import com.tencent.bkrepo.repository.service.fs.impl.FsServiceImpl -import org.springframework.context.annotation.Conditional -import org.springframework.stereotype.Service - -@Service -@Conditional(CommitEdgeCenterCondition::class) -class CommitEdgeCenterFsServiceImpl( - nodeDao: NodeDao, - private val clusterProperties: ClusterProperties -) : FsServiceImpl( - nodeDao -) { - override fun buildTNode(request: NodeCreateRequest): TNode { - val tNode = super.buildTNode(request) - tNode.clusterNames = SecurityUtils.getClusterName()?.let { setOf(it) } - return tNode - } -} diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/fs/impl/edge/EdgeFsServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/fs/impl/edge/EdgeFsServiceImpl.kt index 81a720a9a6..e69de29bb2 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/fs/impl/edge/EdgeFsServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/fs/impl/edge/EdgeFsServiceImpl.kt @@ -1,81 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. - * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. - * - * A copy of the MIT License is included in this file. - * - * - * Terms of the MIT License: - * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT - * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN - * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package com.tencent.bkrepo.repository.service.fs.impl.edge - -import com.tencent.bkrepo.common.metadata.util.ClusterUtils.reportMetadataToCenter -import com.tencent.bkrepo.common.service.cluster.condition.CommitEdgeEdgeCondition -import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties -import com.tencent.bkrepo.common.service.feign.FeignClientFactory -import com.tencent.bkrepo.repository.api.cluster.ClusterFsNodeClient -import com.tencent.bkrepo.common.metadata.dao.node.NodeDao -import com.tencent.bkrepo.common.metadata.model.TNode -import com.tencent.bkrepo.repository.pojo.node.NodeDetail -import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest -import com.tencent.bkrepo.repository.pojo.node.service.NodeSetLengthRequest -import com.tencent.bkrepo.repository.service.fs.impl.FsServiceImpl -import org.springframework.context.annotation.Conditional -import org.springframework.stereotype.Service - -@Service -@Conditional(CommitEdgeEdgeCondition::class) -class EdgeFsServiceImpl( - nodeDao: NodeDao, - private val clusterProperties: ClusterProperties -) : FsServiceImpl( - nodeDao -) { - - private val centerNodeClient: ClusterFsNodeClient - by lazy { FeignClientFactory.create(clusterProperties.center, "repository", clusterProperties.self.name) } - - override fun createNode(createRequest: NodeCreateRequest): NodeDetail { - with(createRequest) { - if (reportMetadataToCenter(projectId, repoName)) { - centerNodeClient.createNode(this) - } - return super.createNode(this) - } - } - - override fun setLength(setLengthRequest: NodeSetLengthRequest) { - with(setLengthRequest) { - if (reportMetadataToCenter(projectId, repoName)) { - centerNodeClient.setLength(this) - } - super.setLength(this) - } - } - - override fun buildTNode(request: NodeCreateRequest): TNode { - val tNode = super.buildTNode(request) - if (reportMetadataToCenter(request.projectId, request.repoName)) { - tNode.clusterNames = setOf(clusterProperties.self.name!!) - } - return tNode - } -} diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/PipelineNodeServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/PipelineNodeServiceImpl.kt index a466987230..39f7d818b2 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/PipelineNodeServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/PipelineNodeServiceImpl.kt @@ -32,7 +32,7 @@ import com.tencent.bkrepo.common.artifact.api.DefaultArtifactInfo import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.repository.pojo.node.NodeInfo import com.tencent.bkrepo.repository.pojo.node.NodeListOption -import com.tencent.bkrepo.repository.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.repository.service.node.PipelineNodeService import org.springframework.stereotype.Service diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/ResourceClearServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/ResourceClearServiceImpl.kt index 62875e5088..377b961714 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/ResourceClearServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/ResourceClearServiceImpl.kt @@ -5,8 +5,8 @@ import com.tencent.bkrepo.common.artifact.api.DefaultArtifactInfo import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.artifact.path.PathUtils.ROOT import com.tencent.bkrepo.common.metadata.model.TRepository +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.metadata.service.repo.ResourceClearService -import com.tencent.bkrepo.repository.service.node.NodeService import com.tencent.bkrepo.repository.service.packages.PackageService import org.slf4j.LoggerFactory import org.springframework.stereotype.Service diff --git a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/FileReferenceServiceTest.kt b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/FileReferenceServiceTest.kt index 75580fccbd..8c5c3ca086 100644 --- a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/FileReferenceServiceTest.kt +++ b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/FileReferenceServiceTest.kt @@ -43,7 +43,7 @@ import com.tencent.bkrepo.common.metadata.dao.node.NodeDao import com.tencent.bkrepo.repository.pojo.credendials.StorageCredentialsCreateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService -import com.tencent.bkrepo.repository.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.metadata.service.project.ProjectService import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService diff --git a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/MetadataServiceTest.kt b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/MetadataServiceTest.kt index e9dc45a132..b7fd1fd1b1 100644 --- a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/MetadataServiceTest.kt +++ b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/MetadataServiceTest.kt @@ -42,7 +42,7 @@ import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import com.tencent.bkrepo.repository.pojo.metadata.MetadataSaveRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import com.tencent.bkrepo.common.metadata.service.metadata.MetadataService -import com.tencent.bkrepo.repository.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.metadata.service.project.ProjectService import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import org.junit.jupiter.api.Assertions diff --git a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/NodeQueryWithoutShardingKeyTest.kt b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/NodeQueryWithoutShardingKeyTest.kt index 950b113cde..b0a5a47d21 100644 --- a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/NodeQueryWithoutShardingKeyTest.kt +++ b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/NodeQueryWithoutShardingKeyTest.kt @@ -37,7 +37,7 @@ import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.NodeListOption import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest -import com.tencent.bkrepo.repository.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.metadata.service.project.ProjectService import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import org.junit.jupiter.api.Assertions diff --git a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/NodeSearchServiceTest.kt b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/NodeSearchServiceTest.kt index 7864f845d5..e363fb5c3d 100644 --- a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/NodeSearchServiceTest.kt +++ b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/NodeSearchServiceTest.kt @@ -50,12 +50,12 @@ import com.tencent.bkrepo.repository.pojo.node.NodeInfo import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import com.tencent.bkrepo.repository.pojo.repo.RepoCreateRequest import com.tencent.bkrepo.repository.pojo.search.NodeQueryBuilder -import com.tencent.bkrepo.repository.search.common.LocalDatetimeRuleInterceptor -import com.tencent.bkrepo.repository.search.common.RepoNameRuleInterceptor -import com.tencent.bkrepo.repository.search.common.RepoTypeRuleInterceptor -import com.tencent.bkrepo.repository.search.node.NodeQueryInterpreter -import com.tencent.bkrepo.repository.service.node.NodeSearchService -import com.tencent.bkrepo.repository.service.node.NodeService +import com.tencent.bkrepo.common.metadata.search.common.LocalDatetimeRuleInterceptor +import com.tencent.bkrepo.common.metadata.search.common.RepoNameRuleInterceptor +import com.tencent.bkrepo.common.metadata.search.common.RepoTypeRuleInterceptor +import com.tencent.bkrepo.common.metadata.search.node.NodeQueryInterpreter +import com.tencent.bkrepo.common.metadata.service.node.NodeSearchService +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.metadata.service.project.ProjectService import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import org.junit.jupiter.api.Assertions diff --git a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/NodeServiceTest.kt b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/NodeServiceTest.kt index 0d44517aad..739b25090a 100644 --- a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/NodeServiceTest.kt +++ b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/NodeServiceTest.kt @@ -49,7 +49,7 @@ import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeMoveCopyRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeRenameRequest -import com.tencent.bkrepo.repository.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.metadata.service.project.ProjectService import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import org.junit.jupiter.api.Assertions.assertEquals diff --git a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/RepositoryServiceTest.kt b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/RepositoryServiceTest.kt index d86274a865..40ffc95ba0 100644 --- a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/RepositoryServiceTest.kt +++ b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/RepositoryServiceTest.kt @@ -55,7 +55,7 @@ import com.tencent.bkrepo.repository.pojo.project.ProjectUpdateRequest import com.tencent.bkrepo.repository.pojo.repo.RepoCreateRequest import com.tencent.bkrepo.repository.pojo.repo.RepoDeleteRequest import com.tencent.bkrepo.repository.pojo.repo.RepoUpdateRequest -import com.tencent.bkrepo.repository.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.metadata.service.project.ProjectService import com.tencent.bkrepo.common.metadata.service.repo.ProxyChannelService import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService diff --git a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ServiceBaseTest.kt b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ServiceBaseTest.kt index f840312a8a..657dec0907 100644 --- a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ServiceBaseTest.kt +++ b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ServiceBaseTest.kt @@ -42,9 +42,17 @@ import com.tencent.bkrepo.common.artifact.event.project.ProjectCreatedEvent import com.tencent.bkrepo.common.artifact.pojo.RepositoryCategory import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.pojo.configuration.local.LocalConfiguration -import com.tencent.bkrepo.common.artifact.router.RouterControllerProperties +import com.tencent.bkrepo.common.artifact.properties.RouterControllerProperties +import com.tencent.bkrepo.common.metadata.config.RepositoryProperties +import com.tencent.bkrepo.common.metadata.dao.node.NodeDao +import com.tencent.bkrepo.common.metadata.dao.project.ProjectDao import com.tencent.bkrepo.common.metadata.dao.repo.RepositoryDao +import com.tencent.bkrepo.common.metadata.listener.ResourcePermissionListener +import com.tencent.bkrepo.common.metadata.service.log.OperateLogService +import com.tencent.bkrepo.common.metadata.service.project.ProjectService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.metadata.service.repo.ResourceClearService +import com.tencent.bkrepo.common.metadata.util.RepositoryServiceHelper import com.tencent.bkrepo.common.metadata.util.StorageCredentialHelper import com.tencent.bkrepo.common.security.http.core.HttpAuthProperties import com.tencent.bkrepo.common.security.manager.PermissionManager @@ -60,17 +68,10 @@ import com.tencent.bkrepo.repository.UT_REPO_DESC import com.tencent.bkrepo.repository.UT_REPO_DISPLAY import com.tencent.bkrepo.repository.UT_REPO_NAME import com.tencent.bkrepo.repository.UT_USER -import com.tencent.bkrepo.common.metadata.config.RepositoryProperties -import com.tencent.bkrepo.common.metadata.dao.node.NodeDao -import com.tencent.bkrepo.common.metadata.dao.project.ProjectDao -import com.tencent.bkrepo.common.metadata.listener.ResourcePermissionListener import com.tencent.bkrepo.repository.pojo.project.ProjectCreateRequest import com.tencent.bkrepo.repository.pojo.project.ProjectInfo import com.tencent.bkrepo.repository.pojo.repo.RepoCreateRequest import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail -import com.tencent.bkrepo.common.metadata.service.project.ProjectService -import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService -import com.tencent.bkrepo.common.metadata.util.RepositoryServiceHelper import com.tencent.bkrepo.router.api.RouterControllerClient import io.mockk.every import io.mockk.mockk @@ -149,6 +150,9 @@ open class ServiceBaseTest { @Autowired lateinit var repositoryServiceHelper: RepositoryServiceHelper + @MockBean + lateinit var operateLogService: OperateLogService + fun initMock() { val tracer = mockk() mockkObject(SpringContextUtils.Companion) diff --git a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/node/NodeRestoreSupportTest.kt b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/node/NodeRestoreSupportTest.kt index 85e1b93076..f379943ab4 100644 --- a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/node/NodeRestoreSupportTest.kt +++ b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/node/NodeRestoreSupportTest.kt @@ -40,8 +40,9 @@ import com.tencent.bkrepo.repository.constant.SYSTEM_USER import com.tencent.bkrepo.common.metadata.dao.file.FileReferenceDao import com.tencent.bkrepo.common.metadata.dao.node.NodeDao import com.tencent.bkrepo.common.metadata.model.TNode -import com.tencent.bkrepo.repository.pojo.node.ConflictStrategy -import com.tencent.bkrepo.repository.pojo.node.NodeRestoreOption +import com.tencent.bkrepo.common.metadata.pojo.node.ConflictStrategy +import com.tencent.bkrepo.common.metadata.pojo.node.NodeRestoreOption +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import com.tencent.bkrepo.repository.service.ServiceBaseTest import com.tencent.bkrepo.common.metadata.service.project.ProjectService diff --git a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/query/NodeQueryInterpreterTest.kt b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/query/NodeQueryInterpreterTest.kt index b7fb2aadf6..75c49ff82a 100644 --- a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/query/NodeQueryInterpreterTest.kt +++ b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/query/NodeQueryInterpreterTest.kt @@ -34,10 +34,10 @@ package com.tencent.bkrepo.repository.service.query import com.tencent.bkrepo.common.query.model.Sort import com.tencent.bkrepo.repository.pojo.search.NodeQueryBuilder import com.tencent.bkrepo.repository.pojo.stage.ArtifactStageEnum -import com.tencent.bkrepo.repository.search.common.LocalDatetimeRuleInterceptor -import com.tencent.bkrepo.repository.search.common.RepoNameRuleInterceptor -import com.tencent.bkrepo.repository.search.common.RepoTypeRuleInterceptor -import com.tencent.bkrepo.repository.search.node.NodeQueryInterpreter +import com.tencent.bkrepo.common.metadata.search.common.LocalDatetimeRuleInterceptor +import com.tencent.bkrepo.common.metadata.search.common.RepoNameRuleInterceptor +import com.tencent.bkrepo.common.metadata.search.common.RepoTypeRuleInterceptor +import com.tencent.bkrepo.common.metadata.search.node.NodeQueryInterpreter import com.tencent.bkrepo.repository.service.ServiceBaseTest import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import org.junit.jupiter.api.BeforeAll diff --git a/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/artifact/repository/RpmLocalRepository.kt b/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/artifact/repository/RpmLocalRepository.kt index e9fa9dc295..1bc9efcf8a 100644 --- a/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/artifact/repository/RpmLocalRepository.kt +++ b/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/artifact/repository/RpmLocalRepository.kt @@ -38,6 +38,7 @@ import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.common.api.pojo.Page import com.tencent.bkrepo.common.api.util.toJsonString import com.tencent.bkrepo.common.artifact.api.ArtifactFile +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.hash.md5 import com.tencent.bkrepo.common.artifact.hash.sha1 import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode @@ -151,7 +152,7 @@ class RpmLocalRepository( val overwrite = HeaderUtils.getRpmBooleanHeader("X-BKREPO-OVERWRITE") if (!overwrite) { with(context.artifactInfo) { - val node = nodeClient.getNodeDetail(projectId, repoName, getArtifactFullPath()).data + val node = nodeService.getNodeDetail(this) if (node != null) { throw ErrorCodeException(ArtifactMessageCode.NODE_EXISTED, getArtifactFullPath()) } @@ -335,7 +336,7 @@ class RpmLocalRepository( val artifactUri = context.artifactInfo.getArtifactFullPath() val artifactSha256 = context.getArtifactSha256() return with(context.artifactInfo) { - val node = nodeClient.getNodeDetail(projectId, repoName, artifactUri).data ?: return NONE + val node = nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, artifactUri)) ?: return NONE if (node.sha256 == artifactSha256) { FULLPATH_SHA256 } else { @@ -563,10 +564,8 @@ class RpmLocalRepository( fun removeByUrl(context: ArtifactRemoveContext) { val fullPath = context.artifactInfo.getArtifactFullPath() - val node = with(context) { - nodeClient.getNodeDetail(projectId, repoName, fullPath).data - ?: throw ErrorCodeException(ArtifactMessageCode.NODE_NOT_FOUND, fullPath) - } + val node = nodeService.getNodeDetail(context.artifactInfo) + ?: throw ErrorCodeException(ArtifactMessageCode.NODE_NOT_FOUND, fullPath) val metadata = node.metadata val rpmVersion = metadata.toRpmVersion(fullPath) val rpmPackagePojo = rpmVersion.toRpmPackagePojo(fullPath) @@ -590,8 +589,9 @@ class RpmLocalRepository( for (packageVersion in pages) { val artifactFullPath = packageVersion.contentPath!! - val node = nodeClient.getNodeDetail(context.projectId, context.repoName, artifactFullPath).data - ?: continue + val node = nodeService.getNodeDetail( + ArtifactInfo(context.projectId, context.repoName, artifactFullPath) + ) ?: continue removeRpmArtifact(node, artifactFullPath, context, packageKey, packageVersion.name) } } else { @@ -602,7 +602,8 @@ class RpmLocalRepository( packageKey, version ).data ?: return - val node = nodeClient.getNodeDetail(projectId, repoName, packageVersion.contentPath!!).data ?: return + val node = nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, packageVersion.contentPath!!)) + ?: return removeRpmArtifact(node, packageVersion.contentPath!!, context, packageKey, version) } } @@ -649,7 +650,7 @@ class RpmLocalRepository( with(context) { val nodeDeleteRequest = NodeDeleteRequest(projectId, repoName, artifactFullPath, context.userId) - nodeClient.deleteNode(nodeDeleteRequest) + nodeService.deleteNode(nodeDeleteRequest) logger.info("Success to delete node $nodeDeleteRequest") deleteVersion(projectId, repoName, packageKey, version) logger.info("Success to delete version $projectId | $repoName : $packageKey $version") @@ -685,9 +686,9 @@ class RpmLocalRepository( ).data ?: return null val artifactPath = trueVersion.contentPath ?: return null with(context.artifactInfo) { - val jarNode = nodeClient.getNodeDetail( - projectId, repoName, artifactPath - ).data ?: return null + val jarNode = nodeService.getNodeDetail( + ArtifactInfo(projectId, repoName, artifactPath) + ) ?: return null val stageTag = stageClient.query(projectId, repoName, packageKey, version).data val packageVersion = packageClient.findVersionByName( projectId, repoName, packageKey, version @@ -727,7 +728,7 @@ class RpmLocalRepository( select = mutableListOf("name", "folder"), rule = Rule.NestedRule(rule1, Rule.NestedRule.RelationType.AND) ) - val pages = nodeClient.search(queryModel).data!! + val pages = nodeSearchService.search(queryModel) return Page(page, size, pages.totalRecords, pages.records.map { it["name"] as String }) } @@ -762,8 +763,8 @@ class RpmLocalRepository( deep = false, sort = false ) - val nodeInfoPage = nodeClient.listNodePage(repo.projectId, repo.name, rpmNodePath, nodeListOption) - .data ?: break@loop + val nodeInfoPage = + nodeService.listNodePage(ArtifactInfo(repo.projectId, repo.name, rpmNodePath), nodeListOption) if (nodeInfoPage.records.isEmpty()) break@loop logger.info( "populatePackage: found ${nodeInfoPage.records.size}," + @@ -823,7 +824,7 @@ class RpmLocalRepository( } private fun getRpmRepoConf(context: ArtifactContext): RpmRepoConf { - val repositoryInfo = repositoryClient.getRepoInfo(context.projectId, context.repoName).data + val repositoryInfo = repositoryService.getRepoInfo(context.projectId, context.repoName) ?: throw ErrorCodeException( CommonMessageCode.RESOURCE_NOT_FOUND, "${context.projectId}/${context.repoName}" diff --git a/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/job/JobService.kt b/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/job/JobService.kt index ecb4fab141..dcbb404234 100644 --- a/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/job/JobService.kt +++ b/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/job/JobService.kt @@ -36,17 +36,19 @@ import com.tencent.bkrepo.common.api.pojo.Page import com.tencent.bkrepo.common.api.util.HumanReadable import com.tencent.bkrepo.common.api.util.toJsonString import com.tencent.bkrepo.common.artifact.api.ArtifactFile +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException import com.tencent.bkrepo.common.artifact.hash.sha1 import com.tencent.bkrepo.common.artifact.manager.StorageManager import com.tencent.bkrepo.common.artifact.resolve.file.ArtifactFileFactory +import com.tencent.bkrepo.common.metadata.service.node.NodeSearchService +import com.tencent.bkrepo.common.metadata.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.query.enums.OperationType import com.tencent.bkrepo.common.query.model.PageLimit import com.tencent.bkrepo.common.query.model.QueryModel import com.tencent.bkrepo.common.query.model.Rule import com.tencent.bkrepo.common.query.model.Sort -import com.tencent.bkrepo.repository.api.NodeClient -import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import com.tencent.bkrepo.repository.pojo.node.NodeInfo import com.tencent.bkrepo.repository.pojo.node.NodeListOption @@ -84,13 +86,15 @@ import java.io.File import java.io.FileInputStream import java.io.FileOutputStream import java.io.RandomAccessFile +import java.util.Locale import java.util.concurrent.ConcurrentHashMap import javax.xml.parsers.SAXParserFactory @Component class JobService( - private val nodeClient: NodeClient, - private val repositoryClient: RepositoryClient, + private val nodeService: NodeService, + private val nodeSearchService: NodeSearchService, + private val repositoryService: RepositoryService, private val storageManager: StorageManager ) { @@ -98,9 +102,9 @@ class JobService( * 查询下所有rpm仓库 */ fun getAllRpmRepo(): List? { - val repoPage = repositoryClient.pageByType(0, 10, "RPM").data - val total = (repoPage?.totalRecords ?: 10).toInt() - return repositoryClient.pageByType(0, total + 1, "RPM").data?.records + val repoPage = repositoryService.listRepoPageByType("RPM", 0, 10) + val total = repoPage.totalRecords.toInt() + return repositoryService.listRepoPageByType("RPM", 0, total + 1).records } /** @@ -122,7 +126,7 @@ class JobService( if (logger.isDebugEnabled) { logger.debug("queryRepodata: $queryModel") } - val page = nodeClient.queryWithoutCount(queryModel).data!! + val page = nodeSearchService.searchWithoutCount(queryModel) return page.records.map { it["fullPath"] as String } } @@ -130,7 +134,7 @@ class JobService( * 查询rpm仓库属性 */ private fun getRpmRepoConf(project: String, repoName: String): RpmRepoConf { - val repositoryInfo = repositoryClient.getRepoInfo(project, repoName).data + val repositoryInfo = repositoryService.getRepoInfo(project, repoName) ?: throw RpmConfNotFoundException("can not found $project | $repoName conf") val rpmConfiguration = repositoryInfo.configuration return rpmConfiguration.toRpmRepoConf() @@ -274,7 +278,7 @@ class JobService( if (list.size > 2) { val surplusNodes = list.subList(2, list.size) for (node in surplusNodes) { - nodeClient.deleteNode(NodeDeleteRequest(node.projectId, node.repoName, node.fullPath, node.createdBy)) + nodeService.deleteNode(NodeDeleteRequest(node.projectId, node.repoName, node.fullPath, node.createdBy)) logger.info("Success to delete ${node.projectId}/${node.repoName}/${node.fullPath}") } } @@ -282,8 +286,8 @@ class JobService( fun getIndexTypeList(repo: RepositoryDetail, repodataPath: String, indexType: IndexType): List { val target = "-${indexType.value}.xml.gz" - val indexList = nodeClient.listNodePage( - repo.projectId, repo.name, repodataPath, + val indexList = nodeService.listNodePage( + ArtifactInfo(repo.projectId, repo.name, repodataPath), NodeListOption( 1, 100, @@ -292,7 +296,7 @@ class JobService( deep = false, sort = false ) - ).data?.records ?: return mutableListOf() + ).records return indexList.filter { it.name.endsWith(target) }.sortedByDescending { it.lastModifiedDate } } @@ -331,7 +335,7 @@ class JobService( ), rule = Rule.NestedRule(ruleList, Rule.NestedRule.RelationType.AND) ) - var nodeList = nodeClient.queryWithoutCount(queryModel).data!!.records.map { resolveNode(it) } + var nodeList = nodeSearchService.searchWithoutCount(queryModel).records.map { resolveNode(it) } val regex = Regex( "${IndexType.PRIMARY.value}.xml.gz" + "|${IndexType.OTHER.value}.xml.gz" + @@ -342,11 +346,11 @@ class JobService( } // 如果是索引文件则执行 if (nameSuffix.matches(regex)) { - val indexType = IndexType.valueOf(nameSuffix.removeSuffix(".xml.gz").toUpperCase()) + val indexType = IndexType.valueOf(nameSuffix.removeSuffix(".xml.gz").uppercase(Locale.getDefault())) if (nodeList.isEmpty()) { logger.debug("Init [${repo.projectId}|${repo.name}|$repodataPath|${indexType.value} index] ") initIndex(repo, repodataPath, indexType) - nodeList = nodeClient.queryWithoutCount(queryModel).data!!.records.map { resolveNode(it) } + nodeList = nodeSearchService.searchWithoutCount(queryModel).records.map { resolveNode(it) } } if (nodeList.isEmpty()) { throw NodeNotFoundException( @@ -373,11 +377,13 @@ class JobService( ".edu/metadata/rpm\" packages=\"0\">\n" + "" } + IndexType.FILELISTS -> { "\n" + "\n" + "" } + IndexType.OTHER -> { "\n" + "\n" + @@ -424,12 +430,14 @@ class JobService( val markContent = resolveIndexXml(markNodeInfo, indexType, repo) ?: return 0 return XmlStrUtils.insertPackageIndex(randomAccessFile, markContent) } + ArtifactRepeat.DELETE -> { logger.info("delete index of [${repo.projectId}|${repo.name}|${markNodeInfo.fullPath}]") val rpmVersion = markNodeInfo.metadata!!.toRpmVersion(markNodeInfo.fullPath) val uniqueStr = getLocationStr(indexType, rpmVersion, locationStr) return XmlStrUtils.deletePackageIndex(randomAccessFile, indexType, uniqueStr) } + ArtifactRepeat.FULLPATH -> { logger.info("replace index of [${repo.projectId}|${repo.name}|${markNodeInfo.fullPath}]") val rpmVersion = markNodeInfo.metadata!!.toRpmVersion(markNodeInfo.fullPath) @@ -437,6 +445,7 @@ class JobService( val markContent = resolveIndexXml(markNodeInfo, indexType, repo) ?: return 0 return XmlStrUtils.updatePackageIndex(randomAccessFile, indexType, uniqueStr, markContent) } + ArtifactRepeat.FULLPATH_SHA256 -> { logger.info("skip index of [${repo.projectId}|${repo.name}|${markNodeInfo.fullPath}]") return 0 @@ -459,6 +468,7 @@ class JobService( """ } } + IndexType.PRIMARY -> { """""" } @@ -536,7 +546,7 @@ class JobService( if (logger.isDebugEnabled) { logger.debug("queryModel: $queryModel") } - val resultPage = nodeClient.search(queryModel).data!! + val resultPage = nodeSearchService.search(queryModel) with(resultPage) { return Page(pageNumber, pageSize, totalRecords, records.map { resolveNode(it) }) } } @@ -561,8 +571,10 @@ class JobService( if (!failNodeList.isNullOrEmpty() && errorNodeMap.getOrDefault(failNodeList.first().fullPath, 0) > 2 ) { - logger.error("${failNodeList.first()}, " + - "failed times: ${errorNodeMap[failNodeList.first().fullPath]}") + logger.error( + "${failNodeList.first()}, " + + "failed times: ${errorNodeMap[failNodeList.first().fullPath]}" + ) logger.error( "Single update ${indexType.value}: [${repo.projectId}|${repo.name}|$repodataPath] error" ) @@ -701,7 +713,7 @@ class JobService( return if (repeat == ArtifactRepeat.DELETE) { updateIndex(randomAccessFile, markNode, repeat, repo, repodataPath, locationHref, indexType) } else { - val rpmNode = nodeClient.getNodeDetail(markNode.projectId, markNode.repoName, locationStr).data + val rpmNode = nodeService.getNodeDetail(ArtifactInfo(markNode.projectId, markNode.repoName, locationStr)) if (rpmNode == null) { with(markNode) { logger.info("rpm node[$projectId|$repoName|$locationStr] no found, skip index") @@ -725,7 +737,7 @@ class JobService( nodes.forEach { nodeInfo -> with(nodeInfo) { try { - nodeClient.deleteNode(NodeDeleteRequest(projectId, repoName, fullPath, "system")) + nodeService.deleteNode(NodeDeleteRequest(projectId, repoName, fullPath, "system")) logger.info("node[$projectId|$repoName|$fullPath] deleted") } catch (e: Exception) { logger.info("node[$projectId|$repoName|$fullPath] delete exception, ${e.message}") @@ -739,7 +751,7 @@ class JobService( nodes.forEach { nodeInfo -> with(nodeInfo) { try { - nodeClient.updateNode( + nodeService.updateNode( NodeUpdateRequest(projectId, repoName, fullPath, 0L, "system") ) logger.info("node[$projectId|$repoName|$fullPath] update") diff --git a/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/servcie/RpmService.kt b/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/servcie/RpmService.kt index e326d6fd61..01b6c38021 100644 --- a/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/servcie/RpmService.kt +++ b/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/servcie/RpmService.kt @@ -40,13 +40,13 @@ import com.tencent.bkrepo.common.artifact.pojo.RepositoryCategory import com.tencent.bkrepo.common.artifact.pojo.configuration.RepositoryConfiguration import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext +import com.tencent.bkrepo.common.artifact.repository.context.ArtifactRemoveContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactSearchContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactUploadContext -import com.tencent.bkrepo.common.artifact.repository.context.ArtifactRemoveContext import com.tencent.bkrepo.common.artifact.repository.core.ArtifactRepository import com.tencent.bkrepo.common.artifact.repository.core.ArtifactService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.security.permission.Permission -import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.pojo.repo.RepoUpdateRequest import com.tencent.bkrepo.rpm.FILELISTS_XML import com.tencent.bkrepo.rpm.OTHERS_XML @@ -59,7 +59,7 @@ import org.springframework.stereotype.Service @Service class RpmService( - private val repositoryClient: RepositoryClient + private val repositoryService: RepositoryService ) : ArtifactService() { // groups 中不允许的元素 @@ -105,7 +105,7 @@ class RpmService( oldGroups.addAll(groups) rpmConfiguration.settings["groupXmlSet"] = oldGroups val repoUpdateRequest = createRepoUpdateRequest(context, rpmConfiguration) - repositoryClient.updateRepo(repoUpdateRequest) + repositoryService.updateRepo(repoUpdateRequest) val repository = ArtifactContextHolder.getRepository(RepositoryCategory.LOCAL) (repository as RpmLocalRepository).flushAllRepoData(context) } @@ -121,7 +121,7 @@ class RpmService( oldGroups.removeAll(groups) rpmConfiguration.settings["groupXmlSet"] = oldGroups val repoUpdateRequest = createRepoUpdateRequest(context, rpmConfiguration) - repositoryClient.updateRepo(repoUpdateRequest) + repositoryService.updateRepo(repoUpdateRequest) val repository = ArtifactContextHolder.getRepository(RepositoryCategory.LOCAL) (repository as RpmLocalRepository).flushAllRepoData(context) } @@ -147,7 +147,7 @@ class RpmService( } private fun getRpmRepoConf(project: String, repoName: String): RepositoryConfiguration { - val repositoryInfo = repositoryClient.getRepoInfo(project, repoName).data + val repositoryInfo = repositoryService.getRepoInfo(project, repoName) ?: throw RpmConfNotFoundException("can not found $project | $repoName conf") return repositoryInfo.configuration } diff --git a/src/backend/s3/biz-s3/src/main/kotlin/com/tencent/bkrepo/s3/service/S3ObjectService.kt b/src/backend/s3/biz-s3/src/main/kotlin/com/tencent/bkrepo/s3/service/S3ObjectService.kt index f3e6695951..2b9a4f9b0c 100644 --- a/src/backend/s3/biz-s3/src/main/kotlin/com/tencent/bkrepo/s3/service/S3ObjectService.kt +++ b/src/backend/s3/biz-s3/src/main/kotlin/com/tencent/bkrepo/s3/service/S3ObjectService.kt @@ -36,6 +36,7 @@ import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.api.constant.StringPool.SLASH import com.tencent.bkrepo.common.api.constant.ensureSuffix import com.tencent.bkrepo.common.artifact.api.ArtifactFile +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext @@ -43,10 +44,11 @@ import com.tencent.bkrepo.common.artifact.repository.context.ArtifactUploadConte import com.tencent.bkrepo.common.artifact.repository.core.ArtifactService import com.tencent.bkrepo.common.generic.configuration.AutoIndexRepositorySettings import com.tencent.bkrepo.common.metadata.service.metadata.MetadataService +import com.tencent.bkrepo.common.metadata.service.node.NodeSearchService +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.query.enums.OperationType import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.HeaderUtils -import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import com.tencent.bkrepo.repository.pojo.metadata.MetadataSaveRequest import com.tencent.bkrepo.repository.pojo.node.NodeDetail @@ -73,7 +75,8 @@ import java.net.URLDecoder */ @Service class S3ObjectService( - private val nodeClient: NodeClient, + private val nodeService: NodeService, + private val nodeSearchService: NodeSearchService, private val metadataService: MetadataService ) : ArtifactService() { @@ -127,7 +130,7 @@ class S3ObjectService( val folders = if (delimiter.isNotEmpty()) { val folderQueryBuilder = nodeQueryBuilder.newBuilder().excludeFile().select(NodeDetail::fullPath.name) - nodeClient.queryWithoutCount(folderQueryBuilder.build()).data!!.records + nodeSearchService.searchWithoutCount(folderQueryBuilder.build()).records .map { it[NodeDetail::fullPath.name].toString().removePrefix(SLASH).ensureSuffix(SLASH) } @@ -140,18 +143,19 @@ class S3ObjectService( excludeFolder() } } - val data = nodeClient.search(queryBuilder.build()).data!! + val data = nodeSearchService.search(queryBuilder.build()) val currentNode = if (data.pageNumber.toLong() == data.totalPages || data.totalPages == 0L) { - nodeClient.getNodeDetail(projectId, repoName, PathUtils.normalizeFullPath(queryPrefix)).data?.run { - mapOf( - NodeDetail::createdBy.name to createdBy, - NodeDetail::folder.name to folder, - NodeDetail::fullPath.name to fullPath, - NodeDetail::lastModifiedDate.name to lastModifiedDate, - NodeDetail::md5.name to md5, - NodeDetail::size.name to size - ) - } + nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, PathUtils.normalizeFullPath(queryPrefix))) + ?.run { + mapOf( + NodeDetail::createdBy.name to createdBy, + NodeDetail::folder.name to folder, + NodeDetail::fullPath.name to fullPath, + NodeDetail::lastModifiedDate.name to lastModifiedDate, + NodeDetail::md5.name to md5, + NodeDetail::size.name to size + ) + } } else { null } @@ -173,7 +177,7 @@ class S3ObjectService( overwrite = true, operator = SecurityUtils.getUserId() ) - var dstNode = nodeClient.copyNode(copyRequest).data!! + var dstNode = nodeService.copyNode(copyRequest) dstNode = replaceMetadata(dstNode) return CopyObjectResult( eTag = "\"${dstNode.md5}\"", @@ -187,7 +191,7 @@ class S3ObjectService( fun deleteObject(artifactInfo: S3ArtifactInfo) { with(artifactInfo) { - nodeClient.deleteNode(NodeDeleteRequest( + nodeService.deleteNode(NodeDeleteRequest( projectId = projectId, repoName = repoName, fullPath = getArtifactFullPath(), diff --git a/src/backend/svn/biz-svn/src/main/kotlin/com/tencent/bkrepo/svn/utils/SvnProxyHelper.kt b/src/backend/svn/biz-svn/src/main/kotlin/com/tencent/bkrepo/svn/utils/SvnProxyHelper.kt index da17191126..4a59f00b25 100644 --- a/src/backend/svn/biz-svn/src/main/kotlin/com/tencent/bkrepo/svn/utils/SvnProxyHelper.kt +++ b/src/backend/svn/biz-svn/src/main/kotlin/com/tencent/bkrepo/svn/utils/SvnProxyHelper.kt @@ -27,17 +27,17 @@ package com.tencent.bkrepo.svn.utils -import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder +import com.tencent.bkrepo.common.artifact.pojo.RepositoryId import javax.servlet.http.HttpServletRequest object SvnProxyHelper { - fun getRepoId(request: HttpServletRequest): ArtifactContextHolder.RepositoryId? { + fun getRepoId(request: HttpServletRequest): RepositoryId? { val paths = request.servletPath.split("/") if (paths.size <= 2) { return null } val projectId = paths[1] val repoName = paths[2] - return ArtifactContextHolder.RepositoryId(projectId, repoName) + return RepositoryId(projectId, repoName) } } diff --git a/src/backend/webhook/biz-webhook/src/main/kotlin/com/tencent/bkrepo/webhook/payload/builder/node/NodePayloadBuilder.kt b/src/backend/webhook/biz-webhook/src/main/kotlin/com/tencent/bkrepo/webhook/payload/builder/node/NodePayloadBuilder.kt index 3ce37579e3..7fb48d0f26 100644 --- a/src/backend/webhook/biz-webhook/src/main/kotlin/com/tencent/bkrepo/webhook/payload/builder/node/NodePayloadBuilder.kt +++ b/src/backend/webhook/biz-webhook/src/main/kotlin/com/tencent/bkrepo/webhook/payload/builder/node/NodePayloadBuilder.kt @@ -28,8 +28,9 @@ package com.tencent.bkrepo.webhook.payload.builder.node import com.tencent.bkrepo.common.api.exception.NotFoundException +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.event.base.EventType -import com.tencent.bkrepo.repository.api.NodeClient +import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.webhook.exception.WebHookMessageCode import com.tencent.bkrepo.webhook.payload.builder.EventPayloadBuilder @@ -42,10 +43,10 @@ abstract class NodePayloadBuilder( ) { @Autowired - private lateinit var nodeClient: NodeClient + private lateinit var nodeService: NodeService fun getNode(projectId: String, repoName: String, fullPath: String): NodeDetail { - return nodeClient.getNodeDetail(projectId, repoName, fullPath).data + return nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, fullPath)) ?: throw NotFoundException(WebHookMessageCode.WEBHOOK_NODE_NOT_FOUND) } } diff --git a/src/backend/webhook/biz-webhook/src/main/kotlin/com/tencent/bkrepo/webhook/payload/builder/repo/RepoPayloadBuilder.kt b/src/backend/webhook/biz-webhook/src/main/kotlin/com/tencent/bkrepo/webhook/payload/builder/repo/RepoPayloadBuilder.kt index 626c4aca68..dbd58b7d77 100644 --- a/src/backend/webhook/biz-webhook/src/main/kotlin/com/tencent/bkrepo/webhook/payload/builder/repo/RepoPayloadBuilder.kt +++ b/src/backend/webhook/biz-webhook/src/main/kotlin/com/tencent/bkrepo/webhook/payload/builder/repo/RepoPayloadBuilder.kt @@ -29,7 +29,7 @@ package com.tencent.bkrepo.webhook.payload.builder.repo import com.tencent.bkrepo.common.api.exception.NotFoundException import com.tencent.bkrepo.common.artifact.event.base.EventType -import com.tencent.bkrepo.repository.api.RepositoryClient +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.repository.pojo.repo.RepositoryInfo import com.tencent.bkrepo.webhook.exception.WebHookMessageCode import com.tencent.bkrepo.webhook.payload.builder.EventPayloadBuilder @@ -42,10 +42,10 @@ abstract class RepoPayloadBuilder( ) { @Autowired - private lateinit var repositoryClient: RepositoryClient + private lateinit var repositoryService: RepositoryService fun getRepo(projectId: String, repoName: String): RepositoryInfo { - return repositoryClient.getRepoInfo(projectId, repoName).data + return repositoryService.getRepoInfo(projectId, repoName) ?: throw NotFoundException(WebHookMessageCode.WEBHOOK_REPO_NOT_FOUND) } }