Skip to content

Commit

Permalink
feat: package自定义搜索支持版本名、元数据、checksum搜索 #1717
Browse files Browse the repository at this point in the history
(cherry picked from commit f44cef4faca90b83052243c709d735104154de3e)
  • Loading branch information
scplsy committed Jan 31, 2024
1 parent d71b630 commit f2b7d0d
Show file tree
Hide file tree
Showing 13 changed files with 379 additions and 21 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -52,21 +52,22 @@ open class CommonQueryContext(
var repoList: List<RepositoryInfo>? = null

fun findProjectId(): String {
if (projectId != null) {
return projectId!!
}
return if (projectId != null) projectId!! else find(TNode::projectId.name).apply { projectId = this }
}

protected fun find(field: String): String {
val rule = queryModel.rule
if (rule is Rule.NestedRule && rule.relation == Rule.NestedRule.RelationType.AND) {
findProjectIdRule(rule.rules)?.let {
return it.value.toString().apply { projectId = this }
findRule(rule.rules, field)?.let {
return it.value.toString()
}
}
throw ErrorCodeException(CommonMessageCode.PARAMETER_MISSING, "projectId")
throw ErrorCodeException(CommonMessageCode.PARAMETER_MISSING, field)
}

private fun findProjectIdRule(rules: List<Rule>): Rule.QueryRule? {
private fun findRule(rules: List<Rule>, field: String): Rule.QueryRule? {
for (rule in rules) {
if (rule is Rule.QueryRule && rule.field == TNode::projectId.name) {
if (rule is Rule.QueryRule && rule.field == field) {
return rule
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,12 +39,14 @@ import com.tencent.bkrepo.repository.constant.METADATA_PREFIX
import com.tencent.bkrepo.repository.model.TMetadata
import com.tencent.bkrepo.repository.model.TNode
import org.springframework.data.mongodb.core.query.Criteria
import org.springframework.stereotype.Component

/**
* 元数据规则拦截器
*
* 条件构造器中传入元数据的条件是`metadata.key=value`,需要适配成mongodb的查询条件
*/
@Component
class MetadataRuleInterceptor : QueryRuleInterceptor {

override fun match(rule: Rule): Boolean {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,8 @@ class NodeQueryInterpreter @Autowired @Lazy constructor(
private val permissionManager: PermissionManager,
private val repoNameRuleInterceptor: RepoNameRuleInterceptor,
private val repoTypeRuleInterceptor: RepoTypeRuleInterceptor,
private val localDatetimeRuleInterceptor: LocalDatetimeRuleInterceptor
private val localDatetimeRuleInterceptor: LocalDatetimeRuleInterceptor,
private val metadataRuleInterceptor: MetadataRuleInterceptor
) : CommonQueryInterpreter(permissionManager) {

@PostConstruct
Expand All @@ -60,7 +61,7 @@ class NodeQueryInterpreter @Autowired @Lazy constructor(
addModelInterceptor(SelectFieldInterceptor())
addRuleInterceptor(repoTypeRuleInterceptor)
addRuleInterceptor(repoNameRuleInterceptor)
addRuleInterceptor(MetadataRuleInterceptor())
addRuleInterceptor(metadataRuleInterceptor)
addRuleInterceptor(localDatetimeRuleInterceptor)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,4 +41,13 @@ class PackageQueryContext(
override var permissionChecked: Boolean = false,
override val mongoQuery: Query,
override val interpreter: MongoQueryInterpreter
) : CommonQueryContext(queryModel, permissionChecked, mongoQuery, interpreter)
) : CommonQueryContext(queryModel, permissionChecked, mongoQuery, interpreter) {

private var repoType: String? = null

val matchedVersions: MutableMap<String, MutableSet<String>> = mutableMapOf()

fun findRepoType(): String {
return if (repoType != null) repoType!! else find("repoType").apply { repoType = this }
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@ import com.tencent.bkrepo.common.query.model.QueryModel
import com.tencent.bkrepo.common.security.manager.PermissionManager
import com.tencent.bkrepo.repository.search.common.CommonQueryInterpreter
import com.tencent.bkrepo.repository.search.common.LocalDatetimeRuleInterceptor
import com.tencent.bkrepo.repository.search.common.MetadataRuleInterceptor
import com.tencent.bkrepo.repository.search.common.ModelValidateInterceptor
import com.tencent.bkrepo.repository.search.common.RepoNameRuleInterceptor
import com.tencent.bkrepo.repository.search.common.RepoTypeRuleInterceptor
Expand All @@ -50,7 +49,10 @@ class PackageSearchInterpreter(
private val permissionManager: PermissionManager,
private val repoNameRuleInterceptor: RepoNameRuleInterceptor,
private val repoTypeRuleInterceptor: RepoTypeRuleInterceptor,
private val localDatetimeRuleInterceptor: LocalDatetimeRuleInterceptor
private val localDatetimeRuleInterceptor: LocalDatetimeRuleInterceptor,
private val versionNameRuleInterceptor: VersionNameRuleInterceptor,
private val versionMetadataRuleInterceptor: VersionMetadataRuleInterceptor,
private val versionChecksumRuleInterceptor: VersionChecksumRuleInterceptor
) : CommonQueryInterpreter(permissionManager) {

@PostConstruct
Expand All @@ -59,8 +61,10 @@ class PackageSearchInterpreter(
addModelInterceptor(SelectFieldInterceptor())
addRuleInterceptor(repoTypeRuleInterceptor)
addRuleInterceptor(repoNameRuleInterceptor)
addRuleInterceptor(MetadataRuleInterceptor())
addRuleInterceptor(versionMetadataRuleInterceptor)
addRuleInterceptor(localDatetimeRuleInterceptor)
addRuleInterceptor(versionNameRuleInterceptor)
addRuleInterceptor(versionChecksumRuleInterceptor)
}

override fun initContext(queryModel: QueryModel, mongoQuery: Query): QueryContext {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
/*
* Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
*
* Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved.
*
* BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
*
* A copy of the MIT License is included in this file.
*
*
* Terms of the MIT License:
* ---------------------------------------------------
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/

package com.tencent.bkrepo.repository.search.packages

import com.tencent.bkrepo.common.api.exception.ErrorCodeException
import com.tencent.bkrepo.common.api.message.CommonMessageCode
import com.tencent.bkrepo.common.artifact.pojo.RepositoryType
import com.tencent.bkrepo.common.query.enums.OperationType
import com.tencent.bkrepo.common.query.model.Rule
import com.tencent.bkrepo.common.security.util.SecurityUtils
import com.tencent.bkrepo.repository.dao.NodeDao
import com.tencent.bkrepo.repository.dao.PackageVersionDao
import com.tencent.bkrepo.repository.model.TNode
import com.tencent.bkrepo.repository.model.TPackageVersion
import com.tencent.bkrepo.repository.pojo.repo.RepoListOption
import com.tencent.bkrepo.repository.service.repo.RepositoryService
import org.springframework.data.mongodb.core.query.Criteria
import org.springframework.data.mongodb.core.query.Query
import org.springframework.data.mongodb.core.query.inValues
import org.springframework.data.mongodb.core.query.isEqualTo
import org.springframework.stereotype.Component

/**
* 版本Checksum规则拦截器, 查询包版本Checksum需要在嵌套查询规则列表中指定projectId和repoType条件,且均为EQ操作
*/
@Component
class VersionChecksumRuleInterceptor(
override val packageVersionDao: PackageVersionDao,
private val nodeDao: NodeDao,
private val repositoryService: RepositoryService
) : VersionRuleInterceptor(packageVersionDao) {

override fun match(rule: Rule): Boolean {
return rule is Rule.QueryRule && rule.field in CHECKSUM_FIELDS
}

override fun getVersionCriteria(rule: Rule, context: PackageQueryContext): Criteria {
with(rule as Rule.QueryRule) {
if (operation != OperationType.EQ) {
throw ErrorCodeException(CommonMessageCode.METHOD_NOT_ALLOWED, "$field only support EQ operation type.")
}
val projectId = context.findProjectId()
val repoType = context.findRepoType().toUpperCase()
val userId = SecurityUtils.getUserId()
// 筛选有权限的仓库列表, docker和oci仓库互相兼容
val repoList = if (repoType in listOf(RepositoryType.DOCKER.name, RepositoryType.OCI.name)) {
val dockerRepoList = repositoryService.listPermissionRepo(
userId, projectId, RepoListOption(type = RepositoryType.DOCKER.name)
)
val ociRepoList = repositoryService.listPermissionRepo(
userId, projectId, RepoListOption(type = RepositoryType.OCI.name)
)
dockerRepoList + ociRepoList
} else
repositoryService.listPermissionRepo(userId, projectId, RepoListOption(type = repoType))
// 从有权限的仓库列表查询符合checksum规则的节点
val nodeQuery = Query(
Criteria.where(TNode::projectId.name).isEqualTo(projectId)
.and(TNode::repoName.name).inValues(repoList.map { it.name })
.and(field).isEqualTo(value.toString())
)
val fullPaths = queryRecords(nodeQuery) { q -> nodeDao.find(q) }.map { it.fullPath }
// 转换为package_version查询条件
return if (repoType == RepositoryType.DOCKER.name || repoType == RepositoryType.OCI.name) {
Criteria.where(TPackageVersion::manifestPath.name).inValues(fullPaths)
} else {
Criteria.where(TPackageVersion::artifactPath.name).inValues(fullPaths)
}
}
}

companion object {
private val CHECKSUM_FIELDS = arrayOf(
TNode::sha256.name,
TNode::md5.name
)
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
/*
* Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
*
* Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved.
*
* BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
*
* A copy of the MIT License is included in this file.
*
*
* Terms of the MIT License:
* ---------------------------------------------------
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/

package com.tencent.bkrepo.repository.search.packages

import com.tencent.bkrepo.common.query.model.Rule
import com.tencent.bkrepo.repository.dao.PackageVersionDao
import com.tencent.bkrepo.repository.search.common.MetadataRuleInterceptor
import org.springframework.stereotype.Component

/**
* 版本元数据规则拦截器
*/
@Component
class VersionMetadataRuleInterceptor(
override val packageVersionDao: PackageVersionDao,
private val metadataRuleInterceptor: MetadataRuleInterceptor
) : VersionRuleInterceptor(packageVersionDao) {

override fun match(rule: Rule) = metadataRuleInterceptor.match(rule)
override fun getVersionCriteria(rule: Rule, context: PackageQueryContext) =
metadataRuleInterceptor.intercept(rule, context)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
/*
* Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
*
* Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved.
*
* BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
*
* A copy of the MIT License is included in this file.
*
*
* Terms of the MIT License:
* ---------------------------------------------------
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/

package com.tencent.bkrepo.repository.search.packages

import com.tencent.bkrepo.common.api.exception.ErrorCodeException
import com.tencent.bkrepo.common.api.message.CommonMessageCode
import com.tencent.bkrepo.common.query.enums.OperationType
import com.tencent.bkrepo.common.query.model.Rule
import com.tencent.bkrepo.repository.dao.PackageVersionDao
import com.tencent.bkrepo.repository.model.TPackageVersion
import org.springframework.data.mongodb.core.query.Criteria
import org.springframework.stereotype.Component

/**
* 版本号规则拦截器
*/
@Component
class VersionNameRuleInterceptor(
override val packageVersionDao: PackageVersionDao
) : VersionRuleInterceptor(packageVersionDao) {

override fun match(rule: Rule): Boolean {
return rule is Rule.QueryRule && rule.field == "version"
}

override fun getVersionCriteria(rule: Rule, context: PackageQueryContext): Criteria {
with(rule as Rule.QueryRule) {
val versionQueryRule = when (operation) {
OperationType.IN ->
rule.copy(field = TPackageVersion::name.name, value = (value as List<*>).map { it.toString() })

in SUPPORT_OPERATIONS -> rule.copy(field = TPackageVersion::name.name, value = value.toString())
else -> throw ErrorCodeException(
CommonMessageCode.METHOD_NOT_ALLOWED,
"$field only support ${SUPPORT_OPERATIONS.map { it.name }} operation type."
)
}.toFixed()
return context.interpreter.resolveRule(versionQueryRule, context)
}
}

companion object {
private val SUPPORT_OPERATIONS = arrayOf(
OperationType.EQ,
OperationType.IN,
OperationType.MATCH,
OperationType.MATCH_I
)
}
}
Loading

0 comments on commit f2b7d0d

Please sign in to comment.