Merge pull request #6008 from tangruotian/issue_5922_new

feat: 跨项目使用构建机支持分享给项目组下的所有项目 #5922

src/backend/ci/core/common/common-scm/src/main/kotlin/com/tencent/devops/scm/pojo/GitProjectGroupInfo.kt

@@ -0,0 +1,137 @@
+package com.tencent.devops.scm.pojo
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties
+import com.fasterxml.jackson.annotation.JsonProperty
+import io.swagger.annotations.ApiModel
+
+/**
+{
+"id": 25123,
+"name": "test00",
+"path": "test00",
+"web_url": "https://git.xxx.com/groups/test00",
+"description": "xxx",
+"avatar_url": null,
+"projects": [
+{
+"id": 64612,
+"description": "0711",
+"public": false,
+"archived": false,
+"visibility_level": 0,
+"namespace": {
+"id": 25123,
+"name": "test00",
+"path": "test00",
+"web_url": "https://git.xxx.com/groups/test00",
+"description": "xxx",
+"avatar_url": null
+},
+"name": "test1",
+"name_with_namespace": "test00/test1",
+"path": "test1",
+"path_with_namespace": "test00/test1",
+"default_branch": "master",
+"ssh_url_to_repo": "[email protected]:test00/test1.git",
+"http_url_to_repo": "http://git.xxxxx.com/test00/test1.git",
+"https_url_to_repo": "https://git.xxxxx.com/test00/test1.git",
+"web_url": "https://git.xxxxx.com/test00/test1",
+"tag_list": [],
+"issues_enabled": true,
+"merge_requests_enabled": true,
+"wiki_enabled": true,
+"snippets_enabled": true,
+"review_enabled": true,
+"fork_enabled": false,
+"tag_name_regex": null,
+"tag_create_push_level": 30,
+"created_at": "2017-07-11T06:29:24+0000",
+"last_activity_at": "2017-07-11T06:46:33+0000",
+"creator_id": 18123,
+"avatar_url": "https://git.xxxxx.com/uploads/project/avatar/123",
+"watchs_count": 2,
+"stars_count": 0,
+"forks_count": 0,
+"config_storage": {
+"limit_lfs_file_size": 500,
+"limit_size": 100000,
+"limit_file_size": 100000,
+"limit_lfs_size": 100000
+},
+"forked_from_project": "Forked Project not found",
+"statistics": {
+"commit_count": 3,
+"repository_size": 0.004
+}
+},
+"sub_projects": [{
+"id": 66062,
+"description": "gggg",
+"public": false,
+"archived": false,
+"visibility_level": 0,
+"namespace": {
+"id": 25123,
+"name": "test00",
+"path": "test00",
+"web_url": "https://git.xxxxx.com/groups/test00",
+"description": "xxx",
+"avatar_url": null
+},
+"name": "test02",
+"name_with_namespace": "test00/test02",
+"path": "test02",
+"path_with_namespace": "test00/test02",
+"default_branch": "master",
+"ssh_url_to_repo": "[email protected]:test00/test02.git",
+"http_url_to_repo": "http://git.xxxxx.com/test00/test02.git",
+"https_url_to_repo": "https://git.xxxxx.com/test00/test02.git",
+"web_url": "https://git.xxxxx.com/test00/test02",
+"tag_list": [],
+"issues_enabled": true,
+"merge_requests_enabled": true,
+"wiki_enabled": true,
+"snippets_enabled": true,
+"review_enabled": true,
+"fork_enabled": false,
+"tag_name_regex": null,
+"tag_create_push_level": 30,
+"created_at": "2018-07-19T07:13:22+0000",
+"last_activity_at": "2018-07-19T07:13:23+0000",
+"creator_id": 18604,
+"avatar_url": "https://git.xxxxx.com/uploads/project/avatar/66062",
+"watchs_count": 2,
+"stars_count": 0,
+"forks_count": 0,
+"config_storage": {
+"limit_lfs_file_size": 500,
+"limit_size": 100000,
+"limit_file_size": 100000,
+"limit_lfs_size": 100000
+},
+"forked_from_project": "Forked Project not found",
+"statistics": {
+"commit_count": 0,
+"repository_size": 0
+}
+}]
+]
+}
+ */
+
+@ApiModel("git 项目组信息")
+@JsonIgnoreProperties(ignoreUnknown = true)
+data class GitProjectGroupInfo(
+    val id: Long,
+    val name: String,
+    val path: String,
+    val projects: List<GitProjectGroupProject>,
+    @JsonProperty("sub_projects")
+    val subProjects: List<GitProjectGroupProject>?
+)
+
+@ApiModel("git 项目组项目信息")
+@JsonIgnoreProperties(ignoreUnknown = true)
+data class GitProjectGroupProject(
+    val id: String
+)

src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/dispatcher/agent/ThirdPartyAgentDispatcher.kt

@@ -29,6 +29,8 @@ package com.tencent.devops.dispatch.service.dispatcher.agent
 
 import com.tencent.devops.common.api.enums.AgentStatus
 import com.tencent.devops.common.api.exception.InvalidParamException
+import com.tencent.devops.common.api.exception.RemoteServiceException
+import com.tencent.devops.common.api.pojo.Result
 import com.tencent.devops.common.client.Client
 import com.tencent.devops.common.event.dispatcher.pipeline.PipelineEventDispatcher
 import com.tencent.devops.common.log.utils.BuildLogPrinter
@@ -312,8 +314,24 @@ class ThirdPartyAgentDispatcher @Autowired constructor(
                     .getAgentsByEnvId(event.projectId, dispatchType.envName)
             }
             AgentType.NAME -> {
-                client.get(ServiceThirdPartyAgentResource::class)
-                    .getAgentsByEnvName(event.projectId, dispatchType.envName)
+                try {
+                    client.get(ServiceThirdPartyAgentResource::class)
+                        .getAgentsByEnvName(event.projectId, dispatchType.envName)
+                } catch (e: Exception) {
+                    onFailBuild(
+                        client = client,
+                        buildLogPrinter = buildLogPrinter,
+                        event = event,
+                        errorType = ErrorCodeEnum.GET_VM_ERROR.errorType,
+                        errorCode = ErrorCodeEnum.GET_VM_ERROR.errorCode,
+                        errorMsg = if (e is RemoteServiceException) {
+                            e.errorMessage
+                        } else {
+                            e.message ?: "${ErrorCodeEnum.GET_VM_ERROR.formatErrorMessage}(${dispatchType.envName})"
+                        }
+                    )
+                    Result(null)
+                }
             }
         }
 

src/backend/ci/core/dockerhost/biz-dockerhost/src/main/kotlin/com/tencent/devops/dockerhost/services/AbstractDockerHostBuildService.kt

@@ -361,13 +361,14 @@ abstract class AbstractDockerHostBuildService constructor(
             )?.data
 
             // 针对白名单项目做bazel cache处理
+            // 去掉install目录，只同步cache目录，及其所有子目录
             if (qpcGitProjectList != null && qpcGitProjectList.isNotEmpty()) {
                 val upperDir = "${getWorkspace(pipelineId, vmSeqId, poolNo, dockerHostConfig.bazelUpperPath!!)}upper"
                 CommandLineUtils.execute(
                     command = "time flock -xn ${dockerHostConfig.bazelLowerPath}  " +
-                            "rsync --stats -ah --ignore-errors --include=\"cache/\" " +
-                            "--include=\"install/\" --exclude=\"*/\" " +
-                            " $upperDir/ ${dockerHostConfig.bazelLowerPath}/",
+                        "rsync --stats -ah --ignore-errors  --include=\"cache/\" --include=\"cache/*\" " +
+                        " --exclude=\"/*\" " +
+                        " $upperDir/ ${dockerHostConfig.bazelLowerPath}/",
                     workspace = File(dockerHostConfig.bazelLowerPath!!),
                     print2Logger = true
                 )

src/backend/ci/core/environment/biz-environment/build.gradle.kts

@@ -31,6 +31,7 @@ dependencies {
     api(project(":core:project:api-project"))
     api(project(":core:environment:model-environment"))
     api(project(":core:notify:api-notify"))
+    api(project(":core:repository:api-repository"))
     api(project(":core:common:common-client"))
     api(project(":core:common:common-service"))
     api(project(":core:common:common-web"))

src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/dao/EnvShareProjectDao.kt

@@ -88,6 +88,19 @@ class EnvShareProjectDao {
         }
     }
 
+    fun list(
+        dslContext: DSLContext,
+        envName: String,
+        mainProjectId: String
+    ): List<TEnvShareProjectRecord> {
+        with(TEnvShareProject.T_ENV_SHARE_PROJECT) {
+            return dslContext.selectFrom(this)
+                .where(ENV_NAME.eq(envName))
+                .and(MAIN_PROJECT_ID.eq(mainProjectId))
+                .fetch()
+        }
+    }
+
     @SuppressWarnings("LongParameterList")
     fun batchSave(
         dslContext: DSLContext,

src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdPartyAgent/ThirdPartyAgentMgrService.kt

@@ -31,6 +31,7 @@ import com.fasterxml.jackson.databind.ObjectMapper
 import com.fasterxml.jackson.module.kotlin.readValue
 import com.tencent.devops.common.api.enums.AgentAction
 import com.tencent.devops.common.api.enums.AgentStatus
+import com.tencent.devops.common.api.exception.CustomException
 import com.tencent.devops.common.api.exception.ErrorCodeException
 import com.tencent.devops.common.api.exception.PermissionForbiddenException
 import com.tencent.devops.common.api.pojo.AgentResult
@@ -67,6 +68,7 @@ import com.tencent.devops.environment.permission.EnvironmentPermissionService
 import com.tencent.devops.environment.pojo.EnvVar
 import com.tencent.devops.environment.pojo.enums.NodeStatus
 import com.tencent.devops.environment.pojo.enums.NodeType
+import com.tencent.devops.environment.pojo.enums.SharedEnvType
 import com.tencent.devops.environment.pojo.thirdPartyAgent.AgentBuildDetail
 import com.tencent.devops.environment.pojo.thirdPartyAgent.AgentTask
 import com.tencent.devops.environment.pojo.thirdPartyAgent.HeartbeatResponse
@@ -86,6 +88,9 @@ import com.tencent.devops.environment.utils.FileMD5CacheUtils.getAgentJarFile
 import com.tencent.devops.environment.utils.FileMD5CacheUtils.getFileMD5
 import com.tencent.devops.environment.utils.NodeStringIdUtils
 import com.tencent.devops.model.environment.tables.records.TEnvironmentThirdpartyAgentRecord
+import com.tencent.devops.repository.api.ServiceOauthResource
+import com.tencent.devops.repository.api.scm.ServiceGitResource
+import com.tencent.devops.repository.pojo.enums.TokenTypeEnum
 import org.jooq.DSLContext
 import org.jooq.impl.DSL
 import org.slf4j.LoggerFactory
@@ -95,6 +100,7 @@ import java.time.LocalDateTime
 import java.time.format.DateTimeFormatter
 import java.util.Date
 import javax.ws.rs.NotFoundException
+import javax.ws.rs.core.Response
 
 @Service
 @Suppress("ALL")
@@ -665,17 +671,65 @@ class ThirdPartyAgentMgrService @Autowired(required = false) constructor(
         }
         val sharedProjectId = sharedProjEnv[0]
         val sharedEnvName = sharedProjEnv[1]
-        val sharedEnvRecord = envShareProjectDao.get(dslContext, sharedEnvName, projectId, sharedProjectId)
+
+        envDao.getByEnvName(
+            dslContext = dslContext,
+            projectId = sharedProjectId,
+            envName = sharedEnvName
+        ) ?: throw CustomException(Response.Status.NOT_FOUND, "第三方构建机环境不存在($projectEnvName)")
+
+        val sharedEnvRecord = envShareProjectDao.list(
+            dslContext = dslContext,
+            envName = sharedEnvName,
+            mainProjectId = sharedProjectId
+        )
         if (sharedEnvRecord.isEmpty()) {
             logger.info("env name not exists, envName: $sharedEnvName, projectId：$projectId, " +
                 "mainProjectId: $sharedProjectId")
-            return emptyList()
+            throw CustomException(Response.Status.FORBIDDEN, "无权限使用第三方构建机环境($projectEnvName)")
         }
         logger.info("sharedEnvRecord size: ${sharedEnvRecord.size}")
         val sharedThirdPartyAgents = mutableListOf<ThirdPartyAgent>()
-        sharedEnvRecord.forEach {
-            val envRecord = envDao.getByEnvName(dslContext, it.mainProjectId, sharedEnvName) ?: return@forEach
-            sharedThirdPartyAgents.addAll(getAgentByEnvId(it.mainProjectId, HashUtil.encodeLongId(envRecord.envId)))
+
+        run outSide@{
+            // 优先进行单个项目的匹配
+            sharedEnvRecord.sortedByDescending { it.type }.forEach nextRecord@{
+                // 对于分享的单独项目则查看是否是同一个
+                if (it.type == SharedEnvType.PROJECT.name && it.sharedProjectId != projectId) {
+                    return@nextRecord
+                }
+
+                // 通过项目组获取所有项目，判断当前项目是否处于被分享的项目组中
+                if (it.type == SharedEnvType.GROUP.name) {
+                    val projectsInGroups = try {
+                        val token = client.get(ServiceOauthResource::class).gitGet(it.creator).data?.accessToken
+                            ?: throw NotFoundException("cannot found oauth access token for user(${it.creator})")
+                        client.get(ServiceGitResource::class).getProjectGroupInfo(
+                            id = it.sharedProjectId.removePrefix("git_"),
+                            includeSubgroups = true,
+                            token = token,
+                            tokenType = TokenTypeEnum.OAUTH
+                        ).data
+                    } catch (e: Exception) {
+                        logger.warn("$projectId $projectEnvName get share project error: ${e.message}")
+                        null
+                    }
+                    val gitProjectId = projectId.removePrefix("git_")
+                    projectsInGroups?.projects?.filter { project -> project.id == gitProjectId }?.ifEmpty {
+                        projectsInGroups.subProjects?.filter { subProject -> subProject.id == gitProjectId }?.ifEmpty {
+                            return@nextRecord
+                        }
+                    }
+                }
+
+                val envRecord = envDao.getByEnvName(dslContext, it.mainProjectId, sharedEnvName) ?: return@nextRecord
+                sharedThirdPartyAgents.addAll(getAgentByEnvId(it.mainProjectId, HashUtil.encodeLongId(envRecord.envId)))
+                // 找到了环境可用就可以退出了
+                return@outSide
+            }
+        }
+        if (sharedThirdPartyAgents.isEmpty()) {
+            throw CustomException(Response.Status.FORBIDDEN, "无权限使用第三方构建机环境($projectEnvName)")
         }
         logger.info("sharedThirdPartyAgents size: ${sharedThirdPartyAgents.size}")
         return sharedThirdPartyAgents

src/backend/ci/core/environment/model-environment/build.gradle.kts

@@ -27,5 +27,4 @@
 
 plugins {
     `task-gen-jooq`
-
 }

src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/api/scm/ServiceGitResource.kt

@@ -43,6 +43,7 @@ import com.tencent.devops.repository.pojo.oauth.GitToken
 import com.tencent.devops.scm.code.git.api.GitBranch
 import com.tencent.devops.scm.code.git.api.GitTag
 import com.tencent.devops.scm.pojo.GitCommit
+import com.tencent.devops.scm.pojo.GitProjectGroupInfo
 import com.tencent.devops.scm.pojo.GitRepositoryResp
 import com.tencent.devops.scm.pojo.Project
 import io.swagger.annotations.Api
@@ -442,4 +443,22 @@ interface ServiceGitResource {
         @QueryParam("mrId")
         mrId: Long
     ): Result<Boolean>
+
+    @ApiOperation("获取git项目组的详细信息")
+    @GET
+    @Path("/getProjectGroupInfo")
+    fun getProjectGroupInfo(
+        @ApiParam(value = "git项目组id", required = true)
+        @QueryParam("id")
+        id: String,
+        @ApiParam(value = "是否包含subgroup项目", required = false)
+        @QueryParam("includeSubgroups")
+        includeSubgroups: Boolean?,
+        @ApiParam(value = "token", required = true)
+        @QueryParam("token")
+        token: String,
+        @ApiParam(value = "token类型 0：oauth 1:privateKey", required = true)
+        @QueryParam("tokenType")
+        tokenType: TokenTypeEnum
+    ): Result<GitProjectGroupInfo>
 }

src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/resources/scm/ServiceGitResourceImpl.kt

@@ -46,6 +46,7 @@ import com.tencent.devops.repository.service.scm.IGitService
 import com.tencent.devops.scm.code.git.api.GitBranch
 import com.tencent.devops.scm.code.git.api.GitTag
 import com.tencent.devops.scm.pojo.GitCommit
+import com.tencent.devops.scm.pojo.GitProjectGroupInfo
 import com.tencent.devops.scm.pojo.GitRepositoryResp
 import org.springframework.beans.factory.annotation.Autowired
 import javax.servlet.http.HttpServletResponse
@@ -286,4 +287,20 @@ class ServiceGitResourceImpl @Autowired constructor(
         )
         return Result(true)
     }
+
+    override fun getProjectGroupInfo(
+        id: String,
+        includeSubgroups: Boolean?,
+        token: String,
+        tokenType: TokenTypeEnum
+    ): Result<GitProjectGroupInfo> {
+        return Result(
+            gitService.getProjectGroupInfo(
+                id = id,
+                includeSubgroups = includeSubgroups,
+                token = token,
+                tokenType = tokenType
+            )
+        )
+    }
 }