Skip to content

Commit

Permalink
Merge branch 'master' of https://github.com/Tencent/bk-ci into issue5…
Browse files Browse the repository at this point in the history
…640_github_branch
  • Loading branch information
sawyersong2 committed Dec 9, 2021
2 parents 063d1ef + 3639f65 commit 377647f
Show file tree
Hide file tree
Showing 221 changed files with 4,340 additions and 1,334 deletions.
2 changes: 2 additions & 0 deletions scripts/bkenv.properties
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,8 @@ BK_CI_FQDN_CERT=
BK_CI_NOTIFY_WEWORK_API_URL=https://qyapi.weixin.qq.com
# BK_CI_NOTIFY_WEWORK_SEND_CHANNEL 发送rtx的通道: weworkAgent 企业微信应用,blueking 蓝鲸消息
BK_CI_NOTIFY_WEWORK_SEND_CHANNEL=weworkAgent
# BK_CI_ARTIFACTORY_REALM 默认值为local,按需修改,制品库存储方式,可选值local, bkrepo
BK_CI_ARTIFACTORY_REALM=local
##########
# 4-微服务依赖
##########
Expand Down
6 changes: 5 additions & 1 deletion src/agent/src/pkg/upgrade/upgrade.go
Original file line number Diff line number Diff line change
Expand Up @@ -135,5 +135,9 @@ func downloadUpgradeFiles() (agentChanged bool, workAgentChanged bool, err error
return false, false, errors.New("check agent md5 failed")
}

return agentMd5 != newAgentMd5 || daemonMd5 != newDaemonMd5, workerMd5 != newWorkerMd5,nil
logs.Info("newDaemonMd5=" + newDaemonMd5 + ",daemonMd5=" + daemonMd5)
logs.Info("newAgentMd5=" + newAgentMd5 + ",agentMd5=" + agentMd5)
logs.Info("newWorkerMd5=" + newWorkerMd5 + ",workerMd5=" + workerMd5)
// #4686 devopsDaemon 暂时不考虑单独的替换升级,windows 无法自动升级,仅当devopsAgent有变化时升级。
return agentMd5 != newAgentMd5, workerMd5 != newWorkerMd5,nil
}
1 change: 1 addition & 0 deletions src/backend/ci/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ allprojects {
dependency("org.elasticsearch.client:elasticsearch-rest-client:${Versions.Elasticsearch}")
dependency("org.elasticsearch.client:elasticsearch-rest-high-level-client:${Versions.Elasticsearch}")
dependency("com.github.oshi:oshi-core:${Versions.Oshi}")
dependency("com.tencent.devops.leaf:leaf-boot-starter:${Versions.Leaf}")
dependencySet("io.github.openfeign:${Versions.Feign}") {
entry("feign-core")
entry("feign-jackson")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,5 +34,6 @@ object Versions {
const val Feign = "11.6" // TODO 修复这个BUG:https://github.com/OpenFeign/feign/pull/1347 , 后续SpringCloud升级后可以去掉
const val PinyinPlus = "1.0"
const val Oshi = "5.8.3"
const val Leaf = "1.0.1-RELEASE"
const val p4 = "2021.1.2163843"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
package com.tencent.devops.artifactory.api.builds

import com.tencent.devops.common.api.pojo.Result
import io.swagger.annotations.Api
import io.swagger.annotations.ApiOperation
import javax.ws.rs.Consumes
import javax.ws.rs.GET
import javax.ws.rs.Path
import javax.ws.rs.Produces
import javax.ws.rs.core.MediaType

@Api(tags = ["BUILD_ARTIFACTORY_CONF"], description = "仓库-配置管理")
@Path("/build/artifactories/conf")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
interface BuildArtifactoryConfigResource {

@ApiOperation("获取配置项artifactory.realm")
@GET
@Path("/realm")
fun getRealm(): Result<String>
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
/*
* Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
*
* Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.
*
* BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
*
* A copy of the MIT License is included in this file.
*
*
* Terms of the MIT License:
* ---------------------------------------------------
* Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
* documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all copies or substantial portions of
* the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
* LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
* NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.devops.artifactory.api.builds

import com.tencent.devops.artifactory.pojo.FileGatewayInfo
import com.tencent.devops.common.api.auth.AUTH_HEADER_PROJECT_ID
import com.tencent.devops.common.api.pojo.Result
import io.swagger.annotations.Api
import io.swagger.annotations.ApiOperation
import io.swagger.annotations.ApiParam
import javax.ws.rs.Consumes
import javax.ws.rs.GET
import javax.ws.rs.HeaderParam
import javax.ws.rs.Path
import javax.ws.rs.Produces
import javax.ws.rs.core.MediaType

@Api(tags = ["BUILD_URL"], description = "文件网关")
@Path("/build/fileGateway")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
interface BuildFileGatewayResource {

@ApiOperation("获取项目文件网关配置")
@Path("/get")
@GET
fun getFileGateway(
@ApiParam("项目ID", required = true)
@HeaderParam(AUTH_HEADER_PROJECT_ID)
projectId: String
): Result<FileGatewayInfo>
}
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,9 @@ interface UserFileResource {
@ApiParam("文件路径", required = true)
@QueryParam("filePath")
filePath: String,
@ApiParam("是否为logo文件", required = false)
@QueryParam("logo")
logo: Boolean?,
@Context
response: HttpServletResponse
)
Expand All @@ -111,6 +114,9 @@ interface UserFileResource {
@ApiParam("文件路径", required = true)
@PathParam("filePath")
filePath: String,
@ApiParam("是否为logo文件", required = false)
@QueryParam("logo")
logo: Boolean?,
@Context
response: HttpServletResponse
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,3 +29,6 @@ package com.tencent.devops.artifactory.constant

const val BK_CI_ATOM_DIR = "bk-atom"
const val BK_CI_PLUGIN_FE_DIR = "bk-plugin-fe"

const val REALM_LOCAL = "local"
const val REALM_BK_REPO = "bkrepo"
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
/*
* Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
*
* Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.
*
* BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
*
* A copy of the MIT License is included in this file.
*
*
* Terms of the MIT License:
* ---------------------------------------------------
* Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
* documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all copies or substantial portions of
* the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
* LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
* NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.devops.artifactory.pojo

data class FileGatewayInfo(
val fileDevnetGateway: String,
val fileIdcGateway: String
)
Original file line number Diff line number Diff line change
Expand Up @@ -88,15 +88,15 @@ class UserFileResourceImpl @Autowired constructor(
return archiveFileService.downloadFileToLocal(userId, filePath, response)
}

override fun downloadFile(userId: String, filePath: String, response: HttpServletResponse) {
override fun downloadFile(userId: String, filePath: String, logo: Boolean?, response: HttpServletResponse) {
val validateResult = archiveFileService.validateUserDownloadFilePermission(userId, filePath)
if (!validateResult) {
throw PermissionForbiddenException("no permission")
}
archiveFileService.downloadFile(userId, filePath, response)
archiveFileService.downloadFile(userId, filePath, response, logo)
}

override fun downloadFileExt(userId: String, filePath: String, response: HttpServletResponse) {
downloadFile(userId, filePath, response)
override fun downloadFileExt(userId: String, filePath: String, logo: Boolean?, response: HttpServletResponse) {
downloadFile(userId, filePath, logo, response)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -29,24 +29,37 @@ package com.tencent.devops.artifactory.resources

import com.tencent.devops.artifactory.api.ServiceArchiveAtomResource
import com.tencent.devops.artifactory.constant.BK_CI_ATOM_DIR
import com.tencent.devops.artifactory.constant.REALM_BK_REPO
import com.tencent.devops.artifactory.constant.REALM_LOCAL
import com.tencent.devops.artifactory.service.ArchiveAtomService
import com.tencent.devops.artifactory.service.ArchiveFileService
import com.tencent.devops.artifactory.util.BkRepoUtils.BKREPO_STORE_PROJECT_ID
import com.tencent.devops.artifactory.util.BkRepoUtils.REPO_NAME_PLUGIN
import com.tencent.devops.common.api.pojo.Result
import com.tencent.devops.common.web.RestResource
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.beans.factory.annotation.Value

@RestResource
class ServiceArchiveAtomResourceImpl @Autowired constructor(
private val archiveFileService: ArchiveFileService,
private val archiveAtomService: ArchiveAtomService
) : ServiceArchiveAtomResource {

@Value("\${artifactory.realm:}")
private var artifactoryRealm: String = ""

override fun getAtomFileContent(filePath: String): Result<String> {
return Result(archiveAtomService.getAtomFileContent(filePath))
}

override fun deleteAtomFile(userId: String, projectCode: String, atomCode: String): Result<Boolean> {
archiveFileService.deleteFile(userId, "$BK_CI_ATOM_DIR/$projectCode/$atomCode")
val filePath = when (artifactoryRealm) {
REALM_LOCAL -> "$BK_CI_ATOM_DIR/$projectCode/$atomCode"
REALM_BK_REPO -> "$BKREPO_STORE_PROJECT_ID/$REPO_NAME_PLUGIN/$projectCode/$atomCode"
else -> throw IllegalArgumentException("Unknown artifactory realm")
}
archiveFileService.deleteFile(userId, filePath)
return Result(true)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,7 @@ abstract class ArchiveAtomServiceImpl : ArchiveAtomService {
inputStream.copyTo(it)
}
// 解压到指定目录
val atomArchivePath = "${getAtomArchiveBasePath()}/$BK_CI_ATOM_DIR/$projectCode/$atomCode/$version"
val atomArchivePath = buildAtomArchivePath(projectCode, atomCode, version)
try {
ZipUtil.unZipFile(file, atomArchivePath, false)
// 判断解压目录下面是否有自定义UI前端文件
Expand All @@ -230,7 +230,7 @@ abstract class ArchiveAtomServiceImpl : ArchiveAtomService {
// 把前端文件拷贝到指定目录
FileUtils.copyDirectory(
frontendFileDir,
File("${getAtomArchiveBasePath()}/$STATIC/$BK_CI_PLUGIN_FE_DIR/$atomCode/$version")
File(buildAtomFrontendPath(atomCode, version))
)
FileSystemUtils.deleteRecursively(frontendFileDir)
}
Expand All @@ -239,6 +239,12 @@ abstract class ArchiveAtomServiceImpl : ArchiveAtomService {
}
}

protected fun buildAtomFrontendPath(atomCode: String, version: String) =
"${getAtomArchiveBasePath()}/$STATIC/$BK_CI_PLUGIN_FE_DIR/$atomCode/$version"

protected fun buildAtomArchivePath(projectCode: String, atomCode: String, version: String) =
"${getAtomArchiveBasePath()}/$BK_CI_ATOM_DIR/$projectCode/$atomCode/$version"

abstract fun clearServerTmpFile(
projectCode: String,
atomCode: String,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
package com.tencent.devops.artifactory.service.impl

import com.tencent.devops.artifactory.client.bkrepo.DefaultBkRepoClient
import com.tencent.devops.artifactory.constant.BK_CI_ATOM_DIR
import com.tencent.devops.artifactory.constant.BK_CI_PLUGIN_FE_DIR
import com.tencent.devops.artifactory.constant.REALM_BK_REPO
import com.tencent.devops.artifactory.util.BkRepoUtils.BKREPO_DEFAULT_USER
import com.tencent.devops.artifactory.util.BkRepoUtils.BKREPO_STORE_PROJECT_ID
import com.tencent.devops.artifactory.util.BkRepoUtils.REPO_NAME_PLUGIN
import com.tencent.devops.artifactory.util.BkRepoUtils.REPO_NAME_STATIC
import com.tencent.devops.artifactory.util.DefaultPathUtils
import com.tencent.devops.common.api.constant.STATIC
import com.tencent.devops.common.api.exception.RemoteServiceException
import org.glassfish.jersey.media.multipart.FormDataContentDisposition
import org.slf4j.LoggerFactory
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
import org.springframework.stereotype.Service
import java.io.File
import java.io.InputStream
import javax.ws.rs.NotFoundException

@Service
@ConditionalOnProperty(prefix = "artifactory", name = ["realm"], havingValue = REALM_BK_REPO)
class ArchiveAtomToBkRepoServiceImpl(
private val bkRepoClient: DefaultBkRepoClient
) : ArchiveAtomServiceImpl() {

override fun getAtomArchiveBasePath(): String {
return System.getProperty("java.io.tmpdir")
}

override fun handleArchiveFile(
disposition: FormDataContentDisposition,
inputStream: InputStream,
projectCode: String,
atomCode: String,
version: String
) {
unzipFile(
disposition = disposition,
inputStream = inputStream,
projectCode = projectCode,
atomCode = atomCode,
version = version
)
val atomArchivePath = buildAtomArchivePath(projectCode, atomCode, version)
val frontendDir = buildAtomFrontendPath(atomCode, version)
logger.info("atom plugin: $atomArchivePath, $frontendDir")
File(atomArchivePath).walk().filter { it.path != atomArchivePath }.forEach {
val path = it.path.removePrefix("${getAtomArchiveBasePath()}/$BK_CI_ATOM_DIR")
bkRepoClient.uploadLocalFile(
userId = BKREPO_DEFAULT_USER,
projectId = BKREPO_STORE_PROJECT_ID,
repoName = REPO_NAME_PLUGIN,
path = path,
file = it
)
}
File(frontendDir).walk().filter { it.path != frontendDir }.forEach {
val path = it.path.removePrefix("${getAtomArchiveBasePath()}/$STATIC/$BK_CI_PLUGIN_FE_DIR")
bkRepoClient.uploadLocalFile(
userId = BKREPO_DEFAULT_USER,
projectId = BKREPO_STORE_PROJECT_ID,
repoName = REPO_NAME_STATIC,
path = path,
file = it
)
}
}

override fun getAtomFileContent(filePath: String): String {
val tmpFile = DefaultPathUtils.randomFile()
return try {
bkRepoClient.downloadFile(
userId = BKREPO_DEFAULT_USER,
projectId = BKREPO_STORE_PROJECT_ID,
repoName = REPO_NAME_PLUGIN,
fullPath = filePath,
destFile = tmpFile
)
tmpFile.readText(Charsets.UTF_8)
} catch (e: NotFoundException) {
logger.warn("file[$filePath] not exists")
""
} catch (e: RemoteServiceException) {
logger.warn("download file[$filePath] error: $e")
""
} finally {
tmpFile.delete()
}
}

override fun clearServerTmpFile(projectCode: String, atomCode: String, version: String) {
val atomArchivePath = buildAtomArchivePath(projectCode, atomCode, version)
val frontendDir = buildAtomFrontendPath(atomCode, version)
File(atomArchivePath).deleteRecursively()
File(frontendDir).deleteRecursively()
}

companion object {
private val logger = LoggerFactory.getLogger(ArchiveAtomToBkRepoServiceImpl::class.java)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -28,18 +28,21 @@
package com.tencent.devops.artifactory.service.impl

import com.tencent.devops.artifactory.constant.BK_CI_ATOM_DIR
import com.tencent.devops.artifactory.constant.REALM_LOCAL
import com.tencent.devops.common.api.constant.CommonMessageCode
import com.tencent.devops.common.api.exception.ErrorCodeException
import org.apache.commons.io.FileUtils
import org.glassfish.jersey.media.multipart.FormDataContentDisposition
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Value
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
import org.springframework.stereotype.Service
import java.io.File
import java.io.InputStream
import java.net.URLDecoder

@Service
@ConditionalOnProperty(prefix = "artifactory", name = ["realm"], havingValue = REALM_LOCAL)
class ArchiveAtomToLocalServiceImpl : ArchiveAtomServiceImpl() {

private val logger = LoggerFactory.getLogger(ArchiveAtomToLocalServiceImpl::class.java)
Expand Down
Loading

0 comments on commit 377647f

Please sign in to comment.