From 2f7497c44b9e054567f482a209b7bf55b32ba05f Mon Sep 17 00:00:00 2001 From: carlyin Date: Mon, 4 Nov 2024 12:22:23 +0800 Subject: [PATCH] =?UTF-8?q?pref=EF=BC=9A=E7=A0=94=E5=8F=91=E5=95=86?= =?UTF-8?q?=E5=BA=97=E9=80=9A=E7=94=A8=E5=8C=96=E6=8E=A5=E5=8F=A3=E5=B0=81?= =?UTF-8?q?=E8=A3=85=E4=BC=98=E5=8C=96=20#11049?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../impl/ArchiveStorePkgServiceImpl.kt | 71 ++++++++---- .../ArchiveStorePkgToBkRepoServiceImpl.kt | 106 ++++++++++++------ .../impl/ArchiveStorePkgToLocalServiceImpl.kt | 40 +++++-- .../store/pojo/common/StoreConstants.kt | 2 +- 4 files changed, 150 insertions(+), 69 deletions(-) diff --git a/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/store/service/impl/ArchiveStorePkgServiceImpl.kt b/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/store/service/impl/ArchiveStorePkgServiceImpl.kt index 8666512e8b2..94d6d320159 100644 --- a/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/store/service/impl/ArchiveStorePkgServiceImpl.kt +++ b/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/store/service/impl/ArchiveStorePkgServiceImpl.kt @@ -43,7 +43,7 @@ import com.tencent.devops.common.client.Client import com.tencent.devops.common.service.utils.ZipUtil import com.tencent.devops.store.api.common.ServiceStoreArchiveResource import com.tencent.devops.store.api.common.ServiceStoreResource -import com.tencent.devops.store.pojo.common.CONFIG_JSON_NAME +import com.tencent.devops.store.pojo.common.CONFIG_YML_NAME import com.tencent.devops.store.pojo.common.QueryComponentPkgEnvInfoParam import com.tencent.devops.store.pojo.common.enums.StoreTypeEnum import com.tencent.devops.store.pojo.common.publication.StorePkgEnvInfo @@ -101,7 +101,8 @@ abstract class ArchiveStorePkgServiceImpl : ArchiveStorePkgService { val storePkgEnvInfos: List? var packageFileInfos: MutableList? = null try { - handleArchiveFile( + // 解压上传的包 + handlePkgFile( disposition = disposition, inputStream = inputStream, storeType = storeType, @@ -109,20 +110,35 @@ abstract class ArchiveStorePkgServiceImpl : ArchiveStorePkgService { version = version ) val storeArchivePath = buildStoreArchivePath(storeType, storeCode, version) - val bkConfigJsonFile = File(storeArchivePath, CONFIG_JSON_NAME) - storePkgEnvInfos = if (bkConfigJsonFile.exists()) { + val bkConfigFile = File(storeArchivePath, CONFIG_YML_NAME) + storePkgEnvInfos = if (bkConfigFile.exists()) { + // 如果上传的文件是压缩包需要删除原压缩包 + File(storeArchivePath, disposition.fileName).deleteRecursively() client.get(ServiceStoreArchiveResource::class).getComponentPkgEnvInfo( userId = userId, storeType = storeType, storeCode = storeCode, version = version, queryComponentPkgEnvInfoParam = QueryComponentPkgEnvInfoParam( - configFileContent = bkConfigJsonFile.readText() + configFileContent = bkConfigFile.readText() ) ).data } else { - listOf(StorePkgEnvInfo(osName = OSType.WINDOWS.name.lowercase(), defaultFlag = true)) + listOf( + StorePkgEnvInfo( + osName = OSType.WINDOWS.name.lowercase(), + pkgLocalPath = disposition.fileName, + defaultFlag = true + ) + ) } + handleArchiveFile( + storeType = storeType, + storeCode = storeCode, + version = version, + storePkgEnvInfos = storePkgEnvInfos + ) + storePkgEnvInfos?.forEach { storePkgEnvInfo -> var pkgLocalPath = storePkgEnvInfo.pkgLocalPath if (storeType == StoreTypeEnum.ATOM && storePkgEnvInfo.target.isNullOrBlank() && @@ -145,15 +161,14 @@ abstract class ArchiveStorePkgServiceImpl : ArchiveStorePkgService { packageFileSize = packageFile.length(), shaContent = packageFile.inputStream().use { ShaUtils.sha1InputStream(it) } ) - val pkgRepoPathSb = StringBuilder("$storeCode/$version/") - if (!storePkgEnvInfo.osName.isNullOrBlank()) { - pkgRepoPathSb.append(storePkgEnvInfo.osName).append("/") - } - if (!storePkgEnvInfo.osArch.isNullOrBlank()) { - pkgRepoPathSb.append(storePkgEnvInfo.osArch).append("/") - } - pkgRepoPathSb.append(packageFileName) - storePkgEnvInfo.pkgRepoPath = pkgRepoPathSb.toString() + val pkgRepoPath = generatePkgRepoPath( + storeCode = storeCode, + version = version, + pkgFileName = packageFileName, + osName = storePkgEnvInfo.osName, + osArch = storePkgEnvInfo.osArch + ) + storePkgEnvInfo.pkgRepoPath = pkgRepoPath storePkgEnvInfo.shaContent = packageFileInfo.shaContent storePkgEnvInfo.pkgName = packageFileName packageFileInfos!!.add(packageFileInfo) @@ -205,7 +220,26 @@ abstract class ArchiveStorePkgServiceImpl : ArchiveStorePkgService { return true } - protected fun handlePkgFile( + protected fun generatePkgRepoPath( + storeCode: String, + version: String, + pkgFileName: String, + osName: String? = null, + osArch: String? = null + ): String { + val pkgRepoPathSb = StringBuilder("$storeCode/$version/") + if (!osName.isNullOrBlank()) { + pkgRepoPathSb.append(osName).append("/") + } + if (!osArch.isNullOrBlank()) { + pkgRepoPathSb.append(osArch).append("/") + } + pkgRepoPathSb.append(pkgFileName) + val pkgRepoPath = pkgRepoPathSb.toString() + return pkgRepoPath + } + + private fun handlePkgFile( disposition: FormDataContentDisposition, inputStream: InputStream, storeType: StoreTypeEnum, @@ -295,11 +329,10 @@ abstract class ArchiveStorePkgServiceImpl : ArchiveStorePkgService { abstract fun getStoreArchiveBasePath(): String abstract fun handleArchiveFile( - disposition: FormDataContentDisposition, - inputStream: InputStream, storeType: StoreTypeEnum, storeCode: String, - version: String + version: String, + storePkgEnvInfos: List? ) override fun getComponentPkgDownloadUrl( diff --git a/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/store/service/impl/ArchiveStorePkgToBkRepoServiceImpl.kt b/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/store/service/impl/ArchiveStorePkgToBkRepoServiceImpl.kt index c16d9a5fd0c..b09310e4c8a 100644 --- a/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/store/service/impl/ArchiveStorePkgToBkRepoServiceImpl.kt +++ b/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/store/service/impl/ArchiveStorePkgToBkRepoServiceImpl.kt @@ -10,13 +10,12 @@ import com.tencent.devops.common.api.constant.STATIC import com.tencent.devops.common.api.exception.RemoteServiceException import com.tencent.devops.common.archive.client.BkRepoClient import com.tencent.devops.common.archive.config.BkRepoClientConfig -import com.tencent.devops.store.pojo.common.CONFIG_JSON_NAME +import com.tencent.devops.store.pojo.common.CONFIG_YML_NAME import com.tencent.devops.store.pojo.common.enums.StoreTypeEnum -import org.glassfish.jersey.media.multipart.FormDataContentDisposition +import com.tencent.devops.store.pojo.common.publication.StorePkgEnvInfo import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import java.io.File -import java.io.InputStream import javax.ws.rs.NotFoundException abstract class ArchiveStorePkgToBkRepoServiceImpl : ArchiveStorePkgServiceImpl() { @@ -34,34 +33,58 @@ abstract class ArchiveStorePkgToBkRepoServiceImpl : ArchiveStorePkgServiceImpl() return System.getProperty("java.io.tmpdir") } - @Suppress("UNCHECKED_CAST") override fun handleArchiveFile( - disposition: FormDataContentDisposition, - inputStream: InputStream, storeType: StoreTypeEnum, storeCode: String, - version: String + version: String, + storePkgEnvInfos: List? ) { - handlePkgFile( - disposition = disposition, - inputStream = inputStream, - storeType = storeType, - storeCode = storeCode, - version = version - ) val storeArchivePath = buildStoreArchivePath(storeType, storeCode, version) - val bkConfigJsonFile = File(storeArchivePath, CONFIG_JSON_NAME) - if (bkConfigJsonFile.exists()) { - // 删除原压缩包 - File(storeArchivePath, disposition.fileName).deleteRecursively() + val prefix = "${getStoreArchiveBasePath()}/${getPkgFileTypeDir(storeType)}" + if (storePkgEnvInfos.isNullOrEmpty()) { + directoryIteration( + directoryFile = File(storeArchivePath), + prefix = prefix, + directoryPath = storeArchivePath, + repoName = getBkRepoName(storeType), + storeType = storeType + ) + } else { + storePkgEnvInfos.forEach { storePkgEnvInfo -> + val pkgLocalPath = storePkgEnvInfo.pkgLocalPath + if (pkgLocalPath.isNullOrBlank()) { + return@forEach + } + val file = File(storeArchivePath, pkgLocalPath) + if (!file.exists()) { + logger.warn("uploadLocalFile file[$pkgLocalPath] not exist!") + return@forEach + } + val pkgRepoPath = generatePkgRepoPath( + storeCode = storeCode, + version = version, + pkgFileName = file.name, + osName = storePkgEnvInfo.osName, + osArch = storePkgEnvInfo.osArch + ) + uploadLocalFile( + storeType = storeType, + repoName = getBkRepoName(storeType), + path = pkgRepoPath, + file = file + ) + } + // 上传配置文件 + val bkConfigFile = File(storeArchivePath, CONFIG_YML_NAME) + if (bkConfigFile.exists()) { + uploadLocalFile( + storeType = storeType, + repoName = getBkRepoName(storeType), + path = bkConfigFile.path.removePrefix(prefix), + file = bkConfigFile + ) + } } - directoryIteration( - directoryFile = File(storeArchivePath), - prefix = "${getStoreArchiveBasePath()}/${getPkgFileTypeDir(storeType)}", - directoryPath = storeArchivePath, - repoName = getBkRepoName(storeType), - storeType = storeType - ) val frontendDir = buildStoreFrontendPath(storeType, storeCode, version) frontendDir?.let { directoryIteration( @@ -93,22 +116,31 @@ abstract class ArchiveStorePkgToBkRepoServiceImpl : ArchiveStorePkgServiceImpl() } else { val path = file.path.removePrefix(prefix) logger.debug("uploadLocalFile fileName=${file.name}|path=$path") - val uploadRepoName = getUploadRepoName(repoName, storeType) - bkRepoClient.uploadLocalFile( - userId = BKREPO_DEFAULT_USER, - projectId = getBkRepoProjectId(storeType), - repoName = uploadRepoName, - path = path, - file = file, - gatewayFlag = false, - bkrepoApiUrl = "${bkRepoClientConfig.bkRepoIdcHost}/api/generic", - userName = bkRepoStoreConfig.bkrepoStoreUserName, - password = bkRepoStoreConfig.bkrepoStorePassword - ) + uploadLocalFile(storeType = storeType, repoName = repoName, path = path, file = file) } } } + private fun uploadLocalFile( + storeType: StoreTypeEnum, + repoName: String, + path: String, + file: File + ) { + val uploadRepoName = getUploadRepoName(repoName, storeType) + bkRepoClient.uploadLocalFile( + userId = BKREPO_DEFAULT_USER, + projectId = getBkRepoProjectId(storeType), + repoName = uploadRepoName, + path = path, + file = file, + gatewayFlag = false, + bkrepoApiUrl = "${bkRepoClientConfig.bkRepoIdcHost}/api/generic", + userName = bkRepoStoreConfig.bkrepoStoreUserName, + password = bkRepoStoreConfig.bkrepoStorePassword + ) + } + private fun getUploadRepoName( repoName: String, storeType: StoreTypeEnum diff --git a/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/store/service/impl/ArchiveStorePkgToLocalServiceImpl.kt b/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/store/service/impl/ArchiveStorePkgToLocalServiceImpl.kt index 501179abf45..d1d686f321e 100644 --- a/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/store/service/impl/ArchiveStorePkgToLocalServiceImpl.kt +++ b/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/store/service/impl/ArchiveStorePkgToLocalServiceImpl.kt @@ -33,15 +33,15 @@ import com.tencent.devops.common.api.exception.ErrorCodeException import com.tencent.devops.common.service.config.CommonConfig import com.tencent.devops.common.service.utils.HomeHostUtil import com.tencent.devops.store.pojo.common.enums.StoreTypeEnum +import com.tencent.devops.store.pojo.common.publication.StorePkgEnvInfo import org.apache.commons.io.FileUtils -import org.glassfish.jersey.media.multipart.FormDataContentDisposition +import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.beans.factory.annotation.Value import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.stereotype.Service import org.springframework.util.FileSystemUtils import java.io.File -import java.io.InputStream import java.net.URLDecoder @Service @@ -54,24 +54,40 @@ class ArchiveStorePkgToLocalServiceImpl : ArchiveStorePkgServiceImpl() { @Autowired lateinit var commonConfig: CommonConfig + companion object { + private val logger = LoggerFactory.getLogger(ArchiveStorePkgToLocalServiceImpl::class.java) + } + override fun getStoreArchiveBasePath(): String { return storeArchiveLocalBasePath } override fun handleArchiveFile( - disposition: FormDataContentDisposition, - inputStream: InputStream, storeType: StoreTypeEnum, storeCode: String, - version: String + version: String, + storePkgEnvInfos: List? ) { - handlePkgFile( - disposition = disposition, - inputStream = inputStream, - storeType = storeType, - storeCode = storeCode, - version = version - ) + val storeArchivePath = buildStoreArchivePath(storeType, storeCode, version) + storePkgEnvInfos?.forEach { storePkgEnvInfo -> + val pkgLocalPath = storePkgEnvInfo.pkgLocalPath + if (pkgLocalPath.isNullOrBlank()) { + return@forEach + } + val file = File(storeArchivePath, pkgLocalPath) + if (!file.exists()) { + logger.warn("uploadLocalFile file[$pkgLocalPath] not exist!!") + return@forEach + } + val pkgRepoPath = generatePkgRepoPath( + storeCode = storeCode, + version = version, + pkgFileName = file.name, + osName = storePkgEnvInfo.osName, + osArch = storePkgEnvInfo.osArch + ) + file.renameTo(File(storeArchivePath, pkgRepoPath)) + } } override fun getStoreFileContent(filePath: String, storeType: StoreTypeEnum, repoName: String?): String { diff --git a/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/pojo/common/StoreConstants.kt b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/pojo/common/StoreConstants.kt index ffe9f6fee89..cf83620bd76 100644 --- a/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/pojo/common/StoreConstants.kt +++ b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/pojo/common/StoreConstants.kt @@ -36,7 +36,7 @@ const val TASK_JSON_NAME = "task.json" const val QUALITY_JSON_NAME = "quality.json" const val ERROR_JSON_NAME = "errorCodes.json" const val EXTENSION_JSON_NAME = "extension.json" -const val CONFIG_JSON_NAME = "bk-config.json" +const val CONFIG_YML_NAME = "bk-config.yml" const val README = "README.md" const val STORE_ATOM_STATUS = "STORE_ATOM_STATUS" // 插件状态 const val STORE_IMAGE_STATUS = "STORE_IMAGE_STATUS" // 镜像状态