feat: generic支持分块追加上传 TencentBlueKing#1387

* feat: generic支持分块追加上传 TencentBlueKing#1387

* feat: generic分块上传调整 TencentBlueKing#1387

* feat: 代码调整 TencentBlueKing#1387

* feat: 代码调整 TencentBlueKing#1387

* feat: 打印调整 TencentBlueKing#1387

* feat: 支持上传分块上传、下载指标 TencentBlueKing#1387

* feat: 代码调整 TencentBlueKing#1387

src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/util/chunked/ChunkedUploadUtils.kt

@@ -0,0 +1,90 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.common.artifact.util.chunked
+
+import com.tencent.bkrepo.common.api.exception.ErrorCodeException
+import com.tencent.bkrepo.common.api.message.CommonMessageCode
+
+object ChunkedUploadUtils {
+
+    fun chunkedRequestCheck(
+        contentLength: Int,
+        range: String?,
+        lengthOfAppendFile: Long
+    ): RangeStatus {
+        // 当range不存在或者length < 0时
+        if (!validateValue(contentLength, range)) {
+            return RangeStatus.ILLEGAL_RANGE
+        }
+        val (start, end) = getRangeInfo(range!!)
+        // 当上传的长度和range内容不匹配时
+        return if ((end - start) != (contentLength - 1).toLong()) {
+            RangeStatus.ILLEGAL_RANGE
+        } else {
+            // 当追加的文件大小和range的起始大小一致时代表写入正常
+            if (start == lengthOfAppendFile) {
+                RangeStatus.READY_TO_APPEND
+            } else if (start > lengthOfAppendFile) {
+                // 当追加的文件大小比start小时，说明文件写入有误
+                RangeStatus.ILLEGAL_RANGE
+            } else {
+                // 当追加的文件大小==end+1时，可能存在重试导致已经写入一次
+                if (lengthOfAppendFile == end + 1) {
+                    RangeStatus.ALREADY_APPENDED
+                } else {
+                    // 当追加的文件大小大于start时，并且不等于end+1时，文件已损坏
+                    RangeStatus.ILLEGAL_RANGE
+                }
+            }
+        }
+    }
+
+    /**
+     * 从Content-Range头中解析出起始位置
+     * Content-Range 类型为"start-end"
+     */
+    fun getRangeInfo(range: String): Pair<Long, Long> {
+        if (range.isNullOrEmpty()) {
+            throw ErrorCodeException(CommonMessageCode.PARAMETER_MISSING, "range is empty!")
+        }
+        val values = range.split("-")
+        if (values.isEmpty() || values.size < 2)
+            throw ErrorCodeException(CommonMessageCode.PARAMETER_INVALID, range)
+        return Pair(values[0].toLong(), values[1].toLong())
+    }
+
+    private fun validateValue(contentLength: Int, range: String?): Boolean {
+        return !(range.isNullOrEmpty() || contentLength < 0)
+    }
+
+    enum class RangeStatus {
+        ILLEGAL_RANGE,
+        ALREADY_APPENDED,
+        READY_TO_APPEND;
+    }
+}

src/backend/common/common-bksync/src/main/kotlin/com/tencent/bkrepo/common/bksync/transfer/http/HttpBkSyncCall.kt

@@ -46,7 +46,7 @@ import kotlin.system.measureNanoTime
  * bksync http实现
  * */
 @Suppress("UnstableApiUsage")
-class HttpBkSyncCall(
+open class HttpBkSyncCall(
     // http客户端
     private val client: OkHttpClient,
     // 重复率阈值，只有大于该阈值时才会使用增量上传
@@ -366,7 +366,7 @@ class HttpBkSyncCall(
     /**
      * 普通上传
      * */
-    private fun commonUpload(context: UploadContext) {
+    open fun commonUpload(context: UploadContext) {
         val request = context.request
         with(request) {
             logger.info("Start use generic upload.")

src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericArtifactInfo.kt

@@ -43,5 +43,6 @@ class GenericArtifactInfo(
         const val BLOCK_MAPPING_URI = "/block/{projectId}/{repoName}/**"
         const val DELTA_MAPPING_URI = "/delta/{projectId}/{repoName}/**"
         const val BATCH_MAPPING_URI = "/batch/{projectId}/{repoName}"
+        const val CHUNKED_UPLOAD_MAPPING_URI = "/{projectId}/{repoName}/**/chunked/uploads/**"
     }
 }

src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericChunkedArtifactInfo.kt

@@ -0,0 +1,40 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.generic.artifact
+
+import com.tencent.bkrepo.common.artifact.api.ArtifactInfo
+
+class GenericChunkedArtifactInfo(
+    projectId: String,
+    repoName: String,
+    artifactUri: String,
+    val uuid: String? = null,
+    val md5: String? = null,
+    val sha256: String? = null,
+    val size: Long? = null
+) : ArtifactInfo(projectId, repoName, artifactUri)

src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/constant/Constants.kt

@@ -44,3 +44,16 @@ const val HEADER_OLD_FILE_PATH = BKREPO_PREFIX + "OLD-FILE-PATH"
 
 const val BKREPO_META_PREFIX = "X-BKREPO-META-"
 const val BKREPO_META = "X-BKREPO-META"
+
+const val HEADER_UPLOAD_TYPE = "UPLOAD-TYPE"
+
+/**
+ * 分块上传标识
+ */
+const val CHUNKED_UPLOAD = "CHUNKED-UPLOAD"
+
+/**
+ * 分块上传uuid
+ */
+const val CHUNKED_UPLOAD_UUID = "CHUNKED-UPLOAD-UUID"
+const val CHUNKED_UPLOAD_CLIENT = "CHUNKED-UPLOAD-CLIENT"

src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/pojo/ChunkedMetrics.kt

@@ -0,0 +1,52 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.generic.pojo
+
+data class ChunkedMetrics(
+    var tag: String = "ChunkedMetrics",
+    // 类型
+    var type: String = "UPLOAD",
+    // 文件路径
+    var fullPath: String = "",
+    // 文件大小，单位字节
+    var fileSize: Long = 0,
+    // 上传或下载时间，单位毫秒
+    var costTime: Long = 0,
+    // 成功或者失败
+    var success: Boolean = true,
+    //  失败原因
+    var failedReason: String = "",
+    // 上报指标时间戳
+    var reportTimeStamp: Long = System.currentTimeMillis(),
+    var projectId: String = "",
+    var repoName: String = "",
+    var pipelineId: String = "",
+    var buildId: String = "",
+    var taskId: String = "",
+)
+

src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/pojo/ChunkedResponseProperty.kt

@@ -0,0 +1,37 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.generic.pojo
+
+import com.tencent.bkrepo.common.api.constant.HttpStatus
+
+data class ChunkedResponseProperty(
+    val uuid: String? = null,
+    val size: Long? = null,
+    val status: HttpStatus? = null,
+    val contentLength: Long? = null
+)

src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericChunkedArtifactInfoResolver.kt

@@ -0,0 +1,70 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.generic.artifact
+
+import com.tencent.bkrepo.common.api.constant.StringPool
+import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder
+import com.tencent.bkrepo.common.artifact.resolve.path.ArtifactInfoResolver
+import com.tencent.bkrepo.common.artifact.resolve.path.Resolver
+import io.undertow.servlet.spec.HttpServletRequestImpl
+import org.springframework.stereotype.Component
+import javax.servlet.http.HttpServletRequest
+
+
+@Component
+@Resolver(GenericChunkedArtifactInfo::class)
+class GenericChunkedArtifactInfoResolver : ArtifactInfoResolver {
+    override fun resolve(
+        projectId: String,
+        repoName: String,
+        artifactUri: String,
+        request: HttpServletRequest
+    ): GenericChunkedArtifactInfo {
+
+        val requestUrl = ArtifactContextHolder.getUrlPath(this.javaClass.name)!!
+        val artifactUri = requestUrl.replaceAfterLast("/chunked/uploads", StringPool.EMPTY)
+            .removeSuffix("/chunked/uploads")
+            .removePrefix("/temporary/$projectId/$repoName")
+        // 解析UUID
+        val uuid = requestUrl.replaceBefore("/chunked/uploads", StringPool.EMPTY)
+            .removePrefix("/chunked/uploads/").removeSuffix("/")
+        val params = (request as HttpServletRequestImpl).queryParameters
+        val size = params?.get("size")?.first?.toLongOrNull()
+        val sha256 = params?.get("sha256")?.first
+        val md5 = params?.get("md5")?.first
+        return GenericChunkedArtifactInfo(
+            projectId = projectId,
+            repoName = repoName,
+            artifactUri = artifactUri,
+            uuid = uuid,
+            md5 = md5,
+            sha256 = sha256,
+            size = size
+        )
+    }
+}