diff --git a/helm-charts/core/ci/templates/gateway/deployment.yaml b/helm-charts/core/ci/templates/gateway/deployment.yaml index b5329e0cf51..208bfc6c647 100644 --- a/helm-charts/core/ci/templates/gateway/deployment.yaml +++ b/helm-charts/core/ci/templates/gateway/deployment.yaml @@ -53,8 +53,6 @@ spec: initContainers: - name: frontend image: {{ include "bkci-frontend.image" . }} - securityContext: - privileged: true volumeMounts: - mountPath: /tmp/frontend name: frontend diff --git a/src/backend/ci/buildSrc/src/main/kotlin/plugins/task-docker-build.gradle.kts b/src/backend/ci/buildSrc/src/main/kotlin/plugins/task-docker-build.gradle.kts index 92d11089bbf..88e2db5cb2b 100644 --- a/src/backend/ci/buildSrc/src/main/kotlin/plugins/task-docker-build.gradle.kts +++ b/src/backend/ci/buildSrc/src/main/kotlin/plugins/task-docker-build.gradle.kts @@ -27,15 +27,12 @@ plugins { id("com.google.cloud.tools.jib") } - val toImageRepo = System.getProperty("to.image.repo") val toImageTag = System.getProperty("to.image.tag") var toImage = System.getProperty("jib.to.image") - // 加这个判断 , 主要是为了编译kts时不报错 if (toImage.isNullOrBlank() || (toImageRepo.isNullOrBlank() && toImageTag.isNullOrBlank())) { val service = name.replace("boot-", "").replace("-tencent", "") - if (toImage.isNullOrBlank() && !toImageRepo.isNullOrBlank()) { toImage = toImageRepo.let { if (toImageRepo.endsWith("/")) it else it + "/" @@ -44,6 +41,8 @@ if (toImage.isNullOrBlank() || (toImageRepo.isNullOrBlank() && toImageTag.isNull val jvmFlagList = System.getProperty("jvmFlags.file")?.let { File(it).readLines() } ?: emptyList() + val watchAllNamespace = System.getProperty("watch.all.namespace", "true") + val finalJvmFlags = mutableListOf( "-server", "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8080", @@ -73,13 +72,12 @@ if (toImage.isNullOrBlank() || (toImageRepo.isNullOrBlank() && toImageTag.isNull "-Dspring.main.allow-circular-references=true", "-Dspring.cloud.kubernetes.config.sources[0].name=config-bk-ci-common", "-Dspring.cloud.kubernetes.config.sources[1].name=config-bk-ci-$service", - "-Dspring.cloud.kubernetes.discovery.all-namespaces=true", + "-Dspring.cloud.kubernetes.discovery.all-namespaces=$watchAllNamespace", "-Dspring.cloud.kubernetes.config.includeProfileSpecificSources=false", "-Dio.undertow.legacy.cookie.ALLOW_HTTP_SEPARATORS_IN_V0=true", "-Dserver.port=80" ) finalJvmFlags.addAll(jvmFlagList) - jib { // 环境变量 container { @@ -96,4 +94,4 @@ if (toImage.isNullOrBlank() || (toImageRepo.isNullOrBlank() && toImageTag.isNull image = toImage } } -} +} \ No newline at end of file diff --git a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/util/DateTimeUtil.kt b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/util/DateTimeUtil.kt index 0159909c2cc..1244da60ad0 100644 --- a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/util/DateTimeUtil.kt +++ b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/util/DateTimeUtil.kt @@ -74,6 +74,32 @@ object DateTimeUtil { const val ONE_THOUSAND_MS = 1000L + /** + * 获取时间列表里面的最小值 + */ + fun min(vararg localDateTimes: LocalDateTime): LocalDateTime { + var result = localDateTimes[0] + for (i in 1 until localDateTimes.size) { + if (localDateTimes[i].isBefore(result)) { + result = localDateTimes[i] + } + } + return result + } + + /** + * 获取时间列表里面的最大值 + */ + fun max(vararg localDateTimes: LocalDateTime): LocalDateTime { + var result = localDateTimes[0] + for (i in 1 until localDateTimes.size) { + if (localDateTimes[i].isAfter(result)) { + result = localDateTimes[i] + } + } + return result + } + /** * 单位转换，分钟转换秒 */ @@ -197,8 +223,8 @@ object DateTimeUtil { val minute = (time - hour * timeGap * timeGap * million) / (timeGap * million) val second = (time - hour * timeGap * timeGap * million - minute * timeGap * million) / million return (if (hour == zero) "00" else if (hour >= ten) hour.toString() else "0$hour").toString() + "时" + - (if (minute == zero) "00" else if (minute >= ten) minute else "0$minute") + "分" + - (if (second == zero) "00" else if (second >= ten) second.toShort() else "0$second") + "秒" + (if (minute == zero) "00" else if (minute >= ten) minute else "0$minute") + "分" + + (if (second == zero) "00" else if (second >= ten) second.toShort() else "0$second") + "秒" } fun formatMilliTime(time: Long): String { @@ -216,8 +242,8 @@ object DateTimeUtil { val minute = (time - hour * 60 * 60 * 1000) / (60 * 1000) val second = (time - hour * 60 * 60 * 1000 - minute * 60 * 1000) / 1000 return (if (hour == 0L) "00" else if (hour >= 10) hour.toString() else "0$hour").toString() + "时" + - (if (minute == 0L) "00" else if (minute >= 10) minute else "0$minute") + "分" + - (if (second == 0L) "00" else if (second >= 10) second.toShort() else "0$second") + "秒" + (if (minute == 0L) "00" else if (minute >= 10) minute else "0$minute") + "分" + + (if (second == 0L) "00" else if (second >= 10) second.toShort() else "0$second") + "秒" } fun formatMillSecond(mss: Long): String { diff --git a/src/backend/ci/core/common/common-event/src/main/kotlin/com/tencent/devops/common/stream/config/CommonEventConfiguration.kt b/src/backend/ci/core/common/common-event/src/main/kotlin/com/tencent/devops/common/stream/config/CommonEventConfiguration.kt index cf5fc9a4722..7e7f4e1e847 100644 --- a/src/backend/ci/core/common/common-event/src/main/kotlin/com/tencent/devops/common/stream/config/CommonEventConfiguration.kt +++ b/src/backend/ci/core/common/common-event/src/main/kotlin/com/tencent/devops/common/stream/config/CommonEventConfiguration.kt @@ -2,6 +2,7 @@ package com.tencent.devops.common.stream.config import com.tencent.devops.common.event.dispatcher.SampleEventDispatcher import com.tencent.devops.common.stream.config.interceptor.BkChannelInterceptor +import com.tencent.devops.common.stream.customizer.BkProducerMessageHandlerCustomizer import org.springframework.cloud.stream.function.StreamBridge import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration @@ -15,4 +16,7 @@ class CommonEventConfiguration { @Bean @GlobalChannelInterceptor fun bkChannelInterceptor() = BkChannelInterceptor() + + @Bean + fun bkProducerMessageHandlerCustomizer() = BkProducerMessageHandlerCustomizer() } diff --git a/src/backend/ci/core/common/common-event/src/main/kotlin/com/tencent/devops/common/stream/customizer/BkProducerMessageHandlerCustomizer.kt b/src/backend/ci/core/common/common-event/src/main/kotlin/com/tencent/devops/common/stream/customizer/BkProducerMessageHandlerCustomizer.kt new file mode 100644 index 00000000000..ec13861fc33 --- /dev/null +++ b/src/backend/ci/core/common/common-event/src/main/kotlin/com/tencent/devops/common/stream/customizer/BkProducerMessageHandlerCustomizer.kt @@ -0,0 +1,16 @@ +package com.tencent.devops.common.stream.customizer + +import org.slf4j.LoggerFactory +import org.springframework.cloud.stream.config.ProducerMessageHandlerCustomizer +import org.springframework.integration.amqp.outbound.AmqpOutboundEndpoint + +class BkProducerMessageHandlerCustomizer : ProducerMessageHandlerCustomizer { + override fun configure(handler: AmqpOutboundEndpoint, destinationName: String) { + logger.info("handler customizer , destinationName: $destinationName , handler: ${handler.beanName}") + handler.rabbitTemplate?.isUsePublisherConnection = false // 生产者和消费者共享连接池, 提高性能 + } + + companion object { + private val logger = LoggerFactory.getLogger(BkProducerMessageHandlerCustomizer::class.java) + } +} \ No newline at end of file diff --git a/src/gateway/core/lua/util/loadbalance_util.lua b/src/gateway/core/lua/util/loadbalance_util.lua index 9acaaabe5cb..7e54024f7f3 100644 --- a/src/gateway/core/lua/util/loadbalance_util.lua +++ b/src/gateway/core/lua/util/loadbalance_util.lua @@ -20,6 +20,7 @@ _M = {} function _M:getTarget(devops_tag, service_name, cache_tail, ns_config) local in_container = ngx.var.namespace ~= '' and ngx.var.namespace ~= nil local gateway_project = ngx.var.project + local devops_project_id = ngx.var.project_id -- 不走容器化的服务 local no_container = false @@ -42,13 +43,19 @@ function _M:getTarget(devops_tag, service_name, cache_tail, ns_config) if gateway_project == 'codecc' then kubernetes_domain = config.kubernetes.codecc.domain else - kubernetes_domain = config.kubernetes.domain - end + if ngx.var.devops_region == 'DEVNET' then + kubernetes_domain = config.kubernetes.devnetDomain + elseif self:is_recovery_project(devops_project_id) then + kubernetes_domain = config.kubernetes.recovery.domain + else + kubernetes_domain = config.kubernetes.domain + end - -- 特殊处理的域名,优先级最高 - local special_key = gateway_project .. ":" .. devops_tag - if config.kubernetes.special_domain[special_key] ~= nil then - kubernetes_domain = config.kubernetes.special_domain[special_key] + -- 特殊处理的域名,优先级最高 + local special_key = gateway_project .. ":" .. devops_tag + if config.kubernetes.special_domain[special_key] ~= nil then + kubernetes_domain = config.kubernetes.special_domain[special_key] + end end return kubernetes_domain .. "/ms/" .. service_name end @@ -166,4 +173,35 @@ function _M:getTarget(devops_tag, service_name, cache_tail, ns_config) return ips[math.random(#ips)] .. ":" .. port end +function _M:is_recovery_project(devops_project_id) + if config.kubernetes.recovery.switchAll then + return true + end + + local recovery_project_cache = ngx.shared.router_srv_store + local local_cache_key = "ci_recovery_" .. devops_project_id + local is_recovery = recovery_project_cache:get(local_cache_key) + if is_recovery == nil then + -- 从redis获取 + local red, err = redisUtil:new() + if not red then + ngx.log(ngx.ERR, "tag failed to new redis ", err) + return false + end + local red_key = "ci:recovery:project:" .. devops_project_id + is_recovery = red:get(red_key) + if is_recovery ~= "1" then + is_recovery = "0" + end + recovery_project_cache:set(local_cache_key, is_recovery, 30) + red:set_keepalive(config.redis.max_idle_time, config.redis.pool_size) + end + + if is_recovery == "1" then + return true + else + return false + end +end + return _M diff --git a/src/gateway/core/lua/util/tag_util.lua b/src/gateway/core/lua/util/tag_util.lua index 5353f3201b3..5f26c2020a3 100644 --- a/src/gateway/core/lua/util/tag_util.lua +++ b/src/gateway/core/lua/util/tag_util.lua @@ -18,18 +18,18 @@ _M = {} -- 判断当前请求属于哪个tag function _M:get_tag(ns_config) - local devops_project_id = ngx.var.project_id - local devops_project = ngx.var.project + local gateway_project = ngx.var.project local devops_service = ngx.var.service + local devops_project_id = ngx.var.project_id local tag = nil local default_tag = ns_config.tag - if devops_project == 'codecc' then + if gateway_project == 'codecc' then default_tag = ns_config.codecc_tag end if ngx.var.use_default_tag == 'true' then - return default_tag + return "kubernetes-" .. default_tag end -- 根据header强制路由tag @@ -40,14 +40,11 @@ function _M:get_tag(ns_config) if x_gateway_tag ~= nil then tag = x_gateway_tag - if not string.find(tag, '^kubernetes-') and self:switch_kubernetes(devops_project, tag) then - tag = "kubernetes-" .. tag - end else -- 获取本地缓存 local tag_cache = ngx.shared.tag_project_store local tag_cache_key = 'tag_cache_' .. tostring(devops_project_id) .. '_' .. tostring(devops_service) .. '_' .. - tostring(devops_project) + tostring(gateway_project) local tag_cache_value = tag_cache:get(tag_cache_key) -- 如果有缓存 ,则使用缓存变量 @@ -57,18 +54,20 @@ function _M:get_tag(ns_config) local red, err = redisUtil:new() if not red then ngx.log(ngx.ERR, "tag failed to new redis ", err) - return tag + return "kubernetes-" .. default_tag end -- 根据project_id路由 if devops_project_id ~= nil and devops_project_id ~= '' then local redis_key = nil - if devops_project == 'codecc' then + if gateway_project == 'codecc' then redis_key = 'project:setting:tag:codecc:v2' else redis_key = "project:setting:tag:v2" end -- 从redis获取tag - local hash_key = '\xAC\xED\x00\x05t\x00' .. string.char(devops_project_id:len()) .. devops_project_id -- 兼容Spring Redis的hashKey的默认序列化 + local hash_key = '\xAC\xED\x00\x05t\x00' .. + string.char(devops_project_id:len()) .. + devops_project_id -- 兼容Spring Redis的hashKey的默认序列化 local redRes = red:hget(redis_key, hash_key) if redRes and redRes ~= ngx.null then local hash_val = redRes:sub(8) -- 兼容Spring Redis的hashValue的默认序列化 @@ -84,8 +83,8 @@ function _M:get_tag(ns_config) end end -- 根据ngx.var.project路由 - if tag == nil and devops_project then - local project_redis_cache_value = red:get("project:setting:project:tag:" .. devops_project) + if tag == nil and gateway_project then + local project_redis_cache_value = red:get("project:setting:project:tag:" .. gateway_project) if project_redis_cache_value and project_redis_cache_value ~= ngx.null then tag = project_redis_cache_value end @@ -94,23 +93,6 @@ function _M:get_tag(ns_config) if tag == nil then tag = default_tag end - -- 是否使用kubernetes - if not string.find(tag, '^kubernetes-') then - if self:switch_kubernetes(devops_project, tag) then - tag = "kubernetes-" .. tag - else - local k8s_redis_key = nil - if devops_project == 'codecc' then - k8s_redis_key = 'project:setting:k8s:codecc' - else - k8s_redis_key = "project:setting:k8s" - end - local k8s_redRes = red:sismember(k8s_redis_key, devops_project_id) - if k8s_redRes == 1 then - tag = "kubernetes-" .. tag - end - end - end --- 将redis连接放回pool中 red:set_keepalive(config.redis.max_idle_time, config.redis.pool_size) @@ -119,17 +101,18 @@ function _M:get_tag(ns_config) end end - -- 客户端选择路由到容器环境 - if type(tag) == "string" and not string.find(tag, '^kubernetes-') and config.kubernetes.useForceHeader then - if devops_project == 'codecc' then - if ngx.var.http_x_gateway_codecc_force_k8s == 'true' then - tag = "kubernetes-" .. tag - end - else - if ngx.var.http_x_gateway_force_k8s == 'true' then - tag = "kubernetes-" .. tag - end + -- 容器化切换已经完成 + if type(tag) == "string" and not string.find(tag, '^kubernetes-') then + tag = "kubernetes-" .. tag + end + + -- DEVNET区域对tag的转换 + local in_container = ngx.var.namespace ~= '' and ngx.var.namespace ~= nil + if in_container and ngx.var.project ~= 'codecc' and ngx.var.devops_region == 'DEVNET' and not tag.find(tag, '^ieg-codeccsvr-bkci-') then + if string.find(tag, '^kubernetes-') then + tag = string.sub(tag, 12) end + tag = 'ieg-codeccsvr-bkci-' .. tag end -- 设置tag到http请求头 @@ -138,26 +121,6 @@ function _M:get_tag(ns_config) return tag end -function _M:switch_kubernetes(devops_project, tag) - if config.kubernetes.switchAll == true then - return true - end - local isInList = false - local tags = nil - if devops_project == 'codecc' then - tags = config.kubernetes.codeccTags - else - tags = config.kubernetes.tags - end - for _, v in ipairs(tags) do - if v == tag then - isInList = true - break - end - end - return isInList -end - -- 设置tag到http请求头 function _M:set_header(tag) if ngx.var.http_x_gateway_tag == nil then diff --git a/support-files/templates/gateway#core#lua#init.lua b/support-files/templates/gateway#core#lua#init.lua index 770ef9dbdf9..ab80cc0f5c4 100644 --- a/support-files/templates/gateway#core#lua#init.lua +++ b/support-files/templates/gateway#core#lua#init.lua @@ -90,16 +90,16 @@ config = { bkci = { host = "__BK_CI_FQDN__", port = 80 }, kubernetes = { domain = "kubernetes.demo.com", - switchAll = false, codecc = { domain = "kubernetes.demo.com" }, - useForceHeader = false, - tags = {}, - codeccTags = {}, api = { url = "https://127.0.0.1/api/v1/nodes", token = "" }, - special_domain = {} + special_domain = {}, + recovery = { + switchAll = false, + domain = "k8s.demo.com" + } } }