feat: 整理网关的tag路由 #11050

helm-charts/core/ci/templates/gateway/deployment.yaml

@@ -53,8 +53,6 @@ spec:
       initContainers:
         - name: frontend
           image: {{ include "bkci-frontend.image" . }}
-          securityContext:
-            privileged: true
           volumeMounts:
           - mountPath: /tmp/frontend
             name: frontend

src/backend/ci/buildSrc/src/main/kotlin/plugins/task-docker-build.gradle.kts

@@ -27,15 +27,12 @@
 plugins {
     id("com.google.cloud.tools.jib")
 }
-
 val toImageRepo = System.getProperty("to.image.repo")
 val toImageTag = System.getProperty("to.image.tag")
 var toImage = System.getProperty("jib.to.image")
-
 // 加这个判断 , 主要是为了编译kts时不报错
 if (toImage.isNullOrBlank() || (toImageRepo.isNullOrBlank() && toImageTag.isNullOrBlank())) {
     val service = name.replace("boot-", "").replace("-tencent", "")
-
     if (toImage.isNullOrBlank() && !toImageRepo.isNullOrBlank()) {
         toImage = toImageRepo.let {
             if (toImageRepo.endsWith("/")) it else it + "/"
@@ -44,6 +41,8 @@ if (toImage.isNullOrBlank() || (toImageRepo.isNullOrBlank() && toImageTag.isNull
 
     val jvmFlagList = System.getProperty("jvmFlags.file")?.let { File(it).readLines() } ?: emptyList()
 
+    val watchAllNamespace = System.getProperty("watch.all.namespace", "true")
+
     val finalJvmFlags = mutableListOf(
         "-server",
         "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8080",
@@ -73,13 +72,12 @@ if (toImage.isNullOrBlank() || (toImageRepo.isNullOrBlank() && toImageTag.isNull
         "-Dspring.main.allow-circular-references=true",
         "-Dspring.cloud.kubernetes.config.sources[0].name=config-bk-ci-common",
         "-Dspring.cloud.kubernetes.config.sources[1].name=config-bk-ci-$service",
-        "-Dspring.cloud.kubernetes.discovery.all-namespaces=true",
+        "-Dspring.cloud.kubernetes.discovery.all-namespaces=$watchAllNamespace",
         "-Dspring.cloud.kubernetes.config.includeProfileSpecificSources=false",
         "-Dio.undertow.legacy.cookie.ALLOW_HTTP_SEPARATORS_IN_V0=true",
         "-Dserver.port=80"
     )
     finalJvmFlags.addAll(jvmFlagList)
-
     jib {
         // 环境变量
         container {
@@ -96,4 +94,4 @@ if (toImage.isNullOrBlank() || (toImageRepo.isNullOrBlank() && toImageTag.isNull
             image = toImage
         }
     }
-}
+}

src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/util/DateTimeUtil.kt

@@ -74,6 +74,32 @@ object DateTimeUtil {
 
     const val ONE_THOUSAND_MS = 1000L
 
+    /**
+     * 获取时间列表里面的最小值
+     */
+    fun min(vararg localDateTimes: LocalDateTime): LocalDateTime {
+        var result = localDateTimes[0]
+        for (i in 1 until localDateTimes.size) {
+            if (localDateTimes[i].isBefore(result)) {
+                result = localDateTimes[i]
+            }
+        }
+        return result
+    }
+
+    /**
+     * 获取时间列表里面的最大值
+     */
+    fun max(vararg localDateTimes: LocalDateTime): LocalDateTime {
+        var result = localDateTimes[0]
+        for (i in 1 until localDateTimes.size) {
+            if (localDateTimes[i].isAfter(result)) {
+                result = localDateTimes[i]
+            }
+        }
+        return result
+    }
+
     /**
      * 单位转换，分钟转换秒
      */
@@ -197,8 +223,8 @@ object DateTimeUtil {
         val minute = (time - hour * timeGap * timeGap * million) / (timeGap * million)
         val second = (time - hour * timeGap * timeGap * million - minute * timeGap * million) / million
         return (if (hour == zero) "00" else if (hour >= ten) hour.toString() else "0$hour").toString() + "时" +
-            (if (minute == zero) "00" else if (minute >= ten) minute else "0$minute") + "分" +
-            (if (second == zero) "00" else if (second >= ten) second.toShort() else "0$second") + "秒"
+                (if (minute == zero) "00" else if (minute >= ten) minute else "0$minute") + "分" +
+                (if (second == zero) "00" else if (second >= ten) second.toShort() else "0$second") + "秒"
     }
 
     fun formatMilliTime(time: Long): String {
@@ -216,8 +242,8 @@ object DateTimeUtil {
         val minute = (time - hour * 60 * 60 * 1000) / (60 * 1000)
         val second = (time - hour * 60 * 60 * 1000 - minute * 60 * 1000) / 1000
         return (if (hour == 0L) "00" else if (hour >= 10) hour.toString() else "0$hour").toString() + "时" +
-            (if (minute == 0L) "00" else if (minute >= 10) minute else "0$minute") + "分" +
-            (if (second == 0L) "00" else if (second >= 10) second.toShort() else "0$second") + "秒"
+                (if (minute == 0L) "00" else if (minute >= 10) minute else "0$minute") + "分" +
+                (if (second == 0L) "00" else if (second >= 10) second.toShort() else "0$second") + "秒"
     }
 
     fun formatMillSecond(mss: Long): String {

src/backend/ci/core/common/common-event/src/main/kotlin/com/tencent/devops/common/stream/config/CommonEventConfiguration.kt

@@ -2,6 +2,7 @@ package com.tencent.devops.common.stream.config
 
 import com.tencent.devops.common.event.dispatcher.SampleEventDispatcher
 import com.tencent.devops.common.stream.config.interceptor.BkChannelInterceptor
+import com.tencent.devops.common.stream.customizer.BkProducerMessageHandlerCustomizer
 import org.springframework.cloud.stream.function.StreamBridge
 import org.springframework.context.annotation.Bean
 import org.springframework.context.annotation.Configuration
@@ -15,4 +16,7 @@ class CommonEventConfiguration {
     @Bean
     @GlobalChannelInterceptor
     fun bkChannelInterceptor() = BkChannelInterceptor()
+
+    @Bean
+    fun bkProducerMessageHandlerCustomizer() = BkProducerMessageHandlerCustomizer()
 }

src/backend/ci/core/common/common-event/src/main/kotlin/com/tencent/devops/common/stream/customizer/BkProducerMessageHandlerCustomizer.kt

@@ -0,0 +1,16 @@
+package com.tencent.devops.common.stream.customizer
+
+import org.slf4j.LoggerFactory
+import org.springframework.cloud.stream.config.ProducerMessageHandlerCustomizer
+import org.springframework.integration.amqp.outbound.AmqpOutboundEndpoint
+
+class BkProducerMessageHandlerCustomizer : ProducerMessageHandlerCustomizer<AmqpOutboundEndpoint> {
+    override fun configure(handler: AmqpOutboundEndpoint, destinationName: String) {
+        logger.info("handler customizer , destinationName: $destinationName , handler: ${handler.beanName}")
+        handler.rabbitTemplate?.isUsePublisherConnection = false // 生产者和消费者共享连接池, 提高性能
+    }
+
+    companion object {
+        private val logger = LoggerFactory.getLogger(BkProducerMessageHandlerCustomizer::class.java)
+    }
+}

src/gateway/core/lua/util/loadbalance_util.lua

@@ -20,6 +20,7 @@ _M = {}
 function _M:getTarget(devops_tag, service_name, cache_tail, ns_config)
     local in_container = ngx.var.namespace ~= '' and ngx.var.namespace ~= nil
     local gateway_project = ngx.var.project
+    local devops_project_id = ngx.var.project_id
 
     -- 不走容器化的服务
     local no_container = false
@@ -42,13 +43,19 @@ function _M:getTarget(devops_tag, service_name, cache_tail, ns_config)
         if gateway_project == 'codecc' then
             kubernetes_domain = config.kubernetes.codecc.domain
         else
-            kubernetes_domain = config.kubernetes.domain
-        end
+            if ngx.var.devops_region == 'DEVNET' then
+                kubernetes_domain = config.kubernetes.devnetDomain
+            elseif self:is_recovery_project(devops_project_id) then
+                kubernetes_domain = config.kubernetes.recovery.domain
+            else
+                kubernetes_domain = config.kubernetes.domain
+            end
 
-        -- 特殊处理的域名,优先级最高
-        local special_key = gateway_project .. ":" .. devops_tag
-        if config.kubernetes.special_domain[special_key] ~= nil then
-            kubernetes_domain = config.kubernetes.special_domain[special_key]
+            -- 特殊处理的域名,优先级最高
+            local special_key = gateway_project .. ":" .. devops_tag
+            if config.kubernetes.special_domain[special_key] ~= nil then
+                kubernetes_domain = config.kubernetes.special_domain[special_key]
+            end
         end
         return kubernetes_domain .. "/ms/" .. service_name
     end
@@ -166,4 +173,35 @@ function _M:getTarget(devops_tag, service_name, cache_tail, ns_config)
     return ips[math.random(#ips)] .. ":" .. port
 end
 
+function _M:is_recovery_project(devops_project_id)
+    if config.kubernetes.recovery.switchAll then
+        return true
+    end
+
+    local recovery_project_cache = ngx.shared.router_srv_store
+    local local_cache_key = "ci_recovery_" .. devops_project_id
+    local is_recovery = recovery_project_cache:get(local_cache_key)
+    if is_recovery == nil then
+        -- 从redis获取
+        local red, err = redisUtil:new()
+        if not red then
+            ngx.log(ngx.ERR, "tag failed to new redis ", err)
+            return false
+        end
+        local red_key = "ci:recovery:project:" .. devops_project_id
+        is_recovery = red:get(red_key)
+        if is_recovery ~= "1" then
+            is_recovery = "0"
+        end
+        recovery_project_cache:set(local_cache_key, is_recovery, 30)
+        red:set_keepalive(config.redis.max_idle_time, config.redis.pool_size)
+    end
+
+    if is_recovery == "1" then
+        return true
+    else
+        return false
+    end
+end
+
 return _M

src/gateway/core/lua/util/tag_util.lua

@@ -18,18 +18,18 @@
 _M = {}
 -- 判断当前请求属于哪个tag
 function _M:get_tag(ns_config)
-    local devops_project_id = ngx.var.project_id
-    local devops_project = ngx.var.project
+    local gateway_project = ngx.var.project
     local devops_service = ngx.var.service
+    local devops_project_id = ngx.var.project_id
     local tag = nil
 
     local default_tag = ns_config.tag
-    if devops_project == 'codecc' then
+    if gateway_project == 'codecc' then
         default_tag = ns_config.codecc_tag
     end
 
     if ngx.var.use_default_tag == 'true' then
-        return default_tag
+        return "kubernetes-" .. default_tag
     end
 
     -- 根据header强制路由tag
@@ -40,14 +40,11 @@ function _M:get_tag(ns_config)
 
     if x_gateway_tag ~= nil then
         tag = x_gateway_tag
-        if not string.find(tag, '^kubernetes-') and self:switch_kubernetes(devops_project, tag) then
-            tag = "kubernetes-" .. tag
-        end
     else
         -- 获取本地缓存
         local tag_cache = ngx.shared.tag_project_store
         local tag_cache_key = 'tag_cache_' .. tostring(devops_project_id) .. '_' .. tostring(devops_service) .. '_' ..
-                                  tostring(devops_project)
+            tostring(gateway_project)
         local tag_cache_value = tag_cache:get(tag_cache_key)
 
         -- 如果有缓存 ,则使用缓存变量
@@ -57,18 +54,20 @@ function _M:get_tag(ns_config)
             local red, err = redisUtil:new()
             if not red then
                 ngx.log(ngx.ERR, "tag failed to new redis ", err)
-                return tag
+                return "kubernetes-" .. default_tag
             end
             -- 根据project_id路由
             if devops_project_id ~= nil and devops_project_id ~= '' then
                 local redis_key = nil
-                if devops_project == 'codecc' then
+                if gateway_project == 'codecc' then
                     redis_key = 'project:setting:tag:codecc:v2'
                 else
                     redis_key = "project:setting:tag:v2"
                 end
                 -- 从redis获取tag
-                local hash_key = '\xAC\xED\x00\x05t\x00' .. string.char(devops_project_id:len()) .. devops_project_id -- 兼容Spring Redis的hashKey的默认序列化
+                local hash_key = '\xAC\xED\x00\x05t\x00' ..
+                    string.char(devops_project_id:len()) ..
+                    devops_project_id -- 兼容Spring Redis的hashKey的默认序列化
                 local redRes = red:hget(redis_key, hash_key)
                 if redRes and redRes ~= ngx.null then
                     local hash_val = redRes:sub(8) -- 兼容Spring Redis的hashValue的默认序列化
@@ -84,8 +83,8 @@ function _M:get_tag(ns_config)
                 end
             end
             -- 根据ngx.var.project路由
-            if tag == nil and devops_project then
-                local project_redis_cache_value = red:get("project:setting:project:tag:" .. devops_project)
+            if tag == nil and gateway_project then
+                local project_redis_cache_value = red:get("project:setting:project:tag:" .. gateway_project)
                 if project_redis_cache_value and project_redis_cache_value ~= ngx.null then
                     tag = project_redis_cache_value
                 end
@@ -94,23 +93,6 @@ function _M:get_tag(ns_config)
             if tag == nil then
                 tag = default_tag
             end
-            -- 是否使用kubernetes
-            if not string.find(tag, '^kubernetes-') then
-                if self:switch_kubernetes(devops_project, tag) then
-                    tag = "kubernetes-" .. tag
-                else
-                    local k8s_redis_key = nil
-                    if devops_project == 'codecc' then
-                        k8s_redis_key = 'project:setting:k8s:codecc'
-                    else
-                        k8s_redis_key = "project:setting:k8s"
-                    end
-                    local k8s_redRes = red:sismember(k8s_redis_key, devops_project_id)
-                    if k8s_redRes == 1 then
-                        tag = "kubernetes-" .. tag
-                    end
-                end
-            end
             --- 将redis连接放回pool中
             red:set_keepalive(config.redis.max_idle_time, config.redis.pool_size)
 
@@ -119,17 +101,18 @@ function _M:get_tag(ns_config)
         end
     end
 
-    -- 客户端选择路由到容器环境
-    if type(tag) == "string" and not string.find(tag, '^kubernetes-') and config.kubernetes.useForceHeader then
-        if devops_project == 'codecc' then
-            if ngx.var.http_x_gateway_codecc_force_k8s == 'true' then
-                tag = "kubernetes-" .. tag
-            end
-        else
-            if ngx.var.http_x_gateway_force_k8s == 'true' then
-                tag = "kubernetes-" .. tag
-            end
+    -- 容器化切换已经完成
+    if type(tag) == "string" and not string.find(tag, '^kubernetes-') then
+        tag = "kubernetes-" .. tag
+    end
+
+    -- DEVNET区域对tag的转换
+    local in_container = ngx.var.namespace ~= '' and ngx.var.namespace ~= nil
+    if in_container and ngx.var.project ~= 'codecc' and ngx.var.devops_region == 'DEVNET' and not tag.find(tag, '^ieg-codeccsvr-bkci-') then
+        if string.find(tag, '^kubernetes-') then
+            tag = string.sub(tag, 12)
         end
+        tag = 'ieg-codeccsvr-bkci-' .. tag
     end
 
     -- 设置tag到http请求头
@@ -138,26 +121,6 @@ function _M:get_tag(ns_config)
     return tag
 end
 
-function _M:switch_kubernetes(devops_project, tag)
-    if config.kubernetes.switchAll == true then
-        return true
-    end
-    local isInList = false
-    local tags = nil
-    if devops_project == 'codecc' then
-        tags = config.kubernetes.codeccTags
-    else
-        tags = config.kubernetes.tags
-    end
-    for _, v in ipairs(tags) do
-        if v == tag then
-            isInList = true
-            break
-        end
-    end
-    return isInList
-end
-
 -- 设置tag到http请求头
 function _M:set_header(tag)
     if ngx.var.http_x_gateway_tag == nil then

support-files/templates/gateway#core#lua#init.lua

@@ -90,16 +90,16 @@ config = {
     bkci = { host = "__BK_CI_FQDN__", port = 80 },
     kubernetes = {
         domain = "kubernetes.demo.com",
-        switchAll = false,
         codecc = { domain = "kubernetes.demo.com" },
-        useForceHeader = false,
-        tags = {},
-        codeccTags = {},
         api = {
             url = "https://127.0.0.1/api/v1/nodes",
             token = ""
         },
-        special_domain = {}
+        special_domain = {},
+        recovery = {
+            switchAll = false,
+            domain = "k8s.demo.com"
+        }
     }
 }