Merge pull request #5984 from carlyin0801/issue_5267_sub_db

perf:蓝盾引擎模块对应的数据库支持分库 #5267
TencentBlueKing · Jan 11, 2022 · 1bbce98 · 1bbce98
2 parents 53cef19 + f8f57d5
commit 1bbce98
Show file tree

Hide file tree

Showing 9 changed files with 87 additions and 15 deletions.
diff --git a/...ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/DataSource.kt b/...ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/DataSource.kt
@@ -33,6 +33,9 @@ import io.swagger.annotations.ApiModelProperty
 
 @ApiModel("数据源")
 data class DataSource(
+    @ApiModelProperty("集群名称")
+    @field:BkField(minLength = 1, maxLength = 64)
+    val clusterName: String,
     @ApiModelProperty("模块标识")
     val moduleCode: SystemModuleEnum,
     @ApiModelProperty("数据源名称")

diff --git a/...t/api-project/src/main/kotlin/com/tencent/devops/project/pojo/enums/ProjectChannelCode.kt b/...t/api-project/src/main/kotlin/com/tencent/devops/project/pojo/enums/ProjectChannelCode.kt
@@ -36,10 +36,10 @@ enum class ProjectChannelCode {
     BS,
     @ApiModelProperty("PREBULD")
     PREBUILD,
-    @ApiModelProperty("CI")
-    CI,
     @ApiModelProperty("CODECC")
     CODECC,
     @ApiModelProperty("GITCI")
-    GITCI
+    GITCI,
+    @ApiModelProperty("自动化集群项目，区分于CODECC项目")
+    AUTO
 }
diff --git a/.../core/project/biz-project/src/main/kotlin/com/tencent/devops/project/dao/DataSourceDao.kt b/.../core/project/biz-project/src/main/kotlin/com/tencent/devops/project/dao/DataSourceDao.kt
@@ -45,6 +45,7 @@ class DataSourceDao {
             dslContext.insertInto(
                 this,
                 ID,
+                CLUSTER_NAME,
                 MODULE_CODE,
                 DATA_SOURCE_NAME,
                 FULL_FLAG,
@@ -53,6 +54,7 @@ class DataSourceDao {
             )
                 .values(
                     UUIDUtil.generate(),
+                    dataSource.clusterName,
                     dataSource.moduleCode.name,
                     dataSource.dataSourceName,
                     dataSource.fullFlag,
@@ -64,10 +66,14 @@ class DataSourceDao {
         }
     }
 
-    fun countByName(dslContext: DSLContext, moduleCode: String, dataSourceName: String): Int {
+    fun countByName(dslContext: DSLContext, clusterName: String, moduleCode: String, dataSourceName: String): Int {
         with(TDataSource.T_DATA_SOURCE) {
             return dslContext.selectCount().from(this)
-                .where(MODULE_CODE.eq(moduleCode).and(DATA_SOURCE_NAME.eq(dataSourceName)))
+                .where(
+                    CLUSTER_NAME.eq(clusterName)
+                        .and(MODULE_CODE.eq(moduleCode))
+                        .and(DATA_SOURCE_NAME.eq(dataSourceName))
+                )
                 .fetchOne(0, Int::class.java)!!
         }
     }
@@ -90,11 +96,13 @@ class DataSourceDao {
 
     fun listByModule(
         dslContext: DSLContext,
+        clusterName: String,
         moduleCode: String,
         fullFlag: Boolean? = false
     ): Result<TDataSourceRecord>? {
         return with(TDataSource.T_DATA_SOURCE) {
             val conditions = mutableListOf<Condition>()
+            conditions.add(CLUSTER_NAME.eq(clusterName))
             conditions.add(MODULE_CODE.eq(moduleCode))
             if (fullFlag != null) {
                 conditions.add(FULL_FLAG.eq(fullFlag))
@@ -106,6 +114,7 @@ class DataSourceDao {
     fun update(dslContext: DSLContext, id: String, dataSource: DataSource) {
         with(TDataSource.T_DATA_SOURCE) {
             dslContext.update(this)
+                .set(CLUSTER_NAME, dataSource.clusterName)
                 .set(MODULE_CODE, dataSource.moduleCode.name)
                 .set(DATA_SOURCE_NAME, dataSource.dataSourceName)
                 .set(FULL_FLAG, dataSource.fullFlag)

diff --git a/...ject/src/main/kotlin/com/tencent/devops/project/service/ProjectDataSourceAssignService.kt b/...ject/src/main/kotlin/com/tencent/devops/project/service/ProjectDataSourceAssignService.kt
@@ -28,8 +28,13 @@
 package com.tencent.devops.project.service
 
 import com.tencent.devops.common.api.enums.SystemModuleEnum
+import com.tencent.devops.project.pojo.enums.ProjectChannelCode
 
 interface ProjectDataSourceAssignService {
 
-    fun assignDataSource(projectId: String, moduleCodes: List<SystemModuleEnum>): Boolean
+    fun assignDataSource(
+        channelCode: ProjectChannelCode,
+        projectId: String,
+        moduleCodes: List<SystemModuleEnum>
+    ): Boolean
 }
diff --git a/...-project/src/main/kotlin/com/tencent/devops/project/service/impl/AbsProjectServiceImpl.kt b/...-project/src/main/kotlin/com/tencent/devops/project/service/impl/AbsProjectServiceImpl.kt
@@ -215,6 +215,7 @@ abstract class AbsProjectServiceImpl @Autowired constructor(
                 }
                 // 为项目分配数据源
                 projectDataSourceAssignService.assignDataSource(
+                    channelCode = projectChannel,
                     projectId = projectCreateInfo.englishName,
                     moduleCodes = listOf(SystemModuleEnum.PROCESS)
                 )

diff --git a/...-project/src/main/kotlin/com/tencent/devops/project/service/impl/DataSourceServiceImpl.kt b/...-project/src/main/kotlin/com/tencent/devops/project/service/impl/DataSourceServiceImpl.kt
@@ -44,14 +44,20 @@ class DataSourceServiceImpl @Autowired constructor(
 ) : DataSourceService {
 
     override fun addDataSource(userId: String, dataSource: DataSource): Boolean {
+        val clusterName = dataSource.clusterName
         val dataSourceName = dataSource.dataSourceName
         val moduleCode = dataSource.moduleCode.name
-        val nameCount = dataSourceDao.countByName(dslContext, moduleCode, dataSourceName)
+        val nameCount = dataSourceDao.countByName(
+            dslContext = dslContext,
+            clusterName = clusterName,
+            moduleCode = moduleCode,
+            dataSourceName = dataSourceName
+        )
         if (nameCount > 0) {
             // 抛出错误提示
             throw ErrorCodeException(
                 errorCode = CommonMessageCode.PARAMETER_IS_EXIST,
-                params = arrayOf("[$moduleCode]$dataSourceName")
+                params = arrayOf("[$clusterName-$moduleCode]$dataSourceName")
             )
         }
         dataSourceDao.add(dslContext, userId, dataSource)
@@ -64,17 +70,23 @@ class DataSourceServiceImpl @Autowired constructor(
     }
 
     override fun updateDataSource(userId: String, id: String, dataSource: DataSource): Boolean {
+        val clusterName = dataSource.clusterName
         val dataSourceName = dataSource.dataSourceName
         val moduleCode = dataSource.moduleCode.name
-        val nameCount = dataSourceDao.countByName(dslContext, moduleCode, dataSourceName)
+        val nameCount = dataSourceDao.countByName(
+            dslContext = dslContext,
+            clusterName = clusterName,
+            moduleCode = moduleCode,
+            dataSourceName = dataSourceName
+        )
         if (nameCount > 0) {
             // 判断更新的名称是否属于自已
             val obj = dataSourceDao.getById(dslContext, id)
             if (null != obj && moduleCode != obj.moduleCode && dataSourceName != obj.dataSourceName) {
                 // 抛出错误提示
                 throw ErrorCodeException(
                     errorCode = CommonMessageCode.PARAMETER_IS_EXIST,
-                    params = arrayOf("[$moduleCode]$dataSourceName")
+                    params = arrayOf("[$clusterName-$moduleCode]$dataSourceName")
                 )
             }
         }
@@ -85,7 +97,12 @@ class DataSourceServiceImpl @Autowired constructor(
     override fun getDataSourceById(id: String): DataSource? {
         val record = dataSourceDao.getById(dslContext, id)
         return if (record != null) {
-            DataSource(SystemModuleEnum.valueOf(record.moduleCode), record.dataSourceName, record.fullFlag)
+            DataSource(
+                clusterName = record.clusterName,
+                moduleCode = SystemModuleEnum.valueOf(record.moduleCode),
+                dataSourceName = record.dataSourceName,
+                fullFlag = record.fullFlag
+            )
         } else {
             null
         }

diff --git a/...otlin/com/tencent/devops/project/service/impl/ProjectDataSourceRandomAssignServiceImpl.kt b/...otlin/com/tencent/devops/project/service/impl/ProjectDataSourceRandomAssignServiceImpl.kt
@@ -30,10 +30,12 @@ package com.tencent.devops.project.service.impl
 import com.tencent.devops.common.api.enums.SystemModuleEnum
 import com.tencent.devops.project.dao.DataSourceDao
 import com.tencent.devops.common.api.pojo.ShardingRoutingRule
+import com.tencent.devops.project.pojo.enums.ProjectChannelCode
 import com.tencent.devops.project.service.ProjectDataSourceAssignService
 import com.tencent.devops.project.service.ShardingRoutingRuleService
 import org.jooq.DSLContext
 import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.beans.factory.annotation.Value
 import org.springframework.stereotype.Service
 
 @Service
@@ -43,10 +45,39 @@ class ProjectDataSourceRandomAssignServiceImpl @Autowired constructor(
     private val shardingRoutingRuleService: ShardingRoutingRuleService
 ) : ProjectDataSourceAssignService {
 
-    override fun assignDataSource(projectId: String, moduleCodes: List<SystemModuleEnum>): Boolean {
+    @Value("\${tag.prod:prod}")
+    private val prodTag: String = "prod"
+
+    @Value("\${tag.auto:auto}")
+    private val autoTag: String = "auto"
+
+    @Value("\${tag.stream:stream}")
+    private val streamTag: String = "stream"
+
+    override fun assignDataSource(
+        channelCode: ProjectChannelCode,
+        projectId: String,
+        moduleCodes: List<SystemModuleEnum>
+    ): Boolean {
+        // 根据channelCode获取集群名称
+        val clusterName = if (channelCode == ProjectChannelCode.BS || channelCode == ProjectChannelCode.PREBUILD) {
+            prodTag
+        } else if (channelCode == ProjectChannelCode.CODECC || channelCode == ProjectChannelCode.AUTO) {
+            autoTag
+        } else if (channelCode == ProjectChannelCode.GITCI) {
+            streamTag
+        } else {
+            // 其他渠道的项目的接口请求默认路由到正式集群
+            prodTag
+        }
         moduleCodes.forEach { moduleCode ->
             // 根据模块查找还有还有空余容量的数据源
-            val dataSources = dataSourceDao.listByModule(dslContext, moduleCode.name, false)
+            val dataSources = dataSourceDao.listByModule(
+                dslContext = dslContext,
+                clusterName = clusterName,
+                moduleCode = moduleCode.name,
+                fullFlag = false
+            )
             if (dataSources.isNullOrEmpty()) {
                 return@forEach
             }

diff --git a/...src/main/kotlin/com/tencent/devops/project/service/impl/ShardingRoutingRuleServiceImpl.kt b/...src/main/kotlin/com/tencent/devops/project/service/impl/ShardingRoutingRuleServiceImpl.kt
@@ -77,7 +77,6 @@ class ShardingRoutingRuleServiceImpl @Autowired constructor(
             val routingName = shardingRoutingRuleRecord.routingName
             // 删除redis中规则信息
             redisOperation.delete(getShardingRoutingRuleKey(routingName))
-            val routingRule = shardingRoutingRuleRecord.routingRule
         }
         return true
     }
@@ -127,6 +126,12 @@ class ShardingRoutingRuleServiceImpl @Autowired constructor(
             // redis缓存中未取到规则信息则从db查
             val record = shardingRoutingRuleDao.getByName(dslContext, routingName)
             if (record != null) {
+                // 更新redis缓存规则信息
+                redisOperation.set(
+                    key = getShardingRoutingRuleKey(routingName),
+                    value = record.routingRule,
+                    expired = false
+                )
                 ShardingRoutingRule(record.routingName, record.routingRule)
             } else {
                 null

diff --git a/support-files/sql/1001_ci_project_ddl_mysql.sql b/support-files/sql/1001_ci_project_ddl_mysql.sql
@@ -269,6 +269,7 @@ CREATE TABLE IF NOT EXISTS `T_SHARDING_ROUTING_RULE` (
 
 CREATE TABLE IF NOT EXISTS `T_DATA_SOURCE` (
   `ID` varchar(32) NOT NULL DEFAULT '' COMMENT '主键ID',
+  `CLUSTER_NAME` varchar(64) NOT NULL DEFAULT '' COMMENT '集群名称',
   `MODULE_CODE` varchar(64) NOT NULL DEFAULT '' COMMENT '模块标识',
   `DATA_SOURCE_NAME` varchar(128) NOT NULL DEFAULT '' COMMENT '数据源名称',
   `FULL_FLAG` bit(1) DEFAULT b'0' COMMENT '容量是否满标识 true：是，false：否',
@@ -277,7 +278,7 @@ CREATE TABLE IF NOT EXISTS `T_DATA_SOURCE` (
   `UPDATE_TIME` datetime(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) COMMENT '修改时间',
   `CREATE_TIME` datetime(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) COMMENT '创建时间',
   PRIMARY KEY (`ID`),
-  UNIQUE KEY  `uni_inx_tds_module_name` (`MODULE_CODE`,`DATA_SOURCE_NAME`)
+  UNIQUE KEY  `uni_inx_tds_module_name` (`CLUSTER_NAME`, `MODULE_CODE`,`DATA_SOURCE_NAME`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='模块数据源配置';
 
 CREATE TABLE IF NOT EXISTS `T_LEAF_ALLOC` (