From d6e4ae1a03a5b47c6e38412f2dbe7b8c7fc6357c Mon Sep 17 00:00:00 2001 From: octopus <912554887@qq.com> Date: Thu, 22 Aug 2024 16:36:54 +0800 Subject: [PATCH] support multible db datasource config --- settings.gradle | 2 +- wedpr-adm/conf/application-dm.properties | 26 +++ wedpr-adm/conf/application-gauss.properties | 27 +++ .../conf/application-kingbase.properties | 26 +++ wedpr-adm/conf/application-mysql.properties | 27 +++ .../conf/application-postgresql.properties | 26 +++ .../conf/application-shentong.properties | 27 +++ wedpr-adm/conf/application-wedpr.properties | 3 + wedpr-adm/conf/wedpr.properties | 15 +- .../dataset/config/DataSourceTypeConfig.java | 2 +- .../components/mybatis/DataSourceConfig.java | 179 ++++++++++++++---- 11 files changed, 311 insertions(+), 49 deletions(-) create mode 100644 wedpr-adm/conf/application-dm.properties create mode 100644 wedpr-adm/conf/application-gauss.properties create mode 100644 wedpr-adm/conf/application-kingbase.properties create mode 100644 wedpr-adm/conf/application-mysql.properties create mode 100644 wedpr-adm/conf/application-postgresql.properties create mode 100644 wedpr-adm/conf/application-shentong.properties diff --git a/settings.gradle b/settings.gradle index 64e8efac..92b40f75 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,4 +1,4 @@ -rootProject.name = "wedpr"; +rootProject.name = "WeDPR"; // submodules diff --git a/wedpr-adm/conf/application-dm.properties b/wedpr-adm/conf/application-dm.properties new file mode 100644 index 00000000..3ba6c828 --- /dev/null +++ b/wedpr-adm/conf/application-dm.properties @@ -0,0 +1,26 @@ +spring.datasource.url=jdbc:dm://127.0.0.1:3306?genKeyNameCase=0 +spring.datasource.username= +spring.datasource.password= +spring.datasource.driverClassName=dm.jdbc.driver.DmDriver +spring.datasource.type=com.alibaba.druid.pool.DruidDataSource +spring.datasource.defaultAutoCommit=true +spring.datasource.initialSize=30 +spring.datasource.maxActive=128 +spring.datasource.maxIdle=50 +spring.datasource.minIdle=30 +spring.datasource.queryTimeout=60 +spring.datasource.keepAlive=true +spring.datasource.testOnBorrow=true +spring.datasource.testOnReturn=false +spring.datasource.testWhileIdle=true +spring.datasource.validationQuery=SELECT 1 +spring.datasource.validationQueryTimeout=2 +spring.datasource.timeBetweenEvictionRunsMillis=60000 +spring.datasource.minEvictableIdleTimeMillis=300000 +spring.datasource.removeAbandoned=false +spring.datasource.logAbandoned=true +spring.datasource.removeAbandonedTimeout=54 +spring.datasource.maxWait=10000 +spring.datasource.poolPreparedStatements=false +spring.datasource.filters= +spring.datasource.connectionProperties=druid.stat.mergeSql=true;druid.stat.logSlowSql=true;druid.stat.slowSqlLogLevel=warn;druid.stat.slowSqlMillis=500 diff --git a/wedpr-adm/conf/application-gauss.properties b/wedpr-adm/conf/application-gauss.properties new file mode 100644 index 00000000..c4b86686 --- /dev/null +++ b/wedpr-adm/conf/application-gauss.properties @@ -0,0 +1,27 @@ +spring.datasource.url=jdbc:postgresql://127.0.0.1:3306/wedpr3 +spring.datasource.username= +spring.datasource.password= +spring.datasource.driverClassName=org.postgresql.Driver +spring.datasource.type=com.alibaba.druid.pool.DruidDataSource +spring.datasource.defaultAutoCommit=true +spring.datasource.initialSize=30 +spring.datasource.maxActive=128 +spring.datasource.maxIdle=50 +spring.datasource.minIdle=30 +spring.datasource.queryTimeout=60 +spring.datasource.keepAlive=true +spring.datasource.testOnBorrow=true +spring.datasource.testOnReturn=false +spring.datasource.testWhileIdle=true +spring.datasource.validationQuery=SELECT 1 +spring.datasource.validationQueryTimeout=2 +spring.datasource.timeBetweenEvictionRunsMillis=60000 +spring.datasource.minEvictableIdleTimeMillis=300000 +spring.datasource.removeAbandoned=false +spring.datasource.logAbandoned=true +spring.datasource.removeAbandonedTimeout=54 +spring.datasource.maxWait=10000 +spring.datasource.poolPreparedStatements=false +spring.datasource.filters= +spring.datasource.connectionProperties=druid.stat.mergeSql=true;druid.stat.logSlowSql=true;druid.stat.slowSqlLogLevel=warn;druid.stat.slowSqlMillis=500 + diff --git a/wedpr-adm/conf/application-kingbase.properties b/wedpr-adm/conf/application-kingbase.properties new file mode 100644 index 00000000..50deebb7 --- /dev/null +++ b/wedpr-adm/conf/application-kingbase.properties @@ -0,0 +1,26 @@ +spring.datasource.url=jdbc:kingbase8://127.0.0.1:3306/wedpr3 +spring.datasource.username= +spring.datasource.password= +spring.datasource.driverClassName=com.kingbase8.Driver +spring.datasource.type=com.alibaba.druid.pool.DruidDataSource +spring.datasource.defaultAutoCommit=true +spring.datasource.initialSize=30 +spring.datasource.maxActive=128 +spring.datasource.maxIdle=50 +spring.datasource.minIdle=30 +spring.datasource.queryTimeout=60 +spring.datasource.keepAlive=true +spring.datasource.testOnBorrow=true +spring.datasource.testOnReturn=false +spring.datasource.testWhileIdle=true +spring.datasource.validationQuery=SELECT 1 +spring.datasource.validationQueryTimeout=2 +spring.datasource.timeBetweenEvictionRunsMillis=60000 +spring.datasource.minEvictableIdleTimeMillis=300000 +spring.datasource.removeAbandoned=false +spring.datasource.logAbandoned=true +spring.datasource.removeAbandonedTimeout=54 +spring.datasource.maxWait=10000 +spring.datasource.poolPreparedStatements=false +spring.datasource.filters= +spring.datasource.connectionProperties=druid.stat.mergeSql=true;druid.stat.logSlowSql=true;druid.stat.slowSqlLogLevel=warn;druid.stat.slowSqlMillis=500 diff --git a/wedpr-adm/conf/application-mysql.properties b/wedpr-adm/conf/application-mysql.properties new file mode 100644 index 00000000..03fdcc48 --- /dev/null +++ b/wedpr-adm/conf/application-mysql.properties @@ -0,0 +1,27 @@ +spring.datasource.url=jdbc:mysql://127.0.0.1:3306/wedpr3?serverTimezone=GMT%2B8&characterEncoding=UTF-8&connectTimeout=60000&socketTimeout=60000 +spring.datasource.username= +spring.datasource.password= +spring.datasource.driverClassName=com.mysql.cj.jdbc.Driver +spring.datasource.type=com.alibaba.druid.pool.DruidDataSource +spring.datasource.defaultAutoCommit=true +spring.datasource.initialSize=30 +spring.datasource.maxActive=128 +spring.datasource.maxIdle=50 +spring.datasource.minIdle=30 +spring.datasource.queryTimeout=60 +spring.datasource.keepAlive=true +spring.datasource.testOnBorrow=true +spring.datasource.testOnReturn=false +spring.datasource.testWhileIdle=true +spring.datasource.validationQuery=SELECT 1 +spring.datasource.validationQueryTimeout=2 +spring.datasource.timeBetweenEvictionRunsMillis=60000 +spring.datasource.minEvictableIdleTimeMillis=300000 +spring.datasource.removeAbandoned=false +spring.datasource.logAbandoned=true +spring.datasource.removeAbandonedTimeout=54 +spring.datasource.maxWait=10000 +spring.datasource.poolPreparedStatements=false +#spring.datasource.filters=stat,log4j2 +spring.datasource.filters= +spring.datasource.connectionProperties=druid.stat.mergeSql=true;druid.stat.logSlowSql=true;druid.stat.slowSqlLogLevel=warn;druid.stat.slowSqlMillis=500 diff --git a/wedpr-adm/conf/application-postgresql.properties b/wedpr-adm/conf/application-postgresql.properties new file mode 100644 index 00000000..4a76ae41 --- /dev/null +++ b/wedpr-adm/conf/application-postgresql.properties @@ -0,0 +1,26 @@ +spring.datasource.url=jdbc:postgresql://127.0.0.1:3306/wedpr3 +spring.datasource.username= +spring.datasource.password= +spring.datasource.driverClassName=org.postgresql.Driver +spring.datasource.type=com.alibaba.druid.pool.DruidDataSource +spring.datasource.defaultAutoCommit=true +spring.datasource.initialSize=30 +spring.datasource.maxActive=128 +spring.datasource.maxIdle=50 +spring.datasource.minIdle=30 +spring.datasource.queryTimeout=60 +spring.datasource.keepAlive=true +spring.datasource.testOnBorrow=true +spring.datasource.testOnReturn=false +spring.datasource.testWhileIdle=true +spring.datasource.validationQuery=SELECT 1 +spring.datasource.validationQueryTimeout=2 +spring.datasource.timeBetweenEvictionRunsMillis=60000 +spring.datasource.minEvictableIdleTimeMillis=300000 +spring.datasource.removeAbandoned=false +spring.datasource.logAbandoned=true +spring.datasource.removeAbandonedTimeout=54 +spring.datasource.maxWait=10000 +spring.datasource.poolPreparedStatements=false +spring.datasource.filters= +spring.datasource.connectionProperties=druid.stat.mergeSql=true;druid.stat.logSlowSql=true;druid.stat.slowSqlLogLevel=warn;druid.stat.slowSqlMillis=500 diff --git a/wedpr-adm/conf/application-shentong.properties b/wedpr-adm/conf/application-shentong.properties new file mode 100644 index 00000000..52fe836b --- /dev/null +++ b/wedpr-adm/conf/application-shentong.properties @@ -0,0 +1,27 @@ +spring.datasource.url=jdbc:oscar://127.0.0.1:3306/wedpr3 +spring.datasource.username= +spring.datasource.password= +spring.datasource.driverClassName=com.oscar.Driver +spring.datasource.type=com.alibaba.druid.pool.DruidDataSource +spring.datasource.defaultAutoCommit=true +spring.datasource.initialSize=30 +spring.datasource.maxActive=128 +spring.datasource.maxIdle=50 +spring.datasource.minIdle=30 +spring.datasource.queryTimeout=60 +spring.datasource.keepAlive=true +spring.datasource.testOnBorrow=true +spring.datasource.testOnReturn=false +spring.datasource.testWhileIdle=true +spring.datasource.validationQuery=SELECT 1 +spring.datasource.validationQueryTimeout=2 +spring.datasource.timeBetweenEvictionRunsMillis=60000 +spring.datasource.minEvictableIdleTimeMillis=300000 +spring.datasource.removeAbandoned=false +spring.datasource.logAbandoned=true +spring.datasource.removeAbandonedTimeout=54 +spring.datasource.maxWait=10000 +spring.datasource.poolPreparedStatements=false +spring.datasource.filters= +spring.datasource.connectionProperties=druid.stat.mergeSql=true;druid.stat.logSlowSql=true;druid.stat.slowSqlLogLevel=warn;druid.stat.slowSqlMillis=500 + diff --git a/wedpr-adm/conf/application-wedpr.properties b/wedpr-adm/conf/application-wedpr.properties index 9337a90e..5b54a555 100644 --- a/wedpr-adm/conf/application-wedpr.properties +++ b/wedpr-adm/conf/application-wedpr.properties @@ -16,6 +16,9 @@ spring.servlet.multipart.max-request-size = -1 # mybatis configuration #mybatis.mapper-locations=classpath:mapper/*Mapper.xml +# mysql or dm or gauss or kingbase or shentong or postgresql +spring.profiles.include=mysql + ## dataset configuration wedpr.dataset.debugMode=false wedpr.dataset.debugMode.userTokenField=Wedpr-Test-User-Token diff --git a/wedpr-adm/conf/wedpr.properties b/wedpr-adm/conf/wedpr.properties index 7230e93a..18744024 100644 --- a/wedpr-adm/conf/wedpr.properties +++ b/wedpr-adm/conf/wedpr.properties @@ -11,15 +11,14 @@ wedpr.uuid.generator.worker.id=10 wedpr.enable.dpm = false ### the mybatis related configuration + #wedpr.mybatis.config.path=mybatis-config.xml -# TODO: instead use CMDB value -wedpr.mybatis.url=jdbc:mysql://127.0.0.1/wedpr3?characterEncoding=UTF-8&allowMultiQueries=true -#wedpr.mybatis.username=[*user_ppcs_modeladm] -#wedpr.mybatis.password=[*pass_ppcs_modeladm] -wedpr.mybatis.username= -wedpr.mybatis.password= - -wedpr.mybatis.driverClassName=com.mysql.cj.jdbc.Driver +# wedpr.mybatis.url=jdbc:mysql://127.0.0.1/wedpr3?characterEncoding=UTF-8&allowMultiQueries=true +# wedpr.mybatis.username= +# wedpr.mybatis.password= +# wedpr.mybatis.username= +# wedpr.mybatis.password= +# wedpr.mybatis.driverClassName=com.mysql.cj.jdbc.Driver wedpr.mybatis.mapperLocations=classpath*:mapper/*Mapper.xml # Note: the basePackage can't set to com.webank.wedpr simply for the mybatis will scan the Service diff --git a/wedpr-components/dataset/src/main/java/com/webank/wedpr/components/dataset/config/DataSourceTypeConfig.java b/wedpr-components/dataset/src/main/java/com/webank/wedpr/components/dataset/config/DataSourceTypeConfig.java index de9d4307..de3bc866 100644 --- a/wedpr-components/dataset/src/main/java/com/webank/wedpr/components/dataset/config/DataSourceTypeConfig.java +++ b/wedpr-components/dataset/src/main/java/com/webank/wedpr/components/dataset/config/DataSourceTypeConfig.java @@ -53,7 +53,7 @@ public void validateDataSourceConfigurations() throws DatasetException { // check db types for (LabelValue childLabelValue : labelValue.getChildren()) { DBType dbType = DBType.fromStrType(childLabelValue.getValue()); - logger.info(" add one db type: {}", dbType); + logger.info(" add one db datasource type: {}", dbType); } } } diff --git a/wedpr-components/mybatis/src/main/java/com/webank/wedpr/components/mybatis/DataSourceConfig.java b/wedpr-components/mybatis/src/main/java/com/webank/wedpr/components/mybatis/DataSourceConfig.java index 7f3d8427..b75b87d4 100644 --- a/wedpr-components/mybatis/src/main/java/com/webank/wedpr/components/mybatis/DataSourceConfig.java +++ b/wedpr-components/mybatis/src/main/java/com/webank/wedpr/components/mybatis/DataSourceConfig.java @@ -1,39 +1,140 @@ -/* - * Copyright 2017-2025 [webank-wedpr] - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - */ - -package com.webank.wedpr.components.mybatis; - -import javax.sql.DataSource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -public class DataSourceConfig { - private static final Logger logger = LoggerFactory.getLogger(DataSourceConfig.class); - - @Bean(name = "dataSource", destroyMethod = "close") - @ConditionalOnMissingBean - public DataSource dataSource() { - logger.info("Load DataSource"); - if (MybatisConfig.MYBATIS_CONFIG != null) { - DataSource dataSource = DataSourceBuilder.build(MybatisConfig.MYBATIS_CONFIG); - logger.info("Load DataSource from configuration success"); - return dataSource; - } - return DataSourceBuilder.build(); - } -} +package com.webank.wedpr.components.mybatis; + +import com.alibaba.druid.pool.DruidDataSource; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class DataSourceConfig { + private static final Logger logger = LoggerFactory.getLogger(DataSourceConfig.class); + + @Value("${spring.datasource.url}") + private String url; + + @Value("${spring.datasource.username}") + private String username; + + @Value("${spring.datasource.password}") + private String password; + + @Value("${spring.datasource.driverClassName}") + private String driverClassName; + + @Value("${spring.datasource.defaultAutoCommit}") + private boolean defaultAutoCommit; + + @Value("${spring.datasource.initialSize}") + private int initialSize; + + @Value("${spring.datasource.maxActive}") + private int maxActive; + + @Value("${spring.datasource.maxIdle}") + private int maxIdle; + + @Value("${spring.datasource.minIdle}") + private int minIdle; + + @Value("${spring.datasource.queryTimeout}") + private int queryTimeout = 0; + + @Value("${spring.datasource.keepAlive}") + private boolean keepAlive; + + @Value("${spring.datasource.testOnBorrow}") + private boolean testOnBorrow; + + @Value("${spring.datasource.testOnReturn}") + private boolean testOnReturn; + + @Value("${spring.datasource.testWhileIdle}") + private boolean testWhileIdle; + + @Value("${spring.datasource.maxWait}") + private long maxWait; + + @Value("${spring.datasource.timeBetweenEvictionRunsMillis}") + private long timeBetweenEvictionRunsMillis; + + @Value("${spring.datasource.minEvictableIdleTimeMillis}") + private long minEvictableIdleTimeMillis; + + @Value("${spring.datasource.validationQuery}") + private String validationQuery; + + @Value("${spring.datasource.validationQueryTimeout}") + private int validationQueryTimeout; + + @Value("${spring.datasource.removeAbandoned}") + private boolean removeAbandoned; + + @Value("${spring.datasource.logAbandoned}") + private boolean logAbandoned; + + @Value("${spring.datasource.removeAbandonedTimeout}") + private int removeAbandonedTimeout; + + @Value("${spring.datasource.poolPreparedStatements}") + private boolean poolPreparedStatements; + + @Value("${spring.datasource.connectionProperties}") + private String connectionProperties; + + @Value("${spring.datasource.filters}") + private String filters; + + @Bean(name = "dataSource", destroyMethod = "close") + public DruidDataSource dataSource() throws Exception { + + logger.info("dataSource dbUrl:{}", url); + logger.info("dataSource dbUsername:{}", username); + logger.info("dataSource driverClassName:{}", driverClassName); + logger.info("dataSource queryTimeout:{}", queryTimeout); + + // https://github.com/alibaba/druid/wiki/DruidDataSource%E9%85%8D%E7%BD%AE + + DruidDataSource datasource = new DruidDataSource(); + datasource.setUrl(url); + datasource.setUsername(username); + datasource.setPassword(password); + datasource.setDriverClassName(driverClassName); + datasource.setDefaultAutoCommit(defaultAutoCommit); + // configuration + datasource.setInitialSize(initialSize); + datasource.setMaxActive(maxActive); + datasource.setMinIdle(maxIdle); + datasource.setMinIdle(minIdle); + datasource.setTestOnBorrow(testOnBorrow); + datasource.setTestOnReturn(testOnReturn); + datasource.setTestWhileIdle(testWhileIdle); + datasource.setValidationQuery(validationQuery); + datasource.setValidationQueryTimeout(validationQueryTimeout); + datasource.setQueryTimeout(queryTimeout); + // + datasource.setTimeBetweenEvictionRunsMillis(timeBetweenEvictionRunsMillis); + datasource.setMinEvictableIdleTimeMillis(minEvictableIdleTimeMillis); + // datasource.setMaxEvictableIdleTimeMillis(minEvictableIdleTimeMillis); + datasource.setKeepAlive(keepAlive); + + datasource.setRemoveAbandoned(removeAbandoned); + datasource.setLogAbandoned(logAbandoned); + datasource.setRemoveAbandonedTimeout(removeAbandonedTimeout); + + datasource.setMaxWait(maxWait); + datasource.setPoolPreparedStatements(poolPreparedStatements); + datasource.setConnectionProperties(connectionProperties); + datasource.setFilters(filters); + + datasource.init(); + Map statData = datasource.getStatData(); + + logger.info("create druid datasource successfully: {}", statData); + + return datasource; + } +}