Merge pull request #10 from ywy2090/feature-milestone2-datasource

support multible db datasource config

settings.gradle

@@ -1,4 +1,4 @@
-rootProject.name = "wedpr";
+rootProject.name = "WeDPR";
 
 
 // submodules

wedpr-adm/conf/application-dm.properties

@@ -0,0 +1,26 @@
+spring.datasource.url=jdbc:dm://127.0.0.1:3306?genKeyNameCase=0
+spring.datasource.username=
+spring.datasource.password=
+spring.datasource.driverClassName=dm.jdbc.driver.DmDriver
+spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
+spring.datasource.defaultAutoCommit=true
+spring.datasource.initialSize=30
+spring.datasource.maxActive=128
+spring.datasource.maxIdle=50
+spring.datasource.minIdle=30
+spring.datasource.queryTimeout=60
+spring.datasource.keepAlive=true
+spring.datasource.testOnBorrow=true
+spring.datasource.testOnReturn=false
+spring.datasource.testWhileIdle=true
+spring.datasource.validationQuery=SELECT 1
+spring.datasource.validationQueryTimeout=2
+spring.datasource.timeBetweenEvictionRunsMillis=60000
+spring.datasource.minEvictableIdleTimeMillis=300000
+spring.datasource.removeAbandoned=false
+spring.datasource.logAbandoned=true
+spring.datasource.removeAbandonedTimeout=54
+spring.datasource.maxWait=10000
+spring.datasource.poolPreparedStatements=false
+spring.datasource.filters=
+spring.datasource.connectionProperties=druid.stat.mergeSql=true;druid.stat.logSlowSql=true;druid.stat.slowSqlLogLevel=warn;druid.stat.slowSqlMillis=500

wedpr-adm/conf/application-gauss.properties

@@ -0,0 +1,27 @@
+spring.datasource.url=jdbc:postgresql://127.0.0.1:3306/wedpr3
+spring.datasource.username=
+spring.datasource.password=
+spring.datasource.driverClassName=org.postgresql.Driver
+spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
+spring.datasource.defaultAutoCommit=true
+spring.datasource.initialSize=30
+spring.datasource.maxActive=128
+spring.datasource.maxIdle=50
+spring.datasource.minIdle=30
+spring.datasource.queryTimeout=60
+spring.datasource.keepAlive=true
+spring.datasource.testOnBorrow=true
+spring.datasource.testOnReturn=false
+spring.datasource.testWhileIdle=true
+spring.datasource.validationQuery=SELECT 1
+spring.datasource.validationQueryTimeout=2
+spring.datasource.timeBetweenEvictionRunsMillis=60000
+spring.datasource.minEvictableIdleTimeMillis=300000
+spring.datasource.removeAbandoned=false
+spring.datasource.logAbandoned=true
+spring.datasource.removeAbandonedTimeout=54
+spring.datasource.maxWait=10000
+spring.datasource.poolPreparedStatements=false
+spring.datasource.filters=
+spring.datasource.connectionProperties=druid.stat.mergeSql=true;druid.stat.logSlowSql=true;druid.stat.slowSqlLogLevel=warn;druid.stat.slowSqlMillis=500
+

wedpr-adm/conf/application-kingbase.properties

@@ -0,0 +1,26 @@
+spring.datasource.url=jdbc:kingbase8://127.0.0.1:3306/wedpr3
+spring.datasource.username=
+spring.datasource.password=
+spring.datasource.driverClassName=com.kingbase8.Driver
+spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
+spring.datasource.defaultAutoCommit=true
+spring.datasource.initialSize=30
+spring.datasource.maxActive=128
+spring.datasource.maxIdle=50
+spring.datasource.minIdle=30
+spring.datasource.queryTimeout=60
+spring.datasource.keepAlive=true
+spring.datasource.testOnBorrow=true
+spring.datasource.testOnReturn=false
+spring.datasource.testWhileIdle=true
+spring.datasource.validationQuery=SELECT 1
+spring.datasource.validationQueryTimeout=2
+spring.datasource.timeBetweenEvictionRunsMillis=60000
+spring.datasource.minEvictableIdleTimeMillis=300000
+spring.datasource.removeAbandoned=false
+spring.datasource.logAbandoned=true
+spring.datasource.removeAbandonedTimeout=54
+spring.datasource.maxWait=10000
+spring.datasource.poolPreparedStatements=false
+spring.datasource.filters=
+spring.datasource.connectionProperties=druid.stat.mergeSql=true;druid.stat.logSlowSql=true;druid.stat.slowSqlLogLevel=warn;druid.stat.slowSqlMillis=500

wedpr-adm/conf/application-mysql.properties

@@ -0,0 +1,27 @@
+spring.datasource.url=jdbc:mysql://127.0.0.1:3306/wedpr3?serverTimezone=GMT%2B8&characterEncoding=UTF-8&connectTimeout=60000&socketTimeout=60000
+spring.datasource.username=
+spring.datasource.password=
+spring.datasource.driverClassName=com.mysql.cj.jdbc.Driver
+spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
+spring.datasource.defaultAutoCommit=true
+spring.datasource.initialSize=30
+spring.datasource.maxActive=128
+spring.datasource.maxIdle=50
+spring.datasource.minIdle=30
+spring.datasource.queryTimeout=60
+spring.datasource.keepAlive=true
+spring.datasource.testOnBorrow=true
+spring.datasource.testOnReturn=false
+spring.datasource.testWhileIdle=true
+spring.datasource.validationQuery=SELECT 1
+spring.datasource.validationQueryTimeout=2
+spring.datasource.timeBetweenEvictionRunsMillis=60000
+spring.datasource.minEvictableIdleTimeMillis=300000
+spring.datasource.removeAbandoned=false
+spring.datasource.logAbandoned=true
+spring.datasource.removeAbandonedTimeout=54
+spring.datasource.maxWait=10000
+spring.datasource.poolPreparedStatements=false
+#spring.datasource.filters=stat,log4j2
+spring.datasource.filters=
+spring.datasource.connectionProperties=druid.stat.mergeSql=true;druid.stat.logSlowSql=true;druid.stat.slowSqlLogLevel=warn;druid.stat.slowSqlMillis=500

wedpr-adm/conf/application-postgresql.properties

@@ -0,0 +1,26 @@
+spring.datasource.url=jdbc:postgresql://127.0.0.1:3306/wedpr3
+spring.datasource.username=
+spring.datasource.password=
+spring.datasource.driverClassName=org.postgresql.Driver
+spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
+spring.datasource.defaultAutoCommit=true
+spring.datasource.initialSize=30
+spring.datasource.maxActive=128
+spring.datasource.maxIdle=50
+spring.datasource.minIdle=30
+spring.datasource.queryTimeout=60
+spring.datasource.keepAlive=true
+spring.datasource.testOnBorrow=true
+spring.datasource.testOnReturn=false
+spring.datasource.testWhileIdle=true
+spring.datasource.validationQuery=SELECT 1
+spring.datasource.validationQueryTimeout=2
+spring.datasource.timeBetweenEvictionRunsMillis=60000
+spring.datasource.minEvictableIdleTimeMillis=300000
+spring.datasource.removeAbandoned=false
+spring.datasource.logAbandoned=true
+spring.datasource.removeAbandonedTimeout=54
+spring.datasource.maxWait=10000
+spring.datasource.poolPreparedStatements=false
+spring.datasource.filters=
+spring.datasource.connectionProperties=druid.stat.mergeSql=true;druid.stat.logSlowSql=true;druid.stat.slowSqlLogLevel=warn;druid.stat.slowSqlMillis=500

wedpr-adm/conf/application-shentong.properties

@@ -0,0 +1,27 @@
+spring.datasource.url=jdbc:oscar://127.0.0.1:3306/wedpr3
+spring.datasource.username=
+spring.datasource.password=
+spring.datasource.driverClassName=com.oscar.Driver
+spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
+spring.datasource.defaultAutoCommit=true
+spring.datasource.initialSize=30
+spring.datasource.maxActive=128
+spring.datasource.maxIdle=50
+spring.datasource.minIdle=30
+spring.datasource.queryTimeout=60
+spring.datasource.keepAlive=true
+spring.datasource.testOnBorrow=true
+spring.datasource.testOnReturn=false
+spring.datasource.testWhileIdle=true
+spring.datasource.validationQuery=SELECT 1
+spring.datasource.validationQueryTimeout=2
+spring.datasource.timeBetweenEvictionRunsMillis=60000
+spring.datasource.minEvictableIdleTimeMillis=300000
+spring.datasource.removeAbandoned=false
+spring.datasource.logAbandoned=true
+spring.datasource.removeAbandonedTimeout=54
+spring.datasource.maxWait=10000
+spring.datasource.poolPreparedStatements=false
+spring.datasource.filters=
+spring.datasource.connectionProperties=druid.stat.mergeSql=true;druid.stat.logSlowSql=true;druid.stat.slowSqlLogLevel=warn;druid.stat.slowSqlMillis=500
+

wedpr-adm/conf/application-wedpr.properties

@@ -16,6 +16,9 @@ spring.servlet.multipart.max-request-size = -1
 # mybatis configuration
 #mybatis.mapper-locations=classpath:mapper/*Mapper.xml
 
+# mysql or dm or gauss or kingbase or shentong or postgresql
+spring.profiles.include=mysql
+
 ## dataset configuration
 wedpr.dataset.debugMode=false
 wedpr.dataset.debugMode.userTokenField=Wedpr-Test-User-Token

wedpr-adm/conf/wedpr.properties

@@ -11,15 +11,14 @@ wedpr.uuid.generator.worker.id=10
 wedpr.enable.dpm = false
 
 ### the mybatis related configuration
+
 #wedpr.mybatis.config.path=mybatis-config.xml
-# TODO: instead use CMDB value
-wedpr.mybatis.url=jdbc:mysql://127.0.0.1/wedpr3?characterEncoding=UTF-8&allowMultiQueries=true
-#wedpr.mybatis.username=[*user_ppcs_modeladm]
-#wedpr.mybatis.password=[*pass_ppcs_modeladm]
-wedpr.mybatis.username=
-wedpr.mybatis.password=
-
-wedpr.mybatis.driverClassName=com.mysql.cj.jdbc.Driver
+# wedpr.mybatis.url=jdbc:mysql://127.0.0.1/wedpr3?characterEncoding=UTF-8&allowMultiQueries=true
+# wedpr.mybatis.username=
+# wedpr.mybatis.password=
+# wedpr.mybatis.username=
+# wedpr.mybatis.password=
+# wedpr.mybatis.driverClassName=com.mysql.cj.jdbc.Driver
 
 wedpr.mybatis.mapperLocations=classpath*:mapper/*Mapper.xml
 # Note: the basePackage can't set to com.webank.wedpr simply for the mybatis will scan the Service

wedpr-components/dataset/src/main/java/com/webank/wedpr/components/dataset/config/DataSourceTypeConfig.java

@@ -53,7 +53,7 @@ public void validateDataSourceConfigurations() throws DatasetException {
             // check db types
             for (LabelValue childLabelValue : labelValue.getChildren()) {
                 DBType dbType = DBType.fromStrType(childLabelValue.getValue());
-                logger.info(" add one db type: {}", dbType);
+                logger.info(" add one db datasource type: {}", dbType);
             }
         }
     }

wedpr-components/mybatis/src/main/java/com/webank/wedpr/components/mybatis/DataSourceConfig.java

@@ -1,39 +1,140 @@
-/*
- * Copyright 2017-2025 [webank-wedpr]
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
- * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations under the License.
- *
- */
-
-package com.webank.wedpr.components.mybatis;
-
-import javax.sql.DataSource;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-
-@Configuration
-public class DataSourceConfig {
-    private static final Logger logger = LoggerFactory.getLogger(DataSourceConfig.class);
-
-    @Bean(name = "dataSource", destroyMethod = "close")
-    @ConditionalOnMissingBean
-    public DataSource dataSource() {
-        logger.info("Load DataSource");
-        if (MybatisConfig.MYBATIS_CONFIG != null) {
-            DataSource dataSource = DataSourceBuilder.build(MybatisConfig.MYBATIS_CONFIG);
-            logger.info("Load DataSource from configuration success");
-            return dataSource;
-        }
-        return DataSourceBuilder.build();
-    }
-}
+package com.webank.wedpr.components.mybatis;
+
+import com.alibaba.druid.pool.DruidDataSource;
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class DataSourceConfig {
+    private static final Logger logger = LoggerFactory.getLogger(DataSourceConfig.class);
+
+    @Value("${spring.datasource.url}")
+    private String url;
+
+    @Value("${spring.datasource.username}")
+    private String username;
+
+    @Value("${spring.datasource.password}")
+    private String password;
+
+    @Value("${spring.datasource.driverClassName}")
+    private String driverClassName;
+
+    @Value("${spring.datasource.defaultAutoCommit}")
+    private boolean defaultAutoCommit;
+
+    @Value("${spring.datasource.initialSize}")
+    private int initialSize;
+
+    @Value("${spring.datasource.maxActive}")
+    private int maxActive;
+
+    @Value("${spring.datasource.maxIdle}")
+    private int maxIdle;
+
+    @Value("${spring.datasource.minIdle}")
+    private int minIdle;
+
+    @Value("${spring.datasource.queryTimeout}")
+    private int queryTimeout = 0;
+
+    @Value("${spring.datasource.keepAlive}")
+    private boolean keepAlive;
+
+    @Value("${spring.datasource.testOnBorrow}")
+    private boolean testOnBorrow;
+
+    @Value("${spring.datasource.testOnReturn}")
+    private boolean testOnReturn;
+
+    @Value("${spring.datasource.testWhileIdle}")
+    private boolean testWhileIdle;
+
+    @Value("${spring.datasource.maxWait}")
+    private long maxWait;
+
+    @Value("${spring.datasource.timeBetweenEvictionRunsMillis}")
+    private long timeBetweenEvictionRunsMillis;
+
+    @Value("${spring.datasource.minEvictableIdleTimeMillis}")
+    private long minEvictableIdleTimeMillis;
+
+    @Value("${spring.datasource.validationQuery}")
+    private String validationQuery;
+
+    @Value("${spring.datasource.validationQueryTimeout}")
+    private int validationQueryTimeout;
+
+    @Value("${spring.datasource.removeAbandoned}")
+    private boolean removeAbandoned;
+
+    @Value("${spring.datasource.logAbandoned}")
+    private boolean logAbandoned;
+
+    @Value("${spring.datasource.removeAbandonedTimeout}")
+    private int removeAbandonedTimeout;
+
+    @Value("${spring.datasource.poolPreparedStatements}")
+    private boolean poolPreparedStatements;
+
+    @Value("${spring.datasource.connectionProperties}")
+    private String connectionProperties;
+
+    @Value("${spring.datasource.filters}")
+    private String filters;
+
+    @Bean(name = "dataSource", destroyMethod = "close")
+    public DruidDataSource dataSource() throws Exception {
+
+        logger.info("dataSource dbUrl:{}", url);
+        logger.info("dataSource dbUsername:{}", username);
+        logger.info("dataSource driverClassName:{}", driverClassName);
+        logger.info("dataSource queryTimeout:{}", queryTimeout);
+
+        // https://github.com/alibaba/druid/wiki/DruidDataSource%E9%85%8D%E7%BD%AE
+
+        DruidDataSource datasource = new DruidDataSource();
+        datasource.setUrl(url);
+        datasource.setUsername(username);
+        datasource.setPassword(password);
+        datasource.setDriverClassName(driverClassName);
+        datasource.setDefaultAutoCommit(defaultAutoCommit);
+        // configuration
+        datasource.setInitialSize(initialSize);
+        datasource.setMaxActive(maxActive);
+        datasource.setMinIdle(maxIdle);
+        datasource.setMinIdle(minIdle);
+        datasource.setTestOnBorrow(testOnBorrow);
+        datasource.setTestOnReturn(testOnReturn);
+        datasource.setTestWhileIdle(testWhileIdle);
+        datasource.setValidationQuery(validationQuery);
+        datasource.setValidationQueryTimeout(validationQueryTimeout);
+        datasource.setQueryTimeout(queryTimeout);
+        //
+        datasource.setTimeBetweenEvictionRunsMillis(timeBetweenEvictionRunsMillis);
+        datasource.setMinEvictableIdleTimeMillis(minEvictableIdleTimeMillis);
+        // datasource.setMaxEvictableIdleTimeMillis(minEvictableIdleTimeMillis);
+        datasource.setKeepAlive(keepAlive);
+
+        datasource.setRemoveAbandoned(removeAbandoned);
+        datasource.setLogAbandoned(logAbandoned);
+        datasource.setRemoveAbandonedTimeout(removeAbandonedTimeout);
+
+        datasource.setMaxWait(maxWait);
+        datasource.setPoolPreparedStatements(poolPreparedStatements);
+        datasource.setConnectionProperties(connectionProperties);
+        datasource.setFilters(filters);
+
+        datasource.init();
+        Map<String, Object> statData = datasource.getStatData();
+
+        logger.info("create druid datasource successfully: {}", statData);
+
+        return datasource;
+    }
+}