Skip to content

Commit

Permalink
starting with multi-schema support
Browse files Browse the repository at this point in the history
  • Loading branch information
arcuri82 committed Nov 14, 2024
1 parent c652145 commit e07a4ae
Show file tree
Hide file tree
Showing 5 changed files with 182 additions and 28 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@ public class DbInfoDto {

public DatabaseType databaseType;

/**
* Usually, this would be the "catalog"
*/
public String name;

public List<TableDto> tables = new ArrayList<>();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ public static DbInfoDto extract(Connection connection) throws Exception {

Objects.requireNonNull(connection);

DbInfoDto schemaDto = new DbInfoDto();
DbInfoDto dbInfoDto = new DbInfoDto();

DatabaseMetaData md = connection.getMetaData();

Expand All @@ -156,21 +156,17 @@ public static DbInfoDto extract(Connection connection) throws Exception {
//https://dev.mysql.com/doc/refman/8.0/en/connecting-using-uri-or-key-value-pairs.html#connecting-using-uri
dt = DatabaseType.MYSQL;
}
schemaDto.databaseType = dt;
dbInfoDto.databaseType = dt;


/*
schema name
*/
schemaDto.name = getSchemaName(connection, dt);
dbInfoDto.name = connection.getCatalog(); //getSchemaName(connection, dt);

if (dt.equals(DatabaseType.POSTGRES)) {
Map<String, Set<String>> enumLabels = getPostgresEnumTypes(connection);
addPostgresEnumTypesToSchema(schemaDto, enumLabels);
schemaDto.compositeTypes = getPostgresCompositeTypes(connection);
addPostgresEnumTypesToSchema(dbInfoDto, enumLabels);
dbInfoDto.compositeTypes = getPostgresCompositeTypes(connection);
}

ResultSet tables = md.getTables(null, schemaDto.name, null, new String[]{"TABLE"});
ResultSet tables = md.getTables(dbInfoDto.name, null, null, new String[]{"TABLE"});

Set<String> tableNames = new HashSet<>();

Expand All @@ -184,43 +180,43 @@ is empty or not, and only way is to call next()
*/
if (!tables.next()) {
tables.close();
schemaDto.name = schemaDto.name.toLowerCase();
tables = md.getTables(null, schemaDto.name, null, new String[]{"TABLE"});
dbInfoDto.name = dbInfoDto.name.toLowerCase();
tables = md.getTables(null, dbInfoDto.name, null, new String[]{"TABLE"});
if (tables.next()) {
do {
handleTableEntry(connection, schemaDto, md, tables, tableNames);
handleTableEntry(connection, dbInfoDto, md, tables, tableNames);
} while (tables.next());
}
} else {
do {
handleTableEntry(connection, schemaDto, md, tables, tableNames);
handleTableEntry(connection, dbInfoDto, md, tables, tableNames);
} while (tables.next());
}
tables.close();

/*
Mark those columns that are using auto generated values
*/
addForeignKeyToAutoIncrement(schemaDto);
addForeignKeyToAutoIncrement(dbInfoDto);

/*
JDBC MetaData is quite limited.
To check constraints, we need to do SQL queries on the system tables.
Unfortunately, this is database-dependent
*/
addConstraints(connection, dt, schemaDto);
addConstraints(connection, dt, dbInfoDto);

if (dt.equals(DatabaseType.POSTGRES)) {
List<ColumnAttributes> columnAttributes = getPostgresColumnAttributes(connection);
addColumnAttributes(schemaDto, columnAttributes);
addColumnAttributes(dbInfoDto, columnAttributes);
} else if (dt.equals(DatabaseType.H2)) {
List<DbTableConstraint> h2EnumConstraints = getH2EnumTypes(schemaDto.name, md);
addConstraints(schemaDto, h2EnumConstraints);
List<DbTableConstraint> h2EnumConstraints = getH2EnumTypes(dbInfoDto.name, md);
addConstraints(dbInfoDto, h2EnumConstraints);
}

assert validate(schemaDto);
assert validate(dbInfoDto);

return schemaDto;
return dbInfoDto;
}

private static void addColumnAttributes(DbInfoDto schemaDto, List<ColumnAttributes> listOfColumnAttributes) {
Expand Down Expand Up @@ -559,13 +555,14 @@ private static void handleTableEntry(Connection connection, DbInfoDto schemaDto,
tableSchema =tableCatalog;
}

if (tableSchema!=null && !tableSchema.equalsIgnoreCase(schemaDto.name)) {
/**
* If this table does not belong to the current schema under extraction,
* skip adding the table.
*/
return;
}
//no longer done: we extract all schemas for a given catalog
// if (tableSchema!=null && !tableSchema.equalsIgnoreCase(schemaDto.name)) {
// /**
// * If this table does not belong to the current schema under extraction,
// * skip adding the table.
// */
// return;
// }

TableDto tableDto = new TableDto();
schemaDto.tables.add(tableDto);
Expand Down
102 changes: 102 additions & 0 deletions core/src/test/kotlin/org/evomaster/core/sql/multidb/MultiDbUtils.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
package org.evomaster.core.sql.multidb

import org.evomaster.client.java.controller.api.dto.database.schema.DatabaseType
import org.evomaster.client.java.sql.DbCleaner
import org.evomaster.client.java.sql.SqlScriptRunner
import org.evomaster.core.KGenericContainer
import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy
import java.sql.Connection
import java.sql.DriverManager
import java.util.*

object MultiDbUtils {


private const val POSTGRES_VERSION: String = "14";
private const val POSTGRES_PORT: Int = 5432
private val postgres = KGenericContainer("postgres:$POSTGRES_VERSION")
.withExposedPorts(POSTGRES_PORT)
//https://www.postgresql.org/docs/current/auth-trust.html
.withEnv("POSTGRES_HOST_AUTH_METHOD","trust")
.withTmpFs(Collections.singletonMap("/var/lib/postgresql/data", "rw"))


private const val MYSQL_DB_NAME = "test"
private const val MYSQL_PORT = 3306
private const val MYSQL_VERSION = "8.0.27";
private val mysql = KGenericContainer("mysql:$MYSQL_VERSION")
.withEnv(
mutableMapOf(
"MYSQL_ROOT_PASSWORD" to "root",
"MYSQL_DATABASE" to MYSQL_DB_NAME, // TODO can this be removed?
"MYSQL_USER" to "test",
"MYSQL_PASSWORD" to "test"
)
)
.withExposedPorts(MYSQL_PORT)


fun createConnection(name: String, type: DatabaseType) : Connection {

return when (type) {
DatabaseType.MYSQL -> {
val host = mysql.host
val port = mysql.getMappedPort(MYSQL_PORT)
val url = "jdbc:mysql://$host:$port/$name"
DriverManager.getConnection(url, "test", "test")
}
DatabaseType.H2 -> {
DriverManager.getConnection("jdbc:h2:mem:$name", "sa", "")
}
DatabaseType.POSTGRES -> {
val host = postgres.host
val port = postgres.getMappedPort(POSTGRES_PORT)
val url = "jdbc:postgresql://$host:$port/postgres" //TODO check name
/*
* A call to getConnection() when the postgres container is still not ready,
* signals a PSQLException with message "FATAL: the database system is starting up".
* The following issue describes how to avoid this by using a LogMessageWaitStrategy
* https://github.com/testcontainers/testcontainers-java/issues/317
*/
postgres.waitingFor(LogMessageWaitStrategy().withRegEx(".*database system is ready to accept connections.*\\s").withTimes(2))

DriverManager.getConnection(url, "postgres", "")
}
else -> throw IllegalArgumentException("Unsupported database type: ${type.name}")
}
}

fun resetDatabase(type: DatabaseType, connection: Connection) {
when(type) {
DatabaseType.H2 -> SqlScriptRunner.execCommand(connection, "DROP ALL OBJECTS;")
//FIXME schema names need fixing. also is this actually dropping tables???
DatabaseType.MYSQL -> DbCleaner.dropDatabaseTables(connection, MYSQL_DB_NAME, null, DatabaseType.MYSQL)
DatabaseType.POSTGRES -> {
//TODO what about other schemas/catalogs?
SqlScriptRunner.execCommand(connection, "DROP SCHEMA public CASCADE;")
SqlScriptRunner.execCommand(connection, "CREATE SCHEMA public;")
SqlScriptRunner.execCommand(connection, "GRANT ALL ON SCHEMA public TO postgres;")
SqlScriptRunner.execCommand(connection, "GRANT ALL ON SCHEMA public TO public;")
}
else -> throw IllegalArgumentException("Unsupported database type: ${type.name}")
}
}

fun startDatabase(type: DatabaseType) {
when(type) {
DatabaseType.H2 -> { /* nothing to do? started automatically on connection*/}
DatabaseType.MYSQL -> mysql.start()
DatabaseType.POSTGRES -> postgres.start()
else -> throw IllegalArgumentException("Unsupported database type: ${type.name}")
}
}

fun stopDatabase(type: DatabaseType) {
when(type) {
DatabaseType.H2 -> {/* nothing to do*/ }
DatabaseType.MYSQL -> mysql.stop()
DatabaseType.POSTGRES -> postgres.stop()
else -> throw IllegalArgumentException("Unsupported database type: ${type.name}")
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
package org.evomaster.core.sql.multidb

import org.evomaster.client.java.controller.api.dto.database.schema.DatabaseType
import org.evomaster.client.java.sql.SchemaExtractor
import org.evomaster.client.java.sql.SqlScriptRunner
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Test
import java.sql.Connection

class SecondSchemaTest {

//TODO parameterize
@Test
fun test(){
val databaseType = DatabaseType.H2
val name = "dbtest"
val sqlSchemaCommand = this::class.java.getResource(getSchemaLocation()).readText()

MultiDbUtils.startDatabase(databaseType)
try {
val connection = MultiDbUtils.createConnection(name, databaseType)
connection.use {
MultiDbUtils.resetDatabase(databaseType, it)
SqlScriptRunner.execCommand(it, sqlSchemaCommand)
verify(it, name)
}
}finally {
MultiDbUtils.stopDatabase(databaseType)
}
}

private fun verify(connection: Connection, name: String){

val info = SchemaExtractor.extract(connection)
assertEquals(name.lowercase(), info.name.lowercase())
//assertEquals(2, info.tables.size)
//TODO other checks
}

fun getSchemaLocation() = "/sql_schema/multidb/secondschema.sql"
}
11 changes: 11 additions & 0 deletions core/src/test/resources/sql_schema/multidb/secondschema.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
create table Foo (
id bigint not null,
primary key (id)
);

CREATE SCHEMA IF NOT EXISTS other;

create table other.Bar(
id bigint not null,
primary key (id)
);

0 comments on commit e07a4ae

Please sign in to comment.