From de7035171de958b62523a901ed801bde8c3552ee Mon Sep 17 00:00:00 2001 From: Jonathan Pearlin Date: Mon, 2 May 2022 14:08:18 -0400 Subject: [PATCH] Add utility classes for database object creation (#12445) * Add utility classes for database object creation * Remove unused variable --- airbyte-db/lib/build.gradle | 1 + .../main/java/io/airbyte/db/Databases.java | 11 + .../airbyte/db/factory/DSLContextFactory.java | 32 +++ .../airbyte/db/factory/DataSourceFactory.java | 251 ++++++++++++++++++ .../io/airbyte/db/factory/FlywayFactory.java | 53 ++++ .../db/factory/AbstractFactoryTest.java | 34 +++ .../db/factory/DSLContextFactoryTest.java | 31 +++ .../db/factory/DataSourceFactoryTest.java | 103 +++++++ .../airbyte/db/factory/FlywayFactoryTest.java | 39 +++ 9 files changed, 555 insertions(+) create mode 100644 airbyte-db/lib/src/main/java/io/airbyte/db/factory/DSLContextFactory.java create mode 100644 airbyte-db/lib/src/main/java/io/airbyte/db/factory/DataSourceFactory.java create mode 100644 airbyte-db/lib/src/main/java/io/airbyte/db/factory/FlywayFactory.java create mode 100644 airbyte-db/lib/src/test/java/io/airbyte/db/factory/AbstractFactoryTest.java create mode 100644 airbyte-db/lib/src/test/java/io/airbyte/db/factory/DSLContextFactoryTest.java create mode 100644 airbyte-db/lib/src/test/java/io/airbyte/db/factory/DataSourceFactoryTest.java create mode 100644 airbyte-db/lib/src/test/java/io/airbyte/db/factory/FlywayFactoryTest.java diff --git a/airbyte-db/lib/build.gradle b/airbyte-db/lib/build.gradle index 8c5d69903d89..3d52778abfcf 100644 --- a/airbyte-db/lib/build.gradle +++ b/airbyte-db/lib/build.gradle @@ -4,6 +4,7 @@ plugins { dependencies { api 'org.apache.commons:commons-dbcp2:2.7.0' + api 'com.zaxxer:HikariCP:5.0.1' api 'org.jooq:jooq-meta:3.13.4' api 'org.jooq:jooq:3.13.4' api 'org.postgresql:postgresql:42.2.18' diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java b/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java index 1e193409e042..2df0b624f079 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java @@ -24,6 +24,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Provides utility methods to create configured {@link Database} instances. + * + * @deprecated This class has been marked as deprecated as we move to using an application framework + * to manage resources. This class will be removed in a future release. + * + * @see io.airbyte.db.factory.DataSourceFactory + * @see io.airbyte.db.factory.DSLContextFactory + * @see io.airbyte.db.factory.FlywayFactory + */ +@Deprecated(forRemoval = true) public class Databases { private static final Logger LOGGER = LoggerFactory.getLogger(Databases.class); diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/factory/DSLContextFactory.java b/airbyte-db/lib/src/main/java/io/airbyte/db/factory/DSLContextFactory.java new file mode 100644 index 000000000000..eba32e7cb620 --- /dev/null +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/factory/DSLContextFactory.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.factory; + +import javax.sql.DataSource; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.jooq.impl.DSL; + +/** + * Temporary factory class that provides convenience methods for creating a {@link DSLContext} + * instances. This class will be removed once the project has been converted to leverage an + * application framework to manage the creation and injection of {@link DSLContext} objects. + * + * This class replaces direct calls to {@link io.airbyte.db.Databases}. + */ +public class DSLContextFactory { + + /** + * Constructs a configured {@link DSLContext} instance using the provided configuration. + * + * @param dataSource The {@link DataSource} used to connect to the database. + * @param dialect The SQL dialect to use with objects created from this context. + * @return The configured {@link DSLContext}. + */ + public static DSLContext create(final DataSource dataSource, final SQLDialect dialect) { + return DSL.using(dataSource, dialect); + } + +} diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/factory/DataSourceFactory.java b/airbyte-db/lib/src/main/java/io/airbyte/db/factory/DataSourceFactory.java new file mode 100644 index 000000000000..31c659a548cf --- /dev/null +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/factory/DataSourceFactory.java @@ -0,0 +1,251 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.factory; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import java.util.Map; +import javax.sql.DataSource; + +/** + * Temporary factory class that provides convenience methods for creating a {@link DataSource} + * instance. This class will be removed once the project has been converted to leverage an + * application framework to manage the creation and injection of {@link DataSource} objects. + * + * This class replaces direct calls to {@link io.airbyte.db.Databases}. + */ +public class DataSourceFactory { + + /** + * Constructs a new {@link DataSource} using the provided configuration. + * + * @param username The username of the database user. + * @param password The password of the database user. + * @param driverClassName The fully qualified name of the JDBC driver class. + * @param jdbcConnectionString The JDBC connection string. + * @return The configured {@link DataSource}. + */ + public static DataSource create(final String username, + final String password, + final String driverClassName, + final String jdbcConnectionString) { + return new DataSourceBuilder() + .withDriverClassName(driverClassName) + .withJdbcUrl(jdbcConnectionString) + .withPassword(password) + .withUsername(username) + .build(); + } + + /** + * Constructs a new {@link DataSource} using the provided configuration. + * + * @param username The username of the database user. + * @param password The password of the database user. + * @param driverClassName The fully qualified name of the JDBC driver class. + * @param jdbcConnectionString The JDBC connection string. + * @param connectionProperties Additional configuration properties for the underlying driver. + * @return The configured {@link DataSource}. + */ + public static DataSource create(final String username, + final String password, + final String driverClassName, + final String jdbcConnectionString, + final Map connectionProperties) { + return new DataSourceBuilder() + .withConnectionProperties(connectionProperties) + .withDriverClassName(driverClassName) + .withJdbcUrl(jdbcConnectionString) + .withPassword(password) + .withUsername(username) + .build(); + } + + /** + * Constructs a new {@link DataSource} using the provided configuration. + * + * @param username The username of the database user. + * @param password The password of the database user. + * @param host The host address of the database. + * @param port The port of the database. + * @param database The name of the database. + * @param driverClassName The fully qualified name of the JDBC driver class. + * @return The configured {@link DataSource}. + */ + public static DataSource create(final String username, + final String password, + final String host, + final int port, + final String database, + final String driverClassName) { + return new DataSourceBuilder() + .withDatabase(database) + .withDriverClassName(driverClassName) + .withHost(host) + .withPort(port) + .withPassword(password) + .withUsername(username) + .build(); + } + + /** + * Constructs a new {@link DataSource} using the provided configuration. + * + * @param username The username of the database user. + * @param password The password of the database user. + * @param host The host address of the database. + * @param port The port of the database. + * @param database The name of the database. + * @param driverClassName The fully qualified name of the JDBC driver class. + * @param connectionProperties Additional configuration properties for the underlying driver. + * @return The configured {@link DataSource}. + */ + public static DataSource create(final String username, + final String password, + final String host, + final int port, + final String database, + final String driverClassName, + final Map connectionProperties) { + return new DataSourceBuilder() + .withConnectionProperties(connectionProperties) + .withDatabase(database) + .withDriverClassName(driverClassName) + .withHost(host) + .withPort(port) + .withPassword(password) + .withUsername(username) + .build(); + } + + /** + * Convenience method that constructs a new {@link DataSource} for a PostgreSQL database using the + * provided configuration. + * + * @param username The username of the database user. + * @param password The password of the database user. + * @param host The host address of the database. + * @param port The port of the database. + * @param database The name of the database. + * @return The configured {@link DataSource}. + */ + public static DataSource createPostgres(final String username, + final String password, + final String host, + final int port, + final String database) { + return new DataSourceBuilder() + .withDatabase(database) + .withDriverClassName("org.postgresql.Driver") + .withHost(host) + .withPort(port) + .withPassword(password) + .withUsername(username) + .build(); + } + + /** + * Builder class used to configure and construct {@link DataSource} instances. + */ + private static class DataSourceBuilder { + + private static final Map JDBC_URL_FORMATS = Map.of("org.postgresql.Driver", "jdbc:postgresql://%s:%d/%s", + "com.amazon.redshift.jdbc.Driver", "jdbc:redshift://%s:%d/%s", + "com.mysql.cj.jdbc.Driver", "jdbc:mysql://%s:%d/%s", + "com.microsoft.sqlserver.jdbc.SQLServerDriver", "jdbc:sqlserver://%s:%d/%s", + "oracle.jdbc.OracleDriver", "jdbc:oracle:thin:@%s:%d:%s", + "ru.yandex.clickhouse.ClickHouseDriver", "jdbc:ch://%s:%d/%s", + "org.mariadb.jdbc.Driver", "jdbc:mariadb://%s:%d/%s"); + + private Map connectionProperties = Map.of(); + private String database; + private String driverClassName = "org.postgresql.Driver"; + private String host; + private String jdbcUrl; + private Integer maximumPoolSize = 5; + private Integer minimumPoolSize = 0; + private String password; + private Integer port = 5432; + private String username; + + private DataSourceBuilder() {} + + public DataSourceBuilder withConnectionProperties(final Map connectionProperties) { + if (connectionProperties != null) { + this.connectionProperties = connectionProperties; + } + return this; + } + + public DataSourceBuilder withDatabase(final String database) { + this.database = database; + return this; + } + + public DataSourceBuilder withDriverClassName(final String driverClassName) { + this.driverClassName = driverClassName; + return this; + } + + public DataSourceBuilder withHost(final String host) { + this.host = host; + return this; + } + + public DataSourceBuilder withJdbcUrl(final String jdbcUrl) { + this.jdbcUrl = jdbcUrl; + return this; + } + + public DataSourceBuilder withMaximumPoolSize(final Integer maximumPoolSize) { + if (maximumPoolSize != null) { + this.maximumPoolSize = maximumPoolSize; + } + return this; + } + + public DataSourceBuilder withMinimumPoolSize(final Integer minimumPoolSize) { + if (minimumPoolSize != null) { + this.minimumPoolSize = minimumPoolSize; + } + return this; + } + + public DataSourceBuilder withPassword(final String password) { + this.password = password; + return this; + } + + public DataSourceBuilder withPort(final Integer port) { + if (port != null) { + this.port = port; + } + return this; + } + + public DataSourceBuilder withUsername(final String username) { + this.username = username; + return this; + } + + public DataSource build() { + final HikariConfig config = new HikariConfig(); + config.setDriverClassName(driverClassName); + config.setJdbcUrl(jdbcUrl != null ? jdbcUrl : String.format(JDBC_URL_FORMATS.getOrDefault(driverClassName, ""), host, port, database)); + config.setMaximumPoolSize(maximumPoolSize); + config.setMinimumIdle(minimumPoolSize); + config.setPassword(password); + config.setUsername(username); + + connectionProperties.forEach(config::addDataSourceProperty); + + final HikariDataSource dataSource = new HikariDataSource(config); + dataSource.validate(); + return dataSource; + } + + } + +} diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/factory/FlywayFactory.java b/airbyte-db/lib/src/main/java/io/airbyte/db/factory/FlywayFactory.java new file mode 100644 index 000000000000..0e5526745fd9 --- /dev/null +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/factory/FlywayFactory.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.factory; + +import javax.sql.DataSource; +import org.flywaydb.core.Flyway; + +/** + * Temporary factory class that provides convenience methods for creating a {@link Flyway} + * instances. This class will be removed once the project has been converted to leverage an + * application framework to manage the creation and injection of {@link Flyway} objects. + * + * This class replaces direct calls to {@link io.airbyte.db.Databases}. + */ +public class FlywayFactory { + + static final String MIGRATION_TABLE_FORMAT = "airbyte_%s_migrations"; + + // Constants for Flyway baseline. See here for details: + // https://flywaydb.org/documentation/command/baseline + static final String BASELINE_VERSION = "0.29.0.001"; + static final String BASELINE_DESCRIPTION = "Baseline from file-based migration v1"; + static final boolean BASELINE_ON_MIGRATION = true; + + /** + * Constructs a configured {@link Flyway} instance using the provided configuration. + * + * @param dataSource The {@link DataSource} used to connect to the database. + * @param installedBy The name of the module performing the migration. + * @param dbIdentifier The name of the database to be migrated. This is used to name the table to + * hold the migration history for the database. + * @param migrationFileLocations The array of migration files to be used. + * @return The configured {@link Flyway} instance. + */ + public static Flyway create(final DataSource dataSource, + final String installedBy, + final String dbIdentifier, + final String... migrationFileLocations) { + return Flyway.configure() + .dataSource(dataSource) + .baselineVersion(BASELINE_VERSION) + .baselineDescription(BASELINE_DESCRIPTION) + .baselineOnMigrate(BASELINE_ON_MIGRATION) + .installedBy(installedBy) + .table(String.format(MIGRATION_TABLE_FORMAT, dbIdentifier)) + .locations(migrationFileLocations) + .load(); + + } + +} diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/factory/AbstractFactoryTest.java b/airbyte-db/lib/src/test/java/io/airbyte/db/factory/AbstractFactoryTest.java new file mode 100644 index 000000000000..25f8b4c4ca3e --- /dev/null +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/factory/AbstractFactoryTest.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.factory; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.testcontainers.containers.PostgreSQLContainer; + +/** + * Common test suite for the classes found in the {@code io.airbyte.db.factory} package. + */ +public abstract class AbstractFactoryTest { + + private static final String DATABASE_NAME = "airbyte_test_database"; + + protected static PostgreSQLContainer container; + + @BeforeAll + public static void dbSetup() { + container = new PostgreSQLContainer<>("postgres:13-alpine") + .withDatabaseName(DATABASE_NAME) + .withUsername("docker") + .withPassword("docker"); + container.start(); + } + + @AfterAll + public static void dbDown() { + container.close(); + } + +} diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/factory/DSLContextFactoryTest.java b/airbyte-db/lib/src/test/java/io/airbyte/db/factory/DSLContextFactoryTest.java new file mode 100644 index 000000000000..b4bae85c24f9 --- /dev/null +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/factory/DSLContextFactoryTest.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.factory; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import javax.sql.DataSource; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.junit.jupiter.api.Test; +import org.postgresql.Driver; + +/** + * Test suite for the {@link DSLContextFactory} class. + */ +public class DSLContextFactoryTest extends AbstractFactoryTest { + + @Test + void testCreatingADslContext() { + final DataSource dataSource = + DataSourceFactory.create(container.getUsername(), container.getPassword(), Driver.class.getName(), container.getJdbcUrl()); + final SQLDialect dialect = SQLDialect.POSTGRES; + final DSLContext dslContext = DSLContextFactory.create(dataSource, dialect); + assertNotNull(dslContext); + assertEquals(dialect, dslContext.configuration().dialect()); + } + +} diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/factory/DataSourceFactoryTest.java b/airbyte-db/lib/src/test/java/io/airbyte/db/factory/DataSourceFactoryTest.java new file mode 100644 index 000000000000..4cfe7cc14124 --- /dev/null +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/factory/DataSourceFactoryTest.java @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.factory; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.zaxxer.hikari.HikariDataSource; +import java.util.Map; +import javax.sql.DataSource; +import org.junit.jupiter.api.Test; +import org.postgresql.Driver; + +/** + * Test suite for the {@link DataSourceFactory} class. + */ +public class DataSourceFactoryTest extends AbstractFactoryTest { + + @Test + void testCreatingADataSourceWithJdbcUrl() { + final String username = container.getUsername(); + final String password = container.getPassword(); + final String driverClassName = Driver.class.getName(); + final String jdbcUrl = container.getJdbcUrl(); + + final DataSource dataSource = DataSourceFactory.create(username, password, driverClassName, jdbcUrl); + assertNotNull(dataSource); + assertEquals(HikariDataSource.class, dataSource.getClass()); + } + + @Test + void testCreatingADataSourceWithJdbcUrlAndConnectionProperties() { + final String username = container.getUsername(); + final String password = container.getPassword(); + final String driverClassName = Driver.class.getName(); + final String jdbcUrl = container.getJdbcUrl(); + final Map connectionProperties = Map.of("foo", "bar"); + + final DataSource dataSource = DataSourceFactory.create(username, password, driverClassName, jdbcUrl, connectionProperties); + assertNotNull(dataSource); + assertEquals(HikariDataSource.class, dataSource.getClass()); + } + + @Test + void testCreatingADataSourceWithHostAndPort() { + final String username = container.getUsername(); + final String password = container.getPassword(); + final String driverClassName = Driver.class.getName(); + final String host = container.getHost(); + final Integer port = container.getFirstMappedPort(); + final String database = container.getDatabaseName(); + + final DataSource dataSource = DataSourceFactory.create(username, password, host, port, database, driverClassName); + assertNotNull(dataSource); + assertEquals(HikariDataSource.class, dataSource.getClass()); + } + + @Test + void testCreatingADataSourceWithHostPortAndConnectionProperties() { + final String username = container.getUsername(); + final String password = container.getPassword(); + final String driverClassName = Driver.class.getName(); + final String host = container.getHost(); + final Integer port = container.getFirstMappedPort(); + final String database = container.getDatabaseName(); + final Map connectionProperties = Map.of("foo", "bar"); + + final DataSource dataSource = DataSourceFactory.create(username, password, host, port, database, driverClassName, connectionProperties); + assertNotNull(dataSource); + assertEquals(HikariDataSource.class, dataSource.getClass()); + } + + @Test + void testCreatingAnInvalidDataSourceWithHostAndPort() { + final String username = container.getUsername(); + final String password = container.getPassword(); + final String driverClassName = "Unknown"; + final String host = container.getHost(); + final Integer port = container.getFirstMappedPort(); + final String database = container.getDatabaseName(); + + assertThrows(RuntimeException.class, () -> { + DataSourceFactory.create(username, password, host, port, database, driverClassName); + }); + } + + @Test + void testCreatingAPostgresqlDataSource() { + final String username = container.getUsername(); + final String password = container.getPassword(); + final String host = container.getHost(); + final Integer port = container.getFirstMappedPort(); + final String database = container.getDatabaseName(); + + final DataSource dataSource = DataSourceFactory.createPostgres(username, password, host, port, database); + assertNotNull(dataSource); + assertEquals(HikariDataSource.class, dataSource.getClass()); + } + +} diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/factory/FlywayFactoryTest.java b/airbyte-db/lib/src/test/java/io/airbyte/db/factory/FlywayFactoryTest.java new file mode 100644 index 000000000000..2c2913261b28 --- /dev/null +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/factory/FlywayFactoryTest.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.factory; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import javax.sql.DataSource; +import org.flywaydb.core.Flyway; +import org.junit.jupiter.api.Test; +import org.postgresql.Driver; + +/** + * Test suite for the {@link FlywayFactory} class. + */ +public class FlywayFactoryTest extends AbstractFactoryTest { + + @Test + void testCreatingAFlywayInstance() { + final String installedBy = "test"; + final String dbIdentifier = "test"; + final String migrationFileLocation = "classpath:io/airbyte/db/instance/toys/migrations"; + final DataSource dataSource = + DataSourceFactory.create(container.getUsername(), container.getPassword(), Driver.class.getName(), container.getJdbcUrl()); + + final Flyway flyway = FlywayFactory.create(dataSource, installedBy, dbIdentifier, migrationFileLocation); + assertNotNull(flyway); + assertTrue(flyway.getConfiguration().isBaselineOnMigrate()); + assertEquals(FlywayFactory.BASELINE_DESCRIPTION, flyway.getConfiguration().getBaselineDescription()); + assertEquals(FlywayFactory.BASELINE_VERSION, flyway.getConfiguration().getBaselineVersion().getVersion()); + assertEquals(installedBy, flyway.getConfiguration().getInstalledBy()); + assertEquals(String.format(FlywayFactory.MIGRATION_TABLE_FORMAT, dbIdentifier), flyway.getConfiguration().getTable()); + assertEquals(migrationFileLocation, flyway.getConfiguration().getLocations()[0].getDescriptor()); + } + +}