From 56660f11bb39632330c5f70015cdca67232dea73 Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Sun, 11 Dec 2022 14:18:43 +0200 Subject: [PATCH 01/34] JDBC sources: consolidate date/time values mapping --- ...bstractJdbcCompatibleSourceOperations.java | 5 +-- .../source/jdbc/AbstractJdbcSource.java | 1 + .../source/mysql/MySqlSourceOperations.java | 20 +++++++----- .../source/oracle/OracleSource.java | 2 +- .../source/oracle/OracleSourceOperations.java | 24 ++++++++++++++ .../source/postgres/PostgresSource.java | 4 ++- .../postgres/PostgresSourceOperations.java | 22 +++++++------ .../source/relationaldb/AbstractDbSource.java | 9 ++++-- .../SnowflakeSourceOperations.java | 32 ++++++++++--------- 9 files changed, 81 insertions(+), 38 deletions(-) create mode 100644 airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSourceOperations.java diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java index eea2286e8f34..34412f8a2ee0 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java @@ -24,6 +24,7 @@ import java.text.ParseException; import java.time.Instant; import java.time.LocalDate; +import java.time.LocalTime; import java.time.OffsetDateTime; import java.time.OffsetTime; import java.time.chrono.IsoEra; @@ -124,11 +125,11 @@ protected void putString(final ObjectNode node, final String columnName, final R } protected void putDate(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { - node.put(columnName, DataTypeUtils.toISO8601String(resultSet.getDate(index))); + node.put(columnName, DateTimeConverter.convertToDate(getObject(resultSet, index, LocalDate.class))); } protected void putTime(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { - node.put(columnName, DataTypeUtils.toISO8601String(resultSet.getTime(index))); + node.put(columnName, DateTimeConverter.convertToTime(getObject(resultSet, index, LocalTime.class))); } protected void putTimestamp(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { diff --git a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java b/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java index dbde54c5cdb9..3c2461cfae31 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java +++ b/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java @@ -621,4 +621,5 @@ protected List identifyStreamsToSnapshot(final Configur .map(Jsons::clone) .collect(Collectors.toList()); } + } diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSourceOperations.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSourceOperations.java index 8590ea970bff..ef92fd17b4ab 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSourceOperations.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSourceOperations.java @@ -212,15 +212,19 @@ public boolean isCursorType(MysqlType type) { return ALLOWED_CURSOR_TYPES.contains(type); } - @Override - protected void putDate(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { - node.put(columnName, DateTimeConverter.convertToDate(getObject(resultSet, index, LocalDate.class))); - } + // @Override + // protected void putDate(final ObjectNode node, final String columnName, final ResultSet resultSet, + // final int index) throws SQLException { + // node.put(columnName, DateTimeConverter.convertToDate(getObject(resultSet, index, + // LocalDate.class))); + // } - @Override - protected void putTime(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { - node.put(columnName, DateTimeConverter.convertToTime(getObject(resultSet, index, LocalTime.class))); - } + // @Override + // protected void putTime(final ObjectNode node, final String columnName, final ResultSet resultSet, + // final int index) throws SQLException { + // node.put(columnName, DateTimeConverter.convertToTime(getObject(resultSet, index, + // LocalTime.class))); + // } @Override protected void putTimestamp(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { diff --git a/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java b/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java index b0f1ebc2ce26..fdd1a7563af8 100644 --- a/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java +++ b/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java @@ -53,7 +53,7 @@ enum Protocol { } public OracleSource() { - super(DRIVER_CLASS, AdaptiveStreamingQueryConfig::new, JdbcUtils.getDefaultSourceOperations()); + super(DRIVER_CLASS, AdaptiveStreamingQueryConfig::new, new OracleSourceOperations()); } public static Source sshWrappedSource() { diff --git a/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSourceOperations.java b/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSourceOperations.java new file mode 100644 index 000000000000..23bcd0d3f0d3 --- /dev/null +++ b/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSourceOperations.java @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.oracle; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.db.jdbc.DateTimeConverter; +import io.airbyte.db.jdbc.JdbcSourceOperations; +import java.sql.ResultSet; +import java.sql.SQLException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class OracleSourceOperations extends JdbcSourceOperations { + + private static final Logger LOGGER = LoggerFactory.getLogger(OracleSourceOperations.class); + + @Override + protected void putDate(ObjectNode node, String columnName, ResultSet resultSet, int index) throws SQLException { + node.put(columnName, DateTimeConverter.convertToTimestamp(resultSet.getTimestamp(index))); + } + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java index e57254a6e7d8..bb636a0e2c5d 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java @@ -517,7 +517,8 @@ protected static String toSslJdbcParamInternal(final SslMode sslMode) { } @Override - protected boolean verifyCursorColumnValues(final JdbcDatabase database, final String schema, final String tableName, final String columnName) throws SQLException { + protected boolean verifyCursorColumnValues(final JdbcDatabase database, final String schema, final String tableName, final String columnName) + throws SQLException { final String query; final String resultColName = "nullValue"; // Query: Only if cursor column allows null values, query whether it contains one @@ -536,4 +537,5 @@ protected boolean verifyCursorColumnValues(final JdbcDatabase database, final St LOGGER.debug("null value exist: {}", nullValExist); return !nullValExist; } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java index 0976219c8697..79d2bf0dcaff 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java @@ -383,15 +383,19 @@ private void putLongArray(final ObjectNode node, final String columnName, final node.set(columnName, arrayNode); } - @Override - protected void putDate(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { - node.put(columnName, DateTimeConverter.convertToDate(getObject(resultSet, index, LocalDate.class))); - } - - @Override - protected void putTime(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { - node.put(columnName, DateTimeConverter.convertToTime(getObject(resultSet, index, LocalTime.class))); - } + // @Override + // protected void putDate(final ObjectNode node, final String columnName, final ResultSet resultSet, + // final int index) throws SQLException { + // node.put(columnName, DateTimeConverter.convertToDate(getObject(resultSet, index, + // LocalDate.class))); + // } + + // @Override + // protected void putTime(final ObjectNode node, final String columnName, final ResultSet resultSet, + // final int index) throws SQLException { + // node.put(columnName, DateTimeConverter.convertToTime(getObject(resultSet, index, + // LocalTime.class))); + // } @Override protected void putTimestamp(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/AbstractDbSource.java b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/AbstractDbSource.java index 2df3f4b6620f..828fce410313 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/AbstractDbSource.java +++ b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/AbstractDbSource.java @@ -182,7 +182,9 @@ public AutoCloseableIterator read(final JsonNode config, private void validateCursorFieldForIncrementalTables( final Map>> tableNameToTable, - final ConfiguredAirbyteCatalog catalog, final Database database) throws SQLException { + final ConfiguredAirbyteCatalog catalog, + final Database database) + throws SQLException { final List tablesWithInvalidCursor = new ArrayList<>(); for (final ConfiguredAirbyteStream airbyteStream : catalog.getStreams()) { final AirbyteStream stream = airbyteStream.getStream(); @@ -231,14 +233,17 @@ private void validateCursorFieldForIncrementalTables( /** * Verify that cursor column allows syncing to go through. + * * @param database database * @return true if syncing can go through. false otherwise * @throws SQLException exception */ - protected boolean verifyCursorColumnValues(final Database database, final String schema, final String tableName, final String columnName) throws SQLException { + protected boolean verifyCursorColumnValues(final Database database, final String schema, final String tableName, final String columnName) + throws SQLException { /* no-op */ return true; } + private List>> discoverWithoutSystemTables( final Database database) throws Exception { diff --git a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceOperations.java b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceOperations.java index 48043913662a..c66840727f8b 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceOperations.java +++ b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceOperations.java @@ -23,7 +23,6 @@ import java.sql.SQLException; import java.sql.Timestamp; import java.time.LocalDate; -import java.time.LocalTime; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -113,7 +112,8 @@ protected void setDate(final PreparedStatement preparedStatement, final int para } @Override - protected void putTimestampWithTimezone(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { + protected void putTimestampWithTimezone(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) + throws SQLException { final Timestamp timestamp = resultSet.getTimestamp(index); node.put(columnName, DateTimeConverter.convertToTimestampWithTimezone(timestamp)); } @@ -123,18 +123,20 @@ protected void putTimestamp(final ObjectNode node, final String columnName, fina final Timestamp timestamp = resultSet.getTimestamp(index); node.put(columnName, DateTimeConverter.convertToTimestamp(timestamp)); } - - @Override - protected void putDate(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { - final Date date = resultSet.getDate(index); - node.put(columnName, DateTimeConverter.convertToDate(date)); - } - - @Override - protected void putTime(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { - // resultSet.getTime() will lose nanoseconds precision - final LocalTime localTime = resultSet.getTimestamp(index).toLocalDateTime().toLocalTime(); - node.put(columnName, DateTimeConverter.convertToTime(localTime)); - } + // + // @Override + // protected void putDate(final ObjectNode node, final String columnName, final ResultSet resultSet, + // final int index) throws SQLException { + // final Date date = resultSet.getDate(index); + // node.put(columnName, DateTimeConverter.convertToDate(date)); + // } + + // @Override + // protected void putTime(final ObjectNode node, final String columnName, final ResultSet resultSet, + // final int index) throws SQLException { + // // resultSet.getTime() will lose nanoseconds precision + // final LocalTime localTime = resultSet.getTimestamp(index).toLocalDateTime().toLocalTime(); + // node.put(columnName, DateTimeConverter.convertToTime(localTime)); + // } } From f59cbb655de8076bd47e3fb3aba7376b28fae42d Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Sun, 11 Dec 2022 16:24:13 +0200 Subject: [PATCH 02/34] fixed Snowflake acceptance tests --- .../SnowflakeSourceOperations.java | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceOperations.java b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceOperations.java index c66840727f8b..40f8778eb0bf 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceOperations.java +++ b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceOperations.java @@ -23,6 +23,8 @@ import java.sql.SQLException; import java.sql.Timestamp; import java.time.LocalDate; +import java.time.LocalTime; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -123,20 +125,20 @@ protected void putTimestamp(final ObjectNode node, final String columnName, fina final Timestamp timestamp = resultSet.getTimestamp(index); node.put(columnName, DateTimeConverter.convertToTimestamp(timestamp)); } - // - // @Override - // protected void putDate(final ObjectNode node, final String columnName, final ResultSet resultSet, - // final int index) throws SQLException { - // final Date date = resultSet.getDate(index); - // node.put(columnName, DateTimeConverter.convertToDate(date)); - // } - - // @Override - // protected void putTime(final ObjectNode node, final String columnName, final ResultSet resultSet, - // final int index) throws SQLException { - // // resultSet.getTime() will lose nanoseconds precision - // final LocalTime localTime = resultSet.getTimestamp(index).toLocalDateTime().toLocalTime(); - // node.put(columnName, DateTimeConverter.convertToTime(localTime)); - // } + + @Override + protected void putDate(final ObjectNode node, final String columnName, final ResultSet resultSet, + final int index) throws SQLException { + final Date date = resultSet.getDate(index); + node.put(columnName, DateTimeConverter.convertToDate(date)); + } + + @Override + protected void putTime(final ObjectNode node, final String columnName, final ResultSet resultSet, + final int index) throws SQLException { + // resultSet.getTime() will lose nanoseconds precision + final LocalTime localTime = resultSet.getTimestamp(index).toLocalDateTime().toLocalTime(); + node.put(columnName, DateTimeConverter.convertToTime(localTime)); + } } From 042a8cc32dd3ea826c0c406daae9bfe1284d68e7 Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Sun, 11 Dec 2022 17:03:17 +0200 Subject: [PATCH 03/34] fixed TestJdbcUtils --- ...bstractJdbcCompatibleSourceOperations.java | 19 ++++++++++++++----- .../io/airbyte/db/jdbc/TestJdbcUtils.java | 8 ++++---- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java index 34412f8a2ee0..3f9c49e48725 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java @@ -28,6 +28,7 @@ import java.time.OffsetDateTime; import java.time.OffsetTime; import java.time.chrono.IsoEra; +import java.time.format.DateTimeParseException; import java.util.Collections; import java.util.List; import java.util.StringJoiner; @@ -147,7 +148,11 @@ protected void putDefault(final ObjectNode node, final String columnName, final } protected void setTime(final PreparedStatement preparedStatement, final int parameterIndex, final String value) throws SQLException { - setTimestamp(preparedStatement, parameterIndex, value); + try { + preparedStatement.setObject(parameterIndex, LocalTime.parse(value)); + } catch (final DateTimeParseException e) { + setTimestamp(preparedStatement, parameterIndex, value); + } } protected void setTimestamp(final PreparedStatement preparedStatement, final int parameterIndex, final String value) throws SQLException { @@ -179,10 +184,14 @@ protected void setTimestamp(final PreparedStatement preparedStatement, final int protected void setDate(final PreparedStatement preparedStatement, final int parameterIndex, final String value) throws SQLException { try { - final Timestamp from = Timestamp.from(DataTypeUtils.getDateFormat().parse(value).toInstant()); - preparedStatement.setDate(parameterIndex, new Date(from.getTime())); - } catch (final ParseException e) { - throw new RuntimeException(e); + preparedStatement.setObject(parameterIndex, LocalDate.parse(value)); + } catch (final Exception e) { + try { + final Timestamp from = Timestamp.from(DataTypeUtils.getDateFormat().parse(value).toInstant()); + preparedStatement.setDate(parameterIndex, new Date(from.getTime())); + } catch (final ParseException ex) { + throw new RuntimeException(ex); + } } } diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java b/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java index 92435db61ce4..25c008e4d65a 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java +++ b/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java @@ -163,8 +163,8 @@ void testSetStatementField() throws SQLException { sourceOperations.setStatementField(ps, 10, JDBCType.DECIMAL, "1"); sourceOperations.setStatementField(ps, 11, JDBCType.CHAR, "a"); sourceOperations.setStatementField(ps, 12, JDBCType.VARCHAR, "a"); - sourceOperations.setStatementField(ps, 13, JDBCType.DATE, "2020-11-01T00:00:00Z"); - sourceOperations.setStatementField(ps, 14, JDBCType.TIME, "1970-01-01T05:00:00.000Z"); + sourceOperations.setStatementField(ps, 13, JDBCType.DATE, "2020-11-01"); + sourceOperations.setStatementField(ps, 14, JDBCType.TIME, "05:00:00.000"); sourceOperations.setStatementField(ps, 15, JDBCType.TIMESTAMP, "2001-09-29T03:00:00.000Z"); sourceOperations.setStatementField(ps, 16, JDBCType.BINARY, "61616161"); @@ -391,9 +391,9 @@ private ObjectNode expectedValues() { expected.put("char", "a"); expected.put("varchar", "a"); // todo (cgardens) we should parse this to a date string - expected.put("date", "2020-11-01T00:00:00Z"); + expected.put("date", "2020-11-01"); // todo (cgardens) we should parse this to a time string - expected.put("time", "1970-01-01T05:00:00Z"); + expected.put("time", "05:00:00.000000"); expected.put("timestamp", "2001-09-29T03:00:00.000000Z"); expected.put("binary1", "aaaa".getBytes(Charsets.UTF_8)); return expected; From 49a90518501a97f302ff14cfdf4cb3e9afc7cc12 Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Sun, 11 Dec 2022 17:40:41 +0200 Subject: [PATCH 04/34] refactoring --- .../source/mysql/MySqlSourceOperations.java | 14 -------- .../postgres/PostgresSourceOperations.java | 14 -------- .../SnowflakeSourceOperations.java | 35 +++++++++++-------- 3 files changed, 20 insertions(+), 43 deletions(-) diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSourceOperations.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSourceOperations.java index ef92fd17b4ab..00391cabf0c5 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSourceOperations.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSourceOperations.java @@ -212,20 +212,6 @@ public boolean isCursorType(MysqlType type) { return ALLOWED_CURSOR_TYPES.contains(type); } - // @Override - // protected void putDate(final ObjectNode node, final String columnName, final ResultSet resultSet, - // final int index) throws SQLException { - // node.put(columnName, DateTimeConverter.convertToDate(getObject(resultSet, index, - // LocalDate.class))); - // } - - // @Override - // protected void putTime(final ObjectNode node, final String columnName, final ResultSet resultSet, - // final int index) throws SQLException { - // node.put(columnName, DateTimeConverter.convertToTime(getObject(resultSet, index, - // LocalTime.class))); - // } - @Override protected void putTimestamp(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { node.put(columnName, DateTimeConverter.convertToTimestamp(getObject(resultSet, index, LocalDateTime.class))); diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java index 79d2bf0dcaff..077908c63fd8 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java @@ -383,20 +383,6 @@ private void putLongArray(final ObjectNode node, final String columnName, final node.set(columnName, arrayNode); } - // @Override - // protected void putDate(final ObjectNode node, final String columnName, final ResultSet resultSet, - // final int index) throws SQLException { - // node.put(columnName, DateTimeConverter.convertToDate(getObject(resultSet, index, - // LocalDate.class))); - // } - - // @Override - // protected void putTime(final ObjectNode node, final String columnName, final ResultSet resultSet, - // final int index) throws SQLException { - // node.put(columnName, DateTimeConverter.convertToTime(getObject(resultSet, index, - // LocalTime.class))); - // } - @Override protected void putTimestamp(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { node.put(columnName, DateTimeConverter.convertToTimestamp(resultSet.getTimestamp(index))); diff --git a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceOperations.java b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceOperations.java index 40f8778eb0bf..1d889a18bcb2 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceOperations.java +++ b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceOperations.java @@ -24,7 +24,6 @@ import java.sql.Timestamp; import java.time.LocalDate; import java.time.LocalTime; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -126,19 +125,25 @@ protected void putTimestamp(final ObjectNode node, final String columnName, fina node.put(columnName, DateTimeConverter.convertToTimestamp(timestamp)); } - @Override - protected void putDate(final ObjectNode node, final String columnName, final ResultSet resultSet, - final int index) throws SQLException { - final Date date = resultSet.getDate(index); - node.put(columnName, DateTimeConverter.convertToDate(date)); - } - - @Override - protected void putTime(final ObjectNode node, final String columnName, final ResultSet resultSet, - final int index) throws SQLException { - // resultSet.getTime() will lose nanoseconds precision - final LocalTime localTime = resultSet.getTimestamp(index).toLocalDateTime().toLocalTime(); - node.put(columnName, DateTimeConverter.convertToTime(localTime)); - } + @Override + protected void putDate(final ObjectNode node, + final String columnName, + final ResultSet resultSet, + final int index) + throws SQLException { + final Date date = resultSet.getDate(index); + node.put(columnName, DateTimeConverter.convertToDate(date)); + } + + @Override + protected void putTime(final ObjectNode node, + final String columnName, + final ResultSet resultSet, + final int index) + throws SQLException { + // resultSet.getTime() will lose nanoseconds precision + final LocalTime localTime = resultSet.getTimestamp(index).toLocalDateTime().toLocalTime(); + node.put(columnName, DateTimeConverter.convertToTime(localTime)); + } } From 907c25a229567f55c85bbd38dde697eb1c0c2c9f Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Sun, 11 Dec 2022 18:11:37 +0200 Subject: [PATCH 05/34] update setDate for incremental sync --- .../jdbc/AbstractJdbcCompatibleSourceOperations.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java index 3f9c49e48725..8520cc7f3664 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java @@ -184,14 +184,10 @@ protected void setTimestamp(final PreparedStatement preparedStatement, final int protected void setDate(final PreparedStatement preparedStatement, final int parameterIndex, final String value) throws SQLException { try { + final Timestamp from = Timestamp.from(DataTypeUtils.getDateFormat().parse(value).toInstant()); + preparedStatement.setDate(parameterIndex, new Date(from.getTime())); + } catch (final ParseException e) { preparedStatement.setObject(parameterIndex, LocalDate.parse(value)); - } catch (final Exception e) { - try { - final Timestamp from = Timestamp.from(DataTypeUtils.getDateFormat().parse(value).toInstant()); - preparedStatement.setDate(parameterIndex, new Date(from.getTime())); - } catch (final ParseException ex) { - throw new RuntimeException(ex); - } } } From a2a4f05bbf26b104a10f21cd49bb1a33d17e024e Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Mon, 12 Dec 2022 14:26:30 +0200 Subject: [PATCH 06/34] fixed CockroachDbJdbcSourceAcceptanceTest --- .../CockroachDbJdbcSourceAcceptanceTest.java | 19 ++++++++++++++----- .../mssql/MssqlJdbcSourceAcceptanceTest.java | 10 ++++++++++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbJdbcSourceAcceptanceTest.java index 18504b4e7c06..894ee08c0461 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbJdbcSourceAcceptanceTest.java @@ -170,7 +170,7 @@ protected List getTestMessages() { .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_1, COL_NAME, "picard", - COL_UPDATED_AT, "2004-10-19T00:00:00Z")))), + COL_UPDATED_AT, "2004-10-19")))), new AirbyteMessage().withType(Type.RECORD) .withRecord(new AirbyteRecordMessage().withStream(streamName) .withNamespace(getDefaultNamespace()) @@ -178,14 +178,14 @@ protected List getTestMessages() { .of(COL_ID, ID_VALUE_2, COL_NAME, "crusher", COL_UPDATED_AT, - "2005-10-19T00:00:00Z")))), + "2005-10-19")))), new AirbyteMessage().withType(Type.RECORD) .withRecord(new AirbyteRecordMessage().withStream(streamName) .withNamespace(getDefaultNamespace()) .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_3, COL_NAME, "vash", - COL_UPDATED_AT, "2006-10-19T00:00:00Z"))))); + COL_UPDATED_AT, "2006-10-19"))))); } @Test @@ -314,13 +314,13 @@ void testReadOneTableIncrementallyTwice() throws Exception { .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_4, COL_NAME, "riker", - COL_UPDATED_AT, "2006-10-19T00:00:00Z"))))); + COL_UPDATED_AT, "2006-10-19"))))); expectedMessages.add(new AirbyteMessage().withType(Type.RECORD) .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_5, COL_NAME, "data", - COL_UPDATED_AT, "2006-10-19T00:00:00Z"))))); + COL_UPDATED_AT, "2006-10-19"))))); expectedMessages.add(new AirbyteMessage() .withType(Type.STATE) .withState(new AirbyteStateMessage() @@ -543,4 +543,13 @@ void testDiscoverWithMultipleSchemas() throws Exception { assertEquals(expected, filterOutOtherSchemas(actual)); } + @Override + protected void incrementalDateCheck() throws Exception { + incrementalCursorCheck( + COL_UPDATED_AT, + "2005-10-18", + "2006-10-19", + List.of(getTestMessages().get(1), getTestMessages().get(2))); + } + } diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlJdbcSourceAcceptanceTest.java index f4f5cd2aeeef..91cce538ec38 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlJdbcSourceAcceptanceTest.java @@ -21,6 +21,7 @@ import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; import io.airbyte.protocol.models.AirbyteConnectionStatus; import java.sql.JDBCType; +import java.util.List; import javax.sql.DataSource; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; @@ -155,4 +156,13 @@ public void testUserHasNoPermissionToDataBase() throws Exception { assertTrue(status.getMessage().contains("State code: S0001; Error code: 4060;")); } + @Override + protected void incrementalDateCheck() throws Exception { + incrementalCursorCheck( + COL_UPDATED_AT, + "2005-10-18", + "2006-10-19", + List.of(getTestMessages().get(1), getTestMessages().get(2))); + } + } From 193f23c2675b540e3c5395ee5ee22b69e5dfc284 Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Mon, 12 Dec 2022 15:20:16 +0200 Subject: [PATCH 07/34] fixed CockroachDbSourceDatatypeTest --- .../cockroachdb/CockroachDbSourceDatatypeTest.java | 2 +- .../source/jdbc/test/JdbcSourceAcceptanceTest.java | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java b/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java index 4688fabe7a56..625742240eff 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java @@ -217,7 +217,7 @@ protected void initTests() { .sourceType("date") .airbyteType(JsonSchemaType.STRING) .addInsertValues("'1999-01-08'", "null") - .addExpectedValues("1999-01-08T00:00:00Z", null) + .addExpectedValues("1999-01-08", null) .build()); addDataTypeTestData( diff --git a/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java index ffd5451ad33e..ede0da0b75fa 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java @@ -633,8 +633,8 @@ void testIncrementalDateCheckCursor() throws Exception { protected void incrementalDateCheck() throws Exception { incrementalCursorCheck( COL_UPDATED_AT, - "2005-10-18T00:00:00Z", - "2006-10-19T00:00:00Z", + "2005-10-18", + "2006-10-19", List.of(getTestMessages().get(1), getTestMessages().get(2))); } @@ -702,13 +702,13 @@ protected List getExpectedAirbyteMessagesSecondSync(final String .withData(Jsons.jsonNode(Map .of(COL_ID, ID_VALUE_4, COL_NAME, "riker", - COL_UPDATED_AT, "2006-10-19T00:00:00Z"))))); + COL_UPDATED_AT, "2006-10-19"))))); expectedMessages.add(new AirbyteMessage().withType(Type.RECORD) .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) .withData(Jsons.jsonNode(Map .of(COL_ID, ID_VALUE_5, COL_NAME, "data", - COL_UPDATED_AT, "2006-10-19T00:00:00Z"))))); + COL_UPDATED_AT, "2006-10-19"))))); final DbStreamState state = new DbStreamState() .withStreamName(streamName) .withStreamNamespace(namespace) @@ -1033,20 +1033,20 @@ protected List getTestMessages() { .withData(Jsons.jsonNode(Map .of(COL_ID, ID_VALUE_1, COL_NAME, "picard", - COL_UPDATED_AT, "2004-10-19T00:00:00Z")))), + COL_UPDATED_AT, "2004-10-19")))), new AirbyteMessage().withType(Type.RECORD) .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(getDefaultNamespace()) .withData(Jsons.jsonNode(Map .of(COL_ID, ID_VALUE_2, COL_NAME, "crusher", COL_UPDATED_AT, - "2005-10-19T00:00:00Z")))), + "2005-10-19")))), new AirbyteMessage().withType(Type.RECORD) .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(getDefaultNamespace()) .withData(Jsons.jsonNode(Map .of(COL_ID, ID_VALUE_3, COL_NAME, "vash", - COL_UPDATED_AT, "2006-10-19T00:00:00Z"))))); + COL_UPDATED_AT, "2006-10-19"))))); } protected List createExpectedTestMessages(final List states) { From 39e5c9f2b12a0a5a9cc3b6021605a764d5797a59 Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Mon, 12 Dec 2022 16:53:28 +0200 Subject: [PATCH 08/34] fixed CockroachDbSourceDatatypeTest --- .../cockroachdb/CockroachDbSourceDatatypeTest.java | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java b/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java index 625742240eff..ce9723bf3903 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java @@ -313,24 +313,12 @@ protected void initTests() { .addExpectedValues("a", "abc", "Миші йдуть;", "櫻花分店", "", null, "\\xF0\\x9F\\x9A\\x80") .build()); - // JdbcUtils-> DATE_FORMAT is set as ""yyyy-MM-dd'T'HH:mm:ss'Z'"" for both Date and Time types. - // Time (04:05:06) would be represented like "1970-01-01T04:05:06Z" addDataTypeTestData( TestDataHolder.builder() .sourceType("time") .airbyteType(JsonSchemaType.STRING) .addInsertValues("'04:05:06'", null) - .addExpectedValues("1970-01-01T04:05:06Z") - .addNullExpectedValue() - .build()); - - // Time (04:05:06) would be represented like "1970-01-01T04:05:06Z" - addDataTypeTestData( - TestDataHolder.builder() - .sourceType("timetz") - .airbyteType(JsonSchemaType.STRING) - .addInsertValues("'04:05:06Z'", null) - .addExpectedValues("1970-01-01T04:05:06Z") + .addExpectedValues("04:05:06.000000") .addNullExpectedValue() .build()); From 29bb71e471e9f5f8a95253087205b48fdcc0d8b6 Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Mon, 12 Dec 2022 18:03:15 +0200 Subject: [PATCH 09/34] fixed MsSQLSourceDatatypeTest --- .../source/mssql/AbstractMssqlSourceDatatypeTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractMssqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractMssqlSourceDatatypeTest.java index 913066af8c06..99ecc7b4142b 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractMssqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractMssqlSourceDatatypeTest.java @@ -144,7 +144,7 @@ protected void initTests() { .sourceType("date") .airbyteType(JsonSchemaType.STRING) .addInsertValues("'0001-01-01'", "'9999-12-31'", "'1999-01-08'", "null") - .addExpectedValues("0001-01-01T00:00:00Z", "9999-12-31T00:00:00Z", "1999-01-08T00:00:00Z", null) + .addExpectedValues("0001-01-01", "9999-12-31", "1999-01-08", null) .createTablePatternSql(CREATE_TABLE_SQL) .build()); From 50a6657cb168c30a448f36afcf3d4190cf5ddb23 Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Mon, 12 Dec 2022 20:38:21 +0200 Subject: [PATCH 10/34] fixed MsSQL cdc datatype test --- .../integrations/debezium/internals/MSSQLConverter.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/MSSQLConverter.java b/airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/MSSQLConverter.java index 50a5c77057ff..cb7904ba1fdb 100644 --- a/airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/MSSQLConverter.java +++ b/airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/MSSQLConverter.java @@ -8,6 +8,7 @@ import com.microsoft.sqlserver.jdbc.Geometry; import com.microsoft.sqlserver.jdbc.SQLServerException; import io.airbyte.db.DataTypeUtils; +import io.airbyte.db.jdbc.DateTimeConverter; import io.debezium.spi.converter.CustomConverter; import io.debezium.spi.converter.RelationalColumn; import java.math.BigDecimal; @@ -107,7 +108,9 @@ private void registerDate(final RelationalColumn field, if (Objects.isNull(input)) { return DebeziumConverterUtils.convertDefaultValue(field); } - + if (field.typeName().equals("DATE")) { + return DateTimeConverter.convertToDate(input); + } return DebeziumConverterUtils.convertDate(input); }); } From 2d96bca314313ae70e46ebc961224ef613680b1c Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Tue, 13 Dec 2022 11:22:12 +0200 Subject: [PATCH 11/34] refactoring --- .../jdbc/AbstractJdbcCompatibleSourceOperations.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java index 8520cc7f3664..8cd7706c34e1 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java @@ -183,11 +183,19 @@ protected void setTimestamp(final PreparedStatement preparedStatement, final int } protected void setDate(final PreparedStatement preparedStatement, final int parameterIndex, final String value) throws SQLException { + try { + preparedStatement.setObject(parameterIndex, LocalDate.parse(value)); + } catch (final DateTimeParseException e) { + setDateAsTimestamp(preparedStatement, parameterIndex, value); + } + } + + private void setDateAsTimestamp(PreparedStatement preparedStatement, int parameterIndex, String value) throws SQLException { try { final Timestamp from = Timestamp.from(DataTypeUtils.getDateFormat().parse(value).toInstant()); preparedStatement.setDate(parameterIndex, new Date(from.getTime())); } catch (final ParseException e) { - preparedStatement.setObject(parameterIndex, LocalDate.parse(value)); + throw new RuntimeException(e); } } From 89149e71b17dadc03cf9514a9cb0e877d67b2687 Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Tue, 13 Dec 2022 12:49:13 +0200 Subject: [PATCH 12/34] fixed mssql cdc test --- .../airbyte/integrations/debezium/internals/MSSQLConverter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/MSSQLConverter.java b/airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/MSSQLConverter.java index cb7904ba1fdb..09939a2eae9a 100644 --- a/airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/MSSQLConverter.java +++ b/airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/MSSQLConverter.java @@ -108,7 +108,7 @@ private void registerDate(final RelationalColumn field, if (Objects.isNull(input)) { return DebeziumConverterUtils.convertDefaultValue(field); } - if (field.typeName().equals("DATE")) { + if (field.typeName().equalsIgnoreCase("DATE")) { return DateTimeConverter.convertToDate(input); } return DebeziumConverterUtils.convertDate(input); From 73f28fac87c98f1072ed6120535f88b86897d879 Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Tue, 13 Dec 2022 14:08:24 +0200 Subject: [PATCH 13/34] fixed Redshift tests --- .../source/redshift/RedshiftSource.java | 2 +- .../redshift/RedshiftSourceOperations.java | 47 +++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSourceOperations.java diff --git a/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSource.java b/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSource.java index e91a610426a6..03d570ad2c06 100644 --- a/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSource.java +++ b/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSource.java @@ -38,7 +38,7 @@ public class RedshiftSource extends AbstractJdbcSource { // todo (cgardens) - clean up passing the dialect as null versus explicitly adding the case to the // constructor. public RedshiftSource() { - super(DRIVER_CLASS, AdaptiveStreamingQueryConfig::new, JdbcUtils.getDefaultSourceOperations()); + super(DRIVER_CLASS, AdaptiveStreamingQueryConfig::new, new RedshiftSourceOperations()); } @Override diff --git a/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSourceOperations.java b/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSourceOperations.java new file mode 100644 index 000000000000..78199fc2d447 --- /dev/null +++ b/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSourceOperations.java @@ -0,0 +1,47 @@ +package io.airbyte.integrations.source.redshift; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.db.jdbc.DateTimeConverter; +import io.airbyte.db.jdbc.JdbcSourceOperations; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Date; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.LocalDate; +import java.time.LocalTime; + +public class RedshiftSourceOperations extends JdbcSourceOperations { + + private static final Logger LOGGER = LoggerFactory.getLogger(RedshiftSourceOperations.class); + + @Override + protected void putDate(final ObjectNode node, + final String columnName, + final ResultSet resultSet, + final int index) + throws SQLException { + final Date date = resultSet.getDate(index); + node.put(columnName, DateTimeConverter.convertToDate(date)); + } + + @Override + protected void putTime(final ObjectNode node, + final String columnName, + final ResultSet resultSet, + final int index) + throws SQLException { + // resultSet.getTime() will lose nanoseconds precision + final LocalTime localTime = resultSet.getTimestamp(index).toLocalDateTime().toLocalTime(); + node.put(columnName, DateTimeConverter.convertToTime(localTime)); + } + + @Override + protected void setDate(final PreparedStatement preparedStatement, final int parameterIndex, final String value) throws SQLException { + final LocalDate date = LocalDate.parse(value); + // LocalDate must be converted to java.sql.Date. Please see https://docs.aws.amazon.com/redshift/latest/mgmt/jdbc20-data-type-mapping.html + preparedStatement.setDate(parameterIndex, Date.valueOf(date)); + } +} From 7c7d776ee6d2245a6a7d133120ee04776548cfc3 Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Tue, 13 Dec 2022 17:21:07 +0200 Subject: [PATCH 14/34] fixed Redshift tests --- .../source-redshift/integration_tests/expected_records.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-redshift/integration_tests/expected_records.json b/airbyte-integrations/connectors/source-redshift/integration_tests/expected_records.json index 1fc4b9afa423..58c245a5778f 100644 --- a/airbyte-integrations/connectors/source-redshift/integration_tests/expected_records.json +++ b/airbyte-integrations/connectors/source-redshift/integration_tests/expected_records.json @@ -1,2 +1,2 @@ -{"stream": "sat_basic_dataset", "data": {"id":1,"test_column_1":1,"test_column_2":126,"test_column_3":1024,"test_column_4":556,"test_column_5":777.888,"test_column_6":999.0,"test_column_7":true,"test_column_8":"q","test_column_9":"some text","test_column_10":"2008-12-31T00:00:00Z","test_column_11":"2008-06-01T09:59:59.000000Z","test_column_12":"2008-06-01T14:59:59.000000Z","test_column_13":"1970-01-01T04:05:06Z","test_column_14":"09:05:06+00","test_column_15":"787878"}, "emitted_at": 1669734903259 } -{"stream": "sat_basic_dataset", "data": {"id":2,"test_column_1":-5,"test_column_2":-126,"test_column_3":-1024,"test_column_4":-556,"test_column_5":-777.888,"test_column_6":-999.0,"test_column_7":false,"test_column_8":"g","test_column_9":"new text","test_column_10":"1987-10-10T00:00:00Z","test_column_11":"2005-06-21T12:00:59.000000Z","test_column_12":"2003-10-15T14:59:59.000000Z","test_column_13":"1970-01-01T04:05:00Z","test_column_14":"09:05:00+00","test_column_15":"797979"}, "emitted_at": 1669734903259 } +{"stream": "sat_basic_dataset", "data": {"id":1,"test_column_1":1,"test_column_2":126,"test_column_3":1024,"test_column_4":556,"test_column_5":777.888,"test_column_6":999.0,"test_column_7":true,"test_column_8":"q","test_column_9":"some text","test_column_10":"2008-12-31","test_column_11":"2008-06-01T09:59:59.000000Z","test_column_12":"2008-06-01T14:59:59.000000Z","test_column_13":"1970-01-01T04:05:06Z","test_column_14":"09:05:06+00","test_column_15":"787878"}, "emitted_at": 1669734903259 } +{"stream": "sat_basic_dataset", "data": {"id":2,"test_column_1":-5,"test_column_2":-126,"test_column_3":-1024,"test_column_4":-556,"test_column_5":-777.888,"test_column_6":-999.0,"test_column_7":false,"test_column_8":"g","test_column_9":"new text","test_column_10":"1987-10-10","test_column_11":"2005-06-21T12:00:59.000000Z","test_column_12":"2003-10-15T14:59:59.000000Z","test_column_13":"1970-01-01T04:05:00Z","test_column_14":"09:05:00+00","test_column_15":"797979"}, "emitted_at": 1669734903259 } From 57e660466d02aa7e6499fe1728985bfe52b9083b Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Tue, 13 Dec 2022 18:07:50 +0200 Subject: [PATCH 15/34] updated db2 datatype tests --- .../integration_tests/sources/Db2SourceDatatypeTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java index 5ea0a2dc6b9f..3b9168ebc56b 100644 --- a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java @@ -303,7 +303,7 @@ protected void initTests() { .sourceType("DATE") .airbyteType(JsonSchemaType.STRING) .addInsertValues("null", "'0001-01-01'", "'9999-12-31'") - .addExpectedValues(null, "0001-01-01T00:00:00Z", "9999-12-31T00:00:00Z") + .addExpectedValues(null, "0001-01-01", "9999-12-31") .build()); addDataTypeTestData( TestDataHolder.builder() @@ -311,7 +311,7 @@ protected void initTests() { .sourceType("TIME") .airbyteType(JsonSchemaType.STRING) .addInsertValues("null", "'00.00.00'", "'1:59 PM'", "'23.59.59'") - .addExpectedValues(null, "1970-01-01T00:00:00Z", "1970-01-01T13:59:00Z", "1970-01-01T23:59:59Z") + .addExpectedValues(null, "00:00:00", "13:59:00", "23:59:59") .build()); addDataTypeTestData( TestDataHolder.builder() From 427ee73f7efdcd7af01f9fbc1da9521697eb4a22 Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Tue, 13 Dec 2022 18:44:35 +0200 Subject: [PATCH 16/34] fixed DB2 tests --- .../Db2SourceOperations.java | 33 +++++++++++++++++++ .../sources/Db2SourceDatatypeTest.java | 2 +- .../integration_tests/expected_records.json | 4 +-- 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2SourceOperations.java b/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2SourceOperations.java index 3bf50749b4fc..ed9fe91e77ba 100644 --- a/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2SourceOperations.java +++ b/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2SourceOperations.java @@ -7,9 +7,15 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import io.airbyte.commons.json.Jsons; +import io.airbyte.db.jdbc.DateTimeConverter; import io.airbyte.db.jdbc.JdbcSourceOperations; + +import java.sql.Date; +import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.time.LocalDate; +import java.time.LocalTime; import java.util.Collections; import java.util.List; import org.slf4j.Logger; @@ -69,4 +75,31 @@ private void putDecfloat(final ObjectNode node, } } + @Override + protected void putDate(final ObjectNode node, + final String columnName, + final ResultSet resultSet, + final int index) + throws SQLException { + final Date date = resultSet.getDate(index); + node.put(columnName, DateTimeConverter.convertToDate(date)); + } + + @Override + protected void putTime(final ObjectNode node, + final String columnName, + final ResultSet resultSet, + final int index) + throws SQLException { + // resultSet.getTime() will lose nanoseconds precision + final LocalTime localTime = resultSet.getTimestamp(index).toLocalDateTime().toLocalTime(); + node.put(columnName, DateTimeConverter.convertToTime(localTime)); + } + + @Override + protected void setDate(final PreparedStatement preparedStatement, final int parameterIndex, final String value) throws SQLException { + final LocalDate date = LocalDate.parse(value); + preparedStatement.setDate(parameterIndex, Date.valueOf(date)); + } + } diff --git a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java index 3b9168ebc56b..4c01488395a8 100644 --- a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java @@ -311,7 +311,7 @@ protected void initTests() { .sourceType("TIME") .airbyteType(JsonSchemaType.STRING) .addInsertValues("null", "'00.00.00'", "'1:59 PM'", "'23.59.59'") - .addExpectedValues(null, "00:00:00", "13:59:00", "23:59:59") + .addExpectedValues(null, "00:00:00.000000", "13:59:00.000000", "23:59:59.000000") .build()); addDataTypeTestData( TestDataHolder.builder() diff --git a/airbyte-integrations/connectors/source-redshift/integration_tests/expected_records.json b/airbyte-integrations/connectors/source-redshift/integration_tests/expected_records.json index 58c245a5778f..28c55a076ada 100644 --- a/airbyte-integrations/connectors/source-redshift/integration_tests/expected_records.json +++ b/airbyte-integrations/connectors/source-redshift/integration_tests/expected_records.json @@ -1,2 +1,2 @@ -{"stream": "sat_basic_dataset", "data": {"id":1,"test_column_1":1,"test_column_2":126,"test_column_3":1024,"test_column_4":556,"test_column_5":777.888,"test_column_6":999.0,"test_column_7":true,"test_column_8":"q","test_column_9":"some text","test_column_10":"2008-12-31","test_column_11":"2008-06-01T09:59:59.000000Z","test_column_12":"2008-06-01T14:59:59.000000Z","test_column_13":"1970-01-01T04:05:06Z","test_column_14":"09:05:06+00","test_column_15":"787878"}, "emitted_at": 1669734903259 } -{"stream": "sat_basic_dataset", "data": {"id":2,"test_column_1":-5,"test_column_2":-126,"test_column_3":-1024,"test_column_4":-556,"test_column_5":-777.888,"test_column_6":-999.0,"test_column_7":false,"test_column_8":"g","test_column_9":"new text","test_column_10":"1987-10-10","test_column_11":"2005-06-21T12:00:59.000000Z","test_column_12":"2003-10-15T14:59:59.000000Z","test_column_13":"1970-01-01T04:05:00Z","test_column_14":"09:05:00+00","test_column_15":"797979"}, "emitted_at": 1669734903259 } +{"stream": "sat_basic_dataset", "data": {"id":1,"test_column_1":1,"test_column_2":126,"test_column_3":1024,"test_column_4":556,"test_column_5":777.888,"test_column_6":999.0,"test_column_7":true,"test_column_8":"q","test_column_9":"some text","test_column_10":"2008-12-31","test_column_11":"2008-06-01T09:59:59.000000Z","test_column_12":"2008-06-01T14:59:59.000000Z","test_column_13":"04:05:06.000000","test_column_14":"09:05:06+00","test_column_15":"787878"}, "emitted_at": 1669734903259 } +{"stream": "sat_basic_dataset", "data": {"id":2,"test_column_1":-5,"test_column_2":-126,"test_column_3":-1024,"test_column_4":-556,"test_column_5":-777.888,"test_column_6":-999.0,"test_column_7":false,"test_column_8":"g","test_column_9":"new text","test_column_10":"1987-10-10","test_column_11":"2005-06-21T12:00:59.000000Z","test_column_12":"2003-10-15T14:59:59.000000Z","test_column_13":"04:05:00.000000","test_column_14":"09:05:00+00","test_column_15":"797979"}, "emitted_at": 1669734903259 } From f15453ec73c0ce2703ddbb22ef2c6c1c7598821a Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Tue, 13 Dec 2022 21:14:57 +0200 Subject: [PATCH 17/34] fixed DB2 strict-encrypt tests --- .../src/test/resources/expected_spec.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test/resources/expected_spec.json index 557025936b6f..3fe1ab780525 100644 --- a/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test/resources/expected_spec.json @@ -56,9 +56,7 @@ "properties": { "encryption_method": { "type": "string", - "const": "encrypted_verify_certificate", - "enum": ["encrypted_verify_certificate"], - "default": "encrypted_verify_certificate" + "const": "encrypted_verify_certificate" }, "ssl_certificate": { "title": "SSL PEM file", From 048ec93d381c240636f9638c66a64540cfe7b34e Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Wed, 14 Dec 2022 12:26:56 +0200 Subject: [PATCH 18/34] fixed Oracle strict-encrypt tests --- .../src/test/resources/expected_spec.json | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test/resources/expected_spec.json index 17856fc32ad8..61f6a599e4c9 100644 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test/resources/expected_spec.json @@ -35,7 +35,6 @@ "connection_type": { "type": "string", "const": "service_name", - "default": "service_name", "order": 0 }, "service_name": { @@ -53,7 +52,6 @@ "connection_type": { "type": "string", "const": "sid", - "default": "sid", "order": 0 }, "sid": { @@ -108,9 +106,7 @@ "properties": { "encryption_method": { "type": "string", - "const": "client_nne", - "enum": ["client_nne"], - "default": "client_nne" + "const": "client_nne" }, "encryption_algorithm": { "type": "string", @@ -128,9 +124,7 @@ "properties": { "encryption_method": { "type": "string", - "const": "encrypted_verify_certificate", - "enum": ["encrypted_verify_certificate"], - "default": "encrypted_verify_certificate" + "const": "encrypted_verify_certificate" }, "ssl_certificate": { "title": "SSL PEM File", @@ -146,4 +140,4 @@ } } } -} +} \ No newline at end of file From 98caefd880bfd33a2a1e0b22cb0f502a637ee170 Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Wed, 14 Dec 2022 12:31:17 +0200 Subject: [PATCH 19/34] refactoring --- .../cockroachdb/CockroachDbJdbcSourceAcceptanceTest.java | 9 --------- .../source/mssql/MssqlJdbcSourceAcceptanceTest.java | 9 --------- 2 files changed, 18 deletions(-) diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbJdbcSourceAcceptanceTest.java index 894ee08c0461..d41c2d20044f 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbJdbcSourceAcceptanceTest.java @@ -543,13 +543,4 @@ void testDiscoverWithMultipleSchemas() throws Exception { assertEquals(expected, filterOutOtherSchemas(actual)); } - @Override - protected void incrementalDateCheck() throws Exception { - incrementalCursorCheck( - COL_UPDATED_AT, - "2005-10-18", - "2006-10-19", - List.of(getTestMessages().get(1), getTestMessages().get(2))); - } - } diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlJdbcSourceAcceptanceTest.java index 91cce538ec38..2a62a5cab709 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlJdbcSourceAcceptanceTest.java @@ -156,13 +156,4 @@ public void testUserHasNoPermissionToDataBase() throws Exception { assertTrue(status.getMessage().contains("State code: S0001; Error code: 4060;")); } - @Override - protected void incrementalDateCheck() throws Exception { - incrementalCursorCheck( - COL_UPDATED_AT, - "2005-10-18", - "2006-10-19", - List.of(getTestMessages().get(1), getTestMessages().get(2))); - } - } From f1bbb619f0c9afc22cf89051edb26554bb5e48d8 Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Wed, 14 Dec 2022 15:26:33 +0200 Subject: [PATCH 20/34] updated changelog --- docs/integrations/sources/alloydb.md | 1 + docs/integrations/sources/clickhouse.md | 1 + docs/integrations/sources/cockroachdb.md | 1 + docs/integrations/sources/db2.md | 1 + docs/integrations/sources/mssql.md | 1 + docs/integrations/sources/mysql.md | 1 + docs/integrations/sources/oracle.md | 1 + docs/integrations/sources/postgres.md | 1 + docs/integrations/sources/redshift.md | 1 + docs/integrations/sources/snowflake.md | 1 + docs/integrations/sources/tidb.md | 1 + 11 files changed, 11 insertions(+) diff --git a/docs/integrations/sources/alloydb.md b/docs/integrations/sources/alloydb.md index c4c3c4fc58bd..4aa43e7a346b 100644 --- a/docs/integrations/sources/alloydb.md +++ b/docs/integrations/sources/alloydb.md @@ -331,6 +331,7 @@ According to Postgres [documentation](https://www.postgresql.org/docs/14/datatyp | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------| +| 1.0.18 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | | 1.0.17 | 2022-10-31 | [18538](https://github.com/airbytehq/airbyte/pull/18538) | Encode database name | | 1.0.16 | 2022-10-25 | [18256](https://github.com/airbytehq/airbyte/pull/18256) | Disable allow and prefer ssl modes in CDC mode | | | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode | diff --git a/docs/integrations/sources/clickhouse.md b/docs/integrations/sources/clickhouse.md index 07df7cb267ba..01ab908916fd 100644 --- a/docs/integrations/sources/clickhouse.md +++ b/docs/integrations/sources/clickhouse.md @@ -96,6 +96,7 @@ Using this feature requires additional configuration, when creating the source. | Version | Date | Pull Request | Subject | |:---| :--- |:---------------------------------------------------------|:---------------------------------------------------------------------------| +| 0.1.15 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | | | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode | | 0.1.14 | 2022-09-27 | [17031](https://github.com/airbytehq/airbyte/pull/17031) | Added custom jdbc url parameters field | | 0.1.13 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently | diff --git a/docs/integrations/sources/cockroachdb.md b/docs/integrations/sources/cockroachdb.md index b6d6f5cc0c71..025dc06c60be 100644 --- a/docs/integrations/sources/cockroachdb.md +++ b/docs/integrations/sources/cockroachdb.md @@ -95,6 +95,7 @@ Your database user should now be ready for use with Airbyte. | Version | Date | Pull Request | Subject | |:--------|:-----------| :--- | :--- | +| 0.1.19 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | | | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode | | 0.1.18 | 2022-09-01 | [16394](https://github.com/airbytehq/airbyte/pull/16394) | Added custom jdbc properties field | | 0.1.17 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently | diff --git a/docs/integrations/sources/db2.md b/docs/integrations/sources/db2.md index 54207cef61e7..a8449022fea5 100644 --- a/docs/integrations/sources/db2.md +++ b/docs/integrations/sources/db2.md @@ -60,6 +60,7 @@ You can also enter your own password for the keystore, but if you don't, the pas | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.17 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | | | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode | | 0.1.16 | 2022-09-06 | [16354](https://github.com/airbytehq/airbyte/pull/16354) | Add custom JDBC params | | 0.1.15 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently | diff --git a/docs/integrations/sources/mssql.md b/docs/integrations/sources/mssql.md index de179e043f8e..44a86a62e01c 100644 --- a/docs/integrations/sources/mssql.md +++ b/docs/integrations/sources/mssql.md @@ -341,6 +341,7 @@ WHERE actor_definition_id ='b5ea17b1-f170-46dc-bc31-cc744ca984c1' AND (configura | Version | Date | Pull Request | Subject | |:--------|:-----------|:------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.4.27 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | | 0.4.26 | 2022-12-12 | [18959](https://github.com/airbytehq/airbyte/pull/18959) | CDC : Don't timeout if snapshot is not complete. | | 0.4.25 | 2022-11-04 | [18732](https://github.com/airbytehq/airbyte/pull/18732) | Upgrade debezium version to 1.9.6 | | 0.4.24 | 2022-10-25 | [18383](https://github.com/airbytehq/airbyte/pull/18383) | Better SSH error handling + messages | diff --git a/docs/integrations/sources/mysql.md b/docs/integrations/sources/mysql.md index 7c33d526b289..fa892d92a866 100644 --- a/docs/integrations/sources/mysql.md +++ b/docs/integrations/sources/mysql.md @@ -256,6 +256,7 @@ WHERE actor_definition_id ='435bb9a5-7887-4809-aa58-28c27df0d7ad' AND (configura | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 1.0.17 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | | 1.0.16 | 2022-12-12 | [18959](https://github.com/airbytehq/airbyte/pull/18959) | CDC : Don't timeout if snapshot is not complete. | | 1.0.15 | 2022-12-06 | [20000](https://github.com/airbytehq/airbyte/pull/20000) | Add check and better messaging when user does not have permission to access binary log in CDC mode | | 1.0.14 | 2022-11-22 | [19514](https://github.com/airbytehq/airbyte/pull/19514) | Adjust batch selection memory limits databases. | diff --git a/docs/integrations/sources/oracle.md b/docs/integrations/sources/oracle.md index d7935841a078..32d7e329af62 100644 --- a/docs/integrations/sources/oracle.md +++ b/docs/integrations/sources/oracle.md @@ -132,6 +132,7 @@ Airbyte has the ability to connect to the Oracle source with 3 network connectiv | Version | Date | Pull Request | Subject | |:--------|:-----------| :--- |:------------------------------------------------| +| 0.3.22 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | | | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode | | 0.3.21 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently | | 0.3.20 | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field | diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index fbc8a7e7b2df..993f96020ec4 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -415,6 +415,7 @@ The root causes is that the WALs needed for the incremental sync has been remove | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 1.0.34 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | | 1.0.33 | 2022-12-12 | [18959](https://github.com/airbytehq/airbyte/pull/18959) | CDC : Don't timeout if snapshot is not complete. | | 1.0.32 | 2022-12-12 | [20192](https://github.com/airbytehq/airbyte/pull/20192) | Only throw a warning if cursor column contains null values. | | 1.0.31 | 2022-12-02 | [19889](https://github.com/airbytehq/airbyte/pull/19889) | Check before each sync and stop if an incremental sync cursor column contains a null value. | diff --git a/docs/integrations/sources/redshift.md b/docs/integrations/sources/redshift.md index 3b0ba43ac40f..8c7ca149e69d 100644 --- a/docs/integrations/sources/redshift.md +++ b/docs/integrations/sources/redshift.md @@ -54,6 +54,7 @@ All Redshift connections are encrypted using SSL | Version | Date | Pull Request | Subject | |:--------|:-----------| :----- |:----------------------------------------------------------------------------------------------------------| +| 0.3.16 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | | 0.3.15 | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode | | 0.3.14 | 2022-09-01 | [16258](https://github.com/airbytehq/airbyte/pull/16258) | Emit state messages more frequently | | 0.3.13 | 2022-05-25 | | Added JDBC URL params | diff --git a/docs/integrations/sources/snowflake.md b/docs/integrations/sources/snowflake.md index 3b203390cd32..854398626d31 100644 --- a/docs/integrations/sources/snowflake.md +++ b/docs/integrations/sources/snowflake.md @@ -122,6 +122,7 @@ To read more please check official [Snowflake documentation](https://docs.snowfl | Version | Date | Pull Request | Subject | | :------ | :--------- |:---------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------| +| 0.1.27 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | | 0.1.26 | 2022-11-10 | [19314](https://github.com/airbytehq/airbyte/pull/19314) | Set application id in JDBC URL params based on OSS/Cloud environment | | 0.1.25 | 2022-11-10 | [15535](https://github.com/airbytehq/airbyte/pull/15535) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode | | 0.1.24 | 2022-09-26 | [17144](https://github.com/airbytehq/airbyte/pull/17144) | Fixed bug with incorrect date-time datatypes handling | diff --git a/docs/integrations/sources/tidb.md b/docs/integrations/sources/tidb.md index e026f886c618..c9766933036a 100644 --- a/docs/integrations/sources/tidb.md +++ b/docs/integrations/sources/tidb.md @@ -128,6 +128,7 @@ Now that you have set up the TiDB source connector, check out the following TiDB | Version | Date | Pull Request | Subject | | :------ | :--- | :----------- | ------- | +| 0.2.2 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | | | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode | | 0.2.1 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently | | 0.2.0 | 2022-07-26 | [14362](https://github.com/airbytehq/airbyte/pull/14362) | Integral columns are now discovered as int64 fields. | From 982b40c592727876ef2056a22ca9541a4173e53e Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Thu, 15 Dec 2022 13:48:00 +0200 Subject: [PATCH 21/34] refactoring --- .../io/airbyte/db/jdbc/DateTimeConverter.java | 14 ++++ .../io/airbyte/db/jdbc/TestJdbcUtils.java | 2 - .../CockroachJdbcSourceOperations.java | 6 ++ .../CockroachDbSourceDatatypeTest.java | 10 +++ .../Db2SourceOperations.java | 17 ++--- .../mssql/MssqlJdbcSourceAcceptanceTest.java | 1 - .../source/oracle/OracleSourceOperations.java | 3 + .../redshift/RedshiftSourceOperations.java | 71 ++++++++++--------- .../SnowflakeSourceOperations.java | 10 ++- 9 files changed, 81 insertions(+), 53 deletions(-) diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/DateTimeConverter.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/DateTimeConverter.java index 5c8b6a306b07..110a5e93dfb9 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/DateTimeConverter.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/DateTimeConverter.java @@ -12,7 +12,10 @@ import static io.airbyte.db.jdbc.AbstractJdbcCompatibleSourceOperations.resolveEra; import static java.time.ZoneOffset.UTC; +import com.fasterxml.jackson.databind.node.ObjectNode; import java.sql.Date; +import java.sql.ResultSet; +import java.sql.SQLException; import java.sql.Time; import java.sql.Timestamp; import java.time.Duration; @@ -168,4 +171,15 @@ public static String convertToTime(final Object time) { } } + public static void putJavaSQLDate(ObjectNode node, String columnName, ResultSet resultSet, int index) throws SQLException { + final Date date = resultSet.getDate(index); + node.put(columnName, convertToDate(date)); + } + + public static void putJavaSQLTime(ObjectNode node, String columnName, ResultSet resultSet, int index) throws SQLException { + // resultSet.getTime() will lose nanoseconds precision + final LocalTime localTime = resultSet.getTimestamp(index).toLocalDateTime().toLocalTime(); + node.put(columnName, convertToTime(localTime)); + } + } diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java b/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java index 25c008e4d65a..a95edb6964b8 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java +++ b/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java @@ -390,9 +390,7 @@ private ObjectNode expectedValues() { expected.put("decimal", new BigDecimal(1)); expected.put("char", "a"); expected.put("varchar", "a"); - // todo (cgardens) we should parse this to a date string expected.put("date", "2020-11-01"); - // todo (cgardens) we should parse this to a time string expected.put("time", "05:00:00.000000"); expected.put("timestamp", "2001-09-29T03:00:00.000000Z"); expected.put("binary1", "aaaa".getBytes(Charsets.UTF_8)); diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachJdbcSourceOperations.java b/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachJdbcSourceOperations.java index f29eee7df9f1..d7ca6a4e99e6 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachJdbcSourceOperations.java +++ b/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachJdbcSourceOperations.java @@ -4,12 +4,15 @@ package io.airbyte.integrations.source.cockroachdb; +import static io.airbyte.db.DataTypeUtils.TIMETZ_FORMATTER; + import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import io.airbyte.commons.json.Jsons; import io.airbyte.db.jdbc.JdbcSourceOperations; import java.sql.ResultSet; import java.sql.SQLException; +import java.time.OffsetTime; import java.util.Collections; /** @@ -61,6 +64,9 @@ private void putCockroachSpecialDataType(final ResultSet resultSet, final int in if ("numeric".equalsIgnoreCase(columnType)) { final double value = resultSet.getDouble(index); node.put(columnName, value); + } else if ("timetz".equalsIgnoreCase(columnType)) { + final OffsetTime timetz = getObject(resultSet, index, OffsetTime.class); + node.put(columnName, timetz.format(TIMETZ_FORMATTER)); } else { node.put(columnName, (Double) null); } diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java b/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java index ce9723bf3903..768dd5d4affa 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java @@ -322,6 +322,16 @@ protected void initTests() { .addNullExpectedValue() .build()); + // Time (04:05:06) would be represented like "1970-01-01T04:05:06Z" + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("timetz") + .airbyteType(JsonSchemaType.STRING) + .addInsertValues("'04:05:06Z'", null) + .addExpectedValues("04:05:06.000000Z") + .addNullExpectedValue() + .build()); + addDataTypeTestData( TestDataHolder.builder() .sourceType("timestamp") diff --git a/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2SourceOperations.java b/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2SourceOperations.java index ed9fe91e77ba..2c51f3e4f5a8 100644 --- a/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2SourceOperations.java +++ b/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2SourceOperations.java @@ -4,18 +4,18 @@ package io.airbyte.integrations.source.db2; +import static io.airbyte.db.jdbc.DateTimeConverter.putJavaSQLDate; +import static io.airbyte.db.jdbc.DateTimeConverter.putJavaSQLTime; + import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.DateTimeConverter; import io.airbyte.db.jdbc.JdbcSourceOperations; - import java.sql.Date; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.time.LocalDate; -import java.time.LocalTime; import java.util.Collections; import java.util.List; import org.slf4j.Logger; @@ -80,9 +80,8 @@ protected void putDate(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) - throws SQLException { - final Date date = resultSet.getDate(index); - node.put(columnName, DateTimeConverter.convertToDate(date)); + throws SQLException { + putJavaSQLDate(node, columnName, resultSet, index); } @Override @@ -90,10 +89,8 @@ protected void putTime(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) - throws SQLException { - // resultSet.getTime() will lose nanoseconds precision - final LocalTime localTime = resultSet.getTimestamp(index).toLocalDateTime().toLocalTime(); - node.put(columnName, DateTimeConverter.convertToTime(localTime)); + throws SQLException { + putJavaSQLTime(node, columnName, resultSet, index); } @Override diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlJdbcSourceAcceptanceTest.java index 2a62a5cab709..f4f5cd2aeeef 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlJdbcSourceAcceptanceTest.java @@ -21,7 +21,6 @@ import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; import io.airbyte.protocol.models.AirbyteConnectionStatus; import java.sql.JDBCType; -import java.util.List; import javax.sql.DataSource; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; diff --git a/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSourceOperations.java b/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSourceOperations.java index 23bcd0d3f0d3..7cd9ed52a163 100644 --- a/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSourceOperations.java +++ b/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSourceOperations.java @@ -16,6 +16,9 @@ public class OracleSourceOperations extends JdbcSourceOperations { private static final Logger LOGGER = LoggerFactory.getLogger(OracleSourceOperations.class); + /** + * Oracle's DATE type can actually have time values, so we return a full timestamp. + */ @Override protected void putDate(ObjectNode node, String columnName, ResultSet resultSet, int index) throws SQLException { node.put(columnName, DateTimeConverter.convertToTimestamp(resultSet.getTimestamp(index))); diff --git a/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSourceOperations.java b/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSourceOperations.java index 78199fc2d447..48a09d6ef8d1 100644 --- a/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSourceOperations.java +++ b/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSourceOperations.java @@ -1,47 +1,50 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.source.redshift; +import static io.airbyte.db.jdbc.DateTimeConverter.putJavaSQLDate; +import static io.airbyte.db.jdbc.DateTimeConverter.putJavaSQLTime; + import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.db.jdbc.DateTimeConverter; import io.airbyte.db.jdbc.JdbcSourceOperations; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.sql.Date; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.time.LocalDate; -import java.time.LocalTime; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class RedshiftSourceOperations extends JdbcSourceOperations { - private static final Logger LOGGER = LoggerFactory.getLogger(RedshiftSourceOperations.class); - - @Override - protected void putDate(final ObjectNode node, - final String columnName, - final ResultSet resultSet, - final int index) - throws SQLException { - final Date date = resultSet.getDate(index); - node.put(columnName, DateTimeConverter.convertToDate(date)); - } - - @Override - protected void putTime(final ObjectNode node, - final String columnName, - final ResultSet resultSet, - final int index) - throws SQLException { - // resultSet.getTime() will lose nanoseconds precision - final LocalTime localTime = resultSet.getTimestamp(index).toLocalDateTime().toLocalTime(); - node.put(columnName, DateTimeConverter.convertToTime(localTime)); - } - - @Override - protected void setDate(final PreparedStatement preparedStatement, final int parameterIndex, final String value) throws SQLException { - final LocalDate date = LocalDate.parse(value); - // LocalDate must be converted to java.sql.Date. Please see https://docs.aws.amazon.com/redshift/latest/mgmt/jdbc20-data-type-mapping.html - preparedStatement.setDate(parameterIndex, Date.valueOf(date)); - } + private static final Logger LOGGER = LoggerFactory.getLogger(RedshiftSourceOperations.class); + + @Override + protected void putDate(final ObjectNode node, + final String columnName, + final ResultSet resultSet, + final int index) + throws SQLException { + putJavaSQLDate(node, columnName, resultSet, index); + } + + @Override + protected void putTime(final ObjectNode node, + final String columnName, + final ResultSet resultSet, + final int index) + throws SQLException { + putJavaSQLTime(node, columnName, resultSet, index); + } + + @Override + protected void setDate(final PreparedStatement preparedStatement, final int parameterIndex, final String value) throws SQLException { + final LocalDate date = LocalDate.parse(value); + // LocalDate must be converted to java.sql.Date. Please see + // https://docs.aws.amazon.com/redshift/latest/mgmt/jdbc20-data-type-mapping.html + preparedStatement.setDate(parameterIndex, Date.valueOf(date)); + } + } diff --git a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceOperations.java b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceOperations.java index 1d889a18bcb2..2353d67aef99 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceOperations.java +++ b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceOperations.java @@ -4,6 +4,8 @@ package io.airbyte.integrations.source.snowflake; +import static io.airbyte.db.jdbc.DateTimeConverter.putJavaSQLDate; +import static io.airbyte.db.jdbc.DateTimeConverter.putJavaSQLTime; import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_NAME; import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_TYPE; import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_TYPE_NAME; @@ -23,7 +25,6 @@ import java.sql.SQLException; import java.sql.Timestamp; import java.time.LocalDate; -import java.time.LocalTime; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -131,8 +132,7 @@ protected void putDate(final ObjectNode node, final ResultSet resultSet, final int index) throws SQLException { - final Date date = resultSet.getDate(index); - node.put(columnName, DateTimeConverter.convertToDate(date)); + putJavaSQLDate(node, columnName, resultSet, index); } @Override @@ -141,9 +141,7 @@ protected void putTime(final ObjectNode node, final ResultSet resultSet, final int index) throws SQLException { - // resultSet.getTime() will lose nanoseconds precision - final LocalTime localTime = resultSet.getTimestamp(index).toLocalDateTime().toLocalTime(); - node.put(columnName, DateTimeConverter.convertToTime(localTime)); + putJavaSQLTime(node, columnName, resultSet, index); } } From 9441548e36bf13004e160e4a3f194ab3e25c5359 Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Sun, 18 Dec 2022 14:19:06 +0200 Subject: [PATCH 22/34] removed redundant overrides --- .../source/mysql/MySqlSourceOperations.java | 12 ------------ .../source/postgres/PostgresSourceOperations.java | 11 ----------- 2 files changed, 23 deletions(-) diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSourceOperations.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSourceOperations.java index 00391cabf0c5..c625c22694dd 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSourceOperations.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSourceOperations.java @@ -272,16 +272,4 @@ private void setTimestampWithTimezone(final PreparedStatement preparedStatement, } } - @Override - protected void setTime(final PreparedStatement preparedStatement, final int parameterIndex, final String value) throws SQLException { - try { - preparedStatement.setObject(parameterIndex, LocalTime.parse(value)); - } catch (final DateTimeParseException e) { - LOGGER.warn("Exception occurred while trying to parse value for time column the new way, trying the old way", e); - // This is just for backward compatibility for connectors created on versions before PR - // https://github.com/airbytehq/airbyte/pull/15504 - super.setTime(preparedStatement, parameterIndex, value); - } - } - } diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java index 077908c63fd8..632eb0a14bcc 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java @@ -157,17 +157,6 @@ protected void setTimestamp(final PreparedStatement preparedStatement, final int } } - @Override - protected void setTime(final PreparedStatement preparedStatement, final int parameterIndex, final String value) throws SQLException { - try { - preparedStatement.setObject(parameterIndex, LocalTime.parse(value)); - } catch (final DateTimeParseException e) { - // attempt to parse the datetime with timezone. This can be caused by schema created with an older - // version of the connector - preparedStatement.setObject(parameterIndex, OffsetTime.parse(value)); - } - } - @Override protected void setDate(final PreparedStatement preparedStatement, final int parameterIndex, final String value) throws SQLException { preparedStatement.setObject(parameterIndex, LocalDate.parse(value)); From 211c6d1fefbff29cf61360900877204ed391efed Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Sun, 18 Dec 2022 15:53:38 +0200 Subject: [PATCH 23/34] bump versions --- .../connectors/source-alloydb-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/source-alloydb/Dockerfile | 2 +- .../connectors/source-clickhouse-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/source-clickhouse/Dockerfile | 2 +- .../connectors/source-cockroachdb-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/source-cockroachdb/Dockerfile | 2 +- .../connectors/source-db2-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/source-db2/Dockerfile | 2 +- .../connectors/source-mssql-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/source-mssql/Dockerfile | 2 +- .../connectors/source-mysql-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/source-mysql/Dockerfile | 2 +- .../connectors/source-oracle-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/source-oracle/Dockerfile | 2 +- .../connectors/source-postgres-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/source-postgres/Dockerfile | 2 +- airbyte-integrations/connectors/source-redshift/Dockerfile | 2 +- airbyte-integrations/connectors/source-snowflake/Dockerfile | 2 +- airbyte-integrations/connectors/source-tidb/Dockerfile | 2 +- docs/integrations/sources/clickhouse.md | 1 + docs/integrations/sources/db2.md | 2 +- 21 files changed, 21 insertions(+), 20 deletions(-) diff --git a/airbyte-integrations/connectors/source-alloydb-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-alloydb-strict-encrypt/Dockerfile index ea01b05c8865..9e6ce4500d97 100644 --- a/airbyte-integrations/connectors/source-alloydb-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-alloydb-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-alloydb-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.34 +LABEL io.airbyte.version=1.0.35 LABEL io.airbyte.name=airbyte/source-alloydb-strict-encrypt diff --git a/airbyte-integrations/connectors/source-alloydb/Dockerfile b/airbyte-integrations/connectors/source-alloydb/Dockerfile index 725fc5eef5e9..6a1441b16c7f 100644 --- a/airbyte-integrations/connectors/source-alloydb/Dockerfile +++ b/airbyte-integrations/connectors/source-alloydb/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-alloydb COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.34 +LABEL io.airbyte.version=1.0.35 LABEL io.airbyte.name=airbyte/source-alloydb diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile index 582507952ef2..0b2921c352eb 100644 --- a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-clickhouse-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.14 +LABEL io.airbyte.version=0.1.15 LABEL io.airbyte.name=airbyte/source-clickhouse-strict-encrypt diff --git a/airbyte-integrations/connectors/source-clickhouse/Dockerfile b/airbyte-integrations/connectors/source-clickhouse/Dockerfile index 6681cc7af3cb..68797ed45ef4 100644 --- a/airbyte-integrations/connectors/source-clickhouse/Dockerfile +++ b/airbyte-integrations/connectors/source-clickhouse/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-clickhouse COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.14 +LABEL io.airbyte.version=0.1.15 LABEL io.airbyte.name=airbyte/source-clickhouse diff --git a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/Dockerfile index 86fe67b15e31..854e9bd36f0b 100644 --- a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-cockroachdb-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.18 +LABEL io.airbyte.version=0.1.19 LABEL io.airbyte.name=airbyte/source-cockroachdb-strict-encrypt diff --git a/airbyte-integrations/connectors/source-cockroachdb/Dockerfile b/airbyte-integrations/connectors/source-cockroachdb/Dockerfile index 1541c933439b..f9ab5ba8ca5b 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/Dockerfile +++ b/airbyte-integrations/connectors/source-cockroachdb/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-cockroachdb COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.18 +LABEL io.airbyte.version=0.1.19 LABEL io.airbyte.name=airbyte/source-cockroachdb diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile index ab7aecf54702..5b9702f1a42f 100644 --- a/airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-db2-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.16 +LABEL io.airbyte.version=0.1.17 LABEL io.airbyte.name=airbyte/source-db2-strict-encrypt diff --git a/airbyte-integrations/connectors/source-db2/Dockerfile b/airbyte-integrations/connectors/source-db2/Dockerfile index 0bb990e58705..2c7585110df5 100644 --- a/airbyte-integrations/connectors/source-db2/Dockerfile +++ b/airbyte-integrations/connectors/source-db2/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-db2 COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.16 +LABEL io.airbyte.version=0.1.17 LABEL io.airbyte.name=airbyte/source-db2 diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile index b5ecbf766a7e..3ec6c69f7c27 100644 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mssql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.4.26 +LABEL io.airbyte.version=0.4.27 LABEL io.airbyte.name=airbyte/source-mssql-strict-encrypt diff --git a/airbyte-integrations/connectors/source-mssql/Dockerfile b/airbyte-integrations/connectors/source-mssql/Dockerfile index 853c017b29cb..b0624aaf167e 100644 --- a/airbyte-integrations/connectors/source-mssql/Dockerfile +++ b/airbyte-integrations/connectors/source-mssql/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mssql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.4.26 +LABEL io.airbyte.version=0.4.27 LABEL io.airbyte.name=airbyte/source-mssql diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile index a0321888bad3..bb8acc44839f 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile @@ -16,6 +16,6 @@ ENV APPLICATION source-mysql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.18 +LABEL io.airbyte.version=1.0.19 LABEL io.airbyte.name=airbyte/source-mysql-strict-encrypt diff --git a/airbyte-integrations/connectors/source-mysql/Dockerfile b/airbyte-integrations/connectors/source-mysql/Dockerfile index 00e243fb16ef..149618a5dfe6 100644 --- a/airbyte-integrations/connectors/source-mysql/Dockerfile +++ b/airbyte-integrations/connectors/source-mysql/Dockerfile @@ -16,6 +16,6 @@ ENV APPLICATION source-mysql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.18 +LABEL io.airbyte.version=1.0.19 LABEL io.airbyte.name=airbyte/source-mysql diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-oracle-strict-encrypt/Dockerfile index af3a651f6cfb..9ee02828dd28 100644 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/Dockerfile @@ -17,5 +17,5 @@ ENV TZ UTC COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.21 +LABEL io.airbyte.version=0.3.22 LABEL io.airbyte.name=airbyte/source-oracle-strict-encrypt diff --git a/airbyte-integrations/connectors/source-oracle/Dockerfile b/airbyte-integrations/connectors/source-oracle/Dockerfile index 5baa8ea52a26..3ed7c32f81c6 100644 --- a/airbyte-integrations/connectors/source-oracle/Dockerfile +++ b/airbyte-integrations/connectors/source-oracle/Dockerfile @@ -8,5 +8,5 @@ ENV TZ UTC COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.3.21 +LABEL io.airbyte.version=0.3.22 LABEL io.airbyte.name=airbyte/source-oracle diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile index d2c1a6dda415..f7c9b147afed 100644 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.34 +LABEL io.airbyte.version=1.0.35 LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile index 1ec5edecc4c6..793e460515a3 100644 --- a/airbyte-integrations/connectors/source-postgres/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.34 +LABEL io.airbyte.version=1.0.35 LABEL io.airbyte.name=airbyte/source-postgres diff --git a/airbyte-integrations/connectors/source-redshift/Dockerfile b/airbyte-integrations/connectors/source-redshift/Dockerfile index 0a9527c26e36..8a66ebcc16d1 100644 --- a/airbyte-integrations/connectors/source-redshift/Dockerfile +++ b/airbyte-integrations/connectors/source-redshift/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-redshift COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.15 +LABEL io.airbyte.version=0.3.16 LABEL io.airbyte.name=airbyte/source-redshift diff --git a/airbyte-integrations/connectors/source-snowflake/Dockerfile b/airbyte-integrations/connectors/source-snowflake/Dockerfile index 975edf0edf0c..6f9f02f09885 100644 --- a/airbyte-integrations/connectors/source-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/source-snowflake/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-snowflake COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.27 +LABEL io.airbyte.version=0.1.28 LABEL io.airbyte.name=airbyte/source-snowflake diff --git a/airbyte-integrations/connectors/source-tidb/Dockerfile b/airbyte-integrations/connectors/source-tidb/Dockerfile index 4f6993303092..dc04e3438d37 100755 --- a/airbyte-integrations/connectors/source-tidb/Dockerfile +++ b/airbyte-integrations/connectors/source-tidb/Dockerfile @@ -17,5 +17,5 @@ ENV APPLICATION source-tidb COPY --from=build /airbyte /airbyte # Airbyte's build system uses these labels to know what to name and tag the docker images produced by this Dockerfile. -LABEL io.airbyte.version=0.2.1 +LABEL io.airbyte.version=0.2.2 LABEL io.airbyte.name=airbyte/source-tidb diff --git a/docs/integrations/sources/clickhouse.md b/docs/integrations/sources/clickhouse.md index 01ab908916fd..5ff8039ab9d6 100644 --- a/docs/integrations/sources/clickhouse.md +++ b/docs/integrations/sources/clickhouse.md @@ -78,6 +78,7 @@ Using this feature requires additional configuration, when creating the source. | Version | Date | Pull Request | Subject | |:--------| :--- |:---------------------------------------------------------|:-----------------------------------------------------------------| +| 0.1.15 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | | 0.1.14 | 2022-09-27 | [17031](https://github.com/airbytehq/airbyte/pull/17031) | Added custom jdbc url parameters field | | 0.1.13 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently | | 0.1.12 | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field | diff --git a/docs/integrations/sources/db2.md b/docs/integrations/sources/db2.md index a8449022fea5..e38fbc5ec865 100644 --- a/docs/integrations/sources/db2.md +++ b/docs/integrations/sources/db2.md @@ -60,7 +60,7 @@ You can also enter your own password for the keystore, but if you don't, the pas | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | -| 0.1.17 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | +| 0.1.17 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | | | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode | | 0.1.16 | 2022-09-06 | [16354](https://github.com/airbytehq/airbyte/pull/16354) | Add custom JDBC params | | 0.1.15 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently | From cc78ece5b328191ef5e057c611d69f5a2db0651d Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Sun, 18 Dec 2022 14:05:59 +0000 Subject: [PATCH 24/34] auto-bump connector version --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 139c220c58b4..740c38bdc175 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -38,7 +38,7 @@ - name: AlloyDB for PostgreSQL sourceDefinitionId: 1fa90628-2b9e-11ed-a261-0242ac120002 dockerRepository: airbyte/source-alloydb - dockerImageTag: 1.0.34 + dockerImageTag: 1.0.35 documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb icon: alloydb.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 8a61c000b377..0c17107a2c58 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -299,7 +299,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-alloydb:1.0.34" +- dockerImage: "airbyte/source-alloydb:1.0.35" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres" connectionSpecification: From 57f5b4c9135ca4878f7e9e86839b883bd8e99e51 Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Mon, 16 Jan 2023 11:37:18 +0200 Subject: [PATCH 25/34] resolved merge conflicts --- .../connectors/source-postgres-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/source-postgres/Dockerfile | 2 +- airbyte-integrations/connectors/source-snowflake/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile index 0978075777e9..df697ded6fa6 100644 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.36 +LABEL io.airbyte.version=1.0.37 LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile index 6edcec59beb5..291e5c2c7763 100644 --- a/airbyte-integrations/connectors/source-postgres/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.36 +LABEL io.airbyte.version=1.0.37 LABEL io.airbyte.name=airbyte/source-postgres diff --git a/airbyte-integrations/connectors/source-snowflake/Dockerfile b/airbyte-integrations/connectors/source-snowflake/Dockerfile index 6f9f02f09885..f34e7d8f03a3 100644 --- a/airbyte-integrations/connectors/source-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/source-snowflake/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-snowflake COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.28 +LABEL io.airbyte.version=0.1.29 LABEL io.airbyte.name=airbyte/source-snowflake From df82df83c9c151f791d520f65417b62f70d485d5 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Mon, 16 Jan 2023 10:29:45 +0000 Subject: [PATCH 26/34] auto-bump connector version --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 4fb7542904f5..7c5a6d1bbe1d 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1728,7 +1728,7 @@ - name: TiDB sourceDefinitionId: 0dad1a35-ccf8-4d03-b73e-6788c00b13ae dockerRepository: airbyte/source-tidb - dockerImageTag: 0.2.1 + dockerImageTag: 0.2.2 documentationUrl: https://docs.airbyte.com/integrations/sources/tidb icon: tidb.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 3ca8d30aa0ce..5528b3c49f72 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -14557,7 +14557,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-tidb:0.2.1" +- dockerImage: "airbyte/source-tidb:0.2.2" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/tidb" connectionSpecification: From 7bcf4b791b46da397c837a9717d62061b59949c6 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Mon, 16 Jan 2023 10:46:13 +0000 Subject: [PATCH 27/34] auto-bump connector version --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 7c5a6d1bbe1d..d5f7f5505623 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1632,7 +1632,7 @@ - name: Snowflake sourceDefinitionId: e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2 dockerRepository: airbyte/source-snowflake - dockerImageTag: 0.1.28 + dockerImageTag: 0.1.29 documentationUrl: https://docs.airbyte.com/integrations/sources/snowflake icon: snowflake.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 5528b3c49f72..937a68086f9e 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -13709,7 +13709,7 @@ - - "client_secret" oauthFlowOutputParameters: - - "refresh_token" -- dockerImage: "airbyte/source-snowflake:0.1.28" +- dockerImage: "airbyte/source-snowflake:0.1.29" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/snowflake" connectionSpecification: From bb35b6eabc3d3d4945e7820ab7a5535ae95c4f2e Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Mon, 16 Jan 2023 11:34:15 +0000 Subject: [PATCH 28/34] auto-bump connector version --- .../src/main/resources/seed/source_definitions.yaml | 2 +- .../init/src/main/resources/seed/source_specs.yaml | 13 +------------ 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index d5f7f5505623..7e5d9bad5aaa 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1170,7 +1170,7 @@ - name: Oracle DB sourceDefinitionId: b39a7370-74c3-45a6-ac3a-380d48520a83 dockerRepository: airbyte/source-oracle - dockerImageTag: 0.3.21 + dockerImageTag: 0.3.22 documentationUrl: https://docs.airbyte.com/integrations/sources/oracle icon: oracle.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 937a68086f9e..fd9d4f935ffb 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -10135,7 +10135,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-oracle:0.3.21" +- dockerImage: "airbyte/source-oracle:0.3.22" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/oracle" connectionSpecification: @@ -10177,7 +10177,6 @@ connection_type: type: "string" const: "service_name" - default: "service_name" order: 0 service_name: title: "Service name" @@ -10191,7 +10190,6 @@ connection_type: type: "string" const: "sid" - default: "sid" order: 0 sid: title: "System ID (SID)" @@ -10239,9 +10237,6 @@ encryption_method: type: "string" const: "unencrypted" - enum: - - "unencrypted" - default: "unencrypted" - title: "Native Network Encryption (NNE)" description: "The native network encryption gives you the ability to encrypt\ \ database connections, without the configuration overhead of TCP/IP\ @@ -10252,9 +10247,6 @@ encryption_method: type: "string" const: "client_nne" - enum: - - "client_nne" - default: "client_nne" encryption_algorithm: type: "string" description: "This parameter defines what encryption algorithm is\ @@ -10274,9 +10266,6 @@ encryption_method: type: "string" const: "encrypted_verify_certificate" - enum: - - "encrypted_verify_certificate" - default: "encrypted_verify_certificate" ssl_certificate: title: "SSL PEM File" description: "Privacy Enhanced Mail (PEM) files are concatenated certificate\ From defa844d326ee5faf83e788bbd098f3d1e603539 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Tue, 17 Jan 2023 11:51:56 +0000 Subject: [PATCH 29/34] auto-bump connector version --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 681a8d7ff54c..59e87524d6bd 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -249,7 +249,7 @@ - name: ClickHouse sourceDefinitionId: bad83517-5e54-4a3d-9b53-63e85fbd4d7c dockerRepository: airbyte/source-clickhouse - dockerImageTag: 0.1.14 + dockerImageTag: 0.1.15 documentationUrl: https://docs.airbyte.com/integrations/sources/clickhouse icon: clickhouse.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 747575678583..09c48a07338a 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -2185,7 +2185,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-clickhouse:0.1.14" +- dockerImage: "airbyte/source-clickhouse:0.1.15" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/clickhouse" connectionSpecification: From ba380110d4c8a848cabca35354a76f1b5ef274c1 Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Wed, 18 Jan 2023 12:20:03 +0200 Subject: [PATCH 30/34] bump postgres version --- .../connectors/source-postgres-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/source-postgres/Dockerfile | 2 +- docs/integrations/sources/postgres.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile index df697ded6fa6..6e8c951d7a93 100644 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.37 +LABEL io.airbyte.version=1.0.38 LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile index 291e5c2c7763..04a84a8f6750 100644 --- a/airbyte-integrations/connectors/source-postgres/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.37 +LABEL io.airbyte.version=1.0.38 LABEL io.airbyte.name=airbyte/source-postgres diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index ec95b9a356b8..032850cc5cfd 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -411,7 +411,7 @@ The root causes is that the WALs needed for the incremental sync has been remove | Version | Date | Pull Request | Subject | |:--------|:-----------|:----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 1.0.37 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | +| 1.0.38 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | | 1.0.36 | 2023-01-11 | [21003](https://github.com/airbytehq/airbyte/pull/21003) | Handle null values for array data types in CDC mode gracefully. | | 1.0.35 | 2023-01-04 | [20469](https://github.com/airbytehq/airbyte/pull/20469) | Introduce feature to make LSN commit behaviour configurable. | | 1.0.34 | 2022-12-13 | [20378](https://github.com/airbytehq/airbyte/pull/20378) | Improve descriptions | From 9bab7f4ef4ba9fb6041b1c8c74252d62ad4847bd Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Wed, 18 Jan 2023 11:27:35 +0000 Subject: [PATCH 31/34] auto-bump connector version --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index ce91d81e5d36..35a5842d9aa5 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1338,7 +1338,7 @@ - name: Postgres sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 dockerRepository: airbyte/source-postgres - dockerImageTag: 1.0.36 + dockerImageTag: 1.0.38 documentationUrl: https://docs.airbyte.com/integrations/sources/postgres icon: postgresql.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 98dd637cd569..bc5095a66389 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -11384,7 +11384,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-postgres:1.0.36" +- dockerImage: "airbyte/source-postgres:1.0.38" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres" connectionSpecification: From 92443bd336adbc4017dc5e64baaf068dc60f299c Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Wed, 18 Jan 2023 15:08:23 +0200 Subject: [PATCH 32/34] updated seed definitions --- .../main/resources/seed/source_definitions.yaml | 10 +++++----- .../src/main/resources/seed/source_specs.yaml | 16 +++++----------- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 35a5842d9aa5..691137f47c06 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -281,7 +281,7 @@ - name: Cockroachdb sourceDefinitionId: 9fa5862c-da7c-11eb-8d19-0242ac130003 dockerRepository: airbyte/source-cockroachdb - dockerImageTag: 0.1.18 + dockerImageTag: 0.1.19 documentationUrl: https://docs.airbyte.com/integrations/sources/cockroachdb icon: cockroachdb.svg sourceType: database @@ -780,7 +780,7 @@ - name: IBM Db2 sourceDefinitionId: 447e0381-3780-4b46-bb62-00a4e3c8b8e2 dockerRepository: airbyte/source-db2 - dockerImageTag: 0.1.16 + dockerImageTag: 0.1.17 documentationUrl: https://docs.airbyte.com/integrations/sources/db2 icon: db2.svg sourceType: database @@ -1019,7 +1019,7 @@ - name: Microsoft SQL Server (MSSQL) sourceDefinitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1 dockerRepository: airbyte/source-mssql - dockerImageTag: 0.4.26 + dockerImageTag: 0.4.27 documentationUrl: https://docs.airbyte.com/integrations/sources/mssql icon: mssql.svg sourceType: database @@ -1075,7 +1075,7 @@ - name: MySQL sourceDefinitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad dockerRepository: airbyte/source-mysql - dockerImageTag: 1.0.18 + dockerImageTag: 1.0.19 documentationUrl: https://docs.airbyte.com/integrations/sources/mysql icon: mysql.svg sourceType: database @@ -1450,7 +1450,7 @@ - name: Redshift sourceDefinitionId: e87ffa8e-a3b5-f69c-9076-6011339de1f6 dockerRepository: airbyte/source-redshift - dockerImageTag: 0.3.15 + dockerImageTag: 0.3.16 documentationUrl: https://docs.airbyte.com/integrations/sources/redshift icon: redshift.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index bc5095a66389..ff8e5f3333c1 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -2476,7 +2476,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-cockroachdb:0.1.18" +- dockerImage: "airbyte/source-cockroachdb:0.1.19" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/cockroachdb" connectionSpecification: @@ -6428,7 +6428,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-db2:0.1.16" +- dockerImage: "airbyte/source-db2:0.1.17" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/db2" connectionSpecification: @@ -6492,9 +6492,6 @@ encryption_method: type: "string" const: "unencrypted" - enum: - - "unencrypted" - default: "unencrypted" - title: "TLS Encrypted (verify certificate)" description: "Verify and use the cert provided by the server." required: @@ -6504,9 +6501,6 @@ encryption_method: type: "string" const: "encrypted_verify_certificate" - enum: - - "encrypted_verify_certificate" - default: "encrypted_verify_certificate" ssl_certificate: title: "SSL PEM file" description: "Privacy Enhanced Mail (PEM) files are concatenated certificate\ @@ -8021,7 +8015,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-mssql:0.4.26" +- dockerImage: "airbyte/source-mssql:0.4.27" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/mssql" connectionSpecification: @@ -8884,7 +8878,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-mysql:1.0.18" +- dockerImage: "airbyte/source-mysql:1.0.19" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/mysql" connectionSpecification: @@ -12222,7 +12216,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-redshift:0.3.15" +- dockerImage: "airbyte/source-redshift:0.3.16" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/redshift" connectionSpecification: From 24263d8c583ee968a4ad9727cf7fa54f65b27d8c Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Wed, 18 Jan 2023 16:58:29 +0200 Subject: [PATCH 33/34] revert generated definitions for Postgres --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 10ca509b8693..b32a6014728d 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1338,7 +1338,7 @@ - name: Postgres sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 dockerRepository: airbyte/source-postgres - dockerImageTag: 1.0.38 + dockerImageTag: 1.0.37 documentationUrl: https://docs.airbyte.com/integrations/sources/postgres icon: postgresql.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index f411d75623b7..76a2d7322fd5 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -11378,7 +11378,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-postgres:1.0.38" +- dockerImage: "airbyte/source-postgres:1.0.37" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres" connectionSpecification: From 1979b42c2ba5b5d5dfdd2345ce028a375d3da4cd Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Wed, 18 Jan 2023 16:44:31 +0000 Subject: [PATCH 34/34] auto-bump connector version --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index b32a6014728d..10ca509b8693 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1338,7 +1338,7 @@ - name: Postgres sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 dockerRepository: airbyte/source-postgres - dockerImageTag: 1.0.37 + dockerImageTag: 1.0.38 documentationUrl: https://docs.airbyte.com/integrations/sources/postgres icon: postgresql.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 76a2d7322fd5..f411d75623b7 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -11378,7 +11378,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-postgres:1.0.37" +- dockerImage: "airbyte/source-postgres:1.0.38" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres" connectionSpecification: