diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index f4f283b7565b..93af0f4ac89a 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -17,7 +17,7 @@ - name: AlloyDB for PostgreSQL sourceDefinitionId: 1fa90628-2b9e-11ed-a261-0242ac120002 dockerRepository: airbyte/source-alloydb - dockerImageTag: 1.0.15 + dockerImageTag: 1.0.16 documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb icon: alloydb.svg sourceType: database @@ -864,7 +864,7 @@ - name: Postgres sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 dockerRepository: airbyte/source-postgres - dockerImageTag: 1.0.20 + dockerImageTag: 1.0.21 documentationUrl: https://docs.airbyte.com/integrations/sources/postgres icon: postgresql.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index dd388830bbca..e42c740451bf 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -198,7 +198,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-alloydb:1.0.15" +- dockerImage: "airbyte/source-alloydb:1.0.16" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres" connectionSpecification: @@ -8792,7 +8792,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-postgres:1.0.20" +- dockerImage: "airbyte/source-postgres:1.0.21" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-alloydb-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-alloydb-strict-encrypt/Dockerfile index 8410e032a8cd..4eae2bdf2c0a 100644 --- a/airbyte-integrations/connectors/source-alloydb-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-alloydb-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-alloydb-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.15 +LABEL io.airbyte.version=1.0.16 LABEL io.airbyte.name=airbyte/source-alloydb-strict-encrypt diff --git a/airbyte-integrations/connectors/source-alloydb/Dockerfile b/airbyte-integrations/connectors/source-alloydb/Dockerfile index 284a1d21d21f..7bffadf916b1 100644 --- a/airbyte-integrations/connectors/source-alloydb/Dockerfile +++ b/airbyte-integrations/connectors/source-alloydb/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-alloydb COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.15 +LABEL io.airbyte.version=1.0.16 LABEL io.airbyte.name=airbyte/source-alloydb diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile index 41f77a3f95ca..4730a7787898 100644 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.20 +LABEL io.airbyte.version=1.0.21 LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile index 04c905dd55c4..fe732c06f6c5 100644 --- a/airbyte-integrations/connectors/source-postgres/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.20 +LABEL io.airbyte.version=1.0.21 LABEL io.airbyte.name=airbyte/source-postgres diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java index 0facfaadf8f8..ddabe5f727b7 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java @@ -16,6 +16,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.features.FeatureFlags; import io.airbyte.commons.functional.CheckedConsumer; @@ -78,6 +79,7 @@ public class PostgresSource extends AbstractJdbcSource implements Sour private static final int INTERMEDIATE_STATE_EMISSION_FREQUENCY = 10_000; public static final String PARAM_SSLMODE = "sslmode"; + public static final String SSL_MODE = "ssl_mode"; public static final String PARAM_SSL = "ssl"; public static final String PARAM_SSL_TRUE = "true"; public static final String PARAM_SSL_FALSE = "false"; @@ -87,11 +89,13 @@ public class PostgresSource extends AbstractJdbcSource implements Sour public static final String CA_CERTIFICATE_PATH = "ca_certificate_path"; public static final String SSL_KEY = "sslkey"; public static final String SSL_PASSWORD = "sslpassword"; + public static final String MODE = "mode"; static final Map SSL_JDBC_PARAMETERS = ImmutableMap.of( "ssl", "true", "sslmode", "require"); private List schemas; private final FeatureFlags featureFlags; + private static final Set INVALID_CDC_SSL_MODES = ImmutableSet.of("allow", "prefer"); public static Source sshWrappedSource() { return new SshWrappedSource(new PostgresSource(), JdbcUtils.HOST_LIST_KEY, JdbcUtils.PORT_LIST_KEY); @@ -464,6 +468,23 @@ public static void main(final String[] args) throws Exception { LOGGER.info("completed source: {}", PostgresSource.class); } + @Override + public AirbyteConnectionStatus check(final JsonNode config) throws Exception { + if (PostgresUtils.isCdc(config)) { + if (config.has(SSL_MODE) && config.get(SSL_MODE).has(MODE)){ + String sslModeValue = config.get(SSL_MODE).get(MODE).asText(); + if (INVALID_CDC_SSL_MODES.contains(sslModeValue)) { + return new AirbyteConnectionStatus() + .withStatus(Status.FAILED) + .withMessage(String.format( + "In CDC replication mode ssl value '%s' is invalid. Please use one of the following SSL modes: disable, require, verify-ca, verify-full", + sslModeValue)); + } + } + } + return super.check(config); + } + @Override protected String toSslJdbcParam(final SslMode sslMode) { return toSslJdbcParamInternal(sslMode); diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceSSLTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceSSLTest.java index 2b4cdce416a8..b4b7aec93524 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceSSLTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceSSLTest.java @@ -25,6 +25,7 @@ import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.protocol.models.AirbyteCatalog; +import io.airbyte.protocol.models.AirbyteConnectionStatus; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteStream; import io.airbyte.protocol.models.CatalogHelpers; @@ -35,6 +36,7 @@ import io.airbyte.test.utils.PostgreSQLContainerHelper; import java.math.BigDecimal; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -197,4 +199,34 @@ void testIsCdc() { assertTrue(PostgresUtils.isCdc(config)); } + @Test + void testAllowSSLWithCdcReplicationMethod() throws Exception { + + JsonNode config = getCDCAndSslModeConfig("allow"); + + final AirbyteConnectionStatus actual = new PostgresSource().check(config); + assertEquals(AirbyteConnectionStatus.Status.FAILED, actual.getStatus()); + assertTrue(actual.getMessage().contains("In CDC replication mode ssl value 'allow' is invalid")); + } + + @Test + void testPreferSSLWithCdcReplicationMethod() throws Exception { + + JsonNode config = getCDCAndSslModeConfig("prefer"); + + final AirbyteConnectionStatus actual = new PostgresSource().check(config); + assertEquals(AirbyteConnectionStatus.Status.FAILED, actual.getStatus()); + assertTrue(actual.getMessage().contains("In CDC replication mode ssl value 'prefer' is invalid")); + } + + private JsonNode getCDCAndSslModeConfig(String sslMode) { + return Jsons.jsonNode(ImmutableMap.builder() + .put(JdbcUtils.SSL_KEY, true) + .put(JdbcUtils.SSL_MODE_KEY, Map.of(JdbcUtils.MODE_KEY, sslMode)) + .put("replication_method", Map.of("method", "CDC", + "replication_slot", "slot", + "publication", "ab_pub")) + .build()); + } + } diff --git a/docs/integrations/sources/alloydb.md b/docs/integrations/sources/alloydb.md index fce3378b4c40..3b50d9ae8bf1 100644 --- a/docs/integrations/sources/alloydb.md +++ b/docs/integrations/sources/alloydb.md @@ -331,6 +331,7 @@ According to Postgres [documentation](https://www.postgresql.org/docs/14/datatyp | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------| +| 1.0.16 | 2022-10-25 | [18256](https://github.com/airbytehq/airbyte/pull/18256) | Disable allow and prefer ssl modes in CDC mode | | | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode | | 1.0.15 | 2022-10-11 | [17782](https://github.com/airbytehq/airbyte/pull/17782) | Align with Postgres source v.1.0.15 | | 1.0.0 | 2022-09-15 | [16776](https://github.com/airbytehq/airbyte/pull/16776) | Align with strict-encrypt version | diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index a0fb80c91a52..0a603016906f 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -398,6 +398,7 @@ The root causes is that the WALs needed for the incremental sync has been remove | Version | Date | Pull Request | Subject | |:--------|:-----------|:-------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 1.0.21 | 2022-10-25 | [18256](https://github.com/airbytehq/airbyte/pull/18256) | Disable allow and prefer ssl modes in CDC mode | | 1.0.20 | 2022-10-25 | [18383](https://github.com/airbytehq/airbyte/pull/18383) | Better SSH error handling + messages | | 1.0.19 | 2022-10-21 | [18263](https://github.com/airbytehq/airbyte/pull/18263) | Fixes bug introduced in [15833](https://github.com/airbytehq/airbyte/pull/15833) and adds better error messaging for SSH tunnel in Destinations | | 1.0.18 | 2022-10-19 | [18087](https://github.com/airbytehq/airbyte/pull/18087) | Better error messaging for configuration errors (SSH configs, choosing an invalid cursor) |