diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index c10c6f7e08da..079e4cf0b701 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -176,7 +176,7 @@ - name: ClickHouse sourceDefinitionId: bad83517-5e54-4a3d-9b53-63e85fbd4d7c dockerRepository: airbyte/source-clickhouse - dockerImageTag: 0.1.13 + dockerImageTag: 0.1.14 documentationUrl: https://docs.airbyte.io/integrations/sources/clickhouse icon: cliskhouse.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 6e972478498d..50783a94d365 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -1823,7 +1823,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-clickhouse:0.1.13" +- dockerImage: "airbyte/source-clickhouse:0.1.14" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/clickhouse" connectionSpecification: @@ -1840,6 +1840,7 @@ description: "The host endpoint of the Clickhouse cluster." title: "Host" type: "string" + order: 0 port: description: "The port of the database." title: "Port" @@ -1849,26 +1850,40 @@ default: 8123 examples: - "8123" + order: 1 database: description: "The name of the database." title: "Database" type: "string" examples: - "default" + order: 2 username: description: "The username which is used to access the database." title: "Username" type: "string" + order: 3 password: description: "The password associated with this username." title: "Password" type: "string" airbyte_secret: true + order: 4 + jdbc_url_params: + description: "Additional properties to pass to the JDBC URL string when\ + \ connecting to the database formatted as 'key=value' pairs separated\ + \ by the symbol '&'. (Eg. key1=value1&key2=value2&key3=value3). For more\ + \ information read about JDBC URL parameters." + title: "JDBC URL Parameters (Advanced)" + type: "string" + order: 5 ssl: title: "SSL Connection" description: "Encrypt data using SSL." type: "boolean" default: true + order: 6 tunnel_method: type: "object" title: "SSH Tunnel Method" diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile index c22f2e3d2902..582507952ef2 100644 --- a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-clickhouse-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.13 +LABEL io.airbyte.version=0.1.14 LABEL io.airbyte.name=airbyte/source-clickhouse-strict-encrypt diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/resources/expected_spec.json b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/resources/expected_spec.json index 985e302f8b27..b6000d108e35 100644 --- a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/resources/expected_spec.json @@ -9,7 +9,8 @@ "host": { "description": "The host endpoint of the Clickhouse cluster.", "title": "Host", - "type": "string" + "type": "string", + "order": 0 }, "port": { "description": "The port of the database.", @@ -18,24 +19,34 @@ "minimum": 0, "maximum": 65536, "default": 8123, - "examples": ["8123"] + "examples": ["8123"], + "order": 1 }, "database": { "description": "The name of the database.", "title": "Database", "type": "string", - "examples": ["default"] + "examples": ["default"], + "order": 2 }, "username": { "description": "The username which is used to access the database.", "title": "Username", - "type": "string" + "type": "string", + "order": 3 }, "password": { "description": "The password associated with this username.", "title": "Password", "type": "string", - "airbyte_secret": true + "airbyte_secret": true, + "order": 4 + }, + "jdbc_url_params": { + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (Eg. key1=value1&key2=value2&key3=value3). For more information read about JDBC URL parameters.", + "title": "JDBC URL Parameters (Advanced)", + "type": "string", + "order": 5 } } } diff --git a/airbyte-integrations/connectors/source-clickhouse/Dockerfile b/airbyte-integrations/connectors/source-clickhouse/Dockerfile index b41fb06cb776..6681cc7af3cb 100644 --- a/airbyte-integrations/connectors/source-clickhouse/Dockerfile +++ b/airbyte-integrations/connectors/source-clickhouse/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-clickhouse COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.13 +LABEL io.airbyte.version=0.1.14 LABEL io.airbyte.name=airbyte/source-clickhouse diff --git a/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java b/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java index f68824d2c680..31b1cfabec04 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java @@ -20,6 +20,7 @@ import java.sql.JDBCType; import java.sql.PreparedStatement; import java.sql.SQLException; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; @@ -90,9 +91,20 @@ public JsonNode toDatabaseConfig(final JsonNode config) { config.get(JdbcUtils.PORT_KEY).asText(), config.get(JdbcUtils.DATABASE_KEY).asText())); + boolean isAdditionalParamsExists = + config.get(JdbcUtils.JDBC_URL_PARAMS_KEY) != null && !config.get(JdbcUtils.JDBC_URL_PARAMS_KEY).asText().isEmpty(); + List params = new ArrayList<>(); // assume ssl if not explicitly mentioned. if (isSsl) { - jdbcUrl.append("?").append(SSL_MODE); + params.add(SSL_MODE); + } + if (isAdditionalParamsExists) { + params.add(config.get(JdbcUtils.JDBC_URL_PARAMS_KEY).asText()); + } + + if (isSsl || isAdditionalParamsExists) { + jdbcUrl.append("?"); + jdbcUrl.append(String.join("&", params)); } final ImmutableMap.Builder configBuilder = ImmutableMap.builder() diff --git a/airbyte-integrations/connectors/source-clickhouse/src/main/resources/spec.json b/airbyte-integrations/connectors/source-clickhouse/src/main/resources/spec.json index 7b28b58ff359..caf9f40395ed 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-clickhouse/src/main/resources/spec.json @@ -9,7 +9,8 @@ "host": { "description": "The host endpoint of the Clickhouse cluster.", "title": "Host", - "type": "string" + "type": "string", + "order": 0 }, "port": { "description": "The port of the database.", @@ -18,30 +19,41 @@ "minimum": 0, "maximum": 65536, "default": 8123, - "examples": ["8123"] + "examples": ["8123"], + "order": 1 }, "database": { "description": "The name of the database.", "title": "Database", "type": "string", - "examples": ["default"] + "examples": ["default"], + "order": 2 }, "username": { "description": "The username which is used to access the database.", "title": "Username", - "type": "string" + "type": "string", + "order": 3 }, "password": { "description": "The password associated with this username.", "title": "Password", "type": "string", - "airbyte_secret": true + "airbyte_secret": true, + "order": 4 + }, + "jdbc_url_params": { + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (Eg. key1=value1&key2=value2&key3=value3). For more information read about JDBC URL parameters.", + "title": "JDBC URL Parameters (Advanced)", + "type": "string", + "order": 5 }, "ssl": { "title": "SSL Connection", "description": "Encrypt data using SSL.", "type": "boolean", - "default": true + "default": true, + "order": 6 } } } diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java index 8aa75368303c..5a66d267fb11 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java @@ -4,7 +4,11 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; +import static io.airbyte.db.jdbc.JdbcUtils.JDBC_URL_KEY; +import static io.airbyte.integrations.source.clickhouse.ClickHouseSource.SSL_MODE; import static java.time.temporal.ChronoUnit.SECONDS; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNotNull; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; @@ -21,6 +25,7 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.testcontainers.containers.ClickHouseContainer; import org.testcontainers.containers.wait.strategy.Wait; @@ -114,4 +119,65 @@ public AbstractJdbcSource getJdbcSource() { return new ClickHouseSource(); } + @Test + public void testEmptyExtraParamsWithSsl() { + final String extraParam = ""; + JsonNode config = buildConfigWithExtraJdbcParameters(extraParam, true); + final JsonNode jdbcConfig = new ClickHouseSource().toDatabaseConfig(config); + JsonNode jdbcUrlNode = jdbcConfig.get(JDBC_URL_KEY); + assertNotNull(jdbcUrlNode); + String actualJdbcUrl = jdbcUrlNode.asText(); + assertTrue(actualJdbcUrl.endsWith("?" + SSL_MODE)); + } + + @Test + public void testEmptyExtraParamsWithoutSsl() { + final String extraParam = ""; + JsonNode config = buildConfigWithExtraJdbcParameters(extraParam, false); + final JsonNode jdbcConfig = new ClickHouseSource().toDatabaseConfig(config); + JsonNode jdbcUrlNode = jdbcConfig.get(JDBC_URL_KEY); + assertNotNull(jdbcUrlNode); + String actualJdbcUrl = jdbcUrlNode.asText(); + assertTrue(actualJdbcUrl.endsWith(config.get("database").asText())); + } + + @Test + public void testExtraParamsWithSsl() { + final String extraParam = "key1=value1&key2=value2&key3=value3"; + JsonNode config = buildConfigWithExtraJdbcParameters(extraParam, true); + final JsonNode jdbcConfig = new ClickHouseSource().toDatabaseConfig(config); + JsonNode jdbcUrlNode = jdbcConfig.get(JDBC_URL_KEY); + assertNotNull(jdbcUrlNode); + String actualJdbcUrl = jdbcUrlNode.asText(); + assertTrue(actualJdbcUrl.endsWith(getFullExpectedValue(extraParam, SSL_MODE))); + } + + @Test + public void testExtraParamsWithoutSsl() { + final String extraParam = "key1=value1&key2=value2&key3=value3"; + JsonNode config = buildConfigWithExtraJdbcParameters(extraParam, false); + final JsonNode jdbcConfig = new ClickHouseSource().toDatabaseConfig(config); + JsonNode jdbcUrlNode = jdbcConfig.get(JDBC_URL_KEY); + assertNotNull(jdbcUrlNode); + String actualJdbcUrl = jdbcUrlNode.asText(); + assertTrue(actualJdbcUrl.endsWith("?" + extraParam)); + } + + private String getFullExpectedValue(String extraParam, String sslMode) { + StringBuilder expected = new StringBuilder(); + return expected.append("?").append(sslMode).append("&").append(extraParam).toString(); + } + + private JsonNode buildConfigWithExtraJdbcParameters(String extraParam, boolean isSsl) { + + return Jsons.jsonNode(com.google.common.collect.ImmutableMap.of( + "host", "localhost", + "port", 8123, + "database", "db", + "username", "username", + "password", "verysecure", + "jdbc_url_params", extraParam, + "ssl", isSsl)); + } + } diff --git a/docs/integrations/sources/clickhouse.md b/docs/integrations/sources/clickhouse.md index 113132059b69..b9fa131fc2d5 100644 --- a/docs/integrations/sources/clickhouse.md +++ b/docs/integrations/sources/clickhouse.md @@ -78,6 +78,7 @@ Using this feature requires additional configuration, when creating the source. | Version | Date | Pull Request | Subject | |:--------| :--- |:---------------------------------------------------------|:-----------------------------------------------------------------| +| 0.1.14 | 2022-09-27 | [17031](https://github.com/airbytehq/airbyte/pull/17031) | Added custom jdbc url parameters field | | 0.1.13 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently | | 0.1.12 | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field | | 0.1.10 | 2022-04-12 | [11729](https://github.com/airbytehq/airbyte/pull/11514) | Bump mina-sshd from 2.7.0 to 2.8.0 | @@ -95,6 +96,7 @@ Using this feature requires additional configuration, when creating the source. | Version | Date | Pull Request | Subject | |:---| :--- |:---------------------------------------------------------|:---------------------------------------------------------------------------| +| 0.1.14 | 2022-09-27 | [17031](https://github.com/airbytehq/airbyte/pull/17031) | Added custom jdbc url parameters field | | 0.1.13 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently | | 0.1.9 | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field | | 0.1.6 | 2022-02-09 | [\#10214](https://github.com/airbytehq/airbyte/pull/10214) | Fix exception in case `password` field is not provided |