Source ClickHouse: add custom jdbc params (airbytehq#17031)

* Source clickhouse: add jdbc params * Source clickhouse: add jdbc params * Source clickhouse: add jdbc params * Source clickhouse: updated tests * Source clickhouse: format * Source clickhouse: format * Source clickhouse: format * Source clickhouse: format * Source clickhouse: bump version * auto-bump connector version [ci skip] Co-authored-by: Octavia Squidington III <[email protected]>
robbinhan · Sep 29, 2022 · 262de42 · 262de42
1 parent c6cbd4a
commit 262de42
Show file tree

Hide file tree

Showing 9 changed files with 134 additions and 16 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -176,7 +176,7 @@
 - name: ClickHouse
   sourceDefinitionId: bad83517-5e54-4a3d-9b53-63e85fbd4d7c
   dockerRepository: airbyte/source-clickhouse
-  dockerImageTag: 0.1.13
+  dockerImageTag: 0.1.14
   documentationUrl: https://docs.airbyte.io/integrations/sources/clickhouse
   icon: cliskhouse.svg
   sourceType: database

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -1823,7 +1823,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-clickhouse:0.1.13"
+- dockerImage: "airbyte/source-clickhouse:0.1.14"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/destinations/clickhouse"
     connectionSpecification:
@@ -1840,6 +1840,7 @@
           description: "The host endpoint of the Clickhouse cluster."
           title: "Host"
           type: "string"
+          order: 0
         port:
           description: "The port of the database."
           title: "Port"
@@ -1849,26 +1850,40 @@
           default: 8123
           examples:
           - "8123"
+          order: 1
         database:
           description: "The name of the database."
           title: "Database"
           type: "string"
           examples:
           - "default"
+          order: 2
         username:
           description: "The username which is used to access the database."
           title: "Username"
           type: "string"
+          order: 3
         password:
           description: "The password associated with this username."
           title: "Password"
           type: "string"
           airbyte_secret: true
+          order: 4
+        jdbc_url_params:
+          description: "Additional properties to pass to the JDBC URL string when\
+            \ connecting to the database formatted as 'key=value' pairs separated\
+            \ by the symbol '&'. (Eg. key1=value1&key2=value2&key3=value3). For more\
+            \ information read about <a href=\"https://jdbc.postgresql.org/documentation/head/connect.html\"\
+            >JDBC URL parameters</a>."
+          title: "JDBC URL Parameters (Advanced)"
+          type: "string"
+          order: 5
         ssl:
           title: "SSL Connection"
           description: "Encrypt data using SSL."
           type: "boolean"
           default: true
+          order: 6
         tunnel_method:
           type: "object"
           title: "SSH Tunnel Method"

diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION source-clickhouse-strict-encrypt
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.1.13
+LABEL io.airbyte.version=0.1.14
 LABEL io.airbyte.name=airbyte/source-clickhouse-strict-encrypt
diff --git a/...ectors/source-clickhouse-strict-encrypt/src/test-integration/resources/expected_spec.json b/...ectors/source-clickhouse-strict-encrypt/src/test-integration/resources/expected_spec.json
@@ -9,7 +9,8 @@
       "host": {
         "description": "The host endpoint of the Clickhouse cluster.",
         "title": "Host",
-        "type": "string"
+        "type": "string",
+        "order": 0
       },
       "port": {
         "description": "The port of the database.",
@@ -18,24 +19,34 @@
         "minimum": 0,
         "maximum": 65536,
         "default": 8123,
-        "examples": ["8123"]
+        "examples": ["8123"],
+        "order": 1
       },
       "database": {
         "description": "The name of the database.",
         "title": "Database",
         "type": "string",
-        "examples": ["default"]
+        "examples": ["default"],
+        "order": 2
       },
       "username": {
         "description": "The username which is used to access the database.",
         "title": "Username",
-        "type": "string"
+        "type": "string",
+        "order": 3
       },
       "password": {
         "description": "The password associated with this username.",
         "title": "Password",
         "type": "string",
-        "airbyte_secret": true
+        "airbyte_secret": true,
+        "order": 4
+      },
+      "jdbc_url_params": {
+        "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (Eg. key1=value1&key2=value2&key3=value3). For more information read about <a href=\"https://jdbc.postgresql.org/documentation/head/connect.html\">JDBC URL parameters</a>.",
+        "title": "JDBC URL Parameters (Advanced)",
+        "type": "string",
+        "order": 5
       }
     }
   }

diff --git a/airbyte-integrations/connectors/source-clickhouse/Dockerfile b/airbyte-integrations/connectors/source-clickhouse/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION source-clickhouse
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.1.13
+LABEL io.airbyte.version=0.1.14
 LABEL io.airbyte.name=airbyte/source-clickhouse
diff --git a/...-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java b/...-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java
@@ -20,6 +20,7 @@
 import java.sql.JDBCType;
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -90,9 +91,20 @@ public JsonNode toDatabaseConfig(final JsonNode config) {
         config.get(JdbcUtils.PORT_KEY).asText(),
         config.get(JdbcUtils.DATABASE_KEY).asText()));
 
+    boolean isAdditionalParamsExists =
+        config.get(JdbcUtils.JDBC_URL_PARAMS_KEY) != null && !config.get(JdbcUtils.JDBC_URL_PARAMS_KEY).asText().isEmpty();
+    List<String> params = new ArrayList<>();
     // assume ssl if not explicitly mentioned.
     if (isSsl) {
-      jdbcUrl.append("?").append(SSL_MODE);
+      params.add(SSL_MODE);
+    }
+    if (isAdditionalParamsExists) {
+      params.add(config.get(JdbcUtils.JDBC_URL_PARAMS_KEY).asText());
+    }
+
+    if (isSsl || isAdditionalParamsExists) {
+      jdbcUrl.append("?");
+      jdbcUrl.append(String.join("&", params));
     }
 
     final ImmutableMap.Builder<Object, Object> configBuilder = ImmutableMap.builder()

diff --git a/airbyte-integrations/connectors/source-clickhouse/src/main/resources/spec.json b/airbyte-integrations/connectors/source-clickhouse/src/main/resources/spec.json
@@ -9,7 +9,8 @@
       "host": {
         "description": "The host endpoint of the Clickhouse cluster.",
         "title": "Host",
-        "type": "string"
+        "type": "string",
+        "order": 0
       },
       "port": {
         "description": "The port of the database.",
@@ -18,30 +19,41 @@
         "minimum": 0,
         "maximum": 65536,
         "default": 8123,
-        "examples": ["8123"]
+        "examples": ["8123"],
+        "order": 1
       },
       "database": {
         "description": "The name of the database.",
         "title": "Database",
         "type": "string",
-        "examples": ["default"]
+        "examples": ["default"],
+        "order": 2
       },
       "username": {
         "description": "The username which is used to access the database.",
         "title": "Username",
-        "type": "string"
+        "type": "string",
+        "order": 3
       },
       "password": {
         "description": "The password associated with this username.",
         "title": "Password",
         "type": "string",
-        "airbyte_secret": true
+        "airbyte_secret": true,
+        "order": 4
+      },
+      "jdbc_url_params": {
+        "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (Eg. key1=value1&key2=value2&key3=value3). For more information read about <a href=\"https://jdbc.postgresql.org/documentation/head/connect.html\">JDBC URL parameters</a>.",
+        "title": "JDBC URL Parameters (Advanced)",
+        "type": "string",
+        "order": 5
       },
       "ssl": {
         "title": "SSL Connection",
         "description": "Encrypt data using SSL.",
         "type": "boolean",
-        "default": true
+        "default": true,
+        "order": 6
       }
     }
   }

diff --git a/...integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java b/...integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java
@@ -4,7 +4,11 @@
 
 package io.airbyte.integrations.io.airbyte.integration_tests.sources;
 
+import static io.airbyte.db.jdbc.JdbcUtils.JDBC_URL_KEY;
+import static io.airbyte.integrations.source.clickhouse.ClickHouseSource.SSL_MODE;
 import static java.time.temporal.ChronoUnit.SECONDS;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.collect.ImmutableMap;
@@ -21,6 +25,7 @@
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
 import org.testcontainers.containers.ClickHouseContainer;
 import org.testcontainers.containers.wait.strategy.Wait;
 
@@ -114,4 +119,65 @@ public AbstractJdbcSource<JDBCType> getJdbcSource() {
     return new ClickHouseSource();
   }
 
+  @Test
+  public void testEmptyExtraParamsWithSsl() {
+    final String extraParam = "";
+    JsonNode config = buildConfigWithExtraJdbcParameters(extraParam, true);
+    final JsonNode jdbcConfig = new ClickHouseSource().toDatabaseConfig(config);
+    JsonNode jdbcUrlNode = jdbcConfig.get(JDBC_URL_KEY);
+    assertNotNull(jdbcUrlNode);
+    String actualJdbcUrl = jdbcUrlNode.asText();
+    assertTrue(actualJdbcUrl.endsWith("?" + SSL_MODE));
+  }
+
+  @Test
+  public void testEmptyExtraParamsWithoutSsl() {
+    final String extraParam = "";
+    JsonNode config = buildConfigWithExtraJdbcParameters(extraParam, false);
+    final JsonNode jdbcConfig = new ClickHouseSource().toDatabaseConfig(config);
+    JsonNode jdbcUrlNode = jdbcConfig.get(JDBC_URL_KEY);
+    assertNotNull(jdbcUrlNode);
+    String actualJdbcUrl = jdbcUrlNode.asText();
+    assertTrue(actualJdbcUrl.endsWith(config.get("database").asText()));
+  }
+
+  @Test
+  public void testExtraParamsWithSsl() {
+    final String extraParam = "key1=value1&key2=value2&key3=value3";
+    JsonNode config = buildConfigWithExtraJdbcParameters(extraParam, true);
+    final JsonNode jdbcConfig = new ClickHouseSource().toDatabaseConfig(config);
+    JsonNode jdbcUrlNode = jdbcConfig.get(JDBC_URL_KEY);
+    assertNotNull(jdbcUrlNode);
+    String actualJdbcUrl = jdbcUrlNode.asText();
+    assertTrue(actualJdbcUrl.endsWith(getFullExpectedValue(extraParam, SSL_MODE)));
+  }
+
+  @Test
+  public void testExtraParamsWithoutSsl() {
+    final String extraParam = "key1=value1&key2=value2&key3=value3";
+    JsonNode config = buildConfigWithExtraJdbcParameters(extraParam, false);
+    final JsonNode jdbcConfig = new ClickHouseSource().toDatabaseConfig(config);
+    JsonNode jdbcUrlNode = jdbcConfig.get(JDBC_URL_KEY);
+    assertNotNull(jdbcUrlNode);
+    String actualJdbcUrl = jdbcUrlNode.asText();
+    assertTrue(actualJdbcUrl.endsWith("?" + extraParam));
+  }
+
+  private String getFullExpectedValue(String extraParam, String sslMode) {
+    StringBuilder expected = new StringBuilder();
+    return expected.append("?").append(sslMode).append("&").append(extraParam).toString();
+  }
+
+  private JsonNode buildConfigWithExtraJdbcParameters(String extraParam, boolean isSsl) {
+
+    return Jsons.jsonNode(com.google.common.collect.ImmutableMap.of(
+        "host", "localhost",
+        "port", 8123,
+        "database", "db",
+        "username", "username",
+        "password", "verysecure",
+        "jdbc_url_params", extraParam,
+        "ssl", isSsl));
+  }
+
 }
diff --git a/docs/integrations/sources/clickhouse.md b/docs/integrations/sources/clickhouse.md
@@ -78,6 +78,7 @@ Using this feature requires additional configuration, when creating the source.
 
 | Version | Date | Pull Request                                             | Subject                                                          |
 |:--------| :--- |:---------------------------------------------------------|:-----------------------------------------------------------------|
+| 0.1.14  | 2022-09-27 | [17031](https://github.com/airbytehq/airbyte/pull/17031) | Added custom jdbc url parameters field  |
 | 0.1.13  | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently |
 | 0.1.12  | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field |
 | 0.1.10  | 2022-04-12 | [11729](https://github.com/airbytehq/airbyte/pull/11514) | Bump mina-sshd from 2.7.0 to 2.8.0                         |
@@ -95,6 +96,7 @@ Using this feature requires additional configuration, when creating the source.
 
 | Version | Date | Pull Request                                             | Subject                                                                    |
 |:---| :---  |:---------------------------------------------------------|:---------------------------------------------------------------------------|
+| 0.1.14  | 2022-09-27 | [17031](https://github.com/airbytehq/airbyte/pull/17031) | Added custom jdbc url parameters field  |
 | 0.1.13  | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently |
 | 0.1.9 | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field |
 | 0.1.6 | 2022-02-09 | [\#10214](https://github.com/airbytehq/airbyte/pull/10214) | Fix exception in case `password` field is not provided  |