Source and Destination MSSql: add jdbc_url_params in connection config (

#11010) * Add jbdc url params to the connection configuration * bump connectors version * correct spect test * correct spec test * correct dest spec tet Co-authored-by: marcosmarxm <[email protected]>
airbytehq · Mar 29, 2022 · 26fd91a · 26fd91a
1 parent b42639a
commit 26fd91a
Show file tree

Hide file tree

Showing 16 changed files with 82 additions and 28 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml
@@ -132,7 +132,7 @@
 - name: MS SQL Server
   destinationDefinitionId: d4353156-9217-4cad-8dd7-c108fd4f74cf
   dockerRepository: airbyte/destination-mssql
-  dockerImageTag: 0.1.15
+  dockerImageTag: 0.1.16
   documentationUrl: https://docs.airbyte.io/integrations/destinations/mssql
   icon: mssql.svg
 - name: MeiliSearch

diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml
@@ -2092,7 +2092,7 @@
     supportsDBT: false
     supported_destination_sync_modes:
     - "append"
-- dockerImage: "airbyte/destination-mssql:0.1.15"
+- dockerImage: "airbyte/destination-mssql:0.1.16"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/destinations/mssql"
     connectionSpecification:

diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -453,7 +453,7 @@
 - name: Microsoft SQL Server (MSSQL)
   sourceDefinitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1
   dockerRepository: airbyte/source-mssql
-  dockerImageTag: 0.3.17
+  dockerImageTag: 0.3.18
   documentationUrl: https://docs.airbyte.io/integrations/sources/mssql
   icon: mssql.svg
   sourceType: database

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -4605,7 +4605,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-mssql:0.3.17"
+- dockerImage: "airbyte/source-mssql:0.3.18"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/destinations/mssql"
     connectionSpecification:

diff --git a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION destination-mssql-strict-encrypt
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.1.5
+LABEL io.airbyte.version=0.1.6
 LABEL io.airbyte.name=airbyte/destination-mssql-strict-encrypt
diff --git a/...ectors/destination-mssql-strict-encrypt/src/test-integration/resources/expected_spec.json b/...ectors/destination-mssql-strict-encrypt/src/test-integration/resources/expected_spec.json
@@ -54,11 +54,17 @@
         "airbyte_secret": true,
         "order": 5
       },
+      "jdbc_url_params": {
+        "title": "JDBC URL Params",
+        "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).",
+        "type": "string",
+        "order": 6
+      },
       "ssl_method": {
         "title": "SSL Method",
         "type": "object",
         "description": "The encryption method which is used to communicate with the database.",
-        "order": 6,
+        "order": 7,
         "oneOf": [
           {
             "title": "Encrypted (trust server certificate)",
@@ -69,6 +75,7 @@
             "properties": {
               "ssl_method": {
                 "type": "string",
+                "const": "encrypted_trust_server_certificate",
                 "enum": ["encrypted_trust_server_certificate"],
                 "default": "encrypted_trust_server_certificate"
               }
@@ -83,14 +90,15 @@
             "properties": {
               "ssl_method": {
                 "type": "string",
+                "const": "encrypted_verify_certificate",
                 "enum": ["encrypted_verify_certificate"],
                 "default": "encrypted_verify_certificate"
               },
               "hostNameInCertificate": {
                 "title": "Host Name In Certificate",
                 "type": "string",
                 "description": "Specifies the host name of the server. The value of this property must match the subject property of the certificate.",
-                "order": 7
+                "order": 8
               }
             }
           }

diff --git a/airbyte-integrations/connectors/destination-mssql/Dockerfile b/airbyte-integrations/connectors/destination-mssql/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION destination-mssql
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.1.15
+LABEL io.airbyte.version=0.1.16
 LABEL io.airbyte.name=airbyte/destination-mssql
diff --git a/...ation-mssql/src/main/java/io/airbyte/integrations/destination/mssql/MSSQLDestination.java b/...ation-mssql/src/main/java/io/airbyte/integrations/destination/mssql/MSSQLDestination.java
@@ -24,6 +24,7 @@ public class MSSQLDestination extends AbstractJdbcDestination implements Destina
   private static final Logger LOGGER = LoggerFactory.getLogger(MSSQLDestination.class);
 
   public static final String DRIVER_CLASS = "com.microsoft.sqlserver.jdbc.SQLServerDriver";
+  public static final String JDBC_URL_PARAMS_KEY = "jdbc_url_params";
   public static final List<String> HOST_KEY = List.of("host");
   public static final List<String> PORT_KEY = List.of("port");
 
@@ -73,6 +74,10 @@ public JsonNode toJdbcConfig(final JsonNode config) {
         .put("password", config.get("password").asText())
         .put("schema", schema);
 
+    if (config.has(JDBC_URL_PARAMS_KEY)) {
+      configBuilder.put("connection_properties", config.get(JDBC_URL_PARAMS_KEY));
+    }
+
     return Jsons.jsonNode(configBuilder.build());
   }
 

diff --git a/airbyte-integrations/connectors/destination-mssql/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-mssql/src/main/resources/spec.json
@@ -54,11 +54,17 @@
         "airbyte_secret": true,
         "order": 5
       },
+      "jdbc_url_params": {
+        "title": "JDBC URL Params",
+        "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).",
+        "type": "string",
+        "order": 6
+      },
       "ssl_method": {
         "title": "SSL Method",
         "type": "object",
         "description": "The encryption method which is used to communicate with the database.",
-        "order": 6,
+        "order": 7,
         "oneOf": [
           {
             "title": "Unencrypted",
@@ -69,6 +75,7 @@
             "properties": {
               "ssl_method": {
                 "type": "string",
+                "const": "unencrypted",
                 "enum": ["unencrypted"],
                 "default": "unencrypted"
               }
@@ -83,6 +90,7 @@
             "properties": {
               "ssl_method": {
                 "type": "string",
+                "const": "encrypted_trust_server_certificate",
                 "enum": ["encrypted_trust_server_certificate"],
                 "default": "encrypted_trust_server_certificate"
               }
@@ -97,14 +105,15 @@
             "properties": {
               "ssl_method": {
                 "type": "string",
+                "const": "encrypted_verify_certificate",
                 "enum": ["encrypted_verify_certificate"],
                 "default": "encrypted_verify_certificate"
               },
               "hostNameInCertificate": {
                 "title": "Host Name In Certificate",
                 "type": "string",
                 "description": "Specifies the host name of the server. The value of this property must match the subject property of the certificate.",
-                "order": 7
+                "order": 8
               }
             }
           }

diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_api.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_api.py
@@ -1,6 +1,7 @@
 #
 # Copyright (c) 2021 Airbyte, Inc., all rights reserved.
 #
+
 import pendulum
 import pytest
 import source_facebook_marketing

diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION source-mssql-strict-encrypt
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.1.9
+LABEL io.airbyte.version=0.1.10
 LABEL io.airbyte.name=airbyte/source-mssql-strict-encrypt
diff --git a/...integrations/connectors/source-mssql-strict-encrypt/src/test/resources/expected_spec.json b/...integrations/connectors/source-mssql-strict-encrypt/src/test/resources/expected_spec.json
@@ -10,32 +10,43 @@
       "host": {
         "description": "The hostname of the database.",
         "title": "Host",
-        "type": "string"
+        "type": "string",
+        "order": 0
       },
       "port": {
         "description": "The port of the database.",
         "title": "Port",
         "type": "integer",
         "minimum": 0,
         "maximum": 65536,
-        "examples": ["1433"]
+        "examples": ["1433"],
+        "order": 1
       },
       "database": {
         "description": "The name of the database.",
         "title": "Database",
         "type": "string",
-        "examples": ["master"]
+        "examples": ["master"],
+        "order": 2
       },
       "username": {
         "description": "The username which is used to access the database.",
         "title": "Username",
-        "type": "string"
+        "type": "string",
+        "order": 3
       },
       "password": {
         "description": "The password associated with the username.",
         "title": "Password",
         "type": "string",
-        "airbyte_secret": true
+        "airbyte_secret": true,
+        "order": 4
+      },
+      "jdbc_url_params": {
+        "title": "JDBC URL Params",
+        "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).",
+        "type": "string",
+        "order": 5
       },
       "ssl_method": {
         "title": "SSL Method",
@@ -84,7 +95,8 @@
         "title": "Replication Method",
         "description": "The replication method used for extracting data from the database. STANDARD replication requires no setup on the DB side but will not be able to represent deletions incrementally. CDC uses {TBC} to detect inserts, updates, and deletes. This needs to be configured on the source database itself.",
         "default": "STANDARD",
-        "enum": ["STANDARD", "CDC"]
+        "enum": ["STANDARD", "CDC"],
+        "order": 8
       }
     }
   }

diff --git a/airbyte-integrations/connectors/source-mssql/Dockerfile b/airbyte-integrations/connectors/source-mssql/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION source-mssql
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.3.17
+LABEL io.airbyte.version=0.3.18
 LABEL io.airbyte.name=airbyte/source-mssql
diff --git a/...nnectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java b/...nnectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java
@@ -53,6 +53,7 @@ public class MssqlSource extends AbstractJdbcSource<JDBCType> implements Source
   public static final String MSSQL_CDC_OFFSET = "mssql_cdc_offset";
   public static final String MSSQL_DB_HISTORY = "mssql_db_history";
   public static final String CDC_LSN = "_ab_cdc_lsn";
+  public static final String JDBC_URL_PARAMS_KEY = "jdbc_url_params";
   public static final List<String> HOST_KEY = List.of("host");
   public static final List<String> PORT_KEY = List.of("port");
   private static final String HIERARCHYID = "hierarchyid";
@@ -188,11 +189,16 @@ public JsonNode toDatabaseConfig(final JsonNode mssqlConfig) {
       jdbcUrl.append(String.join(";", additionalParameters));
     }
 
-    return Jsons.jsonNode(ImmutableMap.builder()
+    final ImmutableMap.Builder<Object, Object> configBuilder = ImmutableMap.builder()
         .put("username", mssqlConfig.get("username").asText())
         .put("password", mssqlConfig.get("password").asText())
-        .put("jdbc_url", jdbcUrl.toString())
-        .build());
+        .put("jdbc_url", jdbcUrl.toString());
+
+    if (mssqlConfig.has(JDBC_URL_PARAMS_KEY)) {
+      configBuilder.put("connection_properties", mssqlConfig.get(JDBC_URL_PARAMS_KEY));
+    }
+
+    return Jsons.jsonNode(configBuilder.build());
   }
 
   @Override

diff --git a/airbyte-integrations/connectors/source-mssql/src/main/resources/spec.json b/airbyte-integrations/connectors/source-mssql/src/main/resources/spec.json
@@ -10,32 +10,43 @@
       "host": {
         "description": "The hostname of the database.",
         "title": "Host",
-        "type": "string"
+        "type": "string",
+        "order": 0
       },
       "port": {
         "description": "The port of the database.",
         "title": "Port",
         "type": "integer",
         "minimum": 0,
         "maximum": 65536,
-        "examples": ["1433"]
+        "examples": ["1433"],
+        "order": 1
       },
       "database": {
         "description": "The name of the database.",
         "title": "Database",
         "type": "string",
-        "examples": ["master"]
+        "examples": ["master"],
+        "order": 2
       },
       "username": {
         "description": "The username which is used to access the database.",
         "title": "Username",
-        "type": "string"
+        "type": "string",
+        "order": 3
       },
       "password": {
         "description": "The password associated with the username.",
         "title": "Password",
         "type": "string",
-        "airbyte_secret": true
+        "airbyte_secret": true,
+        "order": 4
+      },
+      "jdbc_url_params": {
+        "title": "JDBC URL Params",
+        "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).",
+        "type": "string",
+        "order": 5
       },
       "ssl_method": {
         "title": "SSL Method",
@@ -98,7 +109,8 @@
         "title": "Replication Method",
         "description": "The replication method used for extracting data from the database. STANDARD replication requires no setup on the DB side but will not be able to represent deletions incrementally. CDC uses {TBC} to detect inserts, updates, and deletes. This needs to be configured on the source database itself.",
         "default": "STANDARD",
-        "enum": ["STANDARD", "CDC"]
+        "enum": ["STANDARD", "CDC"],
+        "order": 8
       }
     }
   }

diff --git a/...s/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceTest.java b/...s/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceTest.java
@@ -21,6 +21,7 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
 import io.airbyte.commons.json.Jsons;
 import io.airbyte.commons.string.Strings;
 import io.airbyte.db.Database;
@@ -316,7 +317,7 @@ protected CdcTargetPosition cdcLatestTargetPosition() {
             dbName),
         DRIVER_CLASS,
         new MssqlJdbcStreamingQueryConfiguration(),
-        Map.of(),
+        Maps.newHashMap(),
         new MssqlSourceOperations());
     return MssqlCdcTargetPosition.getTargetPosition(jdbcDatabase, dbName);
   }