From 5ee1b93c8908265a06c225a2d3b83a14c77cc30b Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Fri, 1 Oct 2021 16:19:25 +0300 Subject: [PATCH 01/10] OracleSource add encryption --- .../connectors/source-oracle/Dockerfile | 2 +- .../source/oracle/OracleSource.java | 84 ++++++++++++++-- .../src/main/resources/spec.json | 97 ++++++++++++++++++- 3 files changed, 169 insertions(+), 14 deletions(-) diff --git a/airbyte-integrations/connectors/source-oracle/Dockerfile b/airbyte-integrations/connectors/source-oracle/Dockerfile index c99d8f7af489..3066b0d4c483 100644 --- a/airbyte-integrations/connectors/source-oracle/Dockerfile +++ b/airbyte-integrations/connectors/source-oracle/Dockerfile @@ -6,7 +6,7 @@ ENV APPLICATION source-oracle ENV TZ UTC COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - +#ENV JAVA_TOOL_OPTIONS -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005 RUN tar xf ${APPLICATION}.tar --strip-components=1 LABEL io.airbyte.version=0.3.5 diff --git a/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java b/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java index 81d982545688..32867e7a520e 100644 --- a/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java +++ b/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java @@ -15,11 +15,13 @@ import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.relationaldb.TableInfo; import io.airbyte.protocol.models.CommonField; + +import java.io.*; import java.sql.JDBCType; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; -import java.util.Set; +import java.util.*; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.lang3.RandomStringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,18 +33,33 @@ public class OracleSource extends AbstractJdbcSource implements Source { private List schemas; + private static final String KEY_STORE_FILE_PATH = "clientkeystore.jks"; + private static final String KEY_STORE_PASS = RandomStringUtils.randomAlphanumeric(8); + + enum Protocol{ + TCP,TCPS + } + public OracleSource() { super(DRIVER_CLASS, new OracleJdbcStreamingQueryConfiguration()); } @Override public JsonNode toDatabaseConfig(JsonNode config) { - final ImmutableMap.Builder configBuilder = ImmutableMap.builder() - .put("username", config.get("username").asText()) - .put("jdbc_url", String.format("jdbc:oracle:thin:@//%s:%s/%s", + List additionalParameters = new ArrayList<>(); + + JsonNode encryption = config.get("encryption"); + Protocol protocol = obtainConnectionProtocol(encryption, additionalParameters); + final String connectionString = String.format( + "jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS=(PROTOCOL=%s)(HOST=%s)(PORT=%s))(CONNECT_DATA=(SID=%s)))", + protocol, config.get("host").asText(), config.get("port").asText(), - config.get("sid").asText())); + config.get("sid").asText()); + + final ImmutableMap.Builder configBuilder = ImmutableMap.builder() + .put("username", config.get("username").asText()) + .put("jdbc_url", connectionString); if (config.has("password")) { configBuilder.put("password", config.get("password").asText()); @@ -56,10 +73,61 @@ public JsonNode toDatabaseConfig(JsonNode config) { schemas.add(schema.asText()); } } + if (!additionalParameters.isEmpty()) { + String connectionParams = String.join(";", additionalParameters); + configBuilder.put("connection_properties",connectionParams); + } return Jsons.jsonNode(configBuilder.build()); } + private Protocol obtainConnectionProtocol(JsonNode encryption, List additionalParameters) { + String encryptionMethod = encryption.get("encryption_method").asText(); + switch (encryptionMethod) { + case "unencrypted" -> { + return Protocol.TCP; + } + case "client_nne" -> { + String algorithm = encryption.get("encryption_algorithm").asText(); + additionalParameters.add("oracle.net.encryption_client=REQUIRED"); +// additionalParameters.add("oracle.net.encryption_types_client=( 3DES168 )"); + additionalParameters.add("oracle.net.encryption_types_client=( "+algorithm+" )"); + System.out.println(additionalParameters.toString()); + return Protocol.TCP; + } + case "encrypted_verify_certificate" -> { + try { + convertAndImportCertificate(encryption.get("ssl_certificate").asText()); + } catch (IOException | InterruptedException e) { + throw new RuntimeException("Failed to import certificate into Java Keystore"); + } + additionalParameters.add("javax.net.ssl.trustStore="+KEY_STORE_FILE_PATH); + additionalParameters.add("javax.net.ssl.trustStoreType=JKS"); + additionalParameters.add("javax.net.ssl.trustStorePassword="+KEY_STORE_PASS); + return Protocol.TCPS; + } + } + throw new RuntimeException("Failed to obtain connection protocol from config"+encryption.asText()); + } + + private static void convertAndImportCertificate(String certificate) throws IOException, InterruptedException { + Runtime run = Runtime.getRuntime(); + try (PrintWriter out = new PrintWriter("certificate.pem")) { + out.print(certificate); + } + runProcess("openssl x509 -outform der -in certificate.pem -out certificate.der", run); + runProcess("keytool -import -alias rds-root -keystore "+KEY_STORE_FILE_PATH+" -file certificate.der -storepass "+KEY_STORE_PASS+" -noprompt", run); + } + + private static void runProcess(String cmd, Runtime run) throws IOException, InterruptedException { + Process pr = run.exec(cmd); + if (!pr.waitFor(10, TimeUnit.SECONDS)){ + pr.destroy(); + throw new RuntimeException("Timeout while executing: "+ cmd); + }; + } + + @Override public List>> discoverInternal(JdbcDatabase database) throws Exception { List>> internals = new ArrayList<>(); diff --git a/airbyte-integrations/connectors/source-oracle/src/main/resources/spec.json b/airbyte-integrations/connectors/source-oracle/src/main/resources/spec.json index 3b5a02ef9604..0c9d60330eea 100644 --- a/airbyte-integrations/connectors/source-oracle/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-oracle/src/main/resources/spec.json @@ -4,7 +4,13 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Oracle Source Spec", "type": "object", - "required": ["host", "port", "sid", "username"], + "required": [ + "host", + "port", + "sid", + "username", + "encryption" + ], "additionalProperties": false, "properties": { "host": { @@ -14,12 +20,11 @@ }, "port": { "title": "Port", - "description": "Port of the database.", + "description": "Port of the database.\nOracle Corporations recommends the following port numbers:\n1521 - Default listening port for client connections to the listener. \n2484 - Recommended and officially registered listening port for client connections to the listener using TCP/IP with SSL", "type": "integer", "minimum": 0, "maximum": 65536, - "default": 1521, - "examples": ["1521"] + "default": 1521 }, "sid": { "title": "SID (Oracle System Identifier)", @@ -45,7 +50,89 @@ }, "minItems": 1, "uniqueItems": true + }, + "encryption": { + "title": "Encryption", + "type": "object", + "description": "Encryption method to use when communicating with the database", + "order": 6, + "oneOf": [ + { + "title": "Unencrypted", + "additionalProperties": false, + "description": "Data transfer will not be encrypted.", + "required": [ + "encryption_method" + ], + "properties": { + "encryption_method": { + "type": "string", + "const": "unencrypted", + "enum": [ + "unencrypted" + ], + "default": "unencrypted" + } + } + }, + { + "title": "Native Network Ecryption (NNE)", + "additionalProperties": false, + "description": "Native network encryption gives you the ability to encrypt database connections, without the configuration overhead of TCP/IP and SSL/TLS and without the need to open and listen on different ports.", + "required": [ + "encryption_method" + ], + "properties": { + "encryption_method": { + "type": "string", + "const": "client_nne", + "enum": [ + "client_nne" + ], + "default": "client_nne" + }, + "encryption_algorithm": { + "type": "string", + "description": "The location of the dataset. Warning: Changes made after creation will not be applied.", + "title": "Dataset Location", + "default": "AES256", + "enum": [ + "AES256", + "RC4_56", + "3DES168" + ] + } + } + }, + { + "title": "TLS Encrypted (verify certificate)", + "additionalProperties": false, + "description": "Verify and use the cert provided by the server.", + "required": [ + "encryption_method", + "ssl_certificate" + ], + "properties": { + "encryption_method": { + "type": "string", + "const": "encrypted_verify_certificate", + "enum": [ + "encrypted_verify_certificate" + ], + "default": "encrypted_verify_certificate" + }, + "ssl_certificate": { + "title": "SSL PEM file", + "description": "Privacy Enhanced Mail (PEM) files are concatenated certificate containers frequently used in certificate installations", + "type": "string", + "airbyte_secret": true, + "multiline": true, + "order": 4 + } + } + } + ] } } } -} +} \ No newline at end of file From 0f223b1569b5a33db65aa3a0e9e1d278147e36ce Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Fri, 1 Oct 2021 18:10:21 +0300 Subject: [PATCH 02/10] add tests --- .../source/oracle/OracleSource.java | 1 - ...AbstractSshOracleSourceAcceptanceTest.java | 242 +++++++++--------- .../oracle/OracleSourceAcceptanceTest.java | 9 +- .../oracle/OracleSourceDatatypeTest.java | 3 + .../oracle/OracleSourceNneAcceptanceTest.java | 50 ++++ 5 files changed, 183 insertions(+), 122 deletions(-) create mode 100644 airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceNneAcceptanceTest.java diff --git a/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java b/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java index 32867e7a520e..aa9bb6423206 100644 --- a/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java +++ b/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java @@ -90,7 +90,6 @@ private Protocol obtainConnectionProtocol(JsonNode encryption, List addi case "client_nne" -> { String algorithm = encryption.get("encryption_algorithm").asText(); additionalParameters.add("oracle.net.encryption_client=REQUIRED"); -// additionalParameters.add("oracle.net.encryption_types_client=( 3DES168 )"); additionalParameters.add("oracle.net.encryption_types_client=( "+algorithm+" )"); System.out.println(additionalParameters.toString()); return Protocol.TCP; diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/AbstractSshOracleSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/AbstractSshOracleSourceAcceptanceTest.java index 396bd8934c9c..1e743ee4a349 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/AbstractSshOracleSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/AbstractSshOracleSourceAcceptanceTest.java @@ -16,131 +16,135 @@ import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.*; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.OracleContainer; + import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Objects; -import org.testcontainers.containers.Network; -import org.testcontainers.containers.OracleContainer; public abstract class AbstractSshOracleSourceAcceptanceTest extends SourceAcceptanceTest { - private static final String STREAM_NAME = "JDBC_SPACE.ID_AND_NAME"; - private static final String STREAM_NAME2 = "JDBC_SPACE.STARSHIPS"; - private final SshBastionContainer sshBastionContainer = new SshBastionContainer(); - private OracleContainer db; - - private JsonNode config; - - public abstract SshTunnel.TunnelMethod getTunnelMethod(); - - @Override - protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { - startTestContainers(); - config = sshBastionContainer.getTunnelConfig(getTunnelMethod(), getBasicOracleDbConfigBuider(db)); - populateDatabaseTestData(); - } - - private void populateDatabaseTestData() throws Exception { - JdbcDatabase database = Databases.createJdbcDatabase(config.get("username").asText(), - config.get("password").asText(), - String.format("jdbc:oracle:thin:@//%s:%s/%s", - config.get("host").asText(), - config.get("port").asText(), - config.get("sid").asText()), - "oracle.jdbc.driver.OracleDriver"); - - database.execute(connection -> { - connection.createStatement().execute("CREATE USER JDBC_SPACE IDENTIFIED BY JDBC_SPACE DEFAULT TABLESPACE USERS QUOTA UNLIMITED ON USERS"); - connection.createStatement().execute("CREATE TABLE jdbc_space.id_and_name(id NUMERIC(20, 10), name VARCHAR(200), power BINARY_DOUBLE)"); - connection.createStatement().execute("INSERT INTO jdbc_space.id_and_name (id, name, power) VALUES (1,'goku', BINARY_DOUBLE_INFINITY)"); - connection.createStatement().execute("INSERT INTO jdbc_space.id_and_name (id, name, power) VALUES (2, 'vegeta', 9000.1)"); - connection.createStatement().execute("INSERT INTO jdbc_space.id_and_name (id, name, power) VALUES (NULL, 'piccolo', -BINARY_DOUBLE_INFINITY)"); - connection.createStatement().execute("CREATE TABLE jdbc_space.starships(id INTEGER, name VARCHAR(200))"); - connection.createStatement().execute("INSERT INTO jdbc_space.starships (id, name) VALUES (1,'enterprise-d')"); - connection.createStatement().execute("INSERT INTO jdbc_space.starships (id, name) VALUES (2, 'defiant')"); - connection.createStatement().execute("INSERT INTO jdbc_space.starships (id, name) VALUES (3, 'yamato')"); - }); - - database.close(); - } - - ; - - @Override - protected void tearDown(final TestDestinationEnv testEnv) { - sshBastionContainer.stopAndCloseContainers(db); - } - - private void startTestContainers() { - sshBastionContainer.initAndStartBastion(); - initAndStartJdbcContainer(); - } - - private void initAndStartJdbcContainer() { - db = new OracleContainer("epiclabs/docker-oracle-xe-11g") - .withNetwork(sshBastionContainer.getNetWork()); - db.start(); - } - - @Override - protected String getImageName() { - return "airbyte/source-oracle:dev"; - } - - @Override - protected ConnectorSpecification getSpec() throws Exception { - return SshHelpers.getSpecAndInjectSsh(); - } - - public ImmutableMap.Builder getBasicOracleDbConfigBuider(OracleContainer db) { - return ImmutableMap.builder() - .put("host", Objects.requireNonNull(db.getContainerInfo().getNetworkSettings() - .getNetworks() - .get(((Network.NetworkImpl) sshBastionContainer.getNetWork()).getName()) - .getIpAddress())) - .put("username", db.getUsername()) - .put("password", db.getPassword()) - .put("port", db.getExposedPorts().get(0)) - .put("sid", db.getSid()) - .put("schemas", List.of("JDBC_SPACE")); - } - - @Override - protected JsonNode getConfig() { - return config; - } - - @Override - protected ConfiguredAirbyteCatalog getConfiguredCatalog() { - return new ConfiguredAirbyteCatalog().withStreams(Lists.newArrayList( - new ConfiguredAirbyteStream() - .withSyncMode(SyncMode.INCREMENTAL) - .withCursorField(Lists.newArrayList("ID")) - .withStream(CatalogHelpers.createAirbyteStream( - STREAM_NAME, - Field.of("ID", JsonSchemaPrimitive.NUMBER), - Field.of("NAME", JsonSchemaPrimitive.STRING), - Field.of("POWER", JsonSchemaPrimitive.NUMBER)) - .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))), - new ConfiguredAirbyteStream() - .withSyncMode(SyncMode.INCREMENTAL) - .withCursorField(Lists.newArrayList("ID")) - .withStream(CatalogHelpers.createAirbyteStream( - STREAM_NAME2, - Field.of("ID", JsonSchemaPrimitive.NUMBER), - Field.of("NAME", JsonSchemaPrimitive.STRING)) - .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))))); - } - - @Override - protected List getRegexTests() { - return Collections.emptyList(); - } - - @Override - protected JsonNode getState() { - return Jsons.jsonNode(new HashMap<>()); - } + private static final String STREAM_NAME = "JDBC_SPACE.ID_AND_NAME"; + private static final String STREAM_NAME2 = "JDBC_SPACE.STARSHIPS"; + private final SshBastionContainer sshBastionContainer = new SshBastionContainer(); + private OracleContainer db; + + private JsonNode config; + + public abstract SshTunnel.TunnelMethod getTunnelMethod(); + + @Override + protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { + startTestContainers(); + config = sshBastionContainer.getTunnelConfig(getTunnelMethod(), getBasicOracleDbConfigBuider(db)); + populateDatabaseTestData(); + } + + private void populateDatabaseTestData() throws Exception { + JdbcDatabase database = Databases.createJdbcDatabase(config.get("username").asText(), + config.get("password").asText(), + String.format("jdbc:oracle:thin:@//%s:%s/%s", + config.get("host").asText(), + config.get("port").asText(), + config.get("sid").asText()), + "oracle.jdbc.driver.OracleDriver"); + + database.execute(connection -> { + connection.createStatement().execute("CREATE USER JDBC_SPACE IDENTIFIED BY JDBC_SPACE DEFAULT TABLESPACE USERS QUOTA UNLIMITED ON USERS"); + connection.createStatement().execute("CREATE TABLE jdbc_space.id_and_name(id NUMERIC(20, 10), name VARCHAR(200), power BINARY_DOUBLE)"); + connection.createStatement().execute("INSERT INTO jdbc_space.id_and_name (id, name, power) VALUES (1,'goku', BINARY_DOUBLE_INFINITY)"); + connection.createStatement().execute("INSERT INTO jdbc_space.id_and_name (id, name, power) VALUES (2, 'vegeta', 9000.1)"); + connection.createStatement().execute("INSERT INTO jdbc_space.id_and_name (id, name, power) VALUES (NULL, 'piccolo', -BINARY_DOUBLE_INFINITY)"); + connection.createStatement().execute("CREATE TABLE jdbc_space.starships(id INTEGER, name VARCHAR(200))"); + connection.createStatement().execute("INSERT INTO jdbc_space.starships (id, name) VALUES (1,'enterprise-d')"); + connection.createStatement().execute("INSERT INTO jdbc_space.starships (id, name) VALUES (2, 'defiant')"); + connection.createStatement().execute("INSERT INTO jdbc_space.starships (id, name) VALUES (3, 'yamato')"); + }); + + database.close(); + } + + ; + + @Override + protected void tearDown(final TestDestinationEnv testEnv) { + sshBastionContainer.stopAndCloseContainers(db); + } + + private void startTestContainers() { + sshBastionContainer.initAndStartBastion(); + initAndStartJdbcContainer(); + } + + private void initAndStartJdbcContainer() { + db = new OracleContainer("epiclabs/docker-oracle-xe-11g") + .withNetwork(sshBastionContainer.getNetWork()); + db.start(); + } + + @Override + protected String getImageName() { + return "airbyte/source-oracle:dev"; + } + + @Override + protected ConnectorSpecification getSpec() throws Exception { + return SshHelpers.getSpecAndInjectSsh(); + } + + public ImmutableMap.Builder getBasicOracleDbConfigBuider(OracleContainer db) { + return ImmutableMap.builder() + .put("host", Objects.requireNonNull(db.getContainerInfo().getNetworkSettings() + .getNetworks() + .get(((Network.NetworkImpl) sshBastionContainer.getNetWork()).getName()) + .getIpAddress())) + .put("username", db.getUsername()) + .put("password", db.getPassword()) + .put("port", db.getExposedPorts().get(0)) + .put("sid", db.getSid()) + .put("schemas", List.of("JDBC_SPACE")) + .put("encryption", Jsons.jsonNode(ImmutableMap.builder() + .put("encryption_method", "unencrypted") + .build())); + } + + @Override + protected JsonNode getConfig() { + return config; + } + + @Override + protected ConfiguredAirbyteCatalog getConfiguredCatalog() { + return new ConfiguredAirbyteCatalog().withStreams(Lists.newArrayList( + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(Lists.newArrayList("ID")) + .withStream(CatalogHelpers.createAirbyteStream( + STREAM_NAME, + Field.of("ID", JsonSchemaPrimitive.NUMBER), + Field.of("NAME", JsonSchemaPrimitive.STRING), + Field.of("POWER", JsonSchemaPrimitive.NUMBER)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))), + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(Lists.newArrayList("ID")) + .withStream(CatalogHelpers.createAirbyteStream( + STREAM_NAME2, + Field.of("ID", JsonSchemaPrimitive.NUMBER), + Field.of("NAME", JsonSchemaPrimitive.STRING)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))))); + } + + @Override + protected List getRegexTests() { + return Collections.emptyList(); + } + + @Override + protected JsonNode getState() { + return Jsons.jsonNode(new HashMap<>()); + } } diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceAcceptanceTest.java index 5174f1bae2e1..780d1eab458e 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceAcceptanceTest.java @@ -23,6 +23,8 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; + +import org.junit.jupiter.api.Test; import org.testcontainers.containers.OracleContainer; public class OracleSourceAcceptanceTest extends SourceAcceptanceTest { @@ -30,8 +32,8 @@ public class OracleSourceAcceptanceTest extends SourceAcceptanceTest { private static final String STREAM_NAME = "JDBC_SPACE.ID_AND_NAME"; private static final String STREAM_NAME2 = "JDBC_SPACE.STARSHIPS"; - private OracleContainer container; - private JsonNode config; + protected OracleContainer container; + protected JsonNode config; @Override protected void setupEnvironment(TestDestinationEnv environment) throws Exception { @@ -45,6 +47,9 @@ protected void setupEnvironment(TestDestinationEnv environment) throws Exception .put("username", container.getUsername()) .put("password", container.getPassword()) .put("schemas", List.of("JDBC_SPACE")) + .put("encryption", Jsons.jsonNode(ImmutableMap.builder() + .put("encryption_method", "unencrypted") + .build())) .build()); JdbcDatabase database = Databases.createJdbcDatabase(config.get("username").asText(), diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java index 280cad9e7c81..855a0a63936f 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java @@ -44,6 +44,9 @@ protected Database setupDatabase() throws Exception { .put("username", container.getUsername()) .put("password", container.getPassword()) .put("schemas", List.of("TEST")) + .put("encryption", Jsons.jsonNode(ImmutableMap.builder() + .put("encryption_method", "unencrypted") + .build())) .build()); Database database = Databases.createOracleDatabase(config.get("username").asText(), diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceNneAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceNneAcceptanceTest.java new file mode 100644 index 000000000000..6b4e35c6272b --- /dev/null +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceNneAcceptanceTest.java @@ -0,0 +1,50 @@ +package io.airbyte.integrations.source.oracle; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.db.Databases; +import io.airbyte.db.jdbc.JdbcDatabase; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.sql.SQLException; +import java.util.List; +import java.util.stream.Collectors; + +public class OracleSourceNneAcceptanceTest extends OracleSourceAcceptanceTest{ + @Test + public void testhhgjkgk() throws SQLException { + final JsonNode clone = Jsons.clone(getConfig()); + ((ObjectNode) clone).put("encryption", Jsons.jsonNode(ImmutableMap.builder() + .put("encryption_method", "client_nne") + .put("encryption_algorithm", "3DES168") + .build())); + + String algorithm = clone.get("encryption") + .get("encryption_algorithm").asText(); + + JdbcDatabase database = Databases.createJdbcDatabase(clone.get("username").asText(), + clone.get("password").asText(), + String.format("jdbc:oracle:thin:@//%s:%s/%s", + clone.get("host").asText(), + clone.get("port").asText(), + clone.get("sid").asText()), + "oracle.jdbc.driver.OracleDriver", + "oracle.net.encryption_client=REQUIRED;" + + "oracle.net.encryption_types_client=( " + + algorithm+" )"); + + String network_service_banner = "select network_service_banner from v$session_connect_info where sid in (select distinct sid from v$mystat)"; + List collect = database.query(network_service_banner).collect(Collectors.toList()); + + assertTrue(collect.get(2).get("NETWORK_SERVICE_BANNER").asText() + .contains("Oracle Advanced Security: "+algorithm+" encryption")); + } + +// @Override +// protected JsonNode getConfig() { +// return config; +// } +} From 53a4270b906a7e83b9ef7328857754db2d917f8d Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Fri, 1 Oct 2021 19:09:21 +0300 Subject: [PATCH 03/10] add tests --- .../oracle/OracleSourceNneAcceptanceTest.java | 53 +++++++++++++++++-- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceNneAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceNneAcceptanceTest.java index 6b4e35c6272b..6aacd5074925 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceNneAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceNneAcceptanceTest.java @@ -7,6 +7,8 @@ import io.airbyte.db.Databases; import io.airbyte.db.jdbc.JdbcDatabase; import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import java.sql.SQLException; @@ -15,7 +17,7 @@ public class OracleSourceNneAcceptanceTest extends OracleSourceAcceptanceTest{ @Test - public void testhhgjkgk() throws SQLException { + public void testEncrytion() throws SQLException { final JsonNode clone = Jsons.clone(getConfig()); ((ObjectNode) clone).put("encryption", Jsons.jsonNode(ImmutableMap.builder() .put("encryption_method", "client_nne") @@ -43,8 +45,49 @@ public void testhhgjkgk() throws SQLException { .contains("Oracle Advanced Security: "+algorithm+" encryption")); } -// @Override -// protected JsonNode getConfig() { -// return config; -// } + @Test + public void testNoneEncrytion() throws SQLException { + + JdbcDatabase database = Databases.createJdbcDatabase(config.get("username").asText(), + config.get("password").asText(), + String.format("jdbc:oracle:thin:@//%s:%s/%s", + config.get("host").asText(), + config.get("port").asText(), + config.get("sid").asText()), + "oracle.jdbc.driver.OracleDriver"); + + String network_service_banner = "select network_service_banner from v$session_connect_info where sid in (select distinct sid from v$mystat)"; + List collect = database.query(network_service_banner).collect(Collectors.toList()); + + assertTrue(collect.get(1).get("NETWORK_SERVICE_BANNER").asText() + .contains("Oracle Advanced Security: encryption")); + } + + @Test + public void testCheckProtocol() throws SQLException { + final JsonNode clone = Jsons.clone(getConfig()); + ((ObjectNode) clone).put("encryption", Jsons.jsonNode(ImmutableMap.builder() + .put("encryption_method", "client_nne") + .put("encryption_algorithm", "AES256") + .build())); + + String algorithm = clone.get("encryption") + .get("encryption_algorithm").asText(); + + JdbcDatabase database = Databases.createJdbcDatabase(clone.get("username").asText(), + clone.get("password").asText(), + String.format("jdbc:oracle:thin:@//%s:%s/%s", + clone.get("host").asText(), + clone.get("port").asText(), + clone.get("sid").asText()), + "oracle.jdbc.driver.OracleDriver", + "oracle.net.encryption_client=REQUIRED;" + + "oracle.net.encryption_types_client=( " + + algorithm+" )"); + + String network_service_banner = "SELECT sys_context('USERENV', 'NETWORK_PROTOCOL') as network_protocol FROM dual"; + List collect = database.query(network_service_banner).collect(Collectors.toList()); + + assertEquals("tcp", collect.get(0).get("NETWORK_PROTOCOL").asText()); + } } From 5713cdf816e185ce005066fe2485ecc103d779dd Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Fri, 1 Oct 2021 19:10:12 +0300 Subject: [PATCH 04/10] remove debug option from Dockerfile --- airbyte-integrations/connectors/source-oracle/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-oracle/Dockerfile b/airbyte-integrations/connectors/source-oracle/Dockerfile index 3066b0d4c483..fab58afba1f4 100644 --- a/airbyte-integrations/connectors/source-oracle/Dockerfile +++ b/airbyte-integrations/connectors/source-oracle/Dockerfile @@ -6,7 +6,6 @@ ENV APPLICATION source-oracle ENV TZ UTC COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar -#ENV JAVA_TOOL_OPTIONS -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005 RUN tar xf ${APPLICATION}.tar --strip-components=1 LABEL io.airbyte.version=0.3.5 From 14f9d407e4bb98843c088c54277d1247b94b5eb6 Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Mon, 4 Oct 2021 12:39:50 +0300 Subject: [PATCH 05/10] add docs and fix unit tests --- .../source-oracle/src/main/resources/spec.json | 4 ++-- .../source/oracle/OracleJdbcSourceAcceptanceTest.java | 3 +++ .../integrations/source/oracle/OracleSourceTest.java | 3 +++ docs/integrations/sources/oracle.md | 11 +++++++++++ 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-oracle/src/main/resources/spec.json b/airbyte-integrations/connectors/source-oracle/src/main/resources/spec.json index 0c9d60330eea..de5ed5f6fff4 100644 --- a/airbyte-integrations/connectors/source-oracle/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-oracle/src/main/resources/spec.json @@ -93,8 +93,8 @@ }, "encryption_algorithm": { "type": "string", - "description": "The location of the dataset. Warning: Changes made after creation will not be applied.", - "title": "Dataset Location", + "description": "This parameter defines the encryption algorithm to be used", + "title": "Encryption Algorithm", "default": "AES256", "enum": [ "AES256", diff --git a/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleJdbcSourceAcceptanceTest.java index 4e2e425fbd9c..234cdbf36618 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleJdbcSourceAcceptanceTest.java @@ -75,6 +75,9 @@ public void setup() throws Exception { .put("username", ORACLE_DB.getUsername()) .put("password", ORACLE_DB.getPassword()) .put("schemas", List.of(SCHEMA_NAME, SCHEMA_NAME2)) + .put("encryption", Jsons.jsonNode(ImmutableMap.builder() + .put("encryption_method", "unencrypted") + .build())) .build()); // Because Oracle doesn't let me create database easily I need to clean up diff --git a/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleSourceTest.java b/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleSourceTest.java index 5971c6e376de..ba3fea55d716 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleSourceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleSourceTest.java @@ -67,6 +67,9 @@ void setup() throws Exception { .put("username", ORACLE_DB.getUsername()) .put("password", ORACLE_DB.getPassword()) .put("schemas", List.of("TEST")) + .put("encryption", Jsons.jsonNode(ImmutableMap.builder() + .put("encryption_method", "unencrypted") + .build())) .build()); JdbcDatabase database = Databases.createJdbcDatabase(config.get("username").asText(), diff --git a/docs/integrations/sources/oracle.md b/docs/integrations/sources/oracle.md index 6dfbf7a5c2d3..a5c3b316f7d2 100644 --- a/docs/integrations/sources/oracle.md +++ b/docs/integrations/sources/oracle.md @@ -117,11 +117,22 @@ Using this feature requires additional configuration, when creating the source. 6. If you are using `Password Authentication`, then `SSH Login Username` should be set to the password of the User from the previous step. If you are using `SSH Key Authentication` leave this blank. Again, this is not the Oracle password, but the password for the OS-user that Airbyte is using to perform commands on the bastion. 7. If you are using `SSH Key Authentication`, then `SSH Private Key` should be set to the RSA Private Key that you are using to create the SSH connection. This should be the full contents of the key file starting with `-----BEGIN RSA PRIVATE KEY-----` and ending with `-----END RSA PRIVATE KEY-----`. +## Encryption Options + +Airbite has the ability to connect to the Oracle source with 3 network connectivity options: + +1.`Unencrypted` the connection will be made using the TCP protocol. In this case, all data over the network will be transmitted in unencrypted form. +2.`Native network encryption` gives you the ability to encrypt database connections, without the configuration overhead of TCP / IP and SSL / TLS and without the need to open and listen on different ports. +In this case, the *SQLNET.ENCRYPTION_CLIENT* option will always be set as *REQUIRED* by default: The client or server will only accept encrypted traffic, +but the user has the opportunity to choose an `Encryption algorithm` according to the security policies he needs. +3.`TLS Encrypted` (verify certificate) - if this option is selected, data transfer will be transfered using the TLS protocol, taking into account the handshake procedure and certificate verification. +To use this option, insert the content of the certificate issued by the server into the `SSL PEM file` field ## Changelog | Version | Date | Pull Request | Subject | | :------ | :-------- | :----- | :------ | +| 0.3.6 | 2021-10-01 | [6616](https://github.com/airbytehq/airbyte/pull/6616) | Added network encryption options | | 0.3.5 | 2021-09-22 | [6356](https://github.com/airbytehq/airbyte/pull/6356) | Added option to connect to DB via SSH. | | 0.3.4 | 2021-09-01 | [6038](https://github.com/airbytehq/airbyte/pull/6038) | Remove automatic filtering of system schemas. | | 0.3.3 | 2021-09-01 | [5779](https://github.com/airbytehq/airbyte/pull/5779) | Ability to only discover certain schemas. | From aff5d1ce12092b8e4586ecda80dfe0f29278e400 Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Mon, 4 Oct 2021 12:43:36 +0300 Subject: [PATCH 06/10] remove System.out.println --- .../java/io/airbyte/integrations/source/oracle/OracleSource.java | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java b/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java index aa9bb6423206..a6ac6a770a73 100644 --- a/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java +++ b/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java @@ -91,7 +91,6 @@ private Protocol obtainConnectionProtocol(JsonNode encryption, List addi String algorithm = encryption.get("encryption_algorithm").asText(); additionalParameters.add("oracle.net.encryption_client=REQUIRED"); additionalParameters.add("oracle.net.encryption_types_client=( "+algorithm+" )"); - System.out.println(additionalParameters.toString()); return Protocol.TCP; } case "encrypted_verify_certificate" -> { From 9761e5df646b3797704ebacf98f5981a3f927c59 Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Tue, 5 Oct 2021 19:41:15 +0300 Subject: [PATCH 07/10] fix checkstyle | remove 'encryption' from required fields --- .../source/oracle/OracleSource.java | 49 ++-- .../src/main/resources/spec.json | 53 ++-- ...AbstractSshOracleSourceAcceptanceTest.java | 245 +++++++++--------- .../oracle/OracleSourceAcceptanceTest.java | 8 +- .../oracle/OracleSourceDatatypeTest.java | 3 - .../oracle/OracleSourceNneAcceptanceTest.java | 165 ++++++------ .../OracleJdbcSourceAcceptanceTest.java | 3 - .../source/oracle/OracleSourceTest.java | 3 - 8 files changed, 251 insertions(+), 278 deletions(-) diff --git a/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java b/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java index a6ac6a770a73..2f0480f0329e 100644 --- a/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java +++ b/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java @@ -15,12 +15,14 @@ import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.relationaldb.TableInfo; import io.airbyte.protocol.models.CommonField; - -import java.io.*; +import java.io.IOException; +import java.io.PrintWriter; import java.sql.JDBCType; -import java.util.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Set; import java.util.concurrent.TimeUnit; - import org.apache.commons.lang3.RandomStringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,8 +38,9 @@ public class OracleSource extends AbstractJdbcSource implements Source { private static final String KEY_STORE_FILE_PATH = "clientkeystore.jks"; private static final String KEY_STORE_PASS = RandomStringUtils.randomAlphanumeric(8); - enum Protocol{ - TCP,TCPS + enum Protocol { + TCP, + TCPS } public OracleSource() { @@ -48,14 +51,15 @@ public OracleSource() { public JsonNode toDatabaseConfig(JsonNode config) { List additionalParameters = new ArrayList<>(); - JsonNode encryption = config.get("encryption"); - Protocol protocol = obtainConnectionProtocol(encryption, additionalParameters); + Protocol protocol = config.has("encryption") + ? obtainConnectionProtocol(config.get("encryption"), additionalParameters) + : Protocol.TCP; final String connectionString = String.format( - "jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS=(PROTOCOL=%s)(HOST=%s)(PORT=%s))(CONNECT_DATA=(SID=%s)))", - protocol, - config.get("host").asText(), - config.get("port").asText(), - config.get("sid").asText()); + "jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS=(PROTOCOL=%s)(HOST=%s)(PORT=%s))(CONNECT_DATA=(SID=%s)))", + protocol, + config.get("host").asText(), + config.get("port").asText(), + config.get("sid").asText()); final ImmutableMap.Builder configBuilder = ImmutableMap.builder() .put("username", config.get("username").asText()) @@ -75,7 +79,7 @@ public JsonNode toDatabaseConfig(JsonNode config) { } if (!additionalParameters.isEmpty()) { String connectionParams = String.join(";", additionalParameters); - configBuilder.put("connection_properties",connectionParams); + configBuilder.put("connection_properties", connectionParams); } return Jsons.jsonNode(configBuilder.build()); @@ -90,7 +94,7 @@ private Protocol obtainConnectionProtocol(JsonNode encryption, List addi case "client_nne" -> { String algorithm = encryption.get("encryption_algorithm").asText(); additionalParameters.add("oracle.net.encryption_client=REQUIRED"); - additionalParameters.add("oracle.net.encryption_types_client=( "+algorithm+" )"); + additionalParameters.add("oracle.net.encryption_types_client=( " + algorithm + " )"); return Protocol.TCP; } case "encrypted_verify_certificate" -> { @@ -99,13 +103,13 @@ private Protocol obtainConnectionProtocol(JsonNode encryption, List addi } catch (IOException | InterruptedException e) { throw new RuntimeException("Failed to import certificate into Java Keystore"); } - additionalParameters.add("javax.net.ssl.trustStore="+KEY_STORE_FILE_PATH); + additionalParameters.add("javax.net.ssl.trustStore=" + KEY_STORE_FILE_PATH); additionalParameters.add("javax.net.ssl.trustStoreType=JKS"); - additionalParameters.add("javax.net.ssl.trustStorePassword="+KEY_STORE_PASS); + additionalParameters.add("javax.net.ssl.trustStorePassword=" + KEY_STORE_PASS); return Protocol.TCPS; } } - throw new RuntimeException("Failed to obtain connection protocol from config"+encryption.asText()); + throw new RuntimeException("Failed to obtain connection protocol from config" + encryption.asText()); } private static void convertAndImportCertificate(String certificate) throws IOException, InterruptedException { @@ -114,18 +118,17 @@ private static void convertAndImportCertificate(String certificate) throws IOExc out.print(certificate); } runProcess("openssl x509 -outform der -in certificate.pem -out certificate.der", run); - runProcess("keytool -import -alias rds-root -keystore "+KEY_STORE_FILE_PATH+" -file certificate.der -storepass "+KEY_STORE_PASS+" -noprompt", run); + runProcess("keytool -import -alias rds-root -keystore " + KEY_STORE_FILE_PATH + " -file certificate.der -storepass " + KEY_STORE_PASS + " -noprompt", run); } private static void runProcess(String cmd, Runtime run) throws IOException, InterruptedException { Process pr = run.exec(cmd); - if (!pr.waitFor(10, TimeUnit.SECONDS)){ + if (!pr.waitFor(10, TimeUnit.SECONDS)) { pr.destroy(); - throw new RuntimeException("Timeout while executing: "+ cmd); - }; + throw new RuntimeException("Timeout while executing: " + cmd); + } ; } - @Override public List>> discoverInternal(JdbcDatabase database) throws Exception { List>> internals = new ArrayList<>(); diff --git a/airbyte-integrations/connectors/source-oracle/src/main/resources/spec.json b/airbyte-integrations/connectors/source-oracle/src/main/resources/spec.json index de5ed5f6fff4..0851673e4da7 100644 --- a/airbyte-integrations/connectors/source-oracle/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-oracle/src/main/resources/spec.json @@ -4,13 +4,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Oracle Source Spec", "type": "object", - "required": [ - "host", - "port", - "sid", - "username", - "encryption" - ], + "required": ["host", "port", "sid", "username"], "additionalProperties": false, "properties": { "host": { @@ -61,16 +55,12 @@ "title": "Unencrypted", "additionalProperties": false, "description": "Data transfer will not be encrypted.", - "required": [ - "encryption_method" - ], + "required": ["encryption_method"], "properties": { "encryption_method": { "type": "string", "const": "unencrypted", - "enum": [ - "unencrypted" - ], + "enum": ["unencrypted"], "default": "unencrypted" } } @@ -79,46 +69,33 @@ "title": "Native Network Ecryption (NNE)", "additionalProperties": false, "description": "Native network encryption gives you the ability to encrypt database connections, without the configuration overhead of TCP/IP and SSL/TLS and without the need to open and listen on different ports.", - "required": [ - "encryption_method" - ], + "required": ["encryption_method"], "properties": { "encryption_method": { "type": "string", "const": "client_nne", - "enum": [ - "client_nne" - ], + "enum": ["client_nne"], "default": "client_nne" }, - "encryption_algorithm": { - "type": "string", - "description": "This parameter defines the encryption algorithm to be used", - "title": "Encryption Algorithm", - "default": "AES256", - "enum": [ - "AES256", - "RC4_56", - "3DES168" - ] - } + "encryption_algorithm": { + "type": "string", + "description": "This parameter defines the encryption algorithm to be used", + "title": "Encryption Algorithm", + "default": "AES256", + "enum": ["AES256", "RC4_56", "3DES168"] + } } }, { "title": "TLS Encrypted (verify certificate)", "additionalProperties": false, "description": "Verify and use the cert provided by the server.", - "required": [ - "encryption_method", - "ssl_certificate" - ], + "required": ["encryption_method", "ssl_certificate"], "properties": { "encryption_method": { "type": "string", "const": "encrypted_verify_certificate", - "enum": [ - "encrypted_verify_certificate" - ], + "enum": ["encrypted_verify_certificate"], "default": "encrypted_verify_certificate" }, "ssl_certificate": { @@ -135,4 +112,4 @@ } } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/AbstractSshOracleSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/AbstractSshOracleSourceAcceptanceTest.java index 1e743ee4a349..fcd7b45dfebe 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/AbstractSshOracleSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/AbstractSshOracleSourceAcceptanceTest.java @@ -16,135 +16,134 @@ import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.*; -import org.testcontainers.containers.Network; -import org.testcontainers.containers.OracleContainer; - import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Objects; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.OracleContainer; public abstract class AbstractSshOracleSourceAcceptanceTest extends SourceAcceptanceTest { - private static final String STREAM_NAME = "JDBC_SPACE.ID_AND_NAME"; - private static final String STREAM_NAME2 = "JDBC_SPACE.STARSHIPS"; - private final SshBastionContainer sshBastionContainer = new SshBastionContainer(); - private OracleContainer db; - - private JsonNode config; - - public abstract SshTunnel.TunnelMethod getTunnelMethod(); - - @Override - protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { - startTestContainers(); - config = sshBastionContainer.getTunnelConfig(getTunnelMethod(), getBasicOracleDbConfigBuider(db)); - populateDatabaseTestData(); - } - - private void populateDatabaseTestData() throws Exception { - JdbcDatabase database = Databases.createJdbcDatabase(config.get("username").asText(), - config.get("password").asText(), - String.format("jdbc:oracle:thin:@//%s:%s/%s", - config.get("host").asText(), - config.get("port").asText(), - config.get("sid").asText()), - "oracle.jdbc.driver.OracleDriver"); - - database.execute(connection -> { - connection.createStatement().execute("CREATE USER JDBC_SPACE IDENTIFIED BY JDBC_SPACE DEFAULT TABLESPACE USERS QUOTA UNLIMITED ON USERS"); - connection.createStatement().execute("CREATE TABLE jdbc_space.id_and_name(id NUMERIC(20, 10), name VARCHAR(200), power BINARY_DOUBLE)"); - connection.createStatement().execute("INSERT INTO jdbc_space.id_and_name (id, name, power) VALUES (1,'goku', BINARY_DOUBLE_INFINITY)"); - connection.createStatement().execute("INSERT INTO jdbc_space.id_and_name (id, name, power) VALUES (2, 'vegeta', 9000.1)"); - connection.createStatement().execute("INSERT INTO jdbc_space.id_and_name (id, name, power) VALUES (NULL, 'piccolo', -BINARY_DOUBLE_INFINITY)"); - connection.createStatement().execute("CREATE TABLE jdbc_space.starships(id INTEGER, name VARCHAR(200))"); - connection.createStatement().execute("INSERT INTO jdbc_space.starships (id, name) VALUES (1,'enterprise-d')"); - connection.createStatement().execute("INSERT INTO jdbc_space.starships (id, name) VALUES (2, 'defiant')"); - connection.createStatement().execute("INSERT INTO jdbc_space.starships (id, name) VALUES (3, 'yamato')"); - }); - - database.close(); - } - - ; - - @Override - protected void tearDown(final TestDestinationEnv testEnv) { - sshBastionContainer.stopAndCloseContainers(db); - } - - private void startTestContainers() { - sshBastionContainer.initAndStartBastion(); - initAndStartJdbcContainer(); - } - - private void initAndStartJdbcContainer() { - db = new OracleContainer("epiclabs/docker-oracle-xe-11g") - .withNetwork(sshBastionContainer.getNetWork()); - db.start(); - } - - @Override - protected String getImageName() { - return "airbyte/source-oracle:dev"; - } - - @Override - protected ConnectorSpecification getSpec() throws Exception { - return SshHelpers.getSpecAndInjectSsh(); - } - - public ImmutableMap.Builder getBasicOracleDbConfigBuider(OracleContainer db) { - return ImmutableMap.builder() - .put("host", Objects.requireNonNull(db.getContainerInfo().getNetworkSettings() - .getNetworks() - .get(((Network.NetworkImpl) sshBastionContainer.getNetWork()).getName()) - .getIpAddress())) - .put("username", db.getUsername()) - .put("password", db.getPassword()) - .put("port", db.getExposedPorts().get(0)) - .put("sid", db.getSid()) - .put("schemas", List.of("JDBC_SPACE")) - .put("encryption", Jsons.jsonNode(ImmutableMap.builder() - .put("encryption_method", "unencrypted") - .build())); - } - - @Override - protected JsonNode getConfig() { - return config; - } - - @Override - protected ConfiguredAirbyteCatalog getConfiguredCatalog() { - return new ConfiguredAirbyteCatalog().withStreams(Lists.newArrayList( - new ConfiguredAirbyteStream() - .withSyncMode(SyncMode.INCREMENTAL) - .withCursorField(Lists.newArrayList("ID")) - .withStream(CatalogHelpers.createAirbyteStream( - STREAM_NAME, - Field.of("ID", JsonSchemaPrimitive.NUMBER), - Field.of("NAME", JsonSchemaPrimitive.STRING), - Field.of("POWER", JsonSchemaPrimitive.NUMBER)) - .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))), - new ConfiguredAirbyteStream() - .withSyncMode(SyncMode.INCREMENTAL) - .withCursorField(Lists.newArrayList("ID")) - .withStream(CatalogHelpers.createAirbyteStream( - STREAM_NAME2, - Field.of("ID", JsonSchemaPrimitive.NUMBER), - Field.of("NAME", JsonSchemaPrimitive.STRING)) - .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))))); - } - - @Override - protected List getRegexTests() { - return Collections.emptyList(); - } - - @Override - protected JsonNode getState() { - return Jsons.jsonNode(new HashMap<>()); - } + private static final String STREAM_NAME = "JDBC_SPACE.ID_AND_NAME"; + private static final String STREAM_NAME2 = "JDBC_SPACE.STARSHIPS"; + private final SshBastionContainer sshBastionContainer = new SshBastionContainer(); + private OracleContainer db; + + private JsonNode config; + + public abstract SshTunnel.TunnelMethod getTunnelMethod(); + + @Override + protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { + startTestContainers(); + config = sshBastionContainer.getTunnelConfig(getTunnelMethod(), getBasicOracleDbConfigBuider(db)); + populateDatabaseTestData(); + } + + private void populateDatabaseTestData() throws Exception { + JdbcDatabase database = Databases.createJdbcDatabase(config.get("username").asText(), + config.get("password").asText(), + String.format("jdbc:oracle:thin:@//%s:%s/%s", + config.get("host").asText(), + config.get("port").asText(), + config.get("sid").asText()), + "oracle.jdbc.driver.OracleDriver"); + + database.execute(connection -> { + connection.createStatement().execute("CREATE USER JDBC_SPACE IDENTIFIED BY JDBC_SPACE DEFAULT TABLESPACE USERS QUOTA UNLIMITED ON USERS"); + connection.createStatement().execute("CREATE TABLE jdbc_space.id_and_name(id NUMERIC(20, 10), name VARCHAR(200), power BINARY_DOUBLE)"); + connection.createStatement().execute("INSERT INTO jdbc_space.id_and_name (id, name, power) VALUES (1,'goku', BINARY_DOUBLE_INFINITY)"); + connection.createStatement().execute("INSERT INTO jdbc_space.id_and_name (id, name, power) VALUES (2, 'vegeta', 9000.1)"); + connection.createStatement().execute("INSERT INTO jdbc_space.id_and_name (id, name, power) VALUES (NULL, 'piccolo', -BINARY_DOUBLE_INFINITY)"); + connection.createStatement().execute("CREATE TABLE jdbc_space.starships(id INTEGER, name VARCHAR(200))"); + connection.createStatement().execute("INSERT INTO jdbc_space.starships (id, name) VALUES (1,'enterprise-d')"); + connection.createStatement().execute("INSERT INTO jdbc_space.starships (id, name) VALUES (2, 'defiant')"); + connection.createStatement().execute("INSERT INTO jdbc_space.starships (id, name) VALUES (3, 'yamato')"); + }); + + database.close(); + } + + ; + + @Override + protected void tearDown(final TestDestinationEnv testEnv) { + sshBastionContainer.stopAndCloseContainers(db); + } + + private void startTestContainers() { + sshBastionContainer.initAndStartBastion(); + initAndStartJdbcContainer(); + } + + private void initAndStartJdbcContainer() { + db = new OracleContainer("epiclabs/docker-oracle-xe-11g") + .withNetwork(sshBastionContainer.getNetWork()); + db.start(); + } + + @Override + protected String getImageName() { + return "airbyte/source-oracle:dev"; + } + + @Override + protected ConnectorSpecification getSpec() throws Exception { + return SshHelpers.getSpecAndInjectSsh(); + } + + public ImmutableMap.Builder getBasicOracleDbConfigBuider(OracleContainer db) { + return ImmutableMap.builder() + .put("host", Objects.requireNonNull(db.getContainerInfo().getNetworkSettings() + .getNetworks() + .get(((Network.NetworkImpl) sshBastionContainer.getNetWork()).getName()) + .getIpAddress())) + .put("username", db.getUsername()) + .put("password", db.getPassword()) + .put("port", db.getExposedPorts().get(0)) + .put("sid", db.getSid()) + .put("schemas", List.of("JDBC_SPACE")) + .put("encryption", Jsons.jsonNode(ImmutableMap.builder() + .put("encryption_method", "unencrypted") + .build())); + } + + @Override + protected JsonNode getConfig() { + return config; + } + + @Override + protected ConfiguredAirbyteCatalog getConfiguredCatalog() { + return new ConfiguredAirbyteCatalog().withStreams(Lists.newArrayList( + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(Lists.newArrayList("ID")) + .withStream(CatalogHelpers.createAirbyteStream( + STREAM_NAME, + Field.of("ID", JsonSchemaPrimitive.NUMBER), + Field.of("NAME", JsonSchemaPrimitive.STRING), + Field.of("POWER", JsonSchemaPrimitive.NUMBER)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))), + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(Lists.newArrayList("ID")) + .withStream(CatalogHelpers.createAirbyteStream( + STREAM_NAME2, + Field.of("ID", JsonSchemaPrimitive.NUMBER), + Field.of("NAME", JsonSchemaPrimitive.STRING)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))))); + } + + @Override + protected List getRegexTests() { + return Collections.emptyList(); + } + + @Override + protected JsonNode getState() { + return Jsons.jsonNode(new HashMap<>()); + } } diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceAcceptanceTest.java index 780d1eab458e..7e5836385938 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceAcceptanceTest.java @@ -23,8 +23,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; - -import org.junit.jupiter.api.Test; import org.testcontainers.containers.OracleContainer; public class OracleSourceAcceptanceTest extends SourceAcceptanceTest { @@ -47,9 +45,9 @@ protected void setupEnvironment(TestDestinationEnv environment) throws Exception .put("username", container.getUsername()) .put("password", container.getPassword()) .put("schemas", List.of("JDBC_SPACE")) - .put("encryption", Jsons.jsonNode(ImmutableMap.builder() - .put("encryption_method", "unencrypted") - .build())) + .put("encryption", Jsons.jsonNode(ImmutableMap.builder() + .put("encryption_method", "unencrypted") + .build())) .build()); JdbcDatabase database = Databases.createJdbcDatabase(config.get("username").asText(), diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java index 855a0a63936f..280cad9e7c81 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java @@ -44,9 +44,6 @@ protected Database setupDatabase() throws Exception { .put("username", container.getUsername()) .put("password", container.getPassword()) .put("schemas", List.of("TEST")) - .put("encryption", Jsons.jsonNode(ImmutableMap.builder() - .put("encryption_method", "unencrypted") - .build())) .build()); Database database = Databases.createOracleDatabase(config.get("username").asText(), diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceNneAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceNneAcceptanceTest.java index 6aacd5074925..3942f1f44b09 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceNneAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceNneAcceptanceTest.java @@ -1,93 +1,98 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.source.oracle; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Databases; import io.airbyte.db.jdbc.JdbcDatabase; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - import java.sql.SQLException; import java.util.List; import java.util.stream.Collectors; +import org.junit.jupiter.api.Test; + +public class OracleSourceNneAcceptanceTest extends OracleSourceAcceptanceTest { + + @Test + public void testEncrytion() throws SQLException { + final JsonNode clone = Jsons.clone(getConfig()); + ((ObjectNode) clone).put("encryption", Jsons.jsonNode(ImmutableMap.builder() + .put("encryption_method", "client_nne") + .put("encryption_algorithm", "3DES168") + .build())); + + String algorithm = clone.get("encryption") + .get("encryption_algorithm").asText(); + + JdbcDatabase database = Databases.createJdbcDatabase(clone.get("username").asText(), + clone.get("password").asText(), + String.format("jdbc:oracle:thin:@//%s:%s/%s", + clone.get("host").asText(), + clone.get("port").asText(), + clone.get("sid").asText()), + "oracle.jdbc.driver.OracleDriver", + "oracle.net.encryption_client=REQUIRED;" + + "oracle.net.encryption_types_client=( " + + algorithm + " )"); + + String network_service_banner = "select network_service_banner from v$session_connect_info where sid in (select distinct sid from v$mystat)"; + List collect = database.query(network_service_banner).collect(Collectors.toList()); + + assertTrue(collect.get(2).get("NETWORK_SERVICE_BANNER").asText() + .contains("Oracle Advanced Security: " + algorithm + " encryption")); + } + + @Test + public void testNoneEncrytion() throws SQLException { + + JdbcDatabase database = Databases.createJdbcDatabase(config.get("username").asText(), + config.get("password").asText(), + String.format("jdbc:oracle:thin:@//%s:%s/%s", + config.get("host").asText(), + config.get("port").asText(), + config.get("sid").asText()), + "oracle.jdbc.driver.OracleDriver"); + + String network_service_banner = "select network_service_banner from v$session_connect_info where sid in (select distinct sid from v$mystat)"; + List collect = database.query(network_service_banner).collect(Collectors.toList()); + + assertTrue(collect.get(1).get("NETWORK_SERVICE_BANNER").asText() + .contains("Oracle Advanced Security: encryption")); + } + + @Test + public void testCheckProtocol() throws SQLException { + final JsonNode clone = Jsons.clone(getConfig()); + ((ObjectNode) clone).put("encryption", Jsons.jsonNode(ImmutableMap.builder() + .put("encryption_method", "client_nne") + .put("encryption_algorithm", "AES256") + .build())); + + String algorithm = clone.get("encryption") + .get("encryption_algorithm").asText(); + + JdbcDatabase database = Databases.createJdbcDatabase(clone.get("username").asText(), + clone.get("password").asText(), + String.format("jdbc:oracle:thin:@//%s:%s/%s", + clone.get("host").asText(), + clone.get("port").asText(), + clone.get("sid").asText()), + "oracle.jdbc.driver.OracleDriver", + "oracle.net.encryption_client=REQUIRED;" + + "oracle.net.encryption_types_client=( " + + algorithm + " )"); + + String network_service_banner = "SELECT sys_context('USERENV', 'NETWORK_PROTOCOL') as network_protocol FROM dual"; + List collect = database.query(network_service_banner).collect(Collectors.toList()); + + assertEquals("tcp", collect.get(0).get("NETWORK_PROTOCOL").asText()); + } -public class OracleSourceNneAcceptanceTest extends OracleSourceAcceptanceTest{ - @Test - public void testEncrytion() throws SQLException { - final JsonNode clone = Jsons.clone(getConfig()); - ((ObjectNode) clone).put("encryption", Jsons.jsonNode(ImmutableMap.builder() - .put("encryption_method", "client_nne") - .put("encryption_algorithm", "3DES168") - .build())); - - String algorithm = clone.get("encryption") - .get("encryption_algorithm").asText(); - - JdbcDatabase database = Databases.createJdbcDatabase(clone.get("username").asText(), - clone.get("password").asText(), - String.format("jdbc:oracle:thin:@//%s:%s/%s", - clone.get("host").asText(), - clone.get("port").asText(), - clone.get("sid").asText()), - "oracle.jdbc.driver.OracleDriver", - "oracle.net.encryption_client=REQUIRED;" + - "oracle.net.encryption_types_client=( " - + algorithm+" )"); - - String network_service_banner = "select network_service_banner from v$session_connect_info where sid in (select distinct sid from v$mystat)"; - List collect = database.query(network_service_banner).collect(Collectors.toList()); - - assertTrue(collect.get(2).get("NETWORK_SERVICE_BANNER").asText() - .contains("Oracle Advanced Security: "+algorithm+" encryption")); - } - - @Test - public void testNoneEncrytion() throws SQLException { - - JdbcDatabase database = Databases.createJdbcDatabase(config.get("username").asText(), - config.get("password").asText(), - String.format("jdbc:oracle:thin:@//%s:%s/%s", - config.get("host").asText(), - config.get("port").asText(), - config.get("sid").asText()), - "oracle.jdbc.driver.OracleDriver"); - - String network_service_banner = "select network_service_banner from v$session_connect_info where sid in (select distinct sid from v$mystat)"; - List collect = database.query(network_service_banner).collect(Collectors.toList()); - - assertTrue(collect.get(1).get("NETWORK_SERVICE_BANNER").asText() - .contains("Oracle Advanced Security: encryption")); - } - - @Test - public void testCheckProtocol() throws SQLException { - final JsonNode clone = Jsons.clone(getConfig()); - ((ObjectNode) clone).put("encryption", Jsons.jsonNode(ImmutableMap.builder() - .put("encryption_method", "client_nne") - .put("encryption_algorithm", "AES256") - .build())); - - String algorithm = clone.get("encryption") - .get("encryption_algorithm").asText(); - - JdbcDatabase database = Databases.createJdbcDatabase(clone.get("username").asText(), - clone.get("password").asText(), - String.format("jdbc:oracle:thin:@//%s:%s/%s", - clone.get("host").asText(), - clone.get("port").asText(), - clone.get("sid").asText()), - "oracle.jdbc.driver.OracleDriver", - "oracle.net.encryption_client=REQUIRED;" + - "oracle.net.encryption_types_client=( " - + algorithm+" )"); - - String network_service_banner = "SELECT sys_context('USERENV', 'NETWORK_PROTOCOL') as network_protocol FROM dual"; - List collect = database.query(network_service_banner).collect(Collectors.toList()); - - assertEquals("tcp", collect.get(0).get("NETWORK_PROTOCOL").asText()); - } } diff --git a/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleJdbcSourceAcceptanceTest.java index 234cdbf36618..4e2e425fbd9c 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleJdbcSourceAcceptanceTest.java @@ -75,9 +75,6 @@ public void setup() throws Exception { .put("username", ORACLE_DB.getUsername()) .put("password", ORACLE_DB.getPassword()) .put("schemas", List.of(SCHEMA_NAME, SCHEMA_NAME2)) - .put("encryption", Jsons.jsonNode(ImmutableMap.builder() - .put("encryption_method", "unencrypted") - .build())) .build()); // Because Oracle doesn't let me create database easily I need to clean up diff --git a/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleSourceTest.java b/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleSourceTest.java index ba3fea55d716..5971c6e376de 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleSourceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleSourceTest.java @@ -67,9 +67,6 @@ void setup() throws Exception { .put("username", ORACLE_DB.getUsername()) .put("password", ORACLE_DB.getPassword()) .put("schemas", List.of("TEST")) - .put("encryption", Jsons.jsonNode(ImmutableMap.builder() - .put("encryption_method", "unencrypted") - .build())) .build()); JdbcDatabase database = Databases.createJdbcDatabase(config.get("username").asText(), From 9bb5d42022d2bbf3114ae641aaa456f5a95badad Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Thu, 7 Oct 2021 17:28:10 +0300 Subject: [PATCH 08/10] update timeout to 30 seconds --- .../io/airbyte/integrations/source/oracle/OracleSource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java b/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java index 2f0480f0329e..69e99f5b5c7c 100644 --- a/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java +++ b/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java @@ -123,7 +123,7 @@ private static void convertAndImportCertificate(String certificate) throws IOExc private static void runProcess(String cmd, Runtime run) throws IOException, InterruptedException { Process pr = run.exec(cmd); - if (!pr.waitFor(10, TimeUnit.SECONDS)) { + if (!pr.waitFor(30, TimeUnit.SECONDS)) { pr.destroy(); throw new RuntimeException("Timeout while executing: " + cmd); } ; From 9c52207263cdb3c4b7b194d556dddecc80353afa Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Fri, 8 Oct 2021 10:42:51 +0300 Subject: [PATCH 09/10] bump version to 0.3.6 --- .../b39a7370-74c3-45a6-ac3a-380d48520a83.json | 2 +- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-integrations/connectors/source-oracle/Dockerfile | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b39a7370-74c3-45a6-ac3a-380d48520a83.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b39a7370-74c3-45a6-ac3a-380d48520a83.json index 46f93f2d2a0d..d9fe2b09bff3 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b39a7370-74c3-45a6-ac3a-380d48520a83.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b39a7370-74c3-45a6-ac3a-380d48520a83.json @@ -2,6 +2,6 @@ "sourceDefinitionId": "b39a7370-74c3-45a6-ac3a-380d48520a83", "name": "Oracle DB", "dockerRepository": "airbyte/source-oracle", - "dockerImageTag": "0.3.5", + "dockerImageTag": "0.3.6", "documentationUrl": "https://docs.airbyte.io/integrations/sources/oracle" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 4ccee03705d0..b7b50b351755 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -326,7 +326,7 @@ - sourceDefinitionId: b39a7370-74c3-45a6-ac3a-380d48520a83 name: Oracle DB dockerRepository: airbyte/source-oracle - dockerImageTag: 0.3.5 + dockerImageTag: 0.3.6 documentationUrl: https://docs.airbyte.io/integrations/sources/oracle sourceType: database - sourceDefinitionId: c8630570-086d-4a40-99ae-ea5b18673071 diff --git a/airbyte-integrations/connectors/source-oracle/Dockerfile b/airbyte-integrations/connectors/source-oracle/Dockerfile index fab58afba1f4..d2382fadbf9a 100644 --- a/airbyte-integrations/connectors/source-oracle/Dockerfile +++ b/airbyte-integrations/connectors/source-oracle/Dockerfile @@ -6,7 +6,8 @@ ENV APPLICATION source-oracle ENV TZ UTC COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar + RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.3.5 +LABEL io.airbyte.version=0.3.6 LABEL io.airbyte.name=airbyte/source-oracle From 539abb1de8328bf5f27217105180f79745af189f Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Fri, 8 Oct 2021 10:52:34 +0300 Subject: [PATCH 10/10] bump version to 0.3.7 --- .../b39a7370-74c3-45a6-ac3a-380d48520a83.json | 2 +- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-integrations/connectors/source-oracle/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b39a7370-74c3-45a6-ac3a-380d48520a83.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b39a7370-74c3-45a6-ac3a-380d48520a83.json index d9fe2b09bff3..aa2f009fdd11 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b39a7370-74c3-45a6-ac3a-380d48520a83.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b39a7370-74c3-45a6-ac3a-380d48520a83.json @@ -2,6 +2,6 @@ "sourceDefinitionId": "b39a7370-74c3-45a6-ac3a-380d48520a83", "name": "Oracle DB", "dockerRepository": "airbyte/source-oracle", - "dockerImageTag": "0.3.6", + "dockerImageTag": "0.3.7", "documentationUrl": "https://docs.airbyte.io/integrations/sources/oracle" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index f1f3dc90ff60..61324d7fade7 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -326,7 +326,7 @@ - sourceDefinitionId: b39a7370-74c3-45a6-ac3a-380d48520a83 name: Oracle DB dockerRepository: airbyte/source-oracle - dockerImageTag: 0.3.6 + dockerImageTag: 0.3.7 documentationUrl: https://docs.airbyte.io/integrations/sources/oracle sourceType: database - sourceDefinitionId: c8630570-086d-4a40-99ae-ea5b18673071 diff --git a/airbyte-integrations/connectors/source-oracle/Dockerfile b/airbyte-integrations/connectors/source-oracle/Dockerfile index d2382fadbf9a..681a7aad2beb 100644 --- a/airbyte-integrations/connectors/source-oracle/Dockerfile +++ b/airbyte-integrations/connectors/source-oracle/Dockerfile @@ -9,5 +9,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.3.6 +LABEL io.airbyte.version=0.3.7 LABEL io.airbyte.name=airbyte/source-oracle