From ae5fe4ba4d8461d4f3d62084d5f80dae889586ed Mon Sep 17 00:00:00 2001 From: ievgeniit Date: Wed, 19 Oct 2022 20:34:47 +0200 Subject: [PATCH 1/7] [11356] Destination-elasticsearch: added custom sertificate support --- ...trictEncryptDestinationAcceptanceTest.java | 35 +++++++++++++++-- .../src/test/resources/expected_spec.json | 5 +++ .../elasticsearch/ConnectorConfiguration.java | 38 +++++++++++++++++-- .../ElasticsearchConnection.java | 13 ++++++- .../destination/elasticsearch/SslUtils.java | 30 +++++++++++++++ .../src/main/resources/spec.json | 5 +++ ...lasticsearchDestinationAcceptanceTest.java | 7 ++-- deps.toml | 3 +- .../destinations/elasticsearch.md | 8 ++++ 9 files changed, 132 insertions(+), 12 deletions(-) create mode 100644 airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/SslUtils.java diff --git a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestinationAcceptanceTest.java index 31232025fcbe..0bfbf914551a 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestinationAcceptanceTest.java @@ -4,30 +4,43 @@ package io.airbyte.integrations.destination.elasticsearch; +import static org.junit.jupiter.api.Assertions.assertEquals; + import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; +import io.airbyte.config.StandardCheckConnectionOutput.Status; import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.time.Duration; import java.util.List; import java.util.Map; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import org.testcontainers.elasticsearch.ElasticsearchContainer; public class ElasticsearchStrictEncryptDestinationAcceptanceTest extends DestinationAcceptanceTest { - private final ObjectMapper mapper = new ObjectMapper(); private static ElasticsearchContainer container; + private static final String IMAGE_NAME = "docker.elastic.co/elasticsearch/elasticsearch:8.3.3"; + private final ObjectMapper mapper = new ObjectMapper(); @BeforeAll public static void beforeAll() { - container = new ElasticsearchContainer("docker.elastic.co/elasticsearch/elasticsearch:7.15.1") - .withPassword("MagicWord"); + container = new ElasticsearchContainer(IMAGE_NAME) + .withEnv("discovery.type", "single-node") + .withEnv("network.host", "0.0.0.0") + .withEnv("logger.org.elasticsearch", "INFO") + .withEnv("ingest.geoip.downloader.enabled", "false") + .withExposedPorts(9200) + .withPassword("s3cret"); container.start(); } @@ -81,6 +94,22 @@ protected TestDataComparator getTestDataComparator() { @Override protected JsonNode getConfig() { + final JsonNode authConfig = Jsons.jsonNode(Map.of( + "method", "basic", + "username", "elastic", + "password", "s3cret")); + + return Jsons.jsonNode(ImmutableMap.builder() + .put("endpoint", String.format("https://%s:%s", container.getHost(), container.getMappedPort(9200))) + .put("authenticationMethod", authConfig) + .put("certAsBytes", container.copyFileFromContainer( + "/usr/share/elasticsearch/config/certs/http_ca.crt", + InputStream::readAllBytes)) + .build()); + } + + protected JsonNode getUnsecureConfig() { + final JsonNode authConfig = Jsons.jsonNode(Map.of( "method", "basic", "username", "elastic", diff --git a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test/resources/expected_spec.json index a951230fe95a..2e99a7f1ff22 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test/resources/expected_spec.json @@ -23,6 +23,11 @@ "description": "If a primary key identifier is defined in the source, an upsert will be performed using the primary key value as the elasticsearch doc id. Does not support composite primary keys.", "default": true }, + "certAsBytes": { + "title": "CA certificate (Base64 encoded)", + "type": "string", + "description": "CA certificate for Elasticsearch server" + }, "authenticationMethod": { "title": "Authentication Method", "type": "object", diff --git a/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/ConnectorConfiguration.java b/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/ConnectorConfiguration.java index 27c2c91eb9fe..83bc470bd434 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/ConnectorConfiguration.java +++ b/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/ConnectorConfiguration.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.Arrays; import java.util.Objects; @JsonIgnoreProperties(ignoreUnknown = true) @@ -14,6 +15,7 @@ public class ConnectorConfiguration { private String endpoint; private boolean upsert; + private byte[] certAsBytes; private AuthenticationMethod authenticationMethod = new AuthenticationMethod(); public ConnectorConfiguration() {} @@ -34,6 +36,15 @@ public AuthenticationMethod getAuthenticationMethod() { return this.authenticationMethod; } + public byte[] getCertAsBytes() { + return certAsBytes; + } + + public ConnectorConfiguration setCertAsBytes(byte[] certAsBytes) { + this.certAsBytes = certAsBytes; + return this; + } + public void setEndpoint(String endpoint) { this.endpoint = endpoint; } @@ -48,17 +59,35 @@ public void setAuthenticationMethod(AuthenticationMethod authenticationMethod) { @Override public boolean equals(Object o) { - if (this == o) + if (this == o) { return true; - if (o == null || getClass() != o.getClass()) + } + if (o == null || getClass() != o.getClass()) { return false; + } + ConnectorConfiguration that = (ConnectorConfiguration) o; - return upsert == that.upsert && Objects.equals(endpoint, that.endpoint) && Objects.equals(authenticationMethod, that.authenticationMethod); + + if (upsert != that.upsert) { + return false; + } + if (endpoint != null ? !endpoint.equals(that.endpoint) : that.endpoint != null) { + return false; + } + if (!Arrays.equals(certAsBytes, that.certAsBytes)) { + return false; + } + return authenticationMethod != null ? authenticationMethod.equals(that.authenticationMethod) + : that.authenticationMethod == null; } @Override public int hashCode() { - return Objects.hash(endpoint, upsert, authenticationMethod); + int result = endpoint != null ? endpoint.hashCode() : 0; + result = 31 * result + (upsert ? 1 : 0); + result = 31 * result + Arrays.hashCode(certAsBytes); + result = 31 * result + (authenticationMethod != null ? authenticationMethod.hashCode() : 0); + return result; } @Override @@ -66,6 +95,7 @@ public String toString() { return "ConnectorConfiguration{" + "endpoint='" + endpoint + '\'' + ", upsert=" + upsert + + ", certAsBytes=" + Arrays.toString(certAsBytes) + ", authenticationMethod=" + authenticationMethod + '}'; } diff --git a/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchConnection.java b/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchConnection.java index 100fb60a03c0..9f6d55c13eb4 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchConnection.java +++ b/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchConnection.java @@ -28,6 +28,7 @@ import org.apache.http.message.BasicHeader; import org.elasticsearch.client.Node; import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,7 +57,17 @@ public ElasticsearchConnection(ConnectorConfiguration config) { // Create the low-level client httpHost = HttpHost.create(config.getEndpoint()); - restClient = RestClient.builder(httpHost) + final RestClientBuilder builder = RestClient.builder(httpHost); + + // Set custom user's certificate if provided + if (config.getCertAsBytes() != null && config.getCertAsBytes().length != 0){ + builder.setHttpClientConfigCallback(clientBuilder -> { + clientBuilder.setSSLContext(SslUtils.createContextFromCaCert(config.getCertAsBytes())); + return clientBuilder; + }); + } + + restClient = builder .setDefaultHeaders(configureHeaders(config)) .setFailureListener(new FailureListener()) .build(); diff --git a/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/SslUtils.java b/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/SslUtils.java new file mode 100644 index 000000000000..2f46303fd1b8 --- /dev/null +++ b/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/SslUtils.java @@ -0,0 +1,30 @@ +package io.airbyte.integrations.destination.elasticsearch; + +import java.io.ByteArrayInputStream; +import java.security.KeyStore; +import java.security.cert.Certificate; +import java.security.cert.CertificateFactory; +import javax.net.ssl.SSLContext; +import org.apache.http.ssl.SSLContextBuilder; +import org.apache.http.ssl.SSLContexts; + +public class SslUtils { + + public static SSLContext createContextFromCaCert(byte[] certAsBytes) { + try { + CertificateFactory factory = CertificateFactory.getInstance("X.509"); + Certificate trustedCa = factory.generateCertificate( + new ByteArrayInputStream(certAsBytes) + ); + KeyStore trustStore = KeyStore.getInstance("pkcs12"); + trustStore.load(null, null); + trustStore.setCertificateEntry("ca", trustedCa); + SSLContextBuilder sslContextBuilder = + SSLContexts.custom().loadTrustMaterial(trustStore, null); + return sslContextBuilder.build(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + +} diff --git a/airbyte-integrations/connectors/destination-elasticsearch/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-elasticsearch/src/main/resources/spec.json index 546f962d51b2..f2236c21698f 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-elasticsearch/src/main/resources/spec.json @@ -23,6 +23,11 @@ "description": "If a primary key identifier is defined in the source, an upsert will be performed using the primary key value as the elasticsearch doc id. Does not support composite primary keys.", "default": true }, + "certAsBytes": { + "title": "CA certificate (Base64 encoded)", + "type": "string", + "description": "CA certificate for Elasticsearch server" + }, "authenticationMethod": { "title": "Authentication Method", "type": "object", diff --git a/airbyte-integrations/connectors/destination-elasticsearch/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-elasticsearch/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchDestinationAcceptanceTest.java index ca133e64a54b..9f38e58336b5 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-elasticsearch/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchDestinationAcceptanceTest.java @@ -20,6 +20,7 @@ public class ElasticsearchDestinationAcceptanceTest extends DestinationAcceptanceTest { + private static final String IMAGE_NAME = "docker.elastic.co/elasticsearch/elasticsearch:8.3.3"; private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchDestinationAcceptanceTest.class); private ObjectMapper mapper = new ObjectMapper(); @@ -27,14 +28,14 @@ public class ElasticsearchDestinationAcceptanceTest extends DestinationAcceptanc @BeforeAll public static void beforeAll() { - container = new ElasticsearchContainer("docker.elastic.co/elasticsearch/elasticsearch:7.15.1") - .withEnv("ES_JAVA_OPTS", "-Xms512m -Xms512m") + container = new ElasticsearchContainer(IMAGE_NAME) .withEnv("discovery.type", "single-node") .withEnv("network.host", "0.0.0.0") .withEnv("logger.org.elasticsearch", "INFO") .withEnv("ingest.geoip.downloader.enabled", "false") - .withEnv("xpack.security.enabled", "false") + .withPassword("s3cret") .withExposedPorts(9200) + .withEnv("xpack.security.enabled", "false") .withStartupTimeout(Duration.ofSeconds(60)); container.start(); } diff --git a/deps.toml b/deps.toml index 9caaad7cae1a..e2e71e28a72c 100644 --- a/deps.toml +++ b/deps.toml @@ -21,6 +21,7 @@ connectors-testcontainers-scylla = "1.16.2" connectors-testcontainers-tidb = "1.16.3" connectors-destination-testcontainers-clickhouse = "1.17.3" connectors-destination-testcontainers-oracle-xe = "1.17.3" +connectors-destination-testcontainers-elasticsearch = "1.17.3" connectors-source-testcontainers-clickhouse = "1.17.3" platform-testcontainers = "1.17.3" @@ -53,7 +54,7 @@ connectors-testcontainers = { module = "org.testcontainers:testcontainers", vers connectors-testcontainers-cassandra = { module = "org.testcontainers:cassandra", version.ref = "connectors-testcontainers-cassandra" } connectors-testcontainers-cockroachdb = { module = "org.testcontainers:cockroachdb", version.ref = "connectors-testcontainers" } connectors-testcontainers-db2 = { module = "org.testcontainers:db2", version.ref = "connectors-testcontainers" } -connectors-testcontainers-elasticsearch = { module = "org.testcontainers:elasticsearch", version.ref = "connectors-testcontainers" } +connectors-testcontainers-elasticsearch = { module = "org.testcontainers:elasticsearch", version.ref = "connectors-destination-testcontainers-elasticsearch" } connectors-testcontainers-jdbc = { module = "org.testcontainers:jdbc", version.ref = "connectors-testcontainers" } connectors-testcontainers-kafka = { module = "org.testcontainers:kafka", version.ref = "connectors-testcontainers" } connectors-testcontainers-mariadb = { module = "org.testcontainers:mariadb", version.ref = "connectors-testcontainers-mariadb" } diff --git a/docs/integrations/destinations/elasticsearch.md b/docs/integrations/destinations/elasticsearch.md index 2964b67cd403..50c6e2ba717a 100644 --- a/docs/integrations/destinations/elasticsearch.md +++ b/docs/integrations/destinations/elasticsearch.md @@ -57,10 +57,18 @@ The connector should be enhanced to support variable batch sizes. * Endpoint URL [ex. https://elasticsearch.savantly.net:9423] * Username [optional] (basic auth) * Password [optional] (basic auth) + * CA certificate (Base64 encoded) [optional] * Api key ID [optional] * Api key secret [optional] * If authentication is used, the user should have permission to create an index if it doesn't exist, and/or be able to `create` documents +### CA certificate +Ca certificate may be fetched from the Elasticsearch server from /usr/share/elasticsearch/config/certs/http_ca.crt +Fetching example from dockerized Elasticsearch: +`docker cp es01:/usr/share/elasticsearch/config/certs/http_ca.crt .` where es01 is a container's name. For more details please visit https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html + +Airbyte accepts only CA certificates encoded in Base64 format. You may use any convertors to encode it locally or +online service like https://www.base64encode.org. It's always better to use local encoders for sensitive data to prevent data leaks. ### Setup guide From e50914b9e2377d182ba40a03c86fa21090e61433 Mon Sep 17 00:00:00 2001 From: ievgeniit Date: Fri, 21 Oct 2022 14:10:02 +0200 Subject: [PATCH 2/7] updated PR as per comments in PR --- .../airbyte/db/util/SSLCertificateUtils.java | 20 +++ ...trictEncryptDestinationAcceptanceTest.java | 13 -- .../src/test/resources/expected_spec.json | 8 +- .../destination-elasticsearch/build.gradle | 1 + .../elasticsearch/ConnectorConfiguration.java | 153 +----------------- .../ElasticsearchConnection.java | 5 +- .../destination/elasticsearch/SslUtils.java | 30 ---- .../src/main/resources/spec.json | 8 +- .../destinations/elasticsearch.md | 7 +- 9 files changed, 42 insertions(+), 203 deletions(-) delete mode 100644 airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/SslUtils.java diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/util/SSLCertificateUtils.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/util/SSLCertificateUtils.java index 29a87ccc52c5..32062cabfe46 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/util/SSLCertificateUtils.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/util/SSLCertificateUtils.java @@ -28,6 +28,9 @@ import java.util.Objects; import java.util.Random; import java.util.concurrent.TimeUnit; +import javax.net.ssl.SSLContext; +import org.apache.http.ssl.SSLContextBuilder; +import org.apache.http.ssl.SSLContexts; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -160,4 +163,21 @@ public static URI keyStoreFromClientCertificate( return keyStoreFromClientCertificate(certString, keyString, keyStorePassword, FileSystems.getDefault(), directory); } + public static SSLContext createContextFromCaCert(String caCertificate) { + try { + CertificateFactory factory = CertificateFactory.getInstance(X509); + Certificate trustedCa = factory.generateCertificate( + new ByteArrayInputStream(caCertificate.getBytes(StandardCharsets.UTF_8)) + ); + KeyStore trustStore = KeyStore.getInstance(PKCS_12); + trustStore.load(null, null); + trustStore.setCertificateEntry("ca", trustedCa); + SSLContextBuilder sslContextBuilder = + SSLContexts.custom().loadTrustMaterial(trustStore, null); + return sslContextBuilder.build(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } diff --git a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestinationAcceptanceTest.java index 0bfbf914551a..05ab625a907f 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestinationAcceptanceTest.java @@ -108,19 +108,6 @@ protected JsonNode getConfig() { .build()); } - protected JsonNode getUnsecureConfig() { - - final JsonNode authConfig = Jsons.jsonNode(Map.of( - "method", "basic", - "username", "elastic", - "password", "MagicWord")); - - return Jsons.jsonNode(ImmutableMap.builder() - .put("endpoint", String.format("http://%s:%s", container.getHost(), container.getMappedPort(9200))) - .put("authenticationMethod", authConfig) - .build()); - } - @Override protected JsonNode getFailCheckConfig() { // should result in a failed connection check diff --git a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test/resources/expected_spec.json index 2e99a7f1ff22..7fd2ba80689d 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test/resources/expected_spec.json @@ -23,10 +23,12 @@ "description": "If a primary key identifier is defined in the source, an upsert will be performed using the primary key value as the elasticsearch doc id. Does not support composite primary keys.", "default": true }, - "certAsBytes": { - "title": "CA certificate (Base64 encoded)", + "ca_certificate": { "type": "string", - "description": "CA certificate for Elasticsearch server" + "title": "CA certificate", + "description": "CA certificate", + "airbyte_secret": true, + "multiline": true }, "authenticationMethod": { "title": "Authentication Method", diff --git a/airbyte-integrations/connectors/destination-elasticsearch/build.gradle b/airbyte-integrations/connectors/destination-elasticsearch/build.gradle index dc5b8e7c8788..469ad4991a16 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch/build.gradle +++ b/airbyte-integrations/connectors/destination-elasticsearch/build.gradle @@ -10,6 +10,7 @@ application { } dependencies { + implementation project(':airbyte-db:db-lib') implementation project(':airbyte-config:config-models') implementation project(':airbyte-protocol:protocol-models') implementation project(':airbyte-integrations:bases:base-java') diff --git a/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/ConnectorConfiguration.java b/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/ConnectorConfiguration.java index 83bc470bd434..a156e1011d80 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/ConnectorConfiguration.java +++ b/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/ConnectorConfiguration.java @@ -5,101 +5,28 @@ package io.airbyte.integrations.destination.elasticsearch; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.Arrays; import java.util.Objects; +import lombok.Data; @JsonIgnoreProperties(ignoreUnknown = true) +@Data public class ConnectorConfiguration { private String endpoint; private boolean upsert; - private byte[] certAsBytes; + @JsonProperty("ca_certificate") + private String caCertificate; private AuthenticationMethod authenticationMethod = new AuthenticationMethod(); - public ConnectorConfiguration() {} - public static ConnectorConfiguration fromJsonNode(JsonNode config) { return new ObjectMapper().convertValue(config, ConnectorConfiguration.class); } - public String getEndpoint() { - return this.endpoint; - } - - public boolean isUpsert() { - return this.upsert; - } - - public AuthenticationMethod getAuthenticationMethod() { - return this.authenticationMethod; - } - - public byte[] getCertAsBytes() { - return certAsBytes; - } - - public ConnectorConfiguration setCertAsBytes(byte[] certAsBytes) { - this.certAsBytes = certAsBytes; - return this; - } - - public void setEndpoint(String endpoint) { - this.endpoint = endpoint; - } - - public void setUpsert(boolean upsert) { - this.upsert = upsert; - } - - public void setAuthenticationMethod(AuthenticationMethod authenticationMethod) { - this.authenticationMethod = authenticationMethod; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - ConnectorConfiguration that = (ConnectorConfiguration) o; - - if (upsert != that.upsert) { - return false; - } - if (endpoint != null ? !endpoint.equals(that.endpoint) : that.endpoint != null) { - return false; - } - if (!Arrays.equals(certAsBytes, that.certAsBytes)) { - return false; - } - return authenticationMethod != null ? authenticationMethod.equals(that.authenticationMethod) - : that.authenticationMethod == null; - } - - @Override - public int hashCode() { - int result = endpoint != null ? endpoint.hashCode() : 0; - result = 31 * result + (upsert ? 1 : 0); - result = 31 * result + Arrays.hashCode(certAsBytes); - result = 31 * result + (authenticationMethod != null ? authenticationMethod.hashCode() : 0); - return result; - } - - @Override - public String toString() { - return "ConnectorConfiguration{" + - "endpoint='" + endpoint + '\'' + - ", upsert=" + upsert + - ", certAsBytes=" + Arrays.toString(certAsBytes) + - ", authenticationMethod=" + authenticationMethod + - '}'; - } + @Data static class AuthenticationMethod { private ElasticsearchAuthenticationMethod method = ElasticsearchAuthenticationMethod.none; @@ -108,46 +35,6 @@ static class AuthenticationMethod { private String apiKeyId; private String apiKeySecret; - public ElasticsearchAuthenticationMethod getMethod() { - return this.method; - } - - public String getUsername() { - return this.username; - } - - public String getPassword() { - return this.password; - } - - public String getApiKeyId() { - return this.apiKeyId; - } - - public String getApiKeySecret() { - return this.apiKeySecret; - } - - public void setMethod(ElasticsearchAuthenticationMethod method) { - this.method = method; - } - - public void setUsername(String username) { - this.username = username; - } - - public void setPassword(String password) { - this.password = password; - } - - public void setApiKeyId(String apiKeyId) { - this.apiKeyId = apiKeyId; - } - - public void setApiKeySecret(String apiKeySecret) { - this.apiKeySecret = apiKeySecret; - } - public boolean isValid() { return switch (this.method) { case none -> true; @@ -156,34 +43,6 @@ public boolean isValid() { }; } - @Override - public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - AuthenticationMethod that = (AuthenticationMethod) o; - return method == that.method && - Objects.equals(username, that.username) && - Objects.equals(password, that.password) && - Objects.equals(apiKeyId, that.apiKeyId) && - Objects.equals(apiKeySecret, that.apiKeySecret); - } - - @Override - public int hashCode() { - return Objects.hash(method, username, password, apiKeyId, apiKeySecret); - } - - @Override - public String toString() { - return "AuthenticationMethod{" + - "method=" + method + - ", username='" + username + '\'' + - ", apiKeyId='" + apiKeyId + '\'' + - '}'; - } - } } diff --git a/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchConnection.java b/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchConnection.java index 9f6d55c13eb4..ecb346207706 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchConnection.java +++ b/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchConnection.java @@ -17,6 +17,7 @@ import com.fasterxml.jackson.core.JsonPointer; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import io.airbyte.db.util.SSLCertificateUtils; import io.airbyte.protocol.models.AirbyteRecordMessage; import jakarta.json.JsonValue; import java.io.IOException; @@ -60,9 +61,9 @@ public ElasticsearchConnection(ConnectorConfiguration config) { final RestClientBuilder builder = RestClient.builder(httpHost); // Set custom user's certificate if provided - if (config.getCertAsBytes() != null && config.getCertAsBytes().length != 0){ + if (config.getCaCertificate() != null && !config.getCaCertificate().isEmpty()){ builder.setHttpClientConfigCallback(clientBuilder -> { - clientBuilder.setSSLContext(SslUtils.createContextFromCaCert(config.getCertAsBytes())); + clientBuilder.setSSLContext(SSLCertificateUtils.createContextFromCaCert(config.getCaCertificate())); return clientBuilder; }); } diff --git a/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/SslUtils.java b/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/SslUtils.java deleted file mode 100644 index 2f46303fd1b8..000000000000 --- a/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/SslUtils.java +++ /dev/null @@ -1,30 +0,0 @@ -package io.airbyte.integrations.destination.elasticsearch; - -import java.io.ByteArrayInputStream; -import java.security.KeyStore; -import java.security.cert.Certificate; -import java.security.cert.CertificateFactory; -import javax.net.ssl.SSLContext; -import org.apache.http.ssl.SSLContextBuilder; -import org.apache.http.ssl.SSLContexts; - -public class SslUtils { - - public static SSLContext createContextFromCaCert(byte[] certAsBytes) { - try { - CertificateFactory factory = CertificateFactory.getInstance("X.509"); - Certificate trustedCa = factory.generateCertificate( - new ByteArrayInputStream(certAsBytes) - ); - KeyStore trustStore = KeyStore.getInstance("pkcs12"); - trustStore.load(null, null); - trustStore.setCertificateEntry("ca", trustedCa); - SSLContextBuilder sslContextBuilder = - SSLContexts.custom().loadTrustMaterial(trustStore, null); - return sslContextBuilder.build(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - -} diff --git a/airbyte-integrations/connectors/destination-elasticsearch/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-elasticsearch/src/main/resources/spec.json index f2236c21698f..9375ab8ed5a2 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-elasticsearch/src/main/resources/spec.json @@ -23,10 +23,12 @@ "description": "If a primary key identifier is defined in the source, an upsert will be performed using the primary key value as the elasticsearch doc id. Does not support composite primary keys.", "default": true }, - "certAsBytes": { - "title": "CA certificate (Base64 encoded)", + "ca_certificate": { "type": "string", - "description": "CA certificate for Elasticsearch server" + "title": "CA certificate", + "description": "CA certificate", + "airbyte_secret": true, + "multiline": true }, "authenticationMethod": { "title": "Authentication Method", diff --git a/docs/integrations/destinations/elasticsearch.md b/docs/integrations/destinations/elasticsearch.md index 0eae990ac1a8..ba27bcaefc0f 100644 --- a/docs/integrations/destinations/elasticsearch.md +++ b/docs/integrations/destinations/elasticsearch.md @@ -57,7 +57,7 @@ The connector should be enhanced to support variable batch sizes. * Endpoint URL [ex. https://elasticsearch.savantly.net:9423] * Username [optional] (basic auth) * Password [optional] (basic auth) - * CA certificate (Base64 encoded) [optional] + * CA certificate [optional] * Api key ID [optional] * Api key secret [optional] * If authentication is used, the user should have permission to create an index if it doesn't exist, and/or be able to `create` documents @@ -66,10 +66,7 @@ The connector should be enhanced to support variable batch sizes. Ca certificate may be fetched from the Elasticsearch server from /usr/share/elasticsearch/config/certs/http_ca.crt Fetching example from dockerized Elasticsearch: `docker cp es01:/usr/share/elasticsearch/config/certs/http_ca.crt .` where es01 is a container's name. For more details please visit https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html - -Airbyte accepts only CA certificates encoded in Base64 format. You may use any convertors to encode it locally or -online service like https://www.base64encode.org. It's always better to use local encoders for sensitive data to prevent data leaks. - + ### Setup guide Enter the endpoint URL, select authentication method, and whether to use 'upsert' method when indexing new documents. From 1ee548eb161c955c0c8e28a1d981adc6fae42b2c Mon Sep 17 00:00:00 2001 From: ievgeniit Date: Fri, 21 Oct 2022 14:32:55 +0200 Subject: [PATCH 3/7] Fixed tests --- .../ElasticsearchStrictEncryptDestinationAcceptanceTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestinationAcceptanceTest.java index 05ab625a907f..68859c436a47 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestinationAcceptanceTest.java @@ -102,9 +102,9 @@ protected JsonNode getConfig() { return Jsons.jsonNode(ImmutableMap.builder() .put("endpoint", String.format("https://%s:%s", container.getHost(), container.getMappedPort(9200))) .put("authenticationMethod", authConfig) - .put("certAsBytes", container.copyFileFromContainer( + .put("ca_certificate", new String(container.copyFileFromContainer( "/usr/share/elasticsearch/config/certs/http_ca.crt", - InputStream::readAllBytes)) + InputStream::readAllBytes), StandardCharsets.UTF_8)) .build()); } From d68f1b487284ed8a457096cd51609c15d08ddfae Mon Sep 17 00:00:00 2001 From: ievgeniit Date: Sun, 23 Oct 2022 18:54:48 +0200 Subject: [PATCH 4/7] [16251] Destination-elasticsearch: enforced ssl connection --- .../build.gradle | 1 + ...ElasticsearchStrictEncryptDestination.java | 29 +++++++++++++++++++ ...trictEncryptDestinationAcceptanceTest.java | 19 ++++++++++++ 3 files changed, 49 insertions(+) diff --git a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/build.gradle index ffc4c797814a..77da0535c523 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/build.gradle @@ -33,6 +33,7 @@ dependencies { testImplementation libs.connectors.testcontainers.elasticsearch integrationTestJavaImplementation libs.connectors.testcontainers.elasticsearch + integrationTestJavaImplementation project(':airbyte-commons-worker') integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-elasticsearch') } diff --git a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestination.java b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestination.java index 44a7bb080e32..69a2e47578c4 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestination.java +++ b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestination.java @@ -4,12 +4,19 @@ package io.airbyte.integrations.destination.elasticsearch; +import static co.elastic.clients.elasticsearch.watcher.Input.HTTP; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.base.Destination; import io.airbyte.integrations.base.IntegrationRunner; import io.airbyte.integrations.base.spec_modification.SpecModifyingDestination; +import io.airbyte.protocol.models.AirbyteConnectionStatus; import io.airbyte.protocol.models.ConnectorSpecification; +import java.net.URL; +import java.util.Objects; import java.util.stream.IntStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -17,6 +24,7 @@ public class ElasticsearchStrictEncryptDestination extends SpecModifyingDestination implements Destination { private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchStrictEncryptDestination.class); + private final ObjectMapper mapper = new ObjectMapper(); public ElasticsearchStrictEncryptDestination() { super(new ElasticsearchDestination()); @@ -38,4 +46,25 @@ public ConnectorSpecification modifySpec(ConnectorSpecification originalSpec) th return spec; } + @Override + public AirbyteConnectionStatus check(JsonNode config) throws Exception { + + final ConnectorConfiguration configObject = convertConfig(config); + if (Objects.isNull(configObject.getEndpoint())) { + return new AirbyteConnectionStatus() + .withStatus(AirbyteConnectionStatus.Status.FAILED).withMessage("endpoint must not be empty"); + } + + if (new URL(configObject.getEndpoint()).getProtocol().equals(HTTP)) { + return new AirbyteConnectionStatus() + .withStatus(AirbyteConnectionStatus.Status.FAILED).withMessage("only https protocol is allowed"); + } + + return super.check(config); + } + + private ConnectorConfiguration convertConfig(JsonNode config) { + return mapper.convertValue(config, ConnectorConfiguration.class); + } + } diff --git a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestinationAcceptanceTest.java index 68859c436a47..e470ecea9347 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestinationAcceptanceTest.java @@ -108,6 +108,19 @@ protected JsonNode getConfig() { .build()); } + protected JsonNode getUnsecureConfig() { + + final JsonNode authConfig = Jsons.jsonNode(Map.of( + "method", "basic", + "username", "elastic", + "password", "MagicWord")); + + return Jsons.jsonNode(ImmutableMap.builder() + .put("endpoint", String.format("http://%s:%s", container.getHost(), container.getMappedPort(9200))) + .put("authenticationMethod", authConfig) + .build()); + } + @Override protected JsonNode getFailCheckConfig() { // should result in a failed connection check @@ -140,4 +153,10 @@ protected void tearDown(DestinationAcceptanceTest.TestDestinationEnv testEnv) { connection.allIndices().forEach(connection::deleteIndexIfPresent); } + @Test + public void testCheckConnectionInvalidHttpProtocol() throws Exception { + assertEquals(Status.FAILED, runCheck(getUnsecureConfig()).getStatus()); + } + + } From bb66ae3c9f461013817494b138bc5aef24c26204 Mon Sep 17 00:00:00 2001 From: ievgeniit Date: Wed, 26 Oct 2022 13:33:13 +0200 Subject: [PATCH 5/7] Minor refactor --- ...ElasticsearchStrictEncryptDestination.java | 8 +++-- ...trictEncryptDestinationAcceptanceTest.java | 36 +++++++++---------- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestination.java b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestination.java index 69a2e47578c4..e2341d77f982 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestination.java +++ b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestination.java @@ -25,6 +25,8 @@ public class ElasticsearchStrictEncryptDestination extends SpecModifyingDestinat private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchStrictEncryptDestination.class); private final ObjectMapper mapper = new ObjectMapper(); + private static final String NON_EMPTY_URL_ERR_MSG = "Server Endpoint is a required field"; + private static final String NON_SECURE_URL_ERR_MSG = "Server Endpoint requires HTTPS"; public ElasticsearchStrictEncryptDestination() { super(new ElasticsearchDestination()); @@ -52,12 +54,14 @@ public AirbyteConnectionStatus check(JsonNode config) throws Exception { final ConnectorConfiguration configObject = convertConfig(config); if (Objects.isNull(configObject.getEndpoint())) { return new AirbyteConnectionStatus() - .withStatus(AirbyteConnectionStatus.Status.FAILED).withMessage("endpoint must not be empty"); + .withStatus(AirbyteConnectionStatus.Status.FAILED) + .withMessage(NON_EMPTY_URL_ERR_MSG); } if (new URL(configObject.getEndpoint()).getProtocol().equals(HTTP)) { return new AirbyteConnectionStatus() - .withStatus(AirbyteConnectionStatus.Status.FAILED).withMessage("only https protocol is allowed"); + .withStatus(AirbyteConnectionStatus.Status.FAILED) + .withMessage(NON_SECURE_URL_ERR_MSG); } return super.check(config); diff --git a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestinationAcceptanceTest.java index 51ab94c4ae38..8bb93434fe4a 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestinationAcceptanceTest.java @@ -4,10 +4,13 @@ package io.airbyte.integrations.destination.elasticsearch; +import static org.junit.Assert.assertEquals; + import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; +import io.airbyte.config.StandardCheckConnectionOutput.Status; import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; @@ -18,6 +21,7 @@ import java.util.Map; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import org.testcontainers.elasticsearch.ElasticsearchContainer; public class ElasticsearchStrictEncryptDestinationAcceptanceTest extends DestinationAcceptanceTest { @@ -88,33 +92,28 @@ protected TestDataComparator getTestDataComparator() { @Override protected JsonNode getConfig() { - - final JsonNode authConfig = Jsons.jsonNode(Map.of( - "method", "basic", - "username", "elastic", - "password", "s3cret")); - return Jsons.jsonNode(ImmutableMap.builder() .put("endpoint", String.format("https://%s:%s", container.getHost(), container.getMappedPort(9200))) - .put("authenticationMethod", authConfig) + .put("authenticationMethod", getAuthConfig()) .put("ca_certificate", new String(container.copyFileFromContainer( "/usr/share/elasticsearch/config/certs/http_ca.crt", InputStream::readAllBytes), StandardCharsets.UTF_8)) .build()); } - protected JsonNode getUnsecureConfig() { - - final JsonNode authConfig = Jsons.jsonNode(Map.of( - "method", "basic", - "username", "elastic", - "password", "MagicWord")); + protected JsonNode getUnsecureConfig() { + return Jsons.jsonNode(ImmutableMap.builder() + .put("endpoint", String.format("http://%s:%s", container.getHost(), container.getMappedPort(9200))) + .put("authenticationMethod", getAuthConfig()) + .build()); + } - return Jsons.jsonNode(ImmutableMap.builder() - .put("endpoint", String.format("http://%s:%s", container.getHost(), container.getMappedPort(9200))) - .put("authenticationMethod", authConfig) - .build()); - } + protected JsonNode getAuthConfig() { + return Jsons.jsonNode(Map.of( + "method", "basic", + "username", "elastic", + "password", "s3cret")); + } @Override protected JsonNode getFailCheckConfig() { @@ -153,5 +152,4 @@ public void testCheckConnectionInvalidHttpProtocol() throws Exception { assertEquals(Status.FAILED, runCheck(getUnsecureConfig()).getStatus()); } - } From d2458911a49a84d75ed0a63cbe1276ce347b5a9e Mon Sep 17 00:00:00 2001 From: ievgeniit Date: Wed, 26 Oct 2022 21:50:41 +0200 Subject: [PATCH 6/7] Bumped version --- .../destination-elasticsearch-strict-encrypt/Dockerfile | 2 +- .../connectors/destination-elasticsearch/Dockerfile | 2 +- docs/integrations/destinations/elasticsearch.md | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/Dockerfile index 97bf1f52f693..a9e390d8feb9 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-elasticsearch-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.5 +LABEL io.airbyte.version=0.1.6 LABEL io.airbyte.name=airbyte/destination-elasticsearch-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-elasticsearch/Dockerfile b/airbyte-integrations/connectors/destination-elasticsearch/Dockerfile index f78e88860504..876c4adb2fdd 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch/Dockerfile +++ b/airbyte-integrations/connectors/destination-elasticsearch/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-elasticsearch COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.5 +LABEL io.airbyte.version=0.1.6 LABEL io.airbyte.name=airbyte/destination-elasticsearch diff --git a/docs/integrations/destinations/elasticsearch.md b/docs/integrations/destinations/elasticsearch.md index 720753d7576c..43cc9f677a62 100644 --- a/docs/integrations/destinations/elasticsearch.md +++ b/docs/integrations/destinations/elasticsearch.md @@ -94,6 +94,7 @@ Using this feature requires additional configuration, when creating the source. | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.6 | 2022-10-26 | [18341](https://github.com/airbytehq/airbyte/pull/18341) | enforce ssl connection on cloud | | 0.1.5 | 2022-10-24 | [18177](https://github.com/airbytehq/airbyte/pull/18177) | add custom CA certificate processing | | 0.1.4 | 2022-10-14 | [17805](https://github.com/airbytehq/airbyte/pull/17805) | add SSH Tunneling | | 0.1.3 | 2022-05-30 | [14640](https://github.com/airbytehq/airbyte/pull/14640) | Include lifecycle management | From 92763015e5e1e5885331821d93f95f2339f5c0d4 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Wed, 26 Oct 2022 20:08:30 +0000 Subject: [PATCH 7/7] auto-bump connector version --- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- .../init/src/main/resources/seed/destination_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 780c0dc5312f..2762a58e7df8 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -100,7 +100,7 @@ - destinationDefinitionId: 68f351a7-2745-4bef-ad7f-996b8e51bb8c name: ElasticSearch dockerRepository: airbyte/destination-elasticsearch - dockerImageTag: 0.1.5 + dockerImageTag: 0.1.6 documentationUrl: https://docs.airbyte.com/integrations/destinations/elasticsearch icon: elasticsearch.svg releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 1f71e6f53c4c..326cf2bc0921 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -1700,7 +1700,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-elasticsearch:0.1.5" +- dockerImage: "airbyte/destination-elasticsearch:0.1.6" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/elasticsearch" connectionSpecification: