From 0395b738675ef911c6ff63f65981d5320713ff52 Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Tue, 14 Sep 2021 00:40:48 -0300 Subject: [PATCH 1/5] correct spec json and data types in config --- .../kafka/KafkaDestinationConfig.java | 30 ++++++++-------- .../src/main/resources/spec.json | 36 +++++++++---------- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/airbyte-integrations/connectors/destination-kafka/src/main/java/io/airbyte/integrations/destination/kafka/KafkaDestinationConfig.java b/airbyte-integrations/connectors/destination-kafka/src/main/java/io/airbyte/integrations/destination/kafka/KafkaDestinationConfig.java index 2c86ad2fbc51..347d5f5027da 100644 --- a/airbyte-integrations/connectors/destination-kafka/src/main/java/io/airbyte/integrations/destination/kafka/KafkaDestinationConfig.java +++ b/airbyte-integrations/connectors/destination-kafka/src/main/java/io/airbyte/integrations/destination/kafka/KafkaDestinationConfig.java @@ -55,7 +55,7 @@ private KafkaDestinationConfig(String topicPattern, boolean sync, JsonNode confi public static KafkaDestinationConfig getKafkaDestinationConfig(JsonNode config) { return new KafkaDestinationConfig( config.get("topic_pattern").asText(), - config.has("sync_producer") && config.get("sync_producer").booleanValue(), + config.has("sync_producer") && config.get("sync_producer").asBoolean(), config); } @@ -66,25 +66,25 @@ private KafkaProducer buildKafkaProducer(JsonNode config) { .put(ProducerConfig.CLIENT_ID_CONFIG, config.has("client_id") ? config.get("client_id").asText() : null) .put(ProducerConfig.ACKS_CONFIG, config.get("acks").asText()) - .put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, config.get("enable_idempotence").booleanValue()) + .put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, config.get("enable_idempotence").asBoolean()) .put(ProducerConfig.COMPRESSION_TYPE_CONFIG, config.get("compression_type").asText()) - .put(ProducerConfig.BATCH_SIZE_CONFIG, config.get("batch_size").intValue()) - .put(ProducerConfig.LINGER_MS_CONFIG, config.get("linger_ms").longValue()) + .put(ProducerConfig.BATCH_SIZE_CONFIG, config.get("batch_size").asInt()) + .put(ProducerConfig.LINGER_MS_CONFIG, config.get("linger_ms").asLong()) .put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, - config.get("max_in_flight_requests_per_connection").intValue()) + config.get("max_in_flight_requests_per_connection").asInt()) .put(ProducerConfig.CLIENT_DNS_LOOKUP_CONFIG, config.get("client_dns_lookup").asText()) - .put(ProducerConfig.BUFFER_MEMORY_CONFIG, config.get("buffer_memory").longValue()) - .put(ProducerConfig.MAX_REQUEST_SIZE_CONFIG, config.get("max_request_size").intValue()) - .put(ProducerConfig.RETRIES_CONFIG, config.get("retries").intValue()) + .put(ProducerConfig.BUFFER_MEMORY_CONFIG, config.get("buffer_memory").asLong()) + .put(ProducerConfig.MAX_REQUEST_SIZE_CONFIG, config.get("max_request_size").asInt()) + .put(ProducerConfig.RETRIES_CONFIG, config.get("retries").asInt()) .put(ProducerConfig.SOCKET_CONNECTION_SETUP_TIMEOUT_MS_CONFIG, - config.get("socket_connection_setup_timeout_ms").longValue()) + config.get("socket_connection_setup_timeout_ms").asLong()) .put(ProducerConfig.SOCKET_CONNECTION_SETUP_TIMEOUT_MAX_MS_CONFIG, - config.get("socket_connection_setup_timeout_max_ms").longValue()) - .put(ProducerConfig.MAX_BLOCK_MS_CONFIG, config.get("max_block_ms").longValue()) - .put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, config.get("request_timeout_ms").intValue()) - .put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, config.get("delivery_timeout_ms").intValue()) - .put(ProducerConfig.SEND_BUFFER_CONFIG, config.get("send_buffer_bytes").intValue()) - .put(ProducerConfig.RECEIVE_BUFFER_CONFIG, config.get("receive_buffer_bytes").intValue()) + config.get("socket_connection_setup_timeout_max_ms").asLong()) + .put(ProducerConfig.MAX_BLOCK_MS_CONFIG, config.get("max_block_ms").asLong()) + .put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, config.get("request_timeout_ms").asInt()) + .put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, config.get("delivery_timeout_ms").asInt()) + .put(ProducerConfig.SEND_BUFFER_CONFIG, config.get("send_buffer_bytes").asInt()) + .put(ProducerConfig.RECEIVE_BUFFER_CONFIG, config.get("receive_buffer_bytes").asInt()) .put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()) .put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, JsonSerializer.class.getName()) .build(); diff --git a/airbyte-integrations/connectors/destination-kafka/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-kafka/src/main/resources/spec.json index 0ffa9eefbaec..2eb35a5b22cd 100644 --- a/airbyte-integrations/connectors/destination-kafka/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-kafka/src/main/resources/spec.json @@ -162,19 +162,19 @@ "title": "Batch size", "description": "The producer will attempt to batch records together into fewer requests whenever multiple records are being sent to the same partition.", "type": "integer", - "default": 16384 + "examples": [16384] }, "linger_ms": { "title": "Linger ms", "description": "The producer groups together any records that arrive in between request transmissions into a single batched request.", - "type": "number", - "default": 0 + "type": "string", + "examples": [0] }, "max_in_flight_requests_per_connection": { "title": "Max in flight requests per connection", "description": "The maximum number of unacknowledged requests the client will send on a single connection before blocking.", "type": "integer", - "default": 5 + "examples": [5] }, "client_dns_lookup": { "title": "Client DNS lookup", @@ -191,62 +191,62 @@ "buffer_memory": { "title": "Buffer memory", "description": "The total bytes of memory the producer can use to buffer records waiting to be sent to the server.", - "type": "number", - "default": 33554432 + "type": "string", + "examples": 33554432 }, "max_request_size": { "title": "Max request size", "description": "The maximum size of a request in bytes.", "type": "integer", - "default": 1048576 + "examples": [1048576] }, "retries": { "title": "Retries", "description": "Setting a value greater than zero will cause the client to resend any record whose send fails with a potentially transient error.", "type": "integer", - "default": 2147483647 + "examples": [2147483647] }, "socket_connection_setup_timeout_ms": { "title": "Socket connection setup timeout", "description": "The amount of time the client will wait for the socket connection to be established.", - "type": "number", - "default": 10000 + "type": "string", + "examples": [10000] }, "socket_connection_setup_timeout_max_ms": { "title": "Socket connection setup max timeout", "description": "The maximum amount of time the client will wait for the socket connection to be established. The connection setup timeout will increase exponentially for each consecutive connection failure up to this maximum.", - "type": "number", - "default": 30000 + "type": "string", + "examples": [30000] }, "max_block_ms": { "title": "Max block ms", "description": "The configuration controls how long the KafkaProducer's send(), partitionsFor(), initTransactions(), sendOffsetsToTransaction(), commitTransaction() and abortTransaction() methods will block.", - "type": "number", - "default": 60000 + "type": "string", + "examples": [60000] }, "request_timeout_ms": { "title": "Request timeout", "description": "The configuration controls the maximum amount of time the client will wait for the response of a request. If the response is not received before the timeout elapses the client will resend the request if necessary or fail the request if retries are exhausted.", "type": "integer", - "default": 30000 + "examples": [30000] }, "delivery_timeout_ms": { "title": "Delivery timeout", "description": "An upper bound on the time to report success or failure after a call to 'send()' returns.", "type": "integer", - "default": 120000 + "examples": [120000] }, "send_buffer_bytes": { "title": "Send buffer bytes", "description": "The size of the TCP send buffer (SO_SNDBUF) to use when sending data. If the value is -1, the OS default will be used.", "type": "integer", - "default": 131072 + "examples": [131072] }, "receive_buffer_bytes": { "title": "Receive buffer bytes", "description": "The size of the TCP receive buffer (SO_RCVBUF) to use when reading data. If the value is -1, the OS default will be used.", "type": "integer", - "default": 32768 + "examples": [32768] } } } From d7c5acad9e1d50254fe158a7732f6936cff8da5d Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Tue, 14 Sep 2021 11:45:09 -0300 Subject: [PATCH 2/5] bump version --- .../9f760101-60ae-462f-9ee6-b7a9dafd454d.json | 2 +- .../seed/destination_definitions.yaml | 2 +- .../connectors/destination-kafka/Dockerfile | 2 +- .../s3/S3DestinationAcceptanceTest.java | 4 +- .../credentials/credentials.json | 2 +- .../integration_tests/catalog.json | 38 ++------- .../integration_tests/configured_catalog.json | 78 +++++-------------- .../integration_tests/invalid_config.json | 5 +- .../sample_files/sample_config.json | 5 +- .../schemas/search_analytics_all_fields.json | 55 +++---------- .../schemas/search_analytics_by_country.json | 40 ++-------- .../schemas/search_analytics_by_date.json | 35 ++------- .../schemas/search_analytics_by_device.json | 40 ++-------- .../schemas/search_analytics_by_page.json | 40 ++-------- .../schemas/search_analytics_by_query.json | 40 ++-------- .../schemas/sitemaps.json | 57 +++----------- .../schemas/sites.json | 10 +-- .../source_google_search_console/spec.json | 24 ++---- docs/integrations/destinations/kafka.md | 3 +- 19 files changed, 105 insertions(+), 377 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/9f760101-60ae-462f-9ee6-b7a9dafd454d.json b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/9f760101-60ae-462f-9ee6-b7a9dafd454d.json index 06ca1971d456..657a83d19b6d 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/9f760101-60ae-462f-9ee6-b7a9dafd454d.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/9f760101-60ae-462f-9ee6-b7a9dafd454d.json @@ -2,6 +2,6 @@ "destinationDefinitionId": "9f760101-60ae-462f-9ee6-b7a9dafd454d", "name": "Kafka", "dockerRepository": "airbyte/destination-kafka", - "dockerImageTag": "0.1.1", + "dockerImageTag": "0.1.2", "documentationUrl": "https://docs.airbyte.io/integrations/destinations/kafka" } diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index b59309044236..62a529de43c2 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -78,7 +78,7 @@ - destinationDefinitionId: 9f760101-60ae-462f-9ee6-b7a9dafd454d name: Kafka dockerRepository: airbyte/destination-kafka - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.2 documentationUrl: https://docs.airbyte.io/integrations/destinations/kafka - destinationDefinitionId: 8ccd8909-4e99-4141-b48d-4984b70b2d89 name: DynamoDB diff --git a/airbyte-integrations/connectors/destination-kafka/Dockerfile b/airbyte-integrations/connectors/destination-kafka/Dockerfile index 4d8b2c0ee5a0..de08194ffbeb 100644 --- a/airbyte-integrations/connectors/destination-kafka/Dockerfile +++ b/airbyte-integrations/connectors/destination-kafka/Dockerfile @@ -8,5 +8,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/destination-kafka diff --git a/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3DestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3DestinationAcceptanceTest.java index 9f0bae2af0fa..20a92927dfa6 100644 --- a/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3DestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3DestinationAcceptanceTest.java @@ -24,6 +24,8 @@ package io.airbyte.integrations.destination.s3; +import static io.airbyte.integrations.destination.s3.S3DestinationConstants.NAME_TRANSFORMER; + import com.amazonaws.ClientConfiguration; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSStaticCredentialsProvider; @@ -53,8 +55,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static io.airbyte.integrations.destination.s3.S3DestinationConstants.NAME_TRANSFORMER; - /** * When adding a new S3 destination acceptance test, extend this class and do the following: *
  • Implement {@link #getFormatConfig} that returns a {@link S3FormatConfig}
    • diff --git a/airbyte-integrations/connectors/source-google-search-console/credentials/credentials.json b/airbyte-integrations/connectors/source-google-search-console/credentials/credentials.json index c24ccd87710b..6b0d67214193 100644 --- a/airbyte-integrations/connectors/source-google-search-console/credentials/credentials.json +++ b/airbyte-integrations/connectors/source-google-search-console/credentials/credentials.json @@ -2,4 +2,4 @@ "client_id": "YOUR_CLIENT_ID", "client_secret": "YOUR_CLIENT_SECRET", "redirect_uri": "YOUR_REDIRECTED_URI" -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-google-search-console/integration_tests/catalog.json b/airbyte-integrations/connectors/source-google-search-console/integration_tests/catalog.json index ada99aded37e..71c9382b7384 100755 --- a/airbyte-integrations/connectors/source-google-search-console/integration_tests/catalog.json +++ b/airbyte-integrations/connectors/source-google-search-console/integration_tests/catalog.json @@ -4,9 +4,7 @@ "stream": { "name": "sites", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ] + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -15,9 +13,7 @@ "stream": { "name": "sitemaps", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ] + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -26,10 +22,7 @@ "stream": { "name": "search_analytics_by_country", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -38,10 +31,7 @@ "stream": { "name": "search_analytics_by_date", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -50,10 +40,7 @@ "stream": { "name": "search_analytics_by_device", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -62,10 +49,7 @@ "stream": { "name": "search_analytics_by_page", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -74,10 +58,7 @@ "stream": { "name": "search_analytics_by_query", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -86,10 +67,7 @@ "stream": { "name": "search_analytics_all_fields", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" diff --git a/airbyte-integrations/connectors/source-google-search-console/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-google-search-console/integration_tests/configured_catalog.json index 501f4f2c844f..4484f24c8e63 100755 --- a/airbyte-integrations/connectors/source-google-search-console/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-google-search-console/integration_tests/configured_catalog.json @@ -4,114 +4,72 @@ "stream": { "name": "search_analytics_by_date", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [ - "date" - ] + "default_cursor_field": ["date"] }, "sync_mode": "incremental", - "cursor_field": [ - "date" - ], + "cursor_field": ["date"], "destination_sync_mode": "append" }, { "stream": { "name": "search_analytics_by_country", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [ - "date" - ] + "default_cursor_field": ["date"] }, "sync_mode": "incremental", - "cursor_field": [ - "date" - ], + "cursor_field": ["date"], "destination_sync_mode": "append" }, { "stream": { "name": "search_analytics_by_device", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [ - "date" - ] + "default_cursor_field": ["date"] }, "sync_mode": "incremental", - "cursor_field": [ - "date" - ], + "cursor_field": ["date"], "destination_sync_mode": "append" }, { "stream": { "name": "search_analytics_by_page", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [ - "date" - ] + "default_cursor_field": ["date"] }, "sync_mode": "incremental", - "cursor_field": [ - "date" - ], + "cursor_field": ["date"], "destination_sync_mode": "append" }, { "stream": { "name": "search_analytics_by_query", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [ - "date" - ] + "default_cursor_field": ["date"] }, "sync_mode": "incremental", - "cursor_field": [ - "date" - ], + "cursor_field": ["date"], "destination_sync_mode": "append" }, { "stream": { "name": "search_analytics_all_fields", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [ - "date" - ] + "default_cursor_field": ["date"] }, "sync_mode": "incremental", - "cursor_field": [ - "date" - ], + "cursor_field": ["date"], "destination_sync_mode": "append" } ] diff --git a/airbyte-integrations/connectors/source-google-search-console/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-google-search-console/integration_tests/invalid_config.json index 115161f4292a..d7f3ac088bc2 100755 --- a/airbyte-integrations/connectors/source-google-search-console/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-google-search-console/integration_tests/invalid_config.json @@ -1,8 +1,5 @@ { - "site_urls": [ - "https://example1.com", - "https://example2.com" - ], + "site_urls": ["https://example1.com", "https://example2.com"], "start_date": "2021-05-01", "end_date": "2021-05-31", "authorization": { diff --git a/airbyte-integrations/connectors/source-google-search-console/sample_files/sample_config.json b/airbyte-integrations/connectors/source-google-search-console/sample_files/sample_config.json index 21730b0fc88f..689fb05a2e85 100755 --- a/airbyte-integrations/connectors/source-google-search-console/sample_files/sample_config.json +++ b/airbyte-integrations/connectors/source-google-search-console/sample_files/sample_config.json @@ -1,8 +1,5 @@ { - "site_urls": [ - "https://example1.com", - "https://example2.com" - ], + "site_urls": ["https://example1.com", "https://example2.com"], "start_date": "2021-05-01", "end_date": "2021-10-10", "authorization": { diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_all_fields.json b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_all_fields.json index ba9604297d6d..89fe0fe5e9a4 100755 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_all_fields.json +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_all_fields.json @@ -3,72 +3,39 @@ "type": "object", "properties": { "site_url": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "search_type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "date": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date" }, "country": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "device": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "page": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "query": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "clicks": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "impressions": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "ctr": { - "type": [ - "null", - "number" - ], + "type": ["null", "number"], "multipleOf": 1e-25 }, "position": { - "type": [ - "null", - "number" - ], + "type": ["null", "number"], "multipleOf": 1e-25 } } diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_by_country.json b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_by_country.json index 570bdc5b9599..9e74ea044ec5 100755 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_by_country.json +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_by_country.json @@ -3,54 +3,30 @@ "type": "object", "properties": { "site_url": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "search_type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "date": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date" }, "country": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "clicks": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "impressions": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "ctr": { - "type": [ - "null", - "number" - ], + "type": ["null", "number"], "multipleOf": 1e-25 }, "position": { - "type": [ - "null", - "number" - ], + "type": ["null", "number"], "multipleOf": 1e-25 } } diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_by_date.json b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_by_date.json index e0126de137cc..76ffa918c9af 100755 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_by_date.json +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_by_date.json @@ -3,48 +3,27 @@ "type": "object", "properties": { "site_url": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "search_type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "date": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date" }, "clicks": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "impressions": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "ctr": { - "type": [ - "null", - "number" - ], + "type": ["null", "number"], "multipleOf": 1e-25 }, "position": { - "type": [ - "null", - "number" - ], + "type": ["null", "number"], "multipleOf": 1e-25 } } diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_by_device.json b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_by_device.json index d49668002f58..4875135b7f07 100755 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_by_device.json +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_by_device.json @@ -3,54 +3,30 @@ "type": "object", "properties": { "site_url": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "search_type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "date": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date" }, "device": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "clicks": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "impressions": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "ctr": { - "type": [ - "null", - "number" - ], + "type": ["null", "number"], "multipleOf": 1e-25 }, "position": { - "type": [ - "null", - "number" - ], + "type": ["null", "number"], "multipleOf": 1e-25 } } diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_by_page.json b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_by_page.json index b5fea4a88f55..2a1a3d9af816 100755 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_by_page.json +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_by_page.json @@ -3,54 +3,30 @@ "type": "object", "properties": { "site_url": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "search_type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "date": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date" }, "page": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "clicks": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "impressions": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "ctr": { - "type": [ - "null", - "number" - ], + "type": ["null", "number"], "multipleOf": 1e-25 }, "position": { - "type": [ - "null", - "number" - ], + "type": ["null", "number"], "multipleOf": 1e-25 } } diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_by_query.json b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_by_query.json index e353d655f923..8e84cbda814c 100755 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_by_query.json +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_by_query.json @@ -3,54 +3,30 @@ "type": "object", "properties": { "site_url": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "search_type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "date": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date" }, "query": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "clicks": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "impressions": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "ctr": { - "type": [ - "null", - "number" - ], + "type": ["null", "number"], "multipleOf": 1e-25 }, "position": { - "type": [ - "null", - "number" - ], + "type": ["null", "number"], "multipleOf": 1e-25 } } diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/sitemaps.json b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/sitemaps.json index d10f41e05dfd..e84568e41879 100755 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/sitemaps.json +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/sitemaps.json @@ -3,54 +3,30 @@ "type": "object", "properties": { "path": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "lastSubmitted": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" }, "isPending": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "isSitemapsIndex": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "lastDownloaded": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" }, "warnings": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "errors": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "contents": { "type": "array", @@ -58,25 +34,16 @@ "type": "object", "properties": { "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "submitted": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "indexed": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/sites.json b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/sites.json index 0d9ad638cf7a..12b94a4dc084 100755 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/sites.json +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/sites.json @@ -3,16 +3,10 @@ "type": "object", "properties": { "siteUrl": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "permissionLevel": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/spec.json b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/spec.json index f6e14e322e1f..e060196070b4 100755 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/spec.json +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/spec.json @@ -5,34 +5,23 @@ "title": "Google Search Console Spec", "type": "object", "additionalProperties": false, - "required": [ - "site_urls", - "start_date", - "authorization" - ], + "required": ["site_urls", "start_date", "authorization"], "properties": { "site_urls": { "type": "array", "description": "Website URLs property; do not include the domain-level property in the list", - "examples": [ - "https://example1.com", - "https://example2.com" - ] + "examples": ["https://example1.com", "https://example2.com"] }, "start_date": { "type": "string", "description": "The date from which you'd like to replicate data in the format YYYY-MM-DD.", - "examples": [ - "2021-01-01" - ], + "examples": ["2021-01-01"], "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" }, "end_date": { "type": "string", "description": "The date from which you'd like to replicate data in the format YYYY-MM-DD. Must be greater or equal start_date field", - "examples": [ - "2021-12-12" - ], + "examples": ["2021-12-12"], "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" }, "authorization": { @@ -70,10 +59,7 @@ }, { "type": "object", - "required": [ - "auth_type", - "service_account_info" - ], + "required": ["auth_type", "service_account_info"], "properties": { "auth_type": { "type": "string", diff --git a/docs/integrations/destinations/kafka.md b/docs/integrations/destinations/kafka.md index d9fb1b902c1f..658e884be51c 100644 --- a/docs/integrations/destinations/kafka.md +++ b/docs/integrations/destinations/kafka.md @@ -114,5 +114,6 @@ More info about this can be found in the [Kafka producer configs documentation s | Version | Date | Pull Request | Subject | | :------ | :-------- | :----- | :------ | +| 0.1.2 | 2021-09-14 | [#6040](https://github.com/airbytehq/airbyte/pull/6040) | Change spec.json and config parser | | 0.1.1 | 2021-07-30 | [#5125](https://github.com/airbytehq/airbyte/pull/5125) | Enable `additionalPropertities` in spec.json | -| 0.1.0 | 2021-07-21 | [3746](https://github.com/airbytehq/airbyte/pull/3746) | Initial Release | +| 0.1.0 | 2021-07-21 | [#3746](https://github.com/airbytehq/airbyte/pull/3746) | Initial Release | From 691f99c2bb0453676b65f6506aeb52901ea81187 Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Tue, 14 Sep 2021 20:39:11 -0300 Subject: [PATCH 3/5] correct tests --- .../destination/kafka/KafkaDestinationConfig.java | 2 +- .../destination-kafka/src/main/resources/spec.json | 2 +- .../kafka/KafkaDestinationAcceptanceTest.java | 10 +++++----- .../destination/kafka/KafkaRecordConsumerTest.java | 12 ++++++------ 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/airbyte-integrations/connectors/destination-kafka/src/main/java/io/airbyte/integrations/destination/kafka/KafkaDestinationConfig.java b/airbyte-integrations/connectors/destination-kafka/src/main/java/io/airbyte/integrations/destination/kafka/KafkaDestinationConfig.java index 347d5f5027da..33b7c7fa5a89 100644 --- a/airbyte-integrations/connectors/destination-kafka/src/main/java/io/airbyte/integrations/destination/kafka/KafkaDestinationConfig.java +++ b/airbyte-integrations/connectors/destination-kafka/src/main/java/io/airbyte/integrations/destination/kafka/KafkaDestinationConfig.java @@ -80,7 +80,7 @@ private KafkaProducer buildKafkaProducer(JsonNode config) { config.get("socket_connection_setup_timeout_ms").asLong()) .put(ProducerConfig.SOCKET_CONNECTION_SETUP_TIMEOUT_MAX_MS_CONFIG, config.get("socket_connection_setup_timeout_max_ms").asLong()) - .put(ProducerConfig.MAX_BLOCK_MS_CONFIG, config.get("max_block_ms").asLong()) + .put(ProducerConfig.MAX_BLOCK_MS_CONFIG, config.get("max_block_ms").asInt()) .put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, config.get("request_timeout_ms").asInt()) .put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, config.get("delivery_timeout_ms").asInt()) .put(ProducerConfig.SEND_BUFFER_CONFIG, config.get("send_buffer_bytes").asInt()) diff --git a/airbyte-integrations/connectors/destination-kafka/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-kafka/src/main/resources/spec.json index 2eb35a5b22cd..6098a2375207 100644 --- a/airbyte-integrations/connectors/destination-kafka/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-kafka/src/main/resources/spec.json @@ -250,4 +250,4 @@ } } } -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-kafka/src/test-integration/java/io/airbyte/integrations/destination/kafka/KafkaDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-kafka/src/test-integration/java/io/airbyte/integrations/destination/kafka/KafkaDestinationAcceptanceTest.java index 0de8df3703cc..12c46825a9f4 100644 --- a/airbyte-integrations/connectors/destination-kafka/src/test-integration/java/io/airbyte/integrations/destination/kafka/KafkaDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-kafka/src/test-integration/java/io/airbyte/integrations/destination/kafka/KafkaDestinationAcceptanceTest.java @@ -75,15 +75,15 @@ protected JsonNode getConfig() { .put("enable_idempotence", true) .put("compression_type", "none") .put("batch_size", 16384) - .put("linger_ms", 0) + .put("linger_ms", "0") .put("max_in_flight_requests_per_connection", 5) .put("client_dns_lookup", "use_all_dns_ips") - .put("buffer_memory", 33554432) + .put("buffer_memory", "33554432") .put("max_request_size", 1048576) .put("retries", 2147483647) - .put("socket_connection_setup_timeout_ms", 10000) - .put("socket_connection_setup_timeout_max_ms", 30000) - .put("max_block_ms", 60000) + .put("socket_connection_setup_timeout_ms", "10000") + .put("socket_connection_setup_timeout_max_ms", "30000") + .put("max_block_ms", "60000") .put("request_timeout_ms", 30000) .put("delivery_timeout_ms", 120000) .put("send_buffer_bytes", -1) diff --git a/airbyte-integrations/connectors/destination-kafka/src/test/java/io/airbyte/integrations/destination/kafka/KafkaRecordConsumerTest.java b/airbyte-integrations/connectors/destination-kafka/src/test/java/io/airbyte/integrations/destination/kafka/KafkaRecordConsumerTest.java index 1898d21ab9f7..22867e782ba2 100644 --- a/airbyte-integrations/connectors/destination-kafka/src/test/java/io/airbyte/integrations/destination/kafka/KafkaRecordConsumerTest.java +++ b/airbyte-integrations/connectors/destination-kafka/src/test/java/io/airbyte/integrations/destination/kafka/KafkaRecordConsumerTest.java @@ -122,16 +122,16 @@ private JsonNode getConfig(String topicPattern) { .put("transactional_id", "txn-id") .put("enable_idempotence", true) .put("compression_type", "none") - .put("batch_size", 16384) - .put("linger_ms", 0) - .put("max_in_flight_requests_per_connection", 5) + .put("batch_size", "16384") + .put("linger_ms", "0") + .put("max_in_flight_requests_per_connection", "5") .put("client_dns_lookup", "use_all_dns_ips") .put("buffer_memory", 33554432) .put("max_request_size", 1048576) .put("retries", 1) - .put("socket_connection_setup_timeout_ms", 10) - .put("socket_connection_setup_timeout_max_ms", 30) - .put("max_block_ms", 100) + .put("socket_connection_setup_timeout_ms", "10") + .put("socket_connection_setup_timeout_max_ms", "30") + .put("max_block_ms", "100") .put("request_timeout_ms", 100) .put("delivery_timeout_ms", 120) .put("send_buffer_bytes", -1) From 974742400ac382461fa75437f7534ce15ccf8908 Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Tue, 14 Sep 2021 21:18:09 -0300 Subject: [PATCH 4/5] correct config parser NPE --- .../integrations/destination/kafka/KafkaDestinationConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-kafka/src/main/java/io/airbyte/integrations/destination/kafka/KafkaDestinationConfig.java b/airbyte-integrations/connectors/destination-kafka/src/main/java/io/airbyte/integrations/destination/kafka/KafkaDestinationConfig.java index 33b7c7fa5a89..0f1be9c3c734 100644 --- a/airbyte-integrations/connectors/destination-kafka/src/main/java/io/airbyte/integrations/destination/kafka/KafkaDestinationConfig.java +++ b/airbyte-integrations/connectors/destination-kafka/src/main/java/io/airbyte/integrations/destination/kafka/KafkaDestinationConfig.java @@ -64,7 +64,7 @@ private KafkaProducer buildKafkaProducer(JsonNode config) { .put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, config.get("bootstrap_servers").asText()) .putAll(propertiesByProtocol(config)) .put(ProducerConfig.CLIENT_ID_CONFIG, - config.has("client_id") ? config.get("client_id").asText() : null) + config.has("client_id") ? config.get("client_id").asText() : "") .put(ProducerConfig.ACKS_CONFIG, config.get("acks").asText()) .put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, config.get("enable_idempotence").asBoolean()) .put(ProducerConfig.COMPRESSION_TYPE_CONFIG, config.get("compression_type").asText()) From 3da7c871663b361e48c3e01ccb04c416d5dd52b3 Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Tue, 14 Sep 2021 23:29:21 -0300 Subject: [PATCH 5/5] format files --- .../destination-kafka/src/main/resources/spec.json | 2 +- .../integrations/destination/keen/KeenTimestampService.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/destination-kafka/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-kafka/src/main/resources/spec.json index 6098a2375207..2eb35a5b22cd 100644 --- a/airbyte-integrations/connectors/destination-kafka/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-kafka/src/main/resources/spec.json @@ -250,4 +250,4 @@ } } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/destination-keen/src/main/java/io/airbyte/integrations/destination/keen/KeenTimestampService.java b/airbyte-integrations/connectors/destination-keen/src/main/java/io/airbyte/integrations/destination/keen/KeenTimestampService.java index 3b1146ec42da..3c33a93dc4cf 100644 --- a/airbyte-integrations/connectors/destination-keen/src/main/java/io/airbyte/integrations/destination/keen/KeenTimestampService.java +++ b/airbyte-integrations/connectors/destination-keen/src/main/java/io/airbyte/integrations/destination/keen/KeenTimestampService.java @@ -75,9 +75,9 @@ public KeenTimestampService(ConfiguredAirbyteCatalog catalog, boolean timestampI /** * Tries to inject keen.timestamp field to the given message data. If the stream contains cursor * field, it's value is tried to be parsed to timestamp. If this procedure fails, stream is removed - * from timestamp-parsable stream map, so parsing is not tried for future messages in the same stream. - * If parsing succeeds, keen.timestamp field is put as a JSON node to the message data and whole data - * is returned. Otherwise, keen.timestamp is set to emittedAt value + * from timestamp-parsable stream map, so parsing is not tried for future messages in the same + * stream. If parsing succeeds, keen.timestamp field is put as a JSON node to the message data and + * whole data is returned. Otherwise, keen.timestamp is set to emittedAt value * * @param message AirbyteRecordMessage containing record data * @return Record data together with keen.timestamp field