From e9ba908e818c63f64a3d6c8c5e0cd10418dd63d0 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 2 Jun 2022 11:00:38 -0700 Subject: [PATCH 01/10] source-delighted: Unix changed to normal timestamp --- .../source-delighted/source_delighted/source.py | 10 +++++++++- .../source-delighted/source_delighted/spec.json | 7 ++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-delighted/source_delighted/source.py b/airbyte-integrations/connectors/source-delighted/source_delighted/source.py index 87d8d0c18130..3089b1e0fdb4 100644 --- a/airbyte-integrations/connectors/source-delighted/source_delighted/source.py +++ b/airbyte-integrations/connectors/source-delighted/source_delighted/source.py @@ -15,6 +15,7 @@ from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +import datetime # Basic full refresh stream class DelightedStream(HttpStream, ABC): @@ -145,6 +146,13 @@ def _get_authenticator(self, config): token = base64.b64encode(f"{config['api_key']}:".encode("utf-8")).decode("utf-8") return TokenAuthenticator(token=token, auth_method="Basic") + #Function to covert a timestamp to UNIX time format + def toUnix (self,config): + timeDate = config["since"] + dateFormat = datetime.datetime.strptime(timeDate, "%m-%d-%Y,%H:%M:%S") + unixDate = datetime.datetime.timestamp(dateFormat) + return unixDate + def check_connection(self, logger, config) -> Tuple[bool, any]: """ @@ -157,7 +165,7 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: try: auth = self._get_authenticator(config) - args = {"authenticator": auth, "since": config["since"]} + args = {"authenticator": auth, "since": self.toUnix(config)} stream = SurveyResponses(**args) records = stream.read_records(sync_mode=SyncMode.full_refresh) next(records) diff --git a/airbyte-integrations/connectors/source-delighted/source_delighted/spec.json b/airbyte-integrations/connectors/source-delighted/source_delighted/spec.json index 5c6e276f36c8..bde6b7a69aeb 100644 --- a/airbyte-integrations/connectors/source-delighted/source_delighted/spec.json +++ b/airbyte-integrations/connectors/source-delighted/source_delighted/spec.json @@ -8,9 +8,10 @@ "additionalProperties": false, "properties": { "since": { - "type": "integer", - "description": "An Unix timestamp to retrieve records created on or after this time.", - "examples": [1625328167] + "type": "string", + "description": "A timestamp to retrieve records created on or after this time.", + "examples": ["2022-05-30,04:50:23"], + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2},([0-9]{2}:[0-9]{2}:[0-9]{2})?$" }, "api_key": { "type": "string", From 83d2f5910c4562e3c9846b5f01bc63a96b2394d6 Mon Sep 17 00:00:00 2001 From: Michael Nguyen <67665595+michaelnguyen26@users.noreply.github.com> Date: Thu, 9 Jun 2022 16:16:59 -0700 Subject: [PATCH 02/10] Update airbyte-integrations/connectors/source-delighted/source_delighted/source.py Co-authored-by: Augustin --- .../connectors/source-delighted/source_delighted/source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-delighted/source_delighted/source.py b/airbyte-integrations/connectors/source-delighted/source_delighted/source.py index 3089b1e0fdb4..398744a408a4 100644 --- a/airbyte-integrations/connectors/source-delighted/source_delighted/source.py +++ b/airbyte-integrations/connectors/source-delighted/source_delighted/source.py @@ -165,7 +165,7 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: try: auth = self._get_authenticator(config) - args = {"authenticator": auth, "since": self.toUnix(config)} + args = {"authenticator": auth, "since": pendulum.parse(config["since"])} stream = SurveyResponses(**args) records = stream.read_records(sync_mode=SyncMode.full_refresh) next(records) From 9f89b5bc4c1ece7adf3c2d60eaa2ad72f701bbd0 Mon Sep 17 00:00:00 2001 From: Michael Nguyen <67665595+michaelnguyen26@users.noreply.github.com> Date: Thu, 9 Jun 2022 16:17:07 -0700 Subject: [PATCH 03/10] Update airbyte-integrations/connectors/source-delighted/source_delighted/source.py Co-authored-by: Augustin --- .../connectors/source-delighted/source_delighted/source.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/airbyte-integrations/connectors/source-delighted/source_delighted/source.py b/airbyte-integrations/connectors/source-delighted/source_delighted/source.py index 398744a408a4..2f0578f18dba 100644 --- a/airbyte-integrations/connectors/source-delighted/source_delighted/source.py +++ b/airbyte-integrations/connectors/source-delighted/source_delighted/source.py @@ -146,13 +146,6 @@ def _get_authenticator(self, config): token = base64.b64encode(f"{config['api_key']}:".encode("utf-8")).decode("utf-8") return TokenAuthenticator(token=token, auth_method="Basic") - #Function to covert a timestamp to UNIX time format - def toUnix (self,config): - timeDate = config["since"] - dateFormat = datetime.datetime.strptime(timeDate, "%m-%d-%Y,%H:%M:%S") - unixDate = datetime.datetime.timestamp(dateFormat) - return unixDate - def check_connection(self, logger, config) -> Tuple[bool, any]: """ From 2693e9017a721375ee020e0b01f9d3e54575161c Mon Sep 17 00:00:00 2001 From: Michael Nguyen <67665595+michaelnguyen26@users.noreply.github.com> Date: Thu, 9 Jun 2022 16:17:18 -0700 Subject: [PATCH 04/10] Update airbyte-integrations/connectors/source-delighted/source_delighted/spec.json Co-authored-by: Augustin --- .../connectors/source-delighted/source_delighted/spec.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-delighted/source_delighted/spec.json b/airbyte-integrations/connectors/source-delighted/source_delighted/spec.json index bde6b7a69aeb..9d4dc1bc71ad 100644 --- a/airbyte-integrations/connectors/source-delighted/source_delighted/spec.json +++ b/airbyte-integrations/connectors/source-delighted/source_delighted/spec.json @@ -9,7 +9,7 @@ "properties": { "since": { "type": "string", - "description": "A timestamp to retrieve records created on or after this time.", + "description": "The date from which you'd like to replicate the data", "examples": ["2022-05-30,04:50:23"], "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2},([0-9]{2}:[0-9]{2}:[0-9]{2})?$" }, From 4c51cd32f84f6e57352674b975867c075744b31c Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 10 Jun 2022 11:12:16 +0200 Subject: [PATCH 05/10] implement suggestion --- .../source-delighted/source_delighted/source.py | 12 ++++++++---- .../source-delighted/source_delighted/spec.json | 10 +++++++--- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/airbyte-integrations/connectors/source-delighted/source_delighted/source.py b/airbyte-integrations/connectors/source-delighted/source_delighted/source.py index 2f0578f18dba..aa1152f98783 100644 --- a/airbyte-integrations/connectors/source-delighted/source_delighted/source.py +++ b/airbyte-integrations/connectors/source-delighted/source_delighted/source.py @@ -8,6 +8,7 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple from urllib.parse import parse_qsl, urlparse +import pendulum import requests from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource @@ -15,7 +16,6 @@ from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator -import datetime # Basic full refresh stream class DelightedStream(HttpStream, ABC): @@ -28,10 +28,14 @@ class DelightedStream(HttpStream, ABC): # Define primary key to all streams as primary key primary_key = "id" - def __init__(self, since: int, **kwargs): + def __init__(self, since: pendulum.datetime.DateTime, **kwargs): super().__init__(**kwargs) self.since = since + @property + def since_ts(self) -> int: + return int(self.since.timestamp()) + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: response_data = response.json() if len(response_data) == self.limit: @@ -41,7 +45,7 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: - params = {"per_page": self.limit, "since": self.since} + params = {"per_page": self.limit, "since": self.since_ts} if next_page_token: params.update(**next_page_token) return params @@ -168,7 +172,7 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: def streams(self, config: Mapping[str, Any]) -> List[Stream]: auth = self._get_authenticator(config) - args = {"authenticator": auth, "since": config["since"]} + args = {"authenticator": auth, "since": pendulum.parse(config["since"])} return [ Bounces(**args), People(**args), diff --git a/airbyte-integrations/connectors/source-delighted/source_delighted/spec.json b/airbyte-integrations/connectors/source-delighted/source_delighted/spec.json index 9d4dc1bc71ad..0292ddfbb50e 100644 --- a/airbyte-integrations/connectors/source-delighted/source_delighted/spec.json +++ b/airbyte-integrations/connectors/source-delighted/source_delighted/spec.json @@ -8,15 +8,19 @@ "additionalProperties": false, "properties": { "since": { + "title": "Since", "type": "string", "description": "The date from which you'd like to replicate the data", - "examples": ["2022-05-30,04:50:23"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2},([0-9]{2}:[0-9]{2}:[0-9]{2})?$" + "examples": ["2022-05-30 04:50:23"], + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2} ([0-9]{2}:[0-9]{2}:[0-9]{2})?$", + "order": 0 }, "api_key": { + "title": "Delighted API Key", "type": "string", "description": "A Delighted API key.", - "airbyte_secret": true + "airbyte_secret": true, + "order": 1 } } } From 5e903f71f039509be03b09cf33b21eee9075194a Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 10 Jun 2022 11:45:59 +0200 Subject: [PATCH 06/10] fix type hint --- .../connectors/source-delighted/source_delighted/source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-delighted/source_delighted/source.py b/airbyte-integrations/connectors/source-delighted/source_delighted/source.py index aa1152f98783..84c358c643cf 100644 --- a/airbyte-integrations/connectors/source-delighted/source_delighted/source.py +++ b/airbyte-integrations/connectors/source-delighted/source_delighted/source.py @@ -28,7 +28,7 @@ class DelightedStream(HttpStream, ABC): # Define primary key to all streams as primary key primary_key = "id" - def __init__(self, since: pendulum.datetime.DateTime, **kwargs): + def __init__(self, since: pendulum.datetime, **kwargs): super().__init__(**kwargs) self.since = since From d0dd9b289e059da4322c7dcc8234e32b2f6b4c99 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 10 Jun 2022 14:58:01 +0200 Subject: [PATCH 07/10] add delighted to list of builds --- airbyte-integrations/builds.md | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/builds.md b/airbyte-integrations/builds.md index 64ab8590280a..1c35f3dbf12b 100644 --- a/airbyte-integrations/builds.md +++ b/airbyte-integrations/builds.md @@ -24,6 +24,7 @@ | Chartmogul | [![source-chartmogul](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-chartmogul%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-chartmogul/) | | Cart.com | [![source-cart](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-cart%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-cart/) | | Close.com | [![source-close-com](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-close-com%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-close-com/) | +| Delighted | [![source-delighted](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-delighted%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-delighted) | | Dixa | [![source-dixa](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-dixa%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-dixa) | | Drift | [![source-drift](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-drift%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-drift) | | End-to-End Testing | [![source-e2e-test](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-e2e-test%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-e2e-test) | From b72441ff5d2504e75e9009d2fb426465d8f0f276 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 10 Jun 2022 15:31:30 +0200 Subject: [PATCH 08/10] fix tests --- .../integration_tests/invalid_config.json | 2 +- .../source-delighted/source_delighted/source.py | 13 ++++++------- .../source-delighted/unit_tests/unit_test.py | 5 +++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/airbyte-integrations/connectors/source-delighted/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-delighted/integration_tests/invalid_config.json index b2856c90e4be..6e2dc7c913db 100644 --- a/airbyte-integrations/connectors/source-delighted/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-delighted/integration_tests/invalid_config.json @@ -1,4 +1,4 @@ { "api_key": "wrong api key", - "since": 1625328197 + "since": "2022-01-01 00:00:00" } diff --git a/airbyte-integrations/connectors/source-delighted/source_delighted/source.py b/airbyte-integrations/connectors/source-delighted/source_delighted/source.py index 84c358c643cf..f584d5c98792 100644 --- a/airbyte-integrations/connectors/source-delighted/source_delighted/source.py +++ b/airbyte-integrations/connectors/source-delighted/source_delighted/source.py @@ -162,8 +162,7 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: try: auth = self._get_authenticator(config) - args = {"authenticator": auth, "since": pendulum.parse(config["since"])} - stream = SurveyResponses(**args) + stream = SurveyResponses(authenticator=auth, since=pendulum.parse(config["since"])) records = stream.read_records(sync_mode=SyncMode.full_refresh) next(records) return True, None @@ -172,10 +171,10 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: def streams(self, config: Mapping[str, Any]) -> List[Stream]: auth = self._get_authenticator(config) - args = {"authenticator": auth, "since": pendulum.parse(config["since"])} + stream_kwargs = {"authenticator": auth, "since": pendulum.parse(config["since"])} return [ - Bounces(**args), - People(**args), - SurveyResponses(**args), - Unsubscribes(**args), + Bounces(**stream_kwargs), + People(**stream_kwargs), + SurveyResponses(**stream_kwargs), + Unsubscribes(**stream_kwargs), ] diff --git a/airbyte-integrations/connectors/source-delighted/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-delighted/unit_tests/unit_test.py index eed97adb87dc..d3d1c3116ab1 100644 --- a/airbyte-integrations/connectors/source-delighted/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-delighted/unit_tests/unit_test.py @@ -2,6 +2,7 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +import pendulum import pytest import responses from airbyte_cdk.models import SyncMode @@ -12,7 +13,7 @@ def test_config(): return { "api_key": "test_api_key", - "since": "1641289584", + "since": "2022-01-01 00:00:00", } @@ -74,7 +75,7 @@ def test_not_output_records_where_cursor_field_equals_state(state, test_config, status=200, ) - stream = stream_class(test_config["since"], authenticator=SourceDelighted()._get_authenticator(config=test_config)) + stream = stream_class(pendulum.parse(test_config["since"]), authenticator=SourceDelighted()._get_authenticator(config=test_config)) records = [r for r in stream.read_records(SyncMode.incremental, stream_state=state[stream.name])] assert not records From aa8d8a7aef875f0031454e8868bccc84bb60dcf3 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 10 Jun 2022 15:47:49 +0200 Subject: [PATCH 09/10] bump version --- airbyte-integrations/connectors/source-delighted/Dockerfile | 2 +- docs/integrations/sources/delighted.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-delighted/Dockerfile b/airbyte-integrations/connectors/source-delighted/Dockerfile index 068fdf381a2e..42986453d08c 100644 --- a/airbyte-integrations/connectors/source-delighted/Dockerfile +++ b/airbyte-integrations/connectors/source-delighted/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/source-delighted diff --git a/docs/integrations/sources/delighted.md b/docs/integrations/sources/delighted.md index 430ba77c977f..3d978520943c 100644 --- a/docs/integrations/sources/delighted.md +++ b/docs/integrations/sources/delighted.md @@ -37,6 +37,7 @@ This connector supports `API PASSWORD` as the authentication method. | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.4 | 2022-06-10 | [13439](https://github.com/airbytehq/airbyte/pull/13439) | Change since parameter input to iso date | | 0.1.3 | 2022-01-31 | [9550](https://github.com/airbytehq/airbyte/pull/9550) | Output only records in which cursor field is greater than the value in state for incremental streams | | 0.1.2 | 2022-01-06 | [9333](https://github.com/airbytehq/airbyte/pull/9333) | Add incremental sync mode to streams in `integration_tests/configured_catalog.json` | | 0.1.1 | 2022-01-04 | [9275](https://github.com/airbytehq/airbyte/pull/9275) | Fix pagination handling for `survey_responses`, `bounces` and `unsubscribes` streams | From 85a82a8e09ab0f91f156f183b26737663f7a5e99 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Fri, 10 Jun 2022 14:01:25 +0000 Subject: [PATCH 10/10] auto-bump connector version --- .../main/resources/seed/source_definitions.yaml | 2 +- .../init/src/main/resources/seed/source_specs.yaml | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index f438766feeb4..b791b8affd0d 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -200,7 +200,7 @@ - name: Delighted sourceDefinitionId: cc88c43f-6f53-4e8a-8c4d-b284baaf9635 dockerRepository: airbyte/source-delighted - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.4 documentationUrl: https://docs.airbyte.io/integrations/sources/delighted icon: delighted.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 5c3280d98b75..99606929ea51 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -1546,7 +1546,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-delighted:0.1.3" +- dockerImage: "airbyte/source-delighted:0.1.4" spec: documentationUrl: "https://docsurl.com" connectionSpecification: @@ -1559,15 +1559,19 @@ additionalProperties: false properties: since: - type: "integer" - description: "An Unix timestamp to retrieve records created on or after\ - \ this time." + title: "Since" + type: "string" + description: "The date from which you'd like to replicate the data" examples: - - 1625328167 + - "2022-05-30 04:50:23" + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2} ([0-9]{2}:[0-9]{2}:[0-9]{2})?$" + order: 0 api_key: + title: "Delighted API Key" type: "string" description: "A Delighted API key." airbyte_secret: true + order: 1 supportsNormalization: false supportsDBT: false supported_destination_sync_modes: []