Source pinterest: Fix type of start_date (#18285)

Signed-off-by: Sergey Chvalyuk <[email protected]>
airbytehq · Oct 22, 2022 · e7bf365 · e7bf365
1 parent f080d4d
commit e7bf365
Show file tree

Hide file tree

Showing 6 changed files with 25 additions and 23 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -817,7 +817,7 @@
 - name: Pinterest
   sourceDefinitionId: 5cb7e5fe-38c2-11ec-8d3d-0242ac130003
   dockerRepository: airbyte/source-pinterest
-  dockerImageTag: 0.1.7
+  dockerImageTag: 0.1.8
   documentationUrl: https://docs.airbyte.com/integrations/sources/pinterest
   icon: pinterest.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -8478,7 +8478,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-pinterest:0.1.7"
+- dockerImage: "airbyte/source-pinterest:0.1.8"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/pinterest"
     connectionSpecification:

diff --git a/airbyte-integrations/connectors/source-pinterest/Dockerfile b/airbyte-integrations/connectors/source-pinterest/Dockerfile
@@ -34,5 +34,5 @@ COPY source_pinterest ./source_pinterest
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.7
+LABEL io.airbyte.version=0.1.8
 LABEL io.airbyte.name=airbyte/source-pinterest
diff --git a/airbyte-integrations/connectors/source-pinterest/setup.py b/airbyte-integrations/connectors/source-pinterest/setup.py
@@ -5,9 +5,7 @@
 
 from setuptools import find_packages, setup
 
-MAIN_REQUIREMENTS = [
-    "airbyte-cdk",
-]
+MAIN_REQUIREMENTS = ["airbyte-cdk~=0.2", "pendulum~=2.1.2"]
 
 TEST_REQUIREMENTS = [
     "pytest~=6.1",

diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py
@@ -128,8 +128,8 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
 
 class IncrementalPinterestStream(PinterestStream, ABC):
     def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]:
-        latest_state = latest_record.get(self.cursor_field, self.start_date)
-        current_state = current_stream_state.get(self.cursor_field, self.start_date)
+        latest_state = latest_record.get(self.cursor_field, self.start_date.format("YYYY-MM-DD"))
+        current_state = current_stream_state.get(self.cursor_field, self.start_date.format("YYYY-MM-DD"))
 
         if isinstance(latest_state, int) and isinstance(current_state, str):
             current_state = datetime.strptime(current_state, "%Y-%m-%d").timestamp()
@@ -152,7 +152,7 @@ def stream_slices(
             ...]
         """
 
-        start_date = pendulum.parse(self.start_date)
+        start_date = self.start_date
         end_date = pendulum.now()
 
         # determine stream_state, if no stream_state we use start_date
@@ -293,6 +293,20 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
 
 
 class SourcePinterest(AbstractSource):
+    def _validate_and_transform(self, config: Mapping[str, Any]):
+        today = pendulum.today()
+        AMOUNT_OF_DAYS_ALLOWED_FOR_LOOKUP = 914
+        latest_date_allowed_by_api = today.subtract(days=AMOUNT_OF_DAYS_ALLOWED_FOR_LOOKUP)
+
+        start_date = config["start_date"]
+        if not start_date:
+            config["start_date"] = latest_date_allowed_by_api
+        else:
+            config["start_date"] = pendulum.from_format(config["start_date"], "YYYY-MM-DD")
+            if (today - config["start_date"]).days > AMOUNT_OF_DAYS_ALLOWED_FOR_LOOKUP:
+                config["start_date"] = latest_date_allowed_by_api
+        return config
+
     @staticmethod
     def get_authenticator(config):
         config = config.get("credentials") or config
@@ -310,6 +324,7 @@ def get_authenticator(config):
         )
 
     def check_connection(self, logger, config) -> Tuple[bool, any]:
+        config = self._validate_and_transform(config)
         authenticator = self.get_authenticator(config)
         url = f"{PinterestStream.url_base}user_account"
         auth_headers = {"Accept": "application/json", **authenticator.get_auth_header()}
@@ -321,19 +336,7 @@ def check_connection(self, logger, config) -> Tuple[bool, any]:
             return False, e
 
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
-        today = pendulum.today()
-        AMOUNT_OF_DAYS_ALLOWED_FOR_LOOKUP = 914
-        latest_date_allowed_by_api = today.subtract(days=AMOUNT_OF_DAYS_ALLOWED_FOR_LOOKUP)
-
-        start_date = config.get("start_date")
-        if not start_date:
-            config["start_date"] = latest_date_allowed_by_api
-        else:
-            start_date_formatted = pendulum.from_format(config["start_date"], "YYYY-MM-DD")
-            delta_today_start_date = today - start_date_formatted
-            if delta_today_start_date.days > AMOUNT_OF_DAYS_ALLOWED_FOR_LOOKUP:
-                config["start_date"] = latest_date_allowed_by_api
-
+        config = self._validate_and_transform(config)
         config["authenticator"] = self.get_authenticator(config)
         return [
             AdAccountAnalytics(AdAccounts(config), config=config),

diff --git a/docs/integrations/sources/pinterest.md b/docs/integrations/sources/pinterest.md
@@ -73,7 +73,8 @@ Boards streams - 10 calls per sec / per user / per app
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                 |
 | :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------------------ |
-| 0.1.7   | 2022-09-29 | [17387](https://github.com/airbytehq/airbyte/pull/17387) | Set `start_date` dynamically based on API restrictions.
+| 0.1.8   | 2022-10-21 | [18285](https://github.com/airbytehq/airbyte/pull/18285) | Fix type of `start_date`                                                                                |
+| 0.1.7   | 2022-09-29 | [17387](https://github.com/airbytehq/airbyte/pull/17387) | Set `start_date` dynamically based on API restrictions.                                                 |
 | 0.1.6   | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Use CDK 0.1.89                                                                                          |
 | 0.1.5   | 2022-09-16 | [16799](https://github.com/airbytehq/airbyte/pull/16799) | Migrate to per-stream state                                                                             |
 | 0.1.4   | 2022-09-06 | [16161](https://github.com/airbytehq/airbyte/pull/16161) | Added ability to handle `429 - Too Many Requests` error with respect to `Max Rate Limit Exceeded Error` |