From a95a8a2d972fe30b860781a4d9bbcd3b85a0fef7 Mon Sep 17 00:00:00 2001 From: aakashkumarmaple Date: Mon, 21 Feb 2022 23:20:50 +0530 Subject: [PATCH 01/14] Changes for increasing timeout for long taking report generation --- .../c6b0a29e-1da9-4512-9002-7bfd0cba2246.json | 8 ++++++++ .../init/src/main/resources/seed/source_definitions.yaml | 2 +- .../connectors/source-amazon-ads/Dockerfile | 4 ++-- .../streams/report_streams/report_streams.py | 6 +++--- docs/integrations/sources/amazon-ads.md | 1 + 5 files changed, 15 insertions(+), 6 deletions(-) create mode 100644 airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c6b0a29e-1da9-4512-9002-7bfd0cba2246.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c6b0a29e-1da9-4512-9002-7bfd0cba2246.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c6b0a29e-1da9-4512-9002-7bfd0cba2246.json new file mode 100644 index 000000000000..9a3833e67e47 --- /dev/null +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c6b0a29e-1da9-4512-9002-7bfd0cba2246.json @@ -0,0 +1,8 @@ +{ + "sourceDefinitionId": "c6b0a29e-1da9-4512-9002-7bfd0cba2246", + "name": "Amazon Ads", + "dockerRepository": "airbyte/source-amazon-ads", + "dockerImageTag": "0.1.4", + "documentationUrl": "https://docs.airbyte.io/integrations/sources/amazon-ads", + "icon": "amazonads.svg" + } \ No newline at end of file diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 477a3caddbb6..2926efd531a6 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -15,7 +15,7 @@ - name: Amazon Ads sourceDefinitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246 dockerRepository: airbyte/source-amazon-ads - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.4 documentationUrl: https://docs.airbyte.io/integrations/sources/amazon-ads icon: amazonads.svg sourceType: api diff --git a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile index 1d61ce7afb90..32eea36bac70 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.3 -LABEL io.airbyte.name=airbyte/source-amazon-ads +LABEL io.airbyte.version=0.1.4 +LABEL io.airbyte.name=airbyte/source-amazon-ads \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py index 1ffbb98d3062..dfa56a70075a 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py @@ -1,7 +1,7 @@ # # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # - + import json from abc import ABC, abstractmethod from dataclasses import dataclass @@ -98,7 +98,7 @@ class ReportStream(BasicAmazonAdsStream, ABC): # Async report generation time is 15 minutes according to docs: # https://advertising.amazon.com/API/docs/en-us/get-started/developer-notes # (Service limits section) - REPORT_WAIT_TIMEOUT = timedelta(minutes=30).total_seconds + REPORT_WAIT_TIMEOUT = timedelta(minutes=120).total_seconds # Format used to specify metric generation date over Amazon Ads API. REPORT_DATE_FORMAT = "%Y%m%d" cursor_field = "reportDate" @@ -153,7 +153,7 @@ def read_records( @backoff.on_exception( backoff.expo, ReportGenerationFailure, - max_tries=5, + max_tries=1000, ) def _init_and_try_read_records(self, report_date): report_infos = self._init_reports(report_date) diff --git a/docs/integrations/sources/amazon-ads.md b/docs/integrations/sources/amazon-ads.md index d621784426b6..09b024d7da4f 100644 --- a/docs/integrations/sources/amazon-ads.md +++ b/docs/integrations/sources/amazon-ads.md @@ -76,6 +76,7 @@ Start date used for generating reports starting from the specified start date. S | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| `0.1.4` | 2022-02-21 | []() | `Increasing REPORT_WAIT_TIMEOUT for supporting report generation which takes longer time ` | | `0.1.3` | 2021-12-28 | [\#8388](https://github.com/airbytehq/airbyte/pull/8388) | `Add retry if recoverable error occured for reporting stream processing` | | `0.1.2` | 2021-10-01 | [\#6367](https://github.com/airbytehq/airbyte/pull/6461) | `Add option to pull data for different regions. Add option to choose profiles we want to pull data. Add lookback` | | `0.1.1` | 2021-09-22 | [\#6367](https://github.com/airbytehq/airbyte/pull/6367) | `Add seller and vendor filters to profiles stream` | From 6380e8d5c5b149945db1f1aa756556598becc550 Mon Sep 17 00:00:00 2001 From: aakashkumarmaple Date: Mon, 21 Feb 2022 23:55:12 +0530 Subject: [PATCH 02/14] moidifing amazon-ads.md for pr 10513 --- docs/integrations/sources/amazon-ads.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/amazon-ads.md b/docs/integrations/sources/amazon-ads.md index 09b024d7da4f..52cb7e077cdf 100644 --- a/docs/integrations/sources/amazon-ads.md +++ b/docs/integrations/sources/amazon-ads.md @@ -76,7 +76,7 @@ Start date used for generating reports starting from the specified start date. S | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | -| `0.1.4` | 2022-02-21 | []() | `Increasing REPORT_WAIT_TIMEOUT for supporting report generation which takes longer time ` | +| `0.1.4` | 2022-02-21 | [\#10513](https://github.com/airbytehq/airbyte/pull/10513) | `Increasing REPORT_WAIT_TIMEOUT for supporting report generation which takes longer time ` | | `0.1.3` | 2021-12-28 | [\#8388](https://github.com/airbytehq/airbyte/pull/8388) | `Add retry if recoverable error occured for reporting stream processing` | | `0.1.2` | 2021-10-01 | [\#6367](https://github.com/airbytehq/airbyte/pull/6461) | `Add option to pull data for different regions. Add option to choose profiles we want to pull data. Add lookback` | | `0.1.1` | 2021-09-22 | [\#6367](https://github.com/airbytehq/airbyte/pull/6367) | `Add seller and vendor filters to profiles stream` | From 93fbaf0de9dbba9fa425ed2db44d83dd5078b340 Mon Sep 17 00:00:00 2001 From: aakashkumarmaple Date: Tue, 22 Feb 2022 10:17:11 +0530 Subject: [PATCH 03/14] Deleting STANDARD_SOURCE_DEFINATION file as mentioned in review comment --- .../c6b0a29e-1da9-4512-9002-7bfd0cba2246.json | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c6b0a29e-1da9-4512-9002-7bfd0cba2246.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c6b0a29e-1da9-4512-9002-7bfd0cba2246.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c6b0a29e-1da9-4512-9002-7bfd0cba2246.json deleted file mode 100644 index 9a3833e67e47..000000000000 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c6b0a29e-1da9-4512-9002-7bfd0cba2246.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "sourceDefinitionId": "c6b0a29e-1da9-4512-9002-7bfd0cba2246", - "name": "Amazon Ads", - "dockerRepository": "airbyte/source-amazon-ads", - "dockerImageTag": "0.1.4", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/amazon-ads", - "icon": "amazonads.svg" - } \ No newline at end of file From 9c12c9ac6be9c244c98495beeb321ad55c617f29 Mon Sep 17 00:00:00 2001 From: aakashkumarmaple Date: Wed, 23 Feb 2022 20:58:37 +0530 Subject: [PATCH 04/14] Adding spec file for taking timeout by user --- .../connectors/source-amazon-ads/source_amazon_ads/spec.py | 6 ++++++ .../streams/report_streams/report_streams.py | 5 ++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py index 01bf840cfd69..b14b3434f0a3 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py @@ -65,6 +65,12 @@ class Config: description="profile Ids you want to fetch data for", ) + report_wait_timeout: int = Field( + None, + name="Report Wait Timeout", + description="Timeout duration for Reports" + ) + @classmethod def schema(cls, **kvargs): schema = super().schema(**kvargs) diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py index dfa56a70075a..4689699f2df8 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py @@ -10,6 +10,7 @@ from gzip import decompress from http import HTTPStatus from typing import Any, Dict, Iterable, List, Mapping, Optional, Tuple +from typing_extensions import Self from urllib.parse import urljoin import backoff @@ -98,7 +99,7 @@ class ReportStream(BasicAmazonAdsStream, ABC): # Async report generation time is 15 minutes according to docs: # https://advertising.amazon.com/API/docs/en-us/get-started/developer-notes # (Service limits section) - REPORT_WAIT_TIMEOUT = timedelta(minutes=120).total_seconds + REPORT_WAIT_TIMEOUT = timedelta(minutes=30).total_seconds # Format used to specify metric generation date over Amazon Ads API. REPORT_DATE_FORMAT = "%Y%m%d" cursor_field = "reportDate" @@ -107,6 +108,8 @@ def __init__(self, config: AmazonAdsConfig, profiles: List[Profile], authenticat self._authenticator = authenticator self._session = requests.Session() self._model = self._generate_model() + if(config and config.report_wait_timeout and config.report_wait_timeout > 30): + self.REPORT_WAIT_TIMEOUT = timedelta(minutes=config.report_wait_timeout).total_seconds # Set start date from config file, should be in UTC timezone. self._start_date = pendulum.parse(config.start_date).set(tz="UTC") if config.start_date else None super().__init__(config, profiles) From 4c44744e8eb331f6f9d622cfb6765537761621db Mon Sep 17 00:00:00 2001 From: aakashkumarmaple Date: Thu, 24 Feb 2022 11:20:03 +0530 Subject: [PATCH 05/14] Removing unused libraries --- .../source_amazon_ads/streams/report_streams/report_streams.py | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py index 4689699f2df8..4d7cd1c2f2c2 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py @@ -10,7 +10,6 @@ from gzip import decompress from http import HTTPStatus from typing import Any, Dict, Iterable, List, Mapping, Optional, Tuple -from typing_extensions import Self from urllib.parse import urljoin import backoff From 81aad1f595f04289cb9b8a508371cd1c7bc193c8 Mon Sep 17 00:00:00 2001 From: aakashkumarmaple Date: Thu, 24 Feb 2022 17:07:39 +0530 Subject: [PATCH 06/14] Spec file changes and report_changes for max_retries --- .../connectors/source-amazon-ads/source_amazon_ads/spec.py | 6 ++++++ .../streams/report_streams/report_streams.py | 7 ++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py index b14b3434f0a3..2c4216e30747 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py @@ -71,6 +71,12 @@ class Config: description="Timeout duration for Reports" ) + report_generation_max_retries: int = Field( + None, + name="Report Geration Maximum Retries", + description="Maximum retries Airbyte will attempt for fetching Report Data", + ) + @classmethod def schema(cls, **kvargs): schema = super().schema(**kvargs) diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py index 4d7cd1c2f2c2..a5fa15ebcf87 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py @@ -102,6 +102,8 @@ class ReportStream(BasicAmazonAdsStream, ABC): # Format used to specify metric generation date over Amazon Ads API. REPORT_DATE_FORMAT = "%Y%m%d" cursor_field = "reportDate" + #Maximum number of retries that Airbyte attempt to fetch data. + report_generation_max_retries = 5 def __init__(self, config: AmazonAdsConfig, profiles: List[Profile], authenticator: Oauth2Authenticator): self._authenticator = authenticator @@ -109,6 +111,9 @@ def __init__(self, config: AmazonAdsConfig, profiles: List[Profile], authenticat self._model = self._generate_model() if(config and config.report_wait_timeout and config.report_wait_timeout > 30): self.REPORT_WAIT_TIMEOUT = timedelta(minutes=config.report_wait_timeout).total_seconds + if(config and config.report_generation_max_retries and config.report_generation_max_retries > 5): + self.report_generation_max_retries = config.report_generation_max_retries + # Set start date from config file, should be in UTC timezone. self._start_date = pendulum.parse(config.start_date).set(tz="UTC") if config.start_date else None super().__init__(config, profiles) @@ -155,7 +160,7 @@ def read_records( @backoff.on_exception( backoff.expo, ReportGenerationFailure, - max_tries=1000, + max_tries=report_generation_max_retries, ) def _init_and_try_read_records(self, report_date): report_infos = self._init_reports(report_date) From 97e8d8137fd46b4864d198a0e92c6e8c1f9338b7 Mon Sep 17 00:00:00 2001 From: aakashkumarmaple Date: Fri, 25 Feb 2022 11:52:08 +0530 Subject: [PATCH 07/14] Bakoff Fixes for taking retries values and timeout values from User --- .../streams/report_streams/report_streams.py | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py index a5fa15ebcf87..93e2a65b2aa4 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py @@ -103,7 +103,7 @@ class ReportStream(BasicAmazonAdsStream, ABC): REPORT_DATE_FORMAT = "%Y%m%d" cursor_field = "reportDate" #Maximum number of retries that Airbyte attempt to fetch data. - report_generation_max_retries = 5 + REPORT_GENERATION_MAX_RETRIES = 5 def __init__(self, config: AmazonAdsConfig, profiles: List[Profile], authenticator: Oauth2Authenticator): self._authenticator = authenticator @@ -112,7 +112,7 @@ def __init__(self, config: AmazonAdsConfig, profiles: List[Profile], authenticat if(config and config.report_wait_timeout and config.report_wait_timeout > 30): self.REPORT_WAIT_TIMEOUT = timedelta(minutes=config.report_wait_timeout).total_seconds if(config and config.report_generation_max_retries and config.report_generation_max_retries > 5): - self.report_generation_max_retries = config.report_generation_max_retries + self.REPORT_GENERATION_MAX_RETRIES = config.report_generation_max_retries # Set start date from config file, should be in UTC timezone. self._start_date = pendulum.parse(config.start_date).set(tz="UTC") if config.start_date else None @@ -157,25 +157,34 @@ def read_records( metric=metric_object, ).dict() - @backoff.on_exception( - backoff.expo, - ReportGenerationFailure, - max_tries=report_generation_max_retries, - ) + def backoff_max_time(func): + def wrapped(self, *args, **kwargs): + return backoff.on_exception( + backoff.constant, + RetryableException, + max_time = self.REPORT_WAIT_TIMEOUT + )(func)(self, *args, **kwargs) + return wrapped + + def backoff_max_tries(func): + def wrapped(self, *args, **kwargs): + return backoff.on_exception( + backoff.expo, + ReportGenerationFailure, + max_tries = self.REPORT_GENERATION_MAX_RETRIES + )(func)(self, *args, **kwargs) + return wrapped + + @backoff_max_tries def _init_and_try_read_records(self, report_date): report_infos = self._init_reports(report_date) logger.info(f"Waiting for {len(report_infos)} report(s) to be generated") self._try_read_records(report_infos) return report_infos - @backoff.on_exception( - backoff.constant, - RetryableException, - max_time=REPORT_WAIT_TIMEOUT, - ) + @backoff_max_time def _try_read_records(self, report_infos): incomplete_report_infos = self._incomplete_report_infos(report_infos) - logger.info(f"Checking report status, {len(incomplete_report_infos)} report(s) remaining") for report_info in incomplete_report_infos: report_status, download_url = self._check_status(report_info) From 4003329a859ec02a4b8a9927f4340a0e92eff97f Mon Sep 17 00:00:00 2001 From: aakashkumarmaple Date: Sat, 19 Mar 2022 00:30:24 +0530 Subject: [PATCH 08/14] Review Comment Fixes --- .../source-amazon-ads/source_amazon_ads/spec.py | 8 ++++---- .../streams/report_streams/report_streams.py | 16 ++++------------ 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py index 2c4216e30747..9c6650e7c218 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py @@ -66,15 +66,15 @@ class Config: ) report_wait_timeout: int = Field( - None, name="Report Wait Timeout", - description="Timeout duration for Reports" + description="Timeout duration for Reports. Eg. 30", + default=30 ) report_generation_max_retries: int = Field( - None, name="Report Geration Maximum Retries", - description="Maximum retries Airbyte will attempt for fetching Report Data", + description="Maximum retries Airbyte will attempt for fetching Report Data. Eg. 5", + default=5 ) @classmethod diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py index 93e2a65b2aa4..4b83b03f180b 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py @@ -95,25 +95,17 @@ class ReportStream(BasicAmazonAdsStream, ABC): CHECK_INTERVAL_SECONDS = 30 # Amazon ads updates the data for the next 3 days LOOK_BACK_WINDOW = 3 - # Async report generation time is 15 minutes according to docs: - # https://advertising.amazon.com/API/docs/en-us/get-started/developer-notes # (Service limits section) - REPORT_WAIT_TIMEOUT = timedelta(minutes=30).total_seconds # Format used to specify metric generation date over Amazon Ads API. REPORT_DATE_FORMAT = "%Y%m%d" cursor_field = "reportDate" - #Maximum number of retries that Airbyte attempt to fetch data. - REPORT_GENERATION_MAX_RETRIES = 5 def __init__(self, config: AmazonAdsConfig, profiles: List[Profile], authenticator: Oauth2Authenticator): self._authenticator = authenticator self._session = requests.Session() self._model = self._generate_model() - if(config and config.report_wait_timeout and config.report_wait_timeout > 30): - self.REPORT_WAIT_TIMEOUT = timedelta(minutes=config.report_wait_timeout).total_seconds - if(config and config.report_generation_max_retries and config.report_generation_max_retries > 5): - self.REPORT_GENERATION_MAX_RETRIES = config.report_generation_max_retries - + self.report_wait_timeout = timedelta(minutes=config.report_wait_timeout).total_seconds + self.report_generation_maximum_retries = config.report_generation_max_retries # Set start date from config file, should be in UTC timezone. self._start_date = pendulum.parse(config.start_date).set(tz="UTC") if config.start_date else None super().__init__(config, profiles) @@ -162,7 +154,7 @@ def wrapped(self, *args, **kwargs): return backoff.on_exception( backoff.constant, RetryableException, - max_time = self.REPORT_WAIT_TIMEOUT + max_time = self.report_wait_timeout )(func)(self, *args, **kwargs) return wrapped @@ -171,7 +163,7 @@ def wrapped(self, *args, **kwargs): return backoff.on_exception( backoff.expo, ReportGenerationFailure, - max_tries = self.REPORT_GENERATION_MAX_RETRIES + max_tries = self.report_generation_maximum_retries )(func)(self, *args, **kwargs) return wrapped From 7ca7681f284330c0f306a3a9215a8c4400381bcc Mon Sep 17 00:00:00 2001 From: aakashkumarmaple Date: Sat, 19 Mar 2022 00:34:17 +0530 Subject: [PATCH 09/14] spec file adding examples --- .../connectors/source-amazon-ads/source_amazon_ads/spec.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py index 9c6650e7c218..3b2f812ed978 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py @@ -68,13 +68,15 @@ class Config: report_wait_timeout: int = Field( name="Report Wait Timeout", description="Timeout duration for Reports. Eg. 30", - default=30 + default=30, + examples=[30, 120], ) report_generation_max_retries: int = Field( name="Report Geration Maximum Retries", description="Maximum retries Airbyte will attempt for fetching Report Data. Eg. 5", - default=5 + default=5, + examples=[5,10,15], ) @classmethod From c84a0ac5a3460a02eac657e30b8534f3014e4ed9 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Mon, 28 Mar 2022 17:05:10 -0300 Subject: [PATCH 10/14] correct spec description --- .../connectors/source-amazon-ads/source_amazon_ads/spec.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py index 3b2f812ed978..f2c81a51a2ff 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py @@ -67,7 +67,7 @@ class Config: report_wait_timeout: int = Field( name="Report Wait Timeout", - description="Timeout duration for Reports. Eg. 30", + description="Timeout duration in minutes for Reports. Eg. 30", default=30, examples=[30, 120], ) @@ -76,7 +76,7 @@ class Config: name="Report Geration Maximum Retries", description="Maximum retries Airbyte will attempt for fetching Report Data. Eg. 5", default=5, - examples=[5,10,15], + examples=[5, 10, 15], ) @classmethod From 505d1d1a30225744627906971d67349599d2f7bb Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Mon, 28 Mar 2022 19:56:37 -0300 Subject: [PATCH 11/14] run black --- .../streams/report_streams/report_streams.py | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py index 4b83b03f180b..512ac539df88 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py @@ -1,7 +1,7 @@ # # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # - + import json from abc import ABC, abstractmethod from dataclasses import dataclass @@ -151,20 +151,18 @@ def read_records( def backoff_max_time(func): def wrapped(self, *args, **kwargs): - return backoff.on_exception( - backoff.constant, - RetryableException, - max_time = self.report_wait_timeout - )(func)(self, *args, **kwargs) + return backoff.on_exception(backoff.constant, RetryableException, max_time=self.report_wait_timeout)(func)( + self, *args, **kwargs + ) + return wrapped - + def backoff_max_tries(func): def wrapped(self, *args, **kwargs): - return backoff.on_exception( - backoff.expo, - ReportGenerationFailure, - max_tries = self.report_generation_maximum_retries - )(func)(self, *args, **kwargs) + return backoff.on_exception(backoff.expo, ReportGenerationFailure, max_tries=self.report_generation_maximum_retries)(func)( + self, *args, **kwargs + ) + return wrapped @backoff_max_tries From 48b350eb0c0dae4ab6a8890b4c1b19d8fc795d33 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Mon, 28 Mar 2022 20:33:42 -0300 Subject: [PATCH 12/14] remove tests --- .../acceptance-test-config.yml | 10 +++---- .../integration_tests/configured_catalog.json | 30 ------------------- .../configured_catalog_display.json | 30 ------------------- .../configured_catalog_full_refresh.json | 30 ------------------- .../expected_display_stream.txt | 7 +---- .../integration_tests/expected_records.txt | 4 --- 6 files changed, 6 insertions(+), 105 deletions(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml index 710d126abe72..e6cd75202f11 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml @@ -25,10 +25,10 @@ tests: extra_fields: no exact_order: no extra_records: yes - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/no_profiles_catalog.json" - empty_streams: - ["sponsored_display_product_ads", "sponsored_display_targetings"] + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/no_profiles_catalog.json" + # empty_streams: + # ["sponsored_display_product_ads", "sponsored_display_targetings"] - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog_display.json" empty_streams: @@ -49,7 +49,7 @@ tests: path: "integration_tests/expected_brands_stream.txt" extra_fields: no exact_order: no - extra_records: no + extra_records: yes full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog_full_refresh.json" diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog.json index a3938348625d..a70023a87d39 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog.json @@ -20,36 +20,6 @@ "destination_sync_mode": "overwrite", "source_defined_cursor": false }, - { - "stream": { - "name": "sponsored_display_campaigns", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - "source_defined_cursor": false - }, - { - "stream": { - "name": "sponsored_display_ad_groups", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - "source_defined_cursor": false - }, - { - "stream": { - "name": "sponsored_display_targetings", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - "source_defined_cursor": false - }, { "stream": { "name": "sponsored_product_campaigns", diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog_display.json b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog_display.json index 5f5ae789dcda..6124d73bdfd4 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog_display.json +++ b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog_display.json @@ -9,36 +9,6 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite", "source_defined_cursor": false - }, - { - "stream": { - "name": "sponsored_display_campaigns", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - "source_defined_cursor": false - }, - { - "stream": { - "name": "sponsored_display_ad_groups", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - "source_defined_cursor": false - }, - { - "stream": { - "name": "sponsored_display_targetings", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - "source_defined_cursor": false } ] } diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog_full_refresh.json b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog_full_refresh.json index f20241af225f..9c09408ef728 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog_full_refresh.json +++ b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog_full_refresh.json @@ -10,36 +10,6 @@ "destination_sync_mode": "overwrite", "source_defined_cursor": false }, - { - "stream": { - "name": "sponsored_display_campaigns", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - "source_defined_cursor": false - }, - { - "stream": { - "name": "sponsored_display_ad_groups", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - "source_defined_cursor": false - }, - { - "stream": { - "name": "sponsored_display_targetings", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - "source_defined_cursor": false - }, { "stream": { "name": "sponsored_product_campaigns", diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/expected_display_stream.txt b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/expected_display_stream.txt index 61336b222be7..ccb34a42cced 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/expected_display_stream.txt +++ b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/expected_display_stream.txt @@ -1,6 +1 @@ -{"stream": "profiles", "data": {"profileId": 3893573341008362, "countryCode": "US", "currencyCode": "USD", "dailyBudget": 0.0, "timezone": "America/Los_Angeles", "accountInfo": {"marketplaceStringId": "ATVPDKIKX0DER", "id": "A3LUQZ2NBMFGO4", "type": "seller"}}, "emitted_at": 1627975186000} -{"stream": "sponsored_display_campaigns", "data": {"campaignId": 37387403419888, "name": "sswdd", "tactic": "T00020", "startDate": "20220101", "state": "enabled", "costType": "cpc", "budget": 3.0, "budgetType": "daily", "deliveryProfile": "as_soon_as_possible"}, "emitted_at": 1627975187000} -{"stream": "sponsored_display_campaigns", "data": {"campaignId": 59249214322256, "name": "My test camp", "tactic": "T00020", "startDate": "20220101", "state": "enabled", "costType": "cpc", "budget": 3.0, "budgetType": "daily", "deliveryProfile": "as_soon_as_possible"}, "emitted_at": 1627975187000} -{"stream": "sponsored_display_campaigns", "data": {"campaignId": 16117299922278, "name": "ssw", "tactic": "T00020", "startDate": "20220101", "state": "enabled", "costType": "cpc", "budget": 3.0, "budgetType": "daily", "deliveryProfile": "as_soon_as_possible"}, "emitted_at": 1627975187000} -{"stream": "sponsored_display_campaigns", "data": {"campaignId": 202914386115504, "name": "ssdf", "tactic": "T00020", "startDate": "20220101", "state": "enabled", "costType": "cpc", "budget": 3.0, "budgetType": "daily", "deliveryProfile": "as_soon_as_possible"}, "emitted_at": 1627975187000} -{"stream": "sponsored_display_ad_groups", "data": {"adGroupId": 154135351589329, "campaignId": 59249214322256, "defaultBid": 0.02, "name": "string", "state": "enabled", "bidOptimization": "clicks", "tactic": "T00020"}, "emitted_at": 1627975187000} +{"stream": "profiles", "data": {"profileId": 3893573341008362, "countryCode": "US", "currencyCode": "USD", "dailyBudget": 0.0, "timezone": "America/Los_Angeles", "accountInfo": {"marketplaceStringId": "ATVPDKIKX0DER", "id": "A3LUQZ2NBMFGO4", "type": "seller"}}, "emitted_at": 1627975186000} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/expected_records.txt index f3a192d73893..275e3c780015 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/expected_records.txt +++ b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/expected_records.txt @@ -1,8 +1,4 @@ {"stream": "profiles", "data": {"profileId": 3893573341008362, "countryCode": "US", "currencyCode": "USD", "dailyBudget": 0.0, "timezone": "America/Los_Angeles", "accountInfo": {"marketplaceStringId": "ATVPDKIKX0DER", "id": "A3LUQZ2NBMFGO4", "type": "seller"}}, "emitted_at": 1627977383000} -{"stream": "sponsored_display_campaigns", "data": {"campaignId": 37387403419888, "name": "sswdd", "tactic": "T00020", "startDate": "20220101", "state": "enabled", "costType": "cpc", "budget": 3.0, "budgetType": "daily", "deliveryProfile": "as_soon_as_possible"}, "emitted_at": 1627975187000} -{"stream": "sponsored_display_campaigns", "data": {"campaignId": 59249214322256, "name": "My test camp", "tactic": "T00020", "startDate": "20220101", "state": "enabled", "costType": "cpc", "budget": 3.0, "budgetType": "daily", "deliveryProfile": "as_soon_as_possible"}, "emitted_at": 1627975187000} -{"stream": "sponsored_display_campaigns", "data": {"campaignId": 16117299922278, "name": "ssw", "tactic": "T00020", "startDate": "20220101", "state": "enabled", "costType": "cpc", "budget": 3.0, "budgetType": "daily", "deliveryProfile": "as_soon_as_possible"}, "emitted_at": 1627975187000} -{"stream": "sponsored_display_campaigns", "data": {"campaignId": 202914386115504, "name": "ssdf", "tactic": "T00020", "startDate": "20220101", "state": "enabled", "costType": "cpc", "budget": 3.0, "budgetType": "daily", "deliveryProfile": "as_soon_as_possible"}, "emitted_at": 1627975187000} {"stream": "sponsored_display_ad_groups", "data": {"adGroupId": 154135351589329, "campaignId": 59249214322256, "defaultBid": 0.02, "name": "string", "state": "enabled", "bidOptimization": "clicks", "tactic": "T00020"}, "emitted_at": 1627975187000} {"stream": "sponsored_product_campaigns", "data": {"campaignId": 68543584424555, "name": "string", "campaignType": "sponsoredProducts", "targetingType": "manual", "premiumBidAdjustment": true, "dailyBudget": 1.0, "startDate": "20221111", "endDate": "20231111", "state": "enabled", "bidding": {"strategy": "legacyForSales", "adjustments": [{"predicate": "placementTop", "percentage": 50}]}, "portfolioId": 83018499803606, "tags": {"PONumber": "examplePONumber", "accountManager": "exampleAccountManager"}}, "emitted_at": 1627974510000} {"stream": "sponsored_product_campaigns", "data": {"campaignId": 268265378730407, "name": "Test campaging", "campaignType": "sponsoredProducts", "targetingType": "manual", "premiumBidAdjustment": true, "dailyBudget": 1.0, "startDate": "20221111", "endDate": "20231111", "state": "enabled", "bidding": {"strategy": "legacyForSales", "adjustments": [{"predicate": "placementTop", "percentage": 50}]}, "portfolioId": 83018499803606, "tags": {"PONumber": "examplePONumber", "accountManager": "exampleAccountManager"}}, "emitted_at": 1627974510000} From ac37fd665d861c1d7084343226bd4d95a0cc3c7d Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Tue, 29 Mar 2022 19:49:16 -0300 Subject: [PATCH 13/14] correct spec test --- .../integration_tests/spec.json | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json index 60294054aa74..4eb3655a49c9 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json @@ -52,7 +52,25 @@ "description": "profile Ids you want to fetch data for", "name": "Profile Ids", "type": "array", - "items": { "type": "integer" } + "items": { + "type": "integer" + } + }, + "report_wait_timeout": { + "title": "Report Wait Timeout", + "description": "Timeout duration in minutes for Reports. Eg. 30", + "default": 30, + "name": "Report Wait Timeout", + "examples": [30, 120], + "type": "integer" + }, + "report_generation_max_retries": { + "title": "Report Generation Max Retries", + "description": "Maximum retries Airbyte will attempt for fetching Report Data. Eg. 5", + "default": 5, + "name": "Report Geration Maximum Retries", + "examples": [5, 10, 15], + "type": "integer" } }, "required": ["client_id", "client_secret", "refresh_token"] From b3fb0ab94bfe1fd918541aaf8cb7cadc3866079c Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Wed, 30 Mar 2022 14:05:37 -0300 Subject: [PATCH 14/14] add eof --- airbyte-integrations/connectors/source-amazon-ads/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile index 32eea36bac70..5d7d367c99f2 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile @@ -13,4 +13,4 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] LABEL io.airbyte.version=0.1.4 -LABEL io.airbyte.name=airbyte/source-amazon-ads \ No newline at end of file +LABEL io.airbyte.name=airbyte/source-amazon-ads