Fix amazon ads unittests and apply formatting

airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json

@@ -38,6 +38,21 @@
         "name": "Start date",
         "examples": ["2022-10-10", "2022-10-22"],
         "type": "string"
+      },
+      "region": {
+        "description": "Region to pull data from (EU/NA/FE/SANDBOX)",
+        "default": "NA",
+        "name": "Region",
+        "title": "AmazonAdsRegion",
+        "enum": ["NA", "EU", "FE", "SANDBOX"],
+        "type": "string"
+      },
+      "profiles": {
+        "title": "Profiles",
+        "description": "profile Ids you want to fetch data for",
+        "name": "Profile Ids",
+        "type": "array",
+        "items": { "type": "integer" }
       }
     },
     "required": ["client_id", "client_secret", "refresh_token"]

airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/constants.py

@@ -1,14 +1,20 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
+
 from enum import Enum
 
+
 class AmazonAdsRegion(str, Enum):
-    NA="NA"
-    EU="EU"
-    FE="FE"
-    SANDBOX="SANDBOX"
+    NA = "NA"
+    EU = "EU"
+    FE = "FE"
+    SANDBOX = "SANDBOX"
+
 
 URL_MAPPING = {
     "NA": "https://advertising-api.amazon.com/",
     "EU": "https://advertising-api-eu.amazon.com/",
     "FE": "https://advertising-api-fe.amazon.com/",
-    "SANDBOX": "https://advertising-api-test.amazon.com/"
-}
+    "SANDBOX": "https://advertising-api-test.amazon.com/",
+}

airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py

@@ -11,13 +11,15 @@
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator
 
+from .schemas import Profile
 from .spec import AmazonAdsConfig
 from .streams import (
     Profiles,
     SponsoredBrandsAdGroups,
     SponsoredBrandsCampaigns,
     SponsoredBrandsKeywords,
     SponsoredBrandsReportStream,
+    SponsoredBrandsVideoReportStream,
     SponsoredDisplayAdGroups,
     SponsoredDisplayCampaigns,
     SponsoredDisplayProductAds,
@@ -30,10 +32,7 @@
     SponsoredProductNegativeKeywords,
     SponsoredProductsReportStream,
     SponsoredProductTargetings,
-    SponsoredBrandsVideoReportStream,
 )
-from .schemas import Profile
-
 
 # Oauth 2.0 authentication URL for amazon
 TOKEN_URL = "https://api.amazon.com/auth/o2/token"
@@ -108,7 +107,6 @@ def _make_authenticator(config: AmazonAdsConfig):
             scopes=[config.scope],
         )
 
-
     @staticmethod
     def _choose_profiles(config: AmazonAdsConfig, profiles: List[Profile]):
         if not config.profiles:

airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py

@@ -2,11 +2,12 @@
 # Copyright (c) 2021 Airbyte, Inc., all rights reserved.
 #
 
-from pydantic import BaseModel, Field
 from typing import List
 
+from pydantic import BaseModel, Field
 from source_amazon_ads.constants import AmazonAdsRegion
 
+
 class AmazonAdsConfig(BaseModel):
     class Config:
         title = "Amazon Ads Spec"
@@ -44,11 +45,7 @@ class Config:
         examples=["2022-10-10", "2022-10-22"],
     )
 
-    region: AmazonAdsRegion = Field(
-        name="Region",
-        description="Region to pull data from (EU/NA/FE/SANDBOX)",
-        default=AmazonAdsRegion.NA
-    )
+    region: AmazonAdsRegion = Field(name="Region", description="Region to pull data from (EU/NA/FE/SANDBOX)", default=AmazonAdsRegion.NA)
 
     profiles: List[int] = Field(
         None,
@@ -63,4 +60,10 @@ def schema(cls, **kvargs):
         # environment for SAT but dont want it to be visible for end users,
         # filter out it from the jsonschema output
         schema["properties"] = {name: desc for name, desc in schema["properties"].items() if not name.startswith("_")}
+        # Transform pydantic generated enum for region
+        schema["definitions"]["AmazonAdsRegion"].pop("description")
+        schema["properties"]["region"].update(schema["definitions"]["AmazonAdsRegion"])
+        schema["properties"]["region"].pop("allOf", None)
+        schema["properties"]["region"].pop("$ref", None)
+        del schema["definitions"]
         return schema

airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/__init__.py

@@ -1,28 +1,13 @@
 #
-# MIT License
-#
-# Copyright (c) 2020 Airbyte
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in all
-# copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
 #
 from .profiles import Profiles
-from .report_streams import SponsoredBrandsReportStream, SponsoredDisplayReportStream, SponsoredProductsReportStream, SponsoredBrandsVideoReportStream
+from .report_streams import (
+    SponsoredBrandsReportStream,
+    SponsoredBrandsVideoReportStream,
+    SponsoredDisplayReportStream,
+    SponsoredProductsReportStream,
+)
 from .sponsored_brands import SponsoredBrandsAdGroups, SponsoredBrandsCampaigns, SponsoredBrandsKeywords
 from .sponsored_display import SponsoredDisplayAdGroups, SponsoredDisplayCampaigns, SponsoredDisplayProductAds, SponsoredDisplayTargetings
 from .sponsored_products import (

airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py

@@ -10,10 +10,10 @@
 from airbyte_cdk.sources.streams.core import Stream
 from airbyte_cdk.sources.streams.http import HttpStream
 from pydantic import BaseModel, ValidationError
+from source_amazon_ads.constants import URL_MAPPING
 from source_amazon_ads.schemas import CatalogModel
 from source_amazon_ads.schemas.profile import Profile
 from source_amazon_ads.spec import AmazonAdsConfig
-from source_amazon_ads.constants import URL_MAPPING
 
 """
 This class hierarchy may seem overcomplicated so here is a visualization of

airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/__init__.py

@@ -1,29 +1,14 @@
 #
-# MIT License
-#
-# Copyright (c) 2020 Airbyte
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in all
-# copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
 #
 from .brands_report import SponsoredBrandsReportStream
+from .brands_video_report import SponsoredBrandsVideoReportStream
 from .display_report import SponsoredDisplayReportStream
 from .products_report import SponsoredProductsReportStream
-from .brands_video_report import SponsoredBrandsVideoReportStream
 
-__all__ = ["SponsoredDisplayReportStream", "SponsoredProductsReportStream", "SponsoredBrandsReportStream", "SponsoredBrandsVideoReportStream"]
+__all__ = [
+    "SponsoredDisplayReportStream",
+    "SponsoredProductsReportStream",
+    "SponsoredBrandsReportStream",
+    "SponsoredBrandsVideoReportStream",
+]

airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/brands_video_report.py

@@ -1,25 +1,5 @@
 #
-# MIT License
-#
-# Copyright (c) 2020 Airbyte
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in all
-# copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
 #
 
 from .report_streams import ReportStream

airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py

@@ -226,10 +226,10 @@ def get_report_date_ranges(start_report_date: Optional[datetime]) -> Iterable[st
         now = datetime.utcnow()
         if not start_report_date:
             start_report_date = now
-        
+
         # You cannot pull data for amazon ads more than 60 days
         if (now - start_report_date).days > (60 - ReportStream.LOOK_BACK_WINDOW):
-            start_report_date = now + timedelta(days=-(60-ReportStream.LOOK_BACK_WINDOW))
+            start_report_date = now + timedelta(days=-(60 - ReportStream.LOOK_BACK_WINDOW))
 
         for days in range(0, (now - start_report_date).days + 1):
             next_date = start_report_date + timedelta(days=days)
@@ -317,14 +317,12 @@ def _calc_report_generation_date(report_date: str, profile) -> str:
         reports from day before specified report date and we should take into
         account timezone for each profile.
         :param report_date requested date that stored in stream state.
-        :return date parameter for Amazon Ads generate report. It equial to day
-        before current day for the profile's timezone.
+        :return date parameter for Amazon Ads generate report.
         """
         report_date = datetime.strptime(report_date, ReportStream.REPORT_DATE_FORMAT)
         profile_tz = pytz.timezone(profile.timezone)
         profile_time = report_date.astimezone(profile_tz)
-        profile_yesterday = profile_time - timedelta(days=1)
-        return profile_yesterday.strftime(ReportStream.REPORT_DATE_FORMAT)
+        return profile_time.strftime(ReportStream.REPORT_DATE_FORMAT)
 
     def _download_report(self, report_info: ReportInfo, url: str) -> List[dict]:
         """

airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py

@@ -14,7 +14,12 @@
 from requests.exceptions import ConnectionError
 from source_amazon_ads.schemas.profile import AccountInfo, Profile
 from source_amazon_ads.spec import AmazonAdsConfig
-from source_amazon_ads.streams import SponsoredBrandsReportStream, SponsoredDisplayReportStream, SponsoredProductsReportStream, SponsoredBrandsVideoReportStream
+from source_amazon_ads.streams import (
+    SponsoredBrandsReportStream,
+    SponsoredBrandsVideoReportStream,
+    SponsoredDisplayReportStream,
+    SponsoredProductsReportStream,
+)
 from source_amazon_ads.streams.report_streams.report_streams import TooManyRequests
 
 """
@@ -157,6 +162,7 @@ def test_brands_report_stream(test_config):
     metrics = [m for m in stream.read_records(SyncMode.incremental, stream_slice=stream_slice)]
     assert len(metrics) == METRICS_COUNT * len(stream.metrics_map)
 
+
 @responses.activate
 def test_brands_video_report_stream(test_config):
     setup_responses(
@@ -284,18 +290,31 @@ def test_display_report_stream_slices_incremental(test_config):
     stream_state = {"reportDate": "20210726"}
     slices = stream.stream_slices(SyncMode.incremental, cursor_field=stream.cursor_field, stream_state=stream_state)
     assert slices == [
+        {"reportDate": "20210723"},
+        {"reportDate": "20210724"},
+        {"reportDate": "20210725"},
+        {"reportDate": "20210726"},
         {"reportDate": "20210727"},
         {"reportDate": "20210728"},
         {"reportDate": "20210729"},
         {"reportDate": "20210730"},
     ]
     stream_state = {"reportDate": "20210730"}
     slices = stream.stream_slices(SyncMode.incremental, cursor_field=stream.cursor_field, stream_state=stream_state)
-    assert slices == [None]
+    assert slices == [
+        {"reportDate": "20210727"},
+        {"reportDate": "20210728"},
+        {"reportDate": "20210729"},
+        {"reportDate": "20210730"},
+    ]
 
     stream_state = {"reportDate": "20210731"}
     slices = stream.stream_slices(SyncMode.incremental, cursor_field=stream.cursor_field, stream_state=stream_state)
-    assert slices == [None]
+    assert slices == [
+        {"reportDate": "20210728"},
+        {"reportDate": "20210729"},
+        {"reportDate": "20210730"},
+    ]
 
     slices = stream.stream_slices(SyncMode.incremental, cursor_field=stream.cursor_field, stream_state={})
     assert slices == [{"reportDate": "20210730"}]

airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py

@@ -51,7 +51,7 @@ def test_source_streams(test_config):
     setup_responses()
     source = SourceAmazonAds()
     streams = source.streams(test_config)
-    assert len(streams) == 17
+    assert len(streams) == 18
     actual_stream_names = {stream.name for stream in streams}
     expected_stream_names = set(
         [