diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json index 82773de37a9d..2fd15ec14bcd 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json @@ -38,6 +38,21 @@ "name": "Start date", "examples": ["2022-10-10", "2022-10-22"], "type": "string" + }, + "region": { + "description": "Region to pull data from (EU/NA/FE/SANDBOX)", + "default": "NA", + "name": "Region", + "title": "AmazonAdsRegion", + "enum": ["NA", "EU", "FE", "SANDBOX"], + "type": "string" + }, + "profiles": { + "title": "Profiles", + "description": "profile Ids you want to fetch data for", + "name": "Profile Ids", + "type": "array", + "items": { "type": "integer" } } }, "required": ["client_id", "client_secret", "refresh_token"] diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/constants.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/constants.py index ef88e9bd1bf3..e5ab6f8b3957 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/constants.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/constants.py @@ -1,14 +1,20 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + from enum import Enum + class AmazonAdsRegion(str, Enum): - NA="NA" - EU="EU" - FE="FE" - SANDBOX="SANDBOX" + NA = "NA" + EU = "EU" + FE = "FE" + SANDBOX = "SANDBOX" + URL_MAPPING = { "NA": "https://advertising-api.amazon.com/", "EU": "https://advertising-api-eu.amazon.com/", "FE": "https://advertising-api-fe.amazon.com/", - "SANDBOX": "https://advertising-api-test.amazon.com/" -} \ No newline at end of file + "SANDBOX": "https://advertising-api-test.amazon.com/", +} diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py index 6e5e8601599b..9604dd7819d9 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py @@ -11,6 +11,7 @@ from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator +from .schemas import Profile from .spec import AmazonAdsConfig from .streams import ( Profiles, @@ -18,6 +19,7 @@ SponsoredBrandsCampaigns, SponsoredBrandsKeywords, SponsoredBrandsReportStream, + SponsoredBrandsVideoReportStream, SponsoredDisplayAdGroups, SponsoredDisplayCampaigns, SponsoredDisplayProductAds, @@ -30,10 +32,7 @@ SponsoredProductNegativeKeywords, SponsoredProductsReportStream, SponsoredProductTargetings, - SponsoredBrandsVideoReportStream, ) -from .schemas import Profile - # Oauth 2.0 authentication URL for amazon TOKEN_URL = "https://api.amazon.com/auth/o2/token" @@ -108,7 +107,6 @@ def _make_authenticator(config: AmazonAdsConfig): scopes=[config.scope], ) - @staticmethod def _choose_profiles(config: AmazonAdsConfig, profiles: List[Profile]): if not config.profiles: diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py index a398b26c0077..ddc33f70c552 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py @@ -2,11 +2,12 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # -from pydantic import BaseModel, Field from typing import List +from pydantic import BaseModel, Field from source_amazon_ads.constants import AmazonAdsRegion + class AmazonAdsConfig(BaseModel): class Config: title = "Amazon Ads Spec" @@ -44,11 +45,7 @@ class Config: examples=["2022-10-10", "2022-10-22"], ) - region: AmazonAdsRegion = Field( - name="Region", - description="Region to pull data from (EU/NA/FE/SANDBOX)", - default=AmazonAdsRegion.NA - ) + region: AmazonAdsRegion = Field(name="Region", description="Region to pull data from (EU/NA/FE/SANDBOX)", default=AmazonAdsRegion.NA) profiles: List[int] = Field( None, @@ -63,4 +60,10 @@ def schema(cls, **kvargs): # environment for SAT but dont want it to be visible for end users, # filter out it from the jsonschema output schema["properties"] = {name: desc for name, desc in schema["properties"].items() if not name.startswith("_")} + # Transform pydantic generated enum for region + schema["definitions"]["AmazonAdsRegion"].pop("description") + schema["properties"]["region"].update(schema["definitions"]["AmazonAdsRegion"]) + schema["properties"]["region"].pop("allOf", None) + schema["properties"]["region"].pop("$ref", None) + del schema["definitions"] return schema diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/__init__.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/__init__.py index 33d4e5cf5d46..2525ffa63579 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/__init__.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/__init__.py @@ -1,28 +1,13 @@ # -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. # from .profiles import Profiles -from .report_streams import SponsoredBrandsReportStream, SponsoredDisplayReportStream, SponsoredProductsReportStream, SponsoredBrandsVideoReportStream +from .report_streams import ( + SponsoredBrandsReportStream, + SponsoredBrandsVideoReportStream, + SponsoredDisplayReportStream, + SponsoredProductsReportStream, +) from .sponsored_brands import SponsoredBrandsAdGroups, SponsoredBrandsCampaigns, SponsoredBrandsKeywords from .sponsored_display import SponsoredDisplayAdGroups, SponsoredDisplayCampaigns, SponsoredDisplayProductAds, SponsoredDisplayTargetings from .sponsored_products import ( diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py index 45388101bee1..cb7fdce764cb 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py @@ -10,10 +10,10 @@ from airbyte_cdk.sources.streams.core import Stream from airbyte_cdk.sources.streams.http import HttpStream from pydantic import BaseModel, ValidationError +from source_amazon_ads.constants import URL_MAPPING from source_amazon_ads.schemas import CatalogModel from source_amazon_ads.schemas.profile import Profile from source_amazon_ads.spec import AmazonAdsConfig -from source_amazon_ads.constants import URL_MAPPING """ This class hierarchy may seem overcomplicated so here is a visualization of diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/__init__.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/__init__.py index d0c216664669..28e70950a759 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/__init__.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/__init__.py @@ -1,29 +1,14 @@ # -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. # from .brands_report import SponsoredBrandsReportStream +from .brands_video_report import SponsoredBrandsVideoReportStream from .display_report import SponsoredDisplayReportStream from .products_report import SponsoredProductsReportStream -from .brands_video_report import SponsoredBrandsVideoReportStream -__all__ = ["SponsoredDisplayReportStream", "SponsoredProductsReportStream", "SponsoredBrandsReportStream", "SponsoredBrandsVideoReportStream"] +__all__ = [ + "SponsoredDisplayReportStream", + "SponsoredProductsReportStream", + "SponsoredBrandsReportStream", + "SponsoredBrandsVideoReportStream", +] diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/brands_video_report.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/brands_video_report.py index c23cdbcad79c..83d42554c4b2 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/brands_video_report.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/brands_video_report.py @@ -1,25 +1,5 @@ # -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. # from .report_streams import ReportStream diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py index 7c0d19d8d88d..6328c1b511e9 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py @@ -226,10 +226,10 @@ def get_report_date_ranges(start_report_date: Optional[datetime]) -> Iterable[st now = datetime.utcnow() if not start_report_date: start_report_date = now - + # You cannot pull data for amazon ads more than 60 days if (now - start_report_date).days > (60 - ReportStream.LOOK_BACK_WINDOW): - start_report_date = now + timedelta(days=-(60-ReportStream.LOOK_BACK_WINDOW)) + start_report_date = now + timedelta(days=-(60 - ReportStream.LOOK_BACK_WINDOW)) for days in range(0, (now - start_report_date).days + 1): next_date = start_report_date + timedelta(days=days) @@ -317,14 +317,12 @@ def _calc_report_generation_date(report_date: str, profile) -> str: reports from day before specified report date and we should take into account timezone for each profile. :param report_date requested date that stored in stream state. - :return date parameter for Amazon Ads generate report. It equial to day - before current day for the profile's timezone. + :return date parameter for Amazon Ads generate report. """ report_date = datetime.strptime(report_date, ReportStream.REPORT_DATE_FORMAT) profile_tz = pytz.timezone(profile.timezone) profile_time = report_date.astimezone(profile_tz) - profile_yesterday = profile_time - timedelta(days=1) - return profile_yesterday.strftime(ReportStream.REPORT_DATE_FORMAT) + return profile_time.strftime(ReportStream.REPORT_DATE_FORMAT) def _download_report(self, report_info: ReportInfo, url: str) -> List[dict]: """ diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py index 787d1c9876c2..cebfbc28b96b 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py @@ -14,7 +14,12 @@ from requests.exceptions import ConnectionError from source_amazon_ads.schemas.profile import AccountInfo, Profile from source_amazon_ads.spec import AmazonAdsConfig -from source_amazon_ads.streams import SponsoredBrandsReportStream, SponsoredDisplayReportStream, SponsoredProductsReportStream, SponsoredBrandsVideoReportStream +from source_amazon_ads.streams import ( + SponsoredBrandsReportStream, + SponsoredBrandsVideoReportStream, + SponsoredDisplayReportStream, + SponsoredProductsReportStream, +) from source_amazon_ads.streams.report_streams.report_streams import TooManyRequests """ @@ -157,6 +162,7 @@ def test_brands_report_stream(test_config): metrics = [m for m in stream.read_records(SyncMode.incremental, stream_slice=stream_slice)] assert len(metrics) == METRICS_COUNT * len(stream.metrics_map) + @responses.activate def test_brands_video_report_stream(test_config): setup_responses( @@ -284,6 +290,10 @@ def test_display_report_stream_slices_incremental(test_config): stream_state = {"reportDate": "20210726"} slices = stream.stream_slices(SyncMode.incremental, cursor_field=stream.cursor_field, stream_state=stream_state) assert slices == [ + {"reportDate": "20210723"}, + {"reportDate": "20210724"}, + {"reportDate": "20210725"}, + {"reportDate": "20210726"}, {"reportDate": "20210727"}, {"reportDate": "20210728"}, {"reportDate": "20210729"}, @@ -291,11 +301,20 @@ def test_display_report_stream_slices_incremental(test_config): ] stream_state = {"reportDate": "20210730"} slices = stream.stream_slices(SyncMode.incremental, cursor_field=stream.cursor_field, stream_state=stream_state) - assert slices == [None] + assert slices == [ + {"reportDate": "20210727"}, + {"reportDate": "20210728"}, + {"reportDate": "20210729"}, + {"reportDate": "20210730"}, + ] stream_state = {"reportDate": "20210731"} slices = stream.stream_slices(SyncMode.incremental, cursor_field=stream.cursor_field, stream_state=stream_state) - assert slices == [None] + assert slices == [ + {"reportDate": "20210728"}, + {"reportDate": "20210729"}, + {"reportDate": "20210730"}, + ] slices = stream.stream_slices(SyncMode.incremental, cursor_field=stream.cursor_field, stream_state={}) assert slices == [{"reportDate": "20210730"}] diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py index a68e7d0fec90..c304d01cdd92 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py @@ -51,7 +51,7 @@ def test_source_streams(test_config): setup_responses() source = SourceAmazonAds() streams = source.streams(test_config) - assert len(streams) == 17 + assert len(streams) == 18 actual_stream_names = {stream.name for stream in streams} expected_stream_names = set( [