From aa89a4d423de8405d19952f88c9d7d3e76a86846 Mon Sep 17 00:00:00 2001 From: Sage Watterworth <83830720+sage-watterworth@users.noreply.github.com> Date: Thu, 20 Oct 2022 12:50:55 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=89=20Source=20Amazon=20Ads:=20filters?= =?UTF-8?q?=20for=20state=20on=20brand,=20product=20and=20display=20campai?= =?UTF-8?q?gns=20(#17475)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * amazon ad status filter * update test report stream with parametrized campaigns * remove config files related to previous acceptance test. revert to master acceptance test set up * amazon ad status filter * update test report stream with parametrized campaigns * remove config files related to previous acceptance test. revert to master acceptance test set up * oct 17 edits * oct 17 edits: 2 * bump dockerfile to 1.24 * fix: match the cdk version to new one * auto-bump connector version Co-authored-by: sajarin Co-authored-by: Harshith Mullapudi Co-authored-by: Octavia Squidington III --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 16 +++++- .../connectors/source-amazon-ads/Dockerfile | 3 +- .../acceptance-test-config.yml | 2 +- .../integration_tests/spec.json | 11 ++++ .../connectors/source-amazon-ads/setup.py | 2 +- .../source_amazon_ads/spec.yaml | 12 ++++ .../streams/sponsored_brands.py | 11 ++++ .../streams/sponsored_display.py | 19 +++++-- .../streams/sponsored_products.py | 11 ++++ .../unit_tests/test_report_streams.py | 56 +++++++++++++++++++ docs/integrations/sources/amazon-ads.md | 1 + 12 files changed, 137 insertions(+), 9 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 5960faafd7bd..6e73e42baae8 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -33,7 +33,7 @@ - name: Amazon Ads sourceDefinitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246 dockerRepository: airbyte/source-amazon-ads - dockerImageTag: 0.1.23 + dockerImageTag: 0.1.24 documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-ads icon: amazonads.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index f1a4ef305424..1a08b6b37450 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -647,7 +647,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-amazon-ads:0.1.23" +- dockerImage: "airbyte/source-amazon-ads:0.1.24" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/amazon-ads" connectionSpecification: @@ -729,6 +729,20 @@ type: "array" items: type: "integer" + state_filter: + title: "State Filter" + description: "Reflects the state of the Display, Product, and Brand Campaign\ + \ streams as enabled, paused, or archived. If you do not populate this\ + \ field, it will be ignored completely." + items: + type: "string" + enum: + - "enabled" + - "paused" + - "archived" + type: "array" + uniqueItems: true + order: 9 required: - "client_id" - "client_secret" diff --git a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile index 69f76987802c..68fcde896d8f 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile @@ -12,5 +12,6 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.23 + +LABEL io.airbyte.version=0.1.24 LABEL io.airbyte.name=airbyte/source-amazon-ads diff --git a/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml index a19c5c48b498..a1e7adb95e7a 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml @@ -46,4 +46,4 @@ tests: configured_catalog_path: "integration_tests/configured_catalog_report.json" ignored_fields: "sponsored_products_report_stream": ["updatedAt"] - timeout_seconds: 3600 + timeout_seconds: 3600 \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json index ab16585e81bd..d1ba1851a4b5 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json @@ -69,6 +69,17 @@ "items": { "type": "integer" } + }, + "state_filter": { + "title": "State Filter", + "description": "Reflects the state of the Display, Product, and Brand Campaign streams as enabled, paused, or archived. If you do not populate this field, it will be ignored completely.", + "items": { + "type": "string", + "enum": ["enabled", "paused", "archived"] + }, + "type": "array", + "uniqueItems": true, + "order": 9 } }, "required": ["client_id", "client_secret", "refresh_token"], diff --git a/airbyte-integrations/connectors/source-amazon-ads/setup.py b/airbyte-integrations/connectors/source-amazon-ads/setup.py index 7f4200df6c1b..b74dd73a61ed 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/setup.py +++ b/airbyte-integrations/connectors/source-amazon-ads/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1.68", "requests_oauthlib~=1.3.1", "pendulum~=2.1.2"] +MAIN_REQUIREMENTS = ["airbyte-cdk~=0.2.0", "requests_oauthlib~=1.3.1", "pendulum~=2.1.2"] TEST_REQUIREMENTS = [ "pytest~=6.1", diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml index b45824964137..5523b774e86f 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml @@ -86,6 +86,18 @@ connectionSpecification: type: array items: type: integer + state_filter: + title: State Filter + description: Reflects the state of the Display, Product, and Brand Campaign streams as enabled, paused, or archived. If you do not populate this field, it will be ignored completely. + items: + type: string + enum: + - enabled + - paused + - archived + type: array + uniqueItems: true + order: 9 required: - client_id - client_secret diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_brands.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_brands.py index d2b69fda42a6..7ab994bca8d3 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_brands.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_brands.py @@ -12,12 +12,23 @@ class SponsoredBrandsCampaigns(SubProfilesStream): https://advertising.amazon.com/API/docs/en-us/sponsored-brands/3-0/openapi#/Campaigns """ + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.state_filter = kwargs.get("config", {}).get("state_filter") + primary_key = "campaignId" + state_filter = None model = BrandsCampaign def path(self, **kvargs) -> str: return "sb/campaigns" + def request_params(self, *args, **kwargs): + params = super().request_params(*args, **kwargs) + if self.state_filter: + params["stateFilter"] = ",".join(self.state_filter) + return params + class SponsoredBrandsAdGroups(SubProfilesStream): """ diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_display.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_display.py index ae3eb1d8ad24..3ce8e264bbcf 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_display.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_display.py @@ -12,12 +12,23 @@ class SponsoredDisplayCampaigns(SubProfilesStream): https://advertising.amazon.com/API/docs/en-us/sponsored-display/3-0/openapi#/Campaigns """ + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.state_filter = kwargs.get("config", {}).get("state_filter") + primary_key = "campaignId" + state_filter = None model = DisplayCampaign - def path(self, **kvargs) -> str: + def path(self, **kwargs) -> str: return "sd/campaigns" + def request_params(self, *args, **kwargs): + params = super().request_params(*args, **kwargs) + if self.state_filter: + params["stateFilter"] = ",".join(self.state_filter) + return params + class SponsoredDisplayAdGroups(SubProfilesStream): """ @@ -28,7 +39,7 @@ class SponsoredDisplayAdGroups(SubProfilesStream): primary_key = "adGroupId" model = DisplayAdGroup - def path(self, **kvargs) -> str: + def path(self, **kwargs) -> str: return "sd/adGroups" @@ -41,7 +52,7 @@ class SponsoredDisplayProductAds(SubProfilesStream): primary_key = "adId" model = DisplayProductAds - def path(self, **kvargs) -> str: + def path(self, **kwargs) -> str: return "sd/productAds" @@ -54,5 +65,5 @@ class SponsoredDisplayTargetings(SubProfilesStream): primary_key = "targetId" model = DisplayTargeting - def path(self, **kvargs) -> str: + def path(self, **kwargs) -> str: return "sd/targets" diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_products.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_products.py index 444b7eb4ff44..b5baa89105c0 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_products.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_products.py @@ -12,12 +12,23 @@ class SponsoredProductCampaigns(SubProfilesStream): https://advertising.amazon.com/API/docs/en-us/sponsored-display/3-0/openapi#/Campaigns """ + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.state_filter = kwargs.get("config", {}).get("state_filter") + primary_key = "campaignId" + state_filter = None model = ProductCampaign def path(self, **kvargs) -> str: return "v2/sp/campaigns" + def request_params(self, *args, **kwargs): + params = super().request_params(*args, **kwargs) + if self.state_filter: + params["stateFilter"] = ",".join(self.state_filter) + return params + class SponsoredProductAdGroups(SubProfilesStream): """ diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py index 6fc2b3d41951..f53a10777ae0 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py @@ -19,9 +19,12 @@ from source_amazon_ads.schemas.profile import AccountInfo, Profile from source_amazon_ads.source import CONFIG_DATE_FORMAT from source_amazon_ads.streams import ( + SponsoredBrandsCampaigns, SponsoredBrandsReportStream, SponsoredBrandsVideoReportStream, + SponsoredDisplayCampaigns, SponsoredDisplayReportStream, + SponsoredProductCampaigns, SponsoredProductsReportStream, ) from source_amazon_ads.streams.report_streams.report_streams import ReportGenerationFailure, ReportGenerationInProgress, TooManyRequests @@ -556,3 +559,56 @@ def test_read_incremental_with_records_start_date(config): records = list(read_incremental(stream, state)) assert state == {"1": {"reportDate": "20210104"}} assert {r["reportDate"] for r in records} == {"20210103", "20210104", "20210105", "20210106"} + + +@pytest.mark.parametrize( + "state_filter, stream_class", + [ + ( + ["enabled", "archived", "paused"], + SponsoredBrandsCampaigns, + ), + ( + ["enabled"], + SponsoredBrandsCampaigns, + ), + ( + None, + SponsoredBrandsCampaigns, + ), + ( + ["enabled", "archived", "paused"], + SponsoredProductCampaigns, + ), + ( + ["enabled"], + SponsoredProductCampaigns, + ), + ( + None, + SponsoredProductCampaigns, + ), + ( + ["enabled", "archived", "paused"], + SponsoredDisplayCampaigns, + ), + ( + ["enabled"], + SponsoredDisplayCampaigns, + ), + ( + None, + SponsoredDisplayCampaigns, + ), + ], +) +def test_streams_state_filter(mocker, config, state_filter, stream_class): + profiles = make_profiles() + mocker.patch.object(stream_class, "state_filter", new_callable=mocker.PropertyMock, return_value=state_filter) + + stream = stream_class(config, profiles) + params = stream.request_params(stream_state=None, stream_slice=None, next_page_token=None) + if "stateFilter" in params: + assert params["stateFilter"] == ",".join(state_filter) + else: + assert state_filter is None diff --git a/docs/integrations/sources/amazon-ads.md b/docs/integrations/sources/amazon-ads.md index 00bd74d74e9a..acdb987934c4 100644 --- a/docs/integrations/sources/amazon-ads.md +++ b/docs/integrations/sources/amazon-ads.md @@ -91,6 +91,7 @@ Information about expected report generation waiting time you may find [here](ht | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------| +| 0.1.24 | 2022-10-19 | [17475](https://github.com/airbytehq/airbyte/pull/17475) | Add filters for state on brand, product and display campaigns | | 0.1.23 | 2022-09-06 | [16342](https://github.com/airbytehq/airbyte/pull/16342) | Add attribution reports | | 0.1.22 | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream state. | | 0.1.21 | 2022-09-27 | [17202](https://github.com/airbytehq/airbyte/pull/17202) | Improved handling if known reporting errors |