🎉 Source Amazon Ads: filters for state on brand, product and display …

…campaigns (airbytehq#17475) * amazon ad status filter * update test report stream with parametrized campaigns * remove config files related to previous acceptance test. revert to master acceptance test set up * amazon ad status filter * update test report stream with parametrized campaigns * remove config files related to previous acceptance test. revert to master acceptance test set up * oct 17 edits * oct 17 edits: 2 * bump dockerfile to 1.24 * fix: match the cdk version to new one * auto-bump connector version Co-authored-by: sajarin <[email protected]> Co-authored-by: Harshith Mullapudi <[email protected]> Co-authored-by: Octavia Squidington III <[email protected]>
jhammarstedt · Oct 31, 2022 · aa89a4d · aa89a4d
1 parent 977351b
commit aa89a4d
Show file tree

Hide file tree

Showing 12 changed files with 137 additions and 9 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -33,7 +33,7 @@
 - name: Amazon Ads
   sourceDefinitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246
   dockerRepository: airbyte/source-amazon-ads
-  dockerImageTag: 0.1.23
+  dockerImageTag: 0.1.24
   documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-ads
   icon: amazonads.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -647,7 +647,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-amazon-ads:0.1.23"
+- dockerImage: "airbyte/source-amazon-ads:0.1.24"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/amazon-ads"
     connectionSpecification:
@@ -729,6 +729,20 @@
           type: "array"
           items:
             type: "integer"
+        state_filter:
+          title: "State Filter"
+          description: "Reflects the state of the Display, Product, and Brand Campaign\
+            \ streams as enabled, paused, or archived. If you do not populate this\
+            \ field, it will be ignored completely."
+          items:
+            type: "string"
+            enum:
+            - "enabled"
+            - "paused"
+            - "archived"
+          type: "array"
+          uniqueItems: true
+          order: 9
       required:
       - "client_id"
       - "client_secret"

diff --git a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile
@@ -12,5 +12,6 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.23
+
+LABEL io.airbyte.version=0.1.24
 LABEL io.airbyte.name=airbyte/source-amazon-ads
diff --git a/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml
@@ -46,4 +46,4 @@ tests:
       configured_catalog_path: "integration_tests/configured_catalog_report.json"
       ignored_fields:
         "sponsored_products_report_stream": ["updatedAt"]
-      timeout_seconds: 3600
+      timeout_seconds: 3600
diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json
@@ -69,6 +69,17 @@
         "items": {
           "type": "integer"
         }
+      },
+      "state_filter": {
+        "title": "State Filter",
+        "description": "Reflects the state of the Display, Product, and Brand Campaign streams as enabled, paused, or archived. If you do not populate this field, it will be ignored completely.",
+        "items": {
+          "type": "string",
+          "enum": ["enabled", "paused", "archived"]
+        },
+        "type": "array",
+        "uniqueItems": true,
+        "order": 9
       }
     },
     "required": ["client_id", "client_secret", "refresh_token"],

diff --git a/airbyte-integrations/connectors/source-amazon-ads/setup.py b/airbyte-integrations/connectors/source-amazon-ads/setup.py
@@ -5,7 +5,7 @@
 
 from setuptools import find_packages, setup
 
-MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1.68", "requests_oauthlib~=1.3.1", "pendulum~=2.1.2"]
+MAIN_REQUIREMENTS = ["airbyte-cdk~=0.2.0", "requests_oauthlib~=1.3.1", "pendulum~=2.1.2"]
 
 TEST_REQUIREMENTS = [
     "pytest~=6.1",

diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml
@@ -86,6 +86,18 @@ connectionSpecification:
       type: array
       items:
         type: integer
+    state_filter:
+      title: State Filter
+      description: Reflects the state of the Display, Product, and Brand Campaign streams as enabled, paused, or archived. If you do not populate this field, it will be ignored completely.
+      items:
+        type: string
+        enum:
+          - enabled
+          - paused
+          - archived
+      type: array
+      uniqueItems: true
+      order: 9
   required:
     - client_id
     - client_secret

diff --git a/...e-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_brands.py b/...e-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_brands.py
@@ -12,12 +12,23 @@ class SponsoredBrandsCampaigns(SubProfilesStream):
     https://advertising.amazon.com/API/docs/en-us/sponsored-brands/3-0/openapi#/Campaigns
     """
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.state_filter = kwargs.get("config", {}).get("state_filter")
+
     primary_key = "campaignId"
+    state_filter = None
     model = BrandsCampaign
 
     def path(self, **kvargs) -> str:
         return "sb/campaigns"
 
+    def request_params(self, *args, **kwargs):
+        params = super().request_params(*args, **kwargs)
+        if self.state_filter:
+            params["stateFilter"] = ",".join(self.state_filter)
+        return params
+
 
 class SponsoredBrandsAdGroups(SubProfilesStream):
     """

diff --git a/...-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_display.py b/...-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_display.py
@@ -12,12 +12,23 @@ class SponsoredDisplayCampaigns(SubProfilesStream):
     https://advertising.amazon.com/API/docs/en-us/sponsored-display/3-0/openapi#/Campaigns
     """
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.state_filter = kwargs.get("config", {}).get("state_filter")
+
     primary_key = "campaignId"
+    state_filter = None
     model = DisplayCampaign
 
-    def path(self, **kvargs) -> str:
+    def path(self, **kwargs) -> str:
         return "sd/campaigns"
 
+    def request_params(self, *args, **kwargs):
+        params = super().request_params(*args, **kwargs)
+        if self.state_filter:
+            params["stateFilter"] = ",".join(self.state_filter)
+        return params
+
 
 class SponsoredDisplayAdGroups(SubProfilesStream):
     """
@@ -28,7 +39,7 @@ class SponsoredDisplayAdGroups(SubProfilesStream):
     primary_key = "adGroupId"
     model = DisplayAdGroup
 
-    def path(self, **kvargs) -> str:
+    def path(self, **kwargs) -> str:
         return "sd/adGroups"
 
 
@@ -41,7 +52,7 @@ class SponsoredDisplayProductAds(SubProfilesStream):
     primary_key = "adId"
     model = DisplayProductAds
 
-    def path(self, **kvargs) -> str:
+    def path(self, **kwargs) -> str:
         return "sd/productAds"
 
 
@@ -54,5 +65,5 @@ class SponsoredDisplayTargetings(SubProfilesStream):
     primary_key = "targetId"
     model = DisplayTargeting
 
-    def path(self, **kvargs) -> str:
+    def path(self, **kwargs) -> str:
         return "sd/targets"
diff --git a/...integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_products.py b/...integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_products.py
@@ -12,12 +12,23 @@ class SponsoredProductCampaigns(SubProfilesStream):
     https://advertising.amazon.com/API/docs/en-us/sponsored-display/3-0/openapi#/Campaigns
     """
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.state_filter = kwargs.get("config", {}).get("state_filter")
+
     primary_key = "campaignId"
+    state_filter = None
     model = ProductCampaign
 
     def path(self, **kvargs) -> str:
         return "v2/sp/campaigns"
 
+    def request_params(self, *args, **kwargs):
+        params = super().request_params(*args, **kwargs)
+        if self.state_filter:
+            params["stateFilter"] = ",".join(self.state_filter)
+        return params
+
 
 class SponsoredProductAdGroups(SubProfilesStream):
     """

diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py
@@ -19,9 +19,12 @@
 from source_amazon_ads.schemas.profile import AccountInfo, Profile
 from source_amazon_ads.source import CONFIG_DATE_FORMAT
 from source_amazon_ads.streams import (
+    SponsoredBrandsCampaigns,
     SponsoredBrandsReportStream,
     SponsoredBrandsVideoReportStream,
+    SponsoredDisplayCampaigns,
     SponsoredDisplayReportStream,
+    SponsoredProductCampaigns,
     SponsoredProductsReportStream,
 )
 from source_amazon_ads.streams.report_streams.report_streams import ReportGenerationFailure, ReportGenerationInProgress, TooManyRequests
@@ -556,3 +559,56 @@ def test_read_incremental_with_records_start_date(config):
         records = list(read_incremental(stream, state))
         assert state == {"1": {"reportDate": "20210104"}}
         assert {r["reportDate"] for r in records} == {"20210103", "20210104", "20210105", "20210106"}
+
+
+@pytest.mark.parametrize(
+    "state_filter, stream_class",
+    [
+        (
+            ["enabled", "archived", "paused"],
+            SponsoredBrandsCampaigns,
+        ),
+        (
+            ["enabled"],
+            SponsoredBrandsCampaigns,
+        ),
+        (
+            None,
+            SponsoredBrandsCampaigns,
+        ),
+        (
+            ["enabled", "archived", "paused"],
+            SponsoredProductCampaigns,
+        ),
+        (
+            ["enabled"],
+            SponsoredProductCampaigns,
+        ),
+        (
+            None,
+            SponsoredProductCampaigns,
+        ),
+        (
+            ["enabled", "archived", "paused"],
+            SponsoredDisplayCampaigns,
+        ),
+        (
+            ["enabled"],
+            SponsoredDisplayCampaigns,
+        ),
+        (
+            None,
+            SponsoredDisplayCampaigns,
+        ),
+    ],
+)
+def test_streams_state_filter(mocker, config, state_filter, stream_class):
+    profiles = make_profiles()
+    mocker.patch.object(stream_class, "state_filter", new_callable=mocker.PropertyMock, return_value=state_filter)
+
+    stream = stream_class(config, profiles)
+    params = stream.request_params(stream_state=None, stream_slice=None, next_page_token=None)
+    if "stateFilter" in params:
+        assert params["stateFilter"] == ",".join(state_filter)
+    else:
+        assert state_filter is None
diff --git a/docs/integrations/sources/amazon-ads.md b/docs/integrations/sources/amazon-ads.md
@@ -91,6 +91,7 @@ Information about expected report generation waiting time you may find [here](ht
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                         |
 |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------|
+| 0.1.24  | 2022-10-19 | [17475](https://github.com/airbytehq/airbyte/pull/17475) | Add filters for state on brand, product and display campaigns |
 | 0.1.23  | 2022-09-06 | [16342](https://github.com/airbytehq/airbyte/pull/16342) | Add attribution reports                                                                                         |
 | 0.1.22  | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream state.                                                                                    |
 | 0.1.21  | 2022-09-27 | [17202](https://github.com/airbytehq/airbyte/pull/17202) | Improved handling if known reporting errors                                                                     |