AIP-47 - Migrate google marketing DAGs to new design apache#22447

docs/apache-airflow-providers-google/operators/marketing_platform/analytics.rst

@@ -35,7 +35,7 @@ List the Accounts
 To list accounts from Analytics you can use the
 :class:`~airflow.providers.google.marketing_platform.operators.analytics.GoogleAnalyticsListAccountsOperator`.
 
-.. exampleinclude:: /../../airflow/providers/google/marketing_platform/example_dags/example_analytics.py
+.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_analytics.py
     :language: python
     :dedent: 4
     :start-after: [START howto_marketing_platform_list_accounts_operator]
@@ -53,7 +53,7 @@ Returns a web property-Google Ads link to which the user has access.
 To list web property-Google Ads link you can use the
 :class:`~airflow.providers.google.marketing_platform.operators.analytics.GoogleAnalyticsGetAdsLinkOperator`.
 
-.. exampleinclude:: /../../airflow/providers/google/marketing_platform/example_dags/example_analytics.py
+.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_analytics.py
     :language: python
     :dedent: 4
     :start-after: [START howto_marketing_platform_get_ads_link_operator]
@@ -71,7 +71,7 @@ Operator returns a list of entity Google Ads links.
 To list Google Ads links you can use the
 :class:`~airflow.providers.google.marketing_platform.operators.analytics.GoogleAnalyticsRetrieveAdsLinksListOperator`.
 
-.. exampleinclude:: /../../airflow/providers/google/marketing_platform/example_dags/example_analytics.py
+.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_analytics.py
     :language: python
     :dedent: 4
     :start-after: [START howto_marketing_platform_retrieve_ads_links_list_operator]

docs/apache-airflow-providers-google/operators/marketing_platform/campaign_manager.rst

@@ -36,7 +36,7 @@ To delete Campaign Manager report you can use the
 :class:`~airflow.providers.google.marketing_platform.operators.campaign_manager.GoogleCampaignManagerDeleteReportOperator`.
 It deletes a report by its unique ID.
 
-.. exampleinclude:: /../../airflow/providers/google/marketing_platform/example_dags/example_campaign_manager.py
+.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_campaign_manager.py
     :language: python
     :dedent: 4
     :start-after: [START howto_campaign_manager_delete_report_operator]
@@ -54,7 +54,7 @@ Downloading a report
 The :class:`~airflow.providers.google.marketing_platform.operators.campaign_manager.GoogleCampaignManagerDownloadReportOperator`.
 allows you to download a Campaign Manager to Google Cloud Storage bucket.
 
-.. exampleinclude:: /../../airflow/providers/google/marketing_platform/example_dags/example_campaign_manager.py
+.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_campaign_manager.py
     :language: python
     :dedent: 4
     :start-after: [START howto_campaign_manager_get_report_operator]
@@ -72,7 +72,7 @@ Waiting for a report
 Report are generated asynchronously. To wait for report to be ready for downloading
 you can use :class:`~airflow.providers.google.marketing_platform.sensors.campaign_manager.GoogleCampaignManagerReportSensor`.
 
-.. exampleinclude:: /../../airflow/providers/google/marketing_platform/example_dags/example_campaign_manager.py
+.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_campaign_manager.py
     :language: python
     :dedent: 4
     :start-after: [START howto_campaign_manager_wait_for_operation]
@@ -91,7 +91,7 @@ To insert a Campaign Manager report you can use the
 :class:`~airflow.providers.google.marketing_platform.operators.campaign_manager.GoogleCampaignManagerInsertReportOperator`.
 Running this operator creates a new report.
 
-.. exampleinclude:: /../../airflow/providers/google/marketing_platform/example_dags/example_campaign_manager.py
+.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_campaign_manager.py
     :language: python
     :dedent: 4
     :start-after: [START howto_campaign_manager_insert_report_operator]
@@ -111,7 +111,7 @@ Running a report
 To run Campaign Manager report you can use the
 :class:`~airflow.providers.google.marketing_platform.operators.campaign_manager.GoogleCampaignManagerRunReportOperator`.
 
-.. exampleinclude:: /../../airflow/providers/google/marketing_platform/example_dags/example_campaign_manager.py
+.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_campaign_manager.py
     :language: python
     :dedent: 4
     :start-after: [START howto_campaign_manager_run_report_operator]
@@ -130,7 +130,7 @@ Inserting a conversions
 To insert Campaign Manager conversions you can use the
 :class:`~airflow.providers.google.marketing_platform.operators.campaign_manager.GoogleCampaignManagerBatchInsertConversionsOperator`.
 
-.. exampleinclude:: /../../airflow/providers/google/marketing_platform/example_dags/example_campaign_manager.py
+.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_campaign_manager.py
     :language: python
     :dedent: 4
     :start-after: [START howto_campaign_manager_insert_conversions]
@@ -149,7 +149,7 @@ Updating a conversions
 To update Campaign Manager conversions you can use the
 :class:`~airflow.providers.google.marketing_platform.operators.campaign_manager.GoogleCampaignManagerBatchUpdateConversionsOperator`.
 
-.. exampleinclude:: /../../airflow/providers/google/marketing_platform/example_dags/example_campaign_manager.py
+.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_campaign_manager.py
     :language: python
     :dedent: 4
     :start-after: [START howto_campaign_manager_update_conversions]

docs/apache-airflow-providers-google/operators/marketing_platform/display_video.rst

@@ -33,7 +33,7 @@ Creating a report
 To create Display&Video 360 report use
 :class:`~airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360CreateReportOperator`.
 
-.. exampleinclude:: /../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
+.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_display_video.py
     :language: python
     :dedent: 4
     :start-after: [START howto_google_display_video_createquery_report_operator]
@@ -53,7 +53,7 @@ Deleting a report
 To delete Display&Video 360 report use
 :class:`~airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360DeleteReportOperator`.
 
-.. exampleinclude:: /../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
+.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_display_video.py
     :language: python
     :dedent: 4
     :start-after: [START howto_google_display_video_deletequery_report_operator]
@@ -71,7 +71,7 @@ Waiting for report
 To wait for the report use
 :class:`~airflow.providers.google.marketing_platform.sensors.display_video.GoogleDisplayVideo360ReportSensor`.
 
-.. exampleinclude:: /../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
+.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_display_video.py
     :language: python
     :dedent: 4
     :start-after: [START howto_google_display_video_wait_report_operator]
@@ -89,7 +89,7 @@ Downloading a report
 To download a report to GCS bucket use
 :class:`~airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360DownloadReportOperator`.
 
-.. exampleinclude:: /../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
+.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_display_video.py
     :language: python
     :dedent: 4
     :start-after: [START howto_google_display_video_getquery_report_operator]
@@ -108,7 +108,7 @@ Running a report
 To run Display&Video 360 report use
 :class:`~airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360RunReportOperator`.
 
-.. exampleinclude:: /../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
+.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_display_video.py
     :language: python
     :dedent: 4
     :start-after: [START howto_google_display_video_runquery_report_operator]
@@ -137,7 +137,7 @@ The operator accepts body request:
 To download line items in CSV format report use
 :class:`~airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360DownloadLineItemsOperator`.
 
-.. exampleinclude:: /../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
+.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_display_video_misc.py
     :language: python
     :dedent: 4
     :start-after: [START howto_google_display_video_download_line_items_operator]
@@ -156,7 +156,7 @@ Upload line items
 To run Display&Video 360 uploading line items use
 :class:`~airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360UploadLineItemsOperator`.
 
-.. exampleinclude:: /../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
+.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_display_video_misc.py
     :language: python
     :dedent: 4
     :start-after: [START howto_google_display_video_upload_line_items_operator]
@@ -174,7 +174,7 @@ Create SDF download task
 To create SDF download task use
 :class:`~airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360CreateSDFDownloadTaskOperator`.
 
-.. exampleinclude:: /../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
+.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_display_video_sdf.py
     :language: python
     :dedent: 4
     :start-after: [START howto_google_display_video_create_sdf_download_task_operator]
@@ -193,7 +193,7 @@ Save SDF files in the Google Cloud Storage
 To save SDF files and save them in the Google Cloud Storage use
 :class:`~airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360SDFtoGCSOperator`.
 
-.. exampleinclude:: /../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
+.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_display_video_sdf.py
     :language: python
     :dedent: 4
     :start-after: [START howto_google_display_video_save_sdf_in_gcs_operator]
@@ -211,7 +211,7 @@ Waiting for SDF operation
 Wait for SDF operation is executed by:
 :class:`~airflow.providers.google.marketing_platform.sensors.display_video.GoogleDisplayVideo360GetSDFDownloadOperationSensor`.
 
-.. exampleinclude:: /../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
+.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_display_video_sdf.py
     :language: python
     :dedent: 4
     :start-after: [START howto_google_display_video_wait_for_operation_sensor]

docs/apache-airflow-providers-google/operators/marketing_platform/index.rst

@@ -29,4 +29,4 @@ Google Marketing Platform Operators
 
 .. note::
     You can learn how to use Google Cloud integrations by analyzing the
-    `source code <https://github.com/apache/airflow/tree/main/airflow/providers/google/marketing_platform/example_dags/>`_ of the particular example DAGs.
+    `source code <https://github.com/apache/airflow/tree/main/tests/system/providers/google/marketing_platform/>`_ of the particular example DAGs.

docs/apache-airflow-providers-google/operators/marketing_platform/search_ads.rst

@@ -34,7 +34,7 @@ Inserting a report
 To insert a Search Ads report use the
 :class:`~airflow.providers.google.marketing_platform.operators.search_ads.GoogleSearchAdsInsertReportOperator`.
 
-.. exampleinclude:: /../../airflow/providers/google/marketing_platform/example_dags/example_search_ads.py
+.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_search_ads.py
     :language: python
     :dedent: 4
     :start-after: [START howto_search_ads_generate_report_operator]
@@ -46,7 +46,7 @@ parameters which allows you to dynamically determine values. You can provide rep
 .json`` file as this operator supports this template extension.
 The result is saved to :ref:`XCom <concepts:xcom>`, which allows it to be used by other operators:
 
-.. exampleinclude:: /../../airflow/providers/google/marketing_platform/example_dags/example_search_ads.py
+.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_search_ads.py
     :language: python
     :dedent: 4
     :start-after: [START howto_search_ads_get_report_id]
@@ -60,7 +60,7 @@ Awaiting for a report
 To wait for a report to be ready for download use
 :class:`~airflow.providers.google.marketing_platform.sensors.search_ads.GoogleSearchAdsReportSensor`.
 
-.. exampleinclude:: /../../airflow/providers/google/marketing_platform/example_dags/example_search_ads.py
+.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_search_ads.py
     :language: python
     :dedent: 4
     :start-after: [START howto_search_ads_get_report_operator]
@@ -78,7 +78,7 @@ Downloading a report
 To download a Search Ads report to Google Cloud Storage bucket use the
 :class:`~airflow.providers.google.marketing_platform.operators.search_ads.GoogleSearchAdsDownloadReportOperator`.
 
-.. exampleinclude:: /../../airflow/providers/google/marketing_platform/example_dags/example_search_ads.py
+.. exampleinclude:: /../../tests/system/providers/google/marketing_platform/example_search_ads.py
     :language: python
     :dedent: 4
     :start-after: [START howto_search_ads_getfile_report_operator]

tests/providers/google/marketing_platform/operators/test_analytics_system.py

@@ -17,8 +17,8 @@
 
 import pytest
 
-from airflow.providers.google.marketing_platform.example_dags.example_analytics import BUCKET, BUCKET_FILENAME
 from tests.providers.google.cloud.utils.gcp_authenticator import GMP_KEY
+from tests.system.providers.google.marketing_platform.example_analytics import BUCKET, BUCKET_FILENAME
 from tests.test_utils.gcp_system_helpers import MARKETING_DAG_FOLDER, GoogleSystemTest, provide_gcp_context
 
 DATA = """pagePath,dimension1

tests/providers/google/marketing_platform/operators/test_campaign_manager_system.py

@@ -17,8 +17,8 @@
 # under the License.
 import pytest
 
-from airflow.providers.google.marketing_platform.example_dags.example_campaign_manager import BUCKET
 from tests.providers.google.cloud.utils.gcp_authenticator import GMP_KEY
+from tests.system.providers.google.marketing_platform.example_campaign_manager import BUCKET
 from tests.test_utils.gcp_system_helpers import MARKETING_DAG_FOLDER, GoogleSystemTest, provide_gcp_context
 
 # Required scopes

tests/providers/google/marketing_platform/operators/test_display_video_system.py

@@ -17,8 +17,8 @@
 import pytest
 
 from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook
-from airflow.providers.google.marketing_platform.example_dags.example_display_video import BUCKET
 from tests.providers.google.cloud.utils.gcp_authenticator import GCP_BIGQUERY_KEY, GMP_KEY
+from tests.system.providers.google.marketing_platform.example_display_video import BUCKET
 from tests.test_utils.gcp_system_helpers import MARKETING_DAG_FOLDER, GoogleSystemTest, provide_gcp_context
 
 # Requires the following scope:

tests/providers/google/marketing_platform/operators/test_search_ads_system.py

@@ -17,8 +17,8 @@
 # under the License.
 import pytest
 
-from airflow.providers.google.marketing_platform.example_dags.example_search_ads import GCS_BUCKET
 from tests.providers.google.cloud.utils.gcp_authenticator import GMP_KEY
+from tests.system.providers.google.marketing_platform.example_search_ads import GCS_BUCKET
 from tests.test_utils.gcp_system_helpers import MARKETING_DAG_FOLDER, GoogleSystemTest, provide_gcp_context
 
 # Requires the following scope:

airflow/providers/google/marketing_platform/example_dags/__init__.py → tests/system/providers/google/__init__.py

@@ -1,3 +1,4 @@
+#
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information

tests/system/providers/google/marketing_platform/__init__.py

@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

airflow/providers/google/marketing_platform/example_dags/example_analytics.py → tests/system/providers/google/marketing_platform/example_analytics.py

@@ -29,7 +29,10 @@
     GoogleAnalyticsModifyFileHeadersDataImportOperator,
     GoogleAnalyticsRetrieveAdsLinksListOperator,
 )
+from airflow.utils.trigger_rule import TriggerRule
 
+ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
+DAG_ID = "example_google_analytics"
 ACCOUNT_ID = os.environ.get("GA_ACCOUNT_ID", "123456789")
 
 BUCKET = os.environ.get("GMP_ANALYTICS_BUCKET", "test-airflow-analytics-bucket")
@@ -39,7 +42,7 @@
 DATA_ID = "kjdDu3_tQa6n8Q1kXFtSmg"
 
 with models.DAG(
-    "example_google_analytics",
+    DAG_ID,
     schedule_interval='@once',  # Override to match your needs,
     start_date=datetime(2021, 1, 1),
     catchup=False,
@@ -77,12 +80,26 @@
         account_id=ACCOUNT_ID,
         web_property_id=WEB_PROPERTY_ID,
         custom_data_source_id=DATA_ID,
+        trigger_rule=TriggerRule.ALL_DONE,
     )
 
     transform = GoogleAnalyticsModifyFileHeadersDataImportOperator(
         task_id="transform",
         storage_bucket=BUCKET,
         storage_name_object=BUCKET_FILENAME,
+        trigger_rule=TriggerRule.ALL_DONE,
     )
 
     upload >> [delete, transform]
+
+    from tests.system.utils.watcher import watcher
+
+    # This test needs watcher in order to properly mark success/failure
+    # when "tearDown" task with trigger rule is part of the DAG
+    list(dag.tasks) >> watcher()
+
+
+from tests.system.utils import get_test_run  # noqa: E402
+
+# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)
+test_run = get_test_run(dag)

airflow/providers/google/marketing_platform/example_dags/example_campaign_manager.py → tests/system/providers/google/marketing_platform/example_campaign_manager.py

@@ -34,7 +34,10 @@
 from airflow.providers.google.marketing_platform.sensors.campaign_manager import (
     GoogleCampaignManagerReportSensor,
 )
+from airflow.utils.trigger_rule import TriggerRule
 
+ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
+DAG_ID = "example_campaign_manager"
 PROFILE_ID = os.environ.get("MARKETING_PROFILE_ID", "123456789")
 FLOODLIGHT_ACTIVITY_ID = int(os.environ.get("FLOODLIGHT_ACTIVITY_ID", 12345))
 FLOODLIGHT_CONFIGURATION_ID = int(os.environ.get("FLOODLIGHT_CONFIGURATION_ID", 12345))
@@ -84,7 +87,7 @@
 }
 
 with models.DAG(
-    "example_campaign_manager",
+    DAG_ID,
     schedule_interval='@once',  # Override to match your needs,
     start_date=datetime(2021, 1, 1),
     catchup=False,
@@ -158,12 +161,25 @@
         encryption_entity_type="DCM_ADVERTISER",
         encryption_entity_id=ENCRYPTION_ENTITY_ID,
         max_failed_updates=1,
+        trigger_rule=TriggerRule.ALL_DONE,
     )
     # [END howto_campaign_manager_update_conversions]
 
     insert_conversion >> update_conversion
 
+    from tests.system.utils.watcher import watcher
+
+    # This test needs watcher in order to properly mark success/failure
+    # when "tearDown" task with trigger rule is part of the DAG
+    list(dag.tasks) >> watcher()
+
 
 if __name__ == "__main__":
     dag.clear()
     dag.run()
+
+
+from tests.system.utils import get_test_run  # noqa: E402
+
+# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)
+test_run = get_test_run(dag)