Merge pull request #23 from fivetran/feature/enable-null-urls

Feature/enable null urls

.buildkite/pipeline.yml

@@ -58,7 +58,7 @@ steps:
     commands: |
       bash .buildkite/scripts/run_models.sh redshift
 
-  - label: ":bricks: Run Tests - Databricks"
+  - label: ":databricks: Run Tests - Databricks"
     key: "run_dbt_databricks"
     plugins:
       - docker#v3.13.0:

.buildkite/scripts/run_models.sh

@@ -18,4 +18,6 @@ cd integration_tests
 dbt deps
 dbt seed --target "$db" --full-refresh
 dbt run --target "$db" --full-refresh
-dbt test --target "$db"
+dbt test --target "$db"
+dbt run --vars '{ad_reporting__url_report__using_null_filter: false}' --target "$db" --full-refresh
+dbt test --vars '{ad_reporting__url_report__using_null_filter: false}' --target "$db"

CHANGELOG.md

@@ -37,6 +37,12 @@
     - `dbt.current_timestamp_in_utc_backcompat`
 - Dependencies on `fivetran/fivetran_utils` have been upgraded, previously `[">=0.3.0", "<0.4.0"]` now `[">=0.4.0", "<0.5.0"]`.
 
+## 🎉 Features 🎉
+- For use in the [dbt_ad_reporting package](https://github.com/fivetran/dbt_ad_reporting), users can now allow records having nulls in url fields to be included in the `ad_reporting__url_report` model. See the [dbt_ad_reporting README](https://github.com/fivetran/dbt_ad_reporting) for more details ([#23](https://github.com/fivetran/dbt_pinterest/pull/23)).
+## 🚘 Under the Hood 🚘
+- Disabled the `not_null` test for `pinterest_ads__url_report` when null urls are allowed ([#23](https://github.com/fivetran/dbt_pinterest/pull/23)).
+
+
 # dbt_pinterest v0.6.0
 PR [#21](https://github.com/fivetran/dbt_pinterest/pull/21) includes the following changes:
 ## 🚨 Breaking Changes 🚨

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: pinterest_integration_tests
+      schema: pinterest_integration_tests_3
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: pinterest_integration_tests
+      schema: pinterest_integration_tests_3
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: pinterest_integration_tests
+      schema: pinterest_integration_tests_3
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: pinterest_integration_tests
+      schema: pinterest_integration_tests_3
       threads: 8
     databricks:
       catalog: null
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: pinterest_integration_tests
+      schema: pinterest_integration_tests_3
       threads: 2
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -4,7 +4,7 @@ profile: 'integration_tests'
 config-version: 2
 
 vars:
-  pinterest_schema: pinterest_integration_tests
+  pinterest_schema: pinterest_integration_tests_3
   pinterest_ad_group_history_identifier: "ad_group_history_data"
   pinterest_campaign_history_identifier: "campaign_history_data"
   pinterest_pin_promotion_history_identifier: "pin_promotion_history_data"

models/pinterest.yml

@@ -11,6 +11,9 @@ models:
             - campaign_id
             - advertiser_id
             - date_day
+          config:
+            # Below is so we don't run this test if the user has allowed urls to have null entries
+            enabled: "{{ var('ad_reporting__url_report__using_null_filter', true) }}"
     columns:
       - name: date_day
         description: "{{ doc('date_day') }}"
@@ -26,6 +29,11 @@ models:
         description: Status of the ad group.
       - name: destination_url
         description: Pin destination URL.
+        tests:
+          - not_null:
+              config:
+                # Below is so we don't run this test if the user has allowed urls to have null entries
+                enabled: "{{ var('ad_reporting__url_report__using_null_filter', true) }}"
       - name: pin_promotion_id
         description: "{{ doc('pin_promotion_id') }}"
         tests:

models/pinterest_ads__url_report.sql

@@ -75,8 +75,10 @@ joined as (
     left join advertisers
         on campaigns.advertiser_id = advertisers.advertiser_id
 
-    -- We only want utm ads to populate this report. Therefore, we filter where url pins are populated.
+    {% if var('ad_reporting__url_report__using_null_filter', True) %}
     where pins.destination_url is not null
+    {% endif %}
+
     {{ dbt_utils.group_by(22) }}
 )