Skip to content

Commit

Permalink
Add alias argument to deduplicate macro (#526)
Browse files Browse the repository at this point in the history
* Add `alias` argument to `deduplicate

* Test `alias` argument

* Rename `alias` to `relation_alias`
  • Loading branch information
judahrand authored Mar 25, 2022
1 parent e2f6ba7 commit 96c461b
Show file tree
Hide file tree
Showing 4 changed files with 27 additions and 10 deletions.
5 changes: 3 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ For compatibility details between versions of dbt-core and dbt-utils, [see this

- [SQL generators](#sql-generators)
- [date_spine](#date_spine-source)
- [dedupe](#dedupe-source)
- [deduplicate](#deduplicate)
- [haversine_distance](#haversine_distance-source)
- [group_by](#group_by-source)
- [star](#star-source)
Expand Down Expand Up @@ -715,7 +715,8 @@ This macro returns the sql required to remove duplicate rows from a model or sou
{{ dbt_utils.deduplicate(
relation=source('my_source', 'my_table'),
group_by="user_id, cast(timestamp as day)",
order_by="timestamp desc"
order_by="timestamp desc",
relation_alias="my_cte"
)
}}
```
Expand Down
1 change: 1 addition & 0 deletions integration_tests/data/sql/data_deduplicate.csv
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
user_id,event,version
1,play,1
1,play,2
2,pause,1
19 changes: 17 additions & 2 deletions integration_tests/models/sql/test_deduplicate.sql
Original file line number Diff line number Diff line change
@@ -1,6 +1,21 @@
with deduped as (
with

{{ dbt_utils.deduplicate(ref('data_deduplicate'), group_by='user_id', order_by='version desc') | indent }}
source as (
select *
from {{ ref('data_deduplicate') }}
where user_id = 1
),

deduped as (

{{
dbt_utils.deduplicate(
ref('data_deduplicate'),
group_by='user_id',
order_by='version desc',
relation_alias="source"
) | indent
}}

)

Expand Down
12 changes: 6 additions & 6 deletions macros/sql/deduplicate.sql
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
{%- macro deduplicate(relation, group_by, order_by=none) -%}
{{ return(adapter.dispatch('deduplicate', 'dbt_utils')(relation, group_by, order_by=order_by)) }}
{%- macro deduplicate(relation, group_by, order_by=none, relation_alias=none) -%}
{{ return(adapter.dispatch('deduplicate', 'dbt_utils')(relation, group_by, order_by=order_by, relation_alias=relation_alias)) }}
{% endmacro %}

{%- macro default__deduplicate(relation, group_by, order_by=none) -%}
{%- macro default__deduplicate(relation, group_by, order_by=none, relation_alias=none) -%}

select
{{ dbt_utils.star(relation, relation_alias='deduped') | indent }}
Expand All @@ -15,7 +15,7 @@
order by {{ order_by }}
{%- endif %}
) as rn
from {{ relation }} as _inner
from {{ relation if relation_alias is none else relation_alias }} as _inner
) as deduped
where deduped.rn = 1

Expand All @@ -26,7 +26,7 @@
-- clause in BigQuery:
-- https://github.com/dbt-labs/dbt-utils/issues/335#issuecomment-788157572
#}
{%- macro bigquery__deduplicate(relation, group_by, order_by=none) -%}
{%- macro bigquery__deduplicate(relation, group_by, order_by=none, relation_alias=none) -%}

select
{{ dbt_utils.star(relation, relation_alias='deduped') | indent }}
Expand All @@ -39,7 +39,7 @@
{%- endif %}
limit 1
)[offset(0)] as deduped
from {{ relation }} as original
from {{ relation if relation_alias is none else relation_alias }} as original
group by {{ group_by }}
)

Expand Down

0 comments on commit 96c461b

Please sign in to comment.