diff --git a/README.md b/README.md index f707a376..65d96865 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ For compatibility details between versions of dbt-core and dbt-utils, [see this - [Introspective macros](#introspective-macros): - [get_column_values](#get_column_values-source) + - [get_filtered_columns_in_relation](#get_filtered_columns_in_relation-source) - [get_relations_by_pattern](#get_relations_by_pattern-source) - [get_relations_by_prefix](#get_relations_by_prefix-source) - [get_query_results_as_dict](#get_query_results_as_dict-source) @@ -544,7 +545,7 @@ These macros run a query and return the results of the query as objects. They ar #### get_column_values ([source](macros/sql/get_column_values.sql)) This macro returns the unique values for a column in a given [relation](https://docs.getdbt.com/docs/writing-code-in-dbt/class-reference/#relation) as an array. -Arguments: +**Args:** - `table` (required): a [Relation](https://docs.getdbt.com/reference/dbt-classes#relation) (a `ref` or `source`) that contains the list of columns you wish to select from - `column` (required): The name of the column you wish to find the column values of - `order_by` (optional, default=`'count(*) desc'`): How the results should be ordered. The default is to order by `count(*) desc`, i.e. decreasing frequency. Setting this as `'my_column'` will sort alphabetically, while `'min(created_at)'` will sort by when thevalue was first observed. @@ -585,6 +586,28 @@ Arguments: ... ``` +#### get_filtered_columns_in_relation ([source](macros/sql/get_filtered_columns_in_relation.sql)) +This macro returns an iterable Jinja list of columns for a given [relation](https://docs.getdbt.com/docs/writing-code-in-dbt/class-reference/#relation), (i.e. not from a CTE) +- optionally exclude columns +- the input values are not case-sensitive (input uppercase or lowercase and it will work!) +> Note: The native [`adapter.get_columns_in_relation` macro](https://docs.getdbt.com/reference/dbt-jinja-functions/adapter#get_columns_in_relation) allows you +to pull column names in a non-filtered fashion, also bringing along with it other (potentially unwanted) information, such as dtype, char_size, numeric_precision, etc. + +**Args:** +- `from` (required): a [Relation](https://docs.getdbt.com/reference/dbt-classes#relation) (a `ref` or `source`) that contains the list of columns you wish to select from +- `except` (optional, default=`[]`): The name of the columns you wish to exclude. (case-insensitive) + +**Usage:** +```sql +-- Returns a list of the columns from a relation, so you can then iterate in a for loop +{% set column_names = dbt_utils.get_filtered_columns_in_relation(from=ref('your_model'), except=["field_1", "field_2"]) %} +... +{% for column_name in column_names %} + max({{ column_name }}) ... as max_'{{ column_name }}', +{% endfor %} +... +``` + #### get_relations_by_pattern ([source](macros/sql/get_relations_by_pattern.sql)) Returns a list of [Relations](https://docs.getdbt.com/docs/writing-code-in-dbt/class-reference/#relation) that match a given schema- or table-name pattern. @@ -748,9 +771,19 @@ group by 1,2,3 ``` #### star ([source](macros/sql/star.sql)) -This macro generates a comma-separated list of all fields that exist in the `from` relation, excluding any fields listed in the `except` argument. The construction is identical to `select * from {{ref('my_model')}}`, replacing star (`*`) with the star macro. This macro also has an optional `relation_alias` argument that will prefix all generated fields with an alias (`relation_alias`.`field_name`). +This macro generates a comma-separated list of all fields that exist in the `from` relation, excluding any fields +listed in the `except` argument. The construction is identical to `select * from {{ref('my_model')}}`, replacing star (`*`) with +the star macro. +This macro also has an optional `relation_alias` argument that will prefix all generated fields with an alias (`relation_alias`.`field_name`). +The macro also has optional `prefix` and `suffix` arguments. When one or both are provided, they will be concatenated onto each field's alias +in the output (`prefix` ~ `field_name` ~ `suffix`). NB: This prevents the output from being used in any context other than a select statement. -The macro also has optional `prefix` and `suffix` arguments. When one or both are provided, they will be concatenated onto each field's alias in the output (`prefix` ~ `field_name` ~ `suffix`). NB: This prevents the output from being used in any context other than a select statement. +**Args:** +- `from` (required): a [Relation](https://docs.getdbt.com/reference/dbt-classes#relation) (a `ref` or `source`) that contains the list of columns you wish to select from +- `except` (optional, default=`[]`): The name of the columns you wish to exclude. (case-insensitive) +- `relation_alias` (optional, default=`''`): will prefix all generated fields with an alias (`relation_alias`.`field_name`). +- `prefix` (optional, default=`''`): will prefix the output `field_name` (`field_name as prefix_field_name`). +- `suffix` (optional, default=`''`): will suffix the output `field_name` (`field_name as field_name_suffix`). **Usage:** ```sql @@ -767,6 +800,13 @@ from {{ ref('my_model') }} ``` +```sql +select +{{ dbt_utils.star(from=ref('my_model'), except=["exclude_field_1", "exclude_field_2"], prefix="max_") }} +from {{ ref('my_model') }} + +``` + #### union_relations ([source](macros/sql/union.sql)) This macro unions together an array of [Relations](https://docs.getdbt.com/docs/writing-code-in-dbt/class-reference/#relation), diff --git a/integration_tests/data/sql/data_filtered_columns_in_relation.csv b/integration_tests/data/sql/data_filtered_columns_in_relation.csv new file mode 100644 index 00000000..9d96143b --- /dev/null +++ b/integration_tests/data/sql/data_filtered_columns_in_relation.csv @@ -0,0 +1,4 @@ +field_1,field_2,field_3 +a,b,c +d,e,f +g,h,i \ No newline at end of file diff --git a/integration_tests/data/sql/data_filtered_columns_in_relation_expected.csv b/integration_tests/data/sql/data_filtered_columns_in_relation_expected.csv new file mode 100644 index 00000000..77ea0563 --- /dev/null +++ b/integration_tests/data/sql/data_filtered_columns_in_relation_expected.csv @@ -0,0 +1,2 @@ +field_2,field_3 +h,i \ No newline at end of file diff --git a/integration_tests/macros/assert_equal_values.sql b/integration_tests/macros/assert_equal_values.sql new file mode 100644 index 00000000..d4f02618 --- /dev/null +++ b/integration_tests/macros/assert_equal_values.sql @@ -0,0 +1,32 @@ +{% macro assert_equal_values(actual_object, expected_object) %} +{% if not execute %} + + {# pass #} + +{% elif actual_object != expected_object %} + + {% set msg %} + Expected did not match actual + + ----------- + Actual: + ----------- + --->{{ actual_object }}<--- + + ----------- + Expected: + ----------- + --->{{ expected_object }}<--- + + {% endset %} + + {{ log(msg, info=True) }} + + select 'fail' + +{% else %} + + select 'ok' {{ limit_zero() }} + +{% endif %} +{% endmacro %} \ No newline at end of file diff --git a/integration_tests/models/sql/schema.yml b/integration_tests/models/sql/schema.yml index 616dd386..e136f127 100644 --- a/integration_tests/models/sql/schema.yml +++ b/integration_tests/models/sql/schema.yml @@ -50,6 +50,11 @@ models: values: - '5' + - name: test_get_filtered_columns_in_relation + tests: + - dbt_utils.equality: + compare_model: ref('data_filtered_columns_in_relation_expected') + - name: test_get_relations_by_prefix_and_union columns: - name: event @@ -121,6 +126,11 @@ models: - dbt_utils.equality: compare_model: ref('data_star_aggregate_expected') + - name: test_star_uppercase + tests: + - dbt_utils.equality: + compare_model: ref('data_star_expected') + - name: test_surrogate_key tests: - assert_equal: diff --git a/integration_tests/models/sql/test_get_filtered_columns_in_relation.sql b/integration_tests/models/sql/test_get_filtered_columns_in_relation.sql new file mode 100644 index 00000000..7b3ca72f --- /dev/null +++ b/integration_tests/models/sql/test_get_filtered_columns_in_relation.sql @@ -0,0 +1,16 @@ +{% set exclude_field = 'field_1' %} +{% set column_names = dbt_utils.get_filtered_columns_in_relation(from= ref('data_filtered_columns_in_relation'), except=[exclude_field]) %} + +with data as ( + + select + + {% for column_name in column_names %} + max({{ column_name }}) as {{ column_name }} {% if not loop.last %},{% endif %} + {% endfor %} + + from {{ ref('data_filtered_columns_in_relation') }} + +) + +select * from data diff --git a/integration_tests/models/sql/test_star_uppercase.sql b/integration_tests/models/sql/test_star_uppercase.sql new file mode 100644 index 00000000..6179e691 --- /dev/null +++ b/integration_tests/models/sql/test_star_uppercase.sql @@ -0,0 +1,13 @@ +{% set exclude_field = 'FIELD_3' %} + + +with data as ( + + select + {{ dbt_utils.star(from=ref('data_star'), except=[exclude_field]) }} + + from {{ ref('data_star') }} + +) + +select * from data diff --git a/macros/sql/get_filtered_columns_in_relation.sql b/macros/sql/get_filtered_columns_in_relation.sql new file mode 100644 index 00000000..7f4af889 --- /dev/null +++ b/macros/sql/get_filtered_columns_in_relation.sql @@ -0,0 +1,25 @@ +{% macro get_filtered_columns_in_relation(from, except=[]) -%} + {{ return(adapter.dispatch('get_filtered_columns_in_relation', 'dbt_utils')(from, except)) }} +{% endmacro %} + +{% macro default__get_filtered_columns_in_relation(from, except=[]) -%} + {%- do dbt_utils._is_relation(from, 'get_filtered_columns_in_relation') -%} + {%- do dbt_utils._is_ephemeral(from, 'get_filtered_columns_in_relation') -%} + + {# -- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} + {%- if not execute -%} + {{ return('') }} + {% endif %} + + {%- set include_cols = [] %} + {%- set cols = adapter.get_columns_in_relation(from) -%} + {%- set except = except | map("lower") | list %} + {%- for col in cols -%} + {%- if col.column|lower not in except -%} + {% do include_cols.append(col.column) %} + {%- endif %} + {%- endfor %} + + {{ return(include_cols) }} + +{%- endmacro %} \ No newline at end of file diff --git a/macros/sql/star.sql b/macros/sql/star.sql index 0bfa8c9a..72778b1f 100644 --- a/macros/sql/star.sql +++ b/macros/sql/star.sql @@ -11,20 +11,9 @@ {{ return('') }} {% endif %} - {%- set include_cols = [] %} - {%- set cols = adapter.get_columns_in_relation(from) -%} - {%- set except = except | map("lower") | list %} - {%- for col in cols -%} + {%- for col in dbt_utils.get_filtered_columns_in_relation(from, except) %} - {%- if col.column|lower not in except -%} - {% do include_cols.append(col.column) %} - - {%- endif %} - {%- endfor %} - - {%- for col in include_cols %} - - {%- if relation_alias %}{{ relation_alias }}.{% else %}{%- endif -%}{{ adapter.quote(col)|trim }} {%- if prefix!='' or suffix!='' -%} as {{ adapter.quote(prefix ~ col ~ suffix)|trim }} {%- endif -%} + {%- if relation_alias %}{{ relation_alias }}.{% else %}{%- endif -%}{{ adapter.quote(col)|trim }} {%- if prefix!='' or suffix!='' %} as {{ adapter.quote(prefix ~ col ~ suffix)|trim }} {%- endif -%} {%- if not loop.last %},{{ '\n ' }}{% endif %} {%- endfor -%}