From aefaa9d169d15ff77c837763a68f3e0e8aa23e97 Mon Sep 17 00:00:00 2001 From: bruno Date: Thu, 24 Nov 2022 10:38:22 +0000 Subject: [PATCH 01/17] add exclude columns to equality test --- CHANGELOG.md | 8 ++++- README.md | 17 ++++++++++- .../schema_tests/data_test_equality_a.csv | 4 +++ .../schema_tests/data_test_equality_b.csv | 4 +++ .../models/generic_tests/schema.yml | 18 ++++++++++++ macros/generic_tests/equality.sql | 29 +++++++++++++++---- 6 files changed, 73 insertions(+), 7 deletions(-) create mode 100644 integration_tests/data/schema_tests/data_test_equality_a.csv create mode 100644 integration_tests/data/schema_tests/data_test_equality_b.csv diff --git a/CHANGELOG.md b/CHANGELOG.md index b7799278..310180bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,12 @@ ## Contributors: ---> +# Unreleased +## New features +- Add option to ignore columns in equality test ([#734](https://github.com/dbt-labs/dbt-utils/issues/734), [#737](https://github.com/dbt-labs/dbt-utils/pull/737)) +## Contributors: +- [@brunocostalopes](https://github.com/brunocostalopes) + # Unreleased ## Fixes - deduplicate macro for Databricks now uses the QUALIFY clause, which fixes NULL columns issues from the default natural join logic @@ -19,7 +25,7 @@ # dbt utils v1.1.1 ## New features -* Improve the performance of the `at_least_one` test by pruning early. This is especially helpful when running against external tables. By @joshuahuntley in https://github.com/dbt-labs/dbt-utils/pull/775 +* ZZZ by @YYY in https://github.com/dbt-labs/dbt-utils/pull/XXX ## Fixes * Fix legacy links in README by @dbeatty10 in https://github.com/dbt-labs/dbt-utils/pull/796 diff --git a/README.md b/README.md index 827626ba..6061c63e 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ This test supports the `group_by_columns` parameter; see [Grouping in tests](#gr ### equality ([source](macros/generic_tests/equality.sql)) -Asserts the equality of two relations. Optionally specify a subset of columns to compare. +Asserts the equality of two relations. Optionally specify a subset of columns to compare or ignore. **Usage:** @@ -122,13 +122,28 @@ Asserts the equality of two relations. Optionally specify a subset of columns to version: 2 models: + # compare the entire table - name: model_name + tests: + - dbt_utils.equality: + compare_model: ref('other_table_name') + + # only compare some of the columns + - name: model_name_compare_columns tests: - dbt_utils.equality: compare_model: ref('other_table_name') compare_columns: - first_column - second_column + + # compare all columns except the ones on the ignore list + - name: model_name_ignore_columns + tests: + - dbt_utils.equality: + compare_model: ref('other_table_name') + ignore_columns: + - third_column ``` ### expression_is_true ([source](macros/generic_tests/expression_is_true.sql)) diff --git a/integration_tests/data/schema_tests/data_test_equality_a.csv b/integration_tests/data/schema_tests/data_test_equality_a.csv new file mode 100644 index 00000000..35fa4ee0 --- /dev/null +++ b/integration_tests/data/schema_tests/data_test_equality_a.csv @@ -0,0 +1,4 @@ +col_a,col_b,col_c +1,1,3 +1,2,1 +2,3,3 diff --git a/integration_tests/data/schema_tests/data_test_equality_b.csv b/integration_tests/data/schema_tests/data_test_equality_b.csv new file mode 100644 index 00000000..c9fda320 --- /dev/null +++ b/integration_tests/data/schema_tests/data_test_equality_b.csv @@ -0,0 +1,4 @@ +col_a,col_b,col_c +1,1,2 +1,2,2 +2,3,2 diff --git a/integration_tests/models/generic_tests/schema.yml b/integration_tests/models/generic_tests/schema.yml index fa0e7441..a7658d3f 100644 --- a/integration_tests/models/generic_tests/schema.yml +++ b/integration_tests/models/generic_tests/schema.yml @@ -142,6 +142,24 @@ seeds: - dbt_utils.not_null_proportion: at_least: 0.9 + - name: data_test_equality_a + tests: + - dbt_utils.equality: + compare_model: ref('data_test_equality_a') + - dbt_utils.equality: + compare_model: ref('data_test_equality_b') + error_if: "<1" #sneaky way to ensure that the test is returning failing rows + warn_if: "<0" + - dbt_utils.equality: + compare_model: ref('data_test_equality_b') + compare_columns: + - col_a + - col_b + - dbt_utils.equality: + compare_model: ref('data_test_equality_b') + ignore_columns: + - col_c + models: - name: recency_time_included tests: diff --git a/macros/generic_tests/equality.sql b/macros/generic_tests/equality.sql index ffc6a2b8..3ef74c10 100644 --- a/macros/generic_tests/equality.sql +++ b/macros/generic_tests/equality.sql @@ -1,10 +1,14 @@ -{% test equality(model, compare_model, compare_columns=None) %} - {{ return(adapter.dispatch('test_equality', 'dbt_utils')(model, compare_model, compare_columns)) }} +{% test equality(model, compare_model, compare_columns=None, ignore_columns=None) %} + {{ return(adapter.dispatch('test_equality', 'dbt_utils')(model, compare_model, compare_columns, ignore_columns)) }} {% endtest %} -{% macro default__test_equality(model, compare_model, compare_columns=None) %} +{% macro default__test_equality(model, compare_model, compare_columns=None, ignore_columns=None) %} -{% set set_diff %} +{%- if compare_columns and ignore_columns -%} + {{ exceptions.raise_compiler_error("Both a compare and an ignore list were provided to the `equality` macro. Only one is allowed") }} +{%- endif -%} + +{% set set_diff %} count(*) + coalesce(abs( sum(case when which_diff = 'a_minus_b' then 1 else 0 end) - sum(case when which_diff = 'b_minus_a' then 1 else 0 end) @@ -29,7 +33,22 @@ information schema — this allows the model to be an ephemeral model {%- if not compare_columns -%} {%- do dbt_utils._is_ephemeral(model, 'test_equality') -%} - {%- set compare_columns = adapter.get_columns_in_relation(model) | map(attribute='quoted') -%} + {%- set compare_columns = adapter.get_columns_in_relation(model) | map(attribute='name') -%} +{%- endif -%} + +{%- if ignore_columns -%} + {#-- Lower case ignore columns for easier comparison --#} + {%- set ignore_columns = ignore_columns | map("lower") | list %} + + {%- set include_columns = [] %} + {%- for column in compare_columns -%} + {%- if column | lower not in ignore_columns -%} + {% do include_columns.append(column) %} + {%- endif %} + {%- endfor %} + + {%- set compare_columns = include_columns %} + {%- endif -%} {% set compare_cols_csv = compare_columns | join(', ') %} From d5853e34cef431516786c31502d656568508c217 Mon Sep 17 00:00:00 2001 From: Ryan Hill Date: Mon, 6 Feb 2023 15:01:15 +0000 Subject: [PATCH 02/17] add precision option to equality test --- CHANGELOG.md | 6 ++ README.md | 3 +- .../data_test_equality_floats.csv | 11 +++ .../data_test_equality_floats_columns.csv | 11 +++ integration_tests/dbt_project.yml | 10 ++- .../models/generic_tests/schema.yml | 34 ++++++++ .../generic_tests/test_equality_floats.sql | 9 ++ .../test_equality_floats_columns.sql | 9 ++ macros/generic_tests/equality.sql | 86 +++++++++++++------ 9 files changed, 149 insertions(+), 30 deletions(-) create mode 100644 integration_tests/data/schema_tests/data_test_equality_floats.csv create mode 100644 integration_tests/data/schema_tests/data_test_equality_floats_columns.csv create mode 100644 integration_tests/models/generic_tests/test_equality_floats.sql create mode 100644 integration_tests/models/generic_tests/test_equality_floats_columns.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index 310180bc..7de08e52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,12 @@ ## Contributors: ---> +# Unreleased +## New features +- The `equality` test now accepts an additional argument, `precision` to aide in comparing floating point numbers ([#757](https://github.com/dbt-labs/dbt-utils/issues/757), [#765](https://github.com/dbt-labs/dbt-utils/pull/765)) +## Contributors: +- [@rlh1994](https://github.com/rlh1994) + # Unreleased ## New features - Add option to ignore columns in equality test ([#734](https://github.com/dbt-labs/dbt-utils/issues/734), [#737](https://github.com/dbt-labs/dbt-utils/pull/737)) diff --git a/README.md b/README.md index 6061c63e..a13b282d 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ This test supports the `group_by_columns` parameter; see [Grouping in tests](#gr ### equality ([source](macros/generic_tests/equality.sql)) -Asserts the equality of two relations. Optionally specify a subset of columns to compare or ignore. +Asserts the equality of two relations. Optionally specify a subset of columns to compare or ignore, and a precision to compare numeric columns on. **Usage:** @@ -136,6 +136,7 @@ models: compare_columns: - first_column - second_column + precision: 4 # compare all columns except the ones on the ignore list - name: model_name_ignore_columns diff --git a/integration_tests/data/schema_tests/data_test_equality_floats.csv b/integration_tests/data/schema_tests/data_test_equality_floats.csv new file mode 100644 index 00000000..85241961 --- /dev/null +++ b/integration_tests/data/schema_tests/data_test_equality_floats.csv @@ -0,0 +1,11 @@ +id,float_number +05ac09c4-f947-45a8-8c14-88f430f8b294,62.3888186 +cfae9054-940b-42a1-84d4-052daae6194f,81.2511656 +6029501d-c274-49f2-a69d-4c75a3d9931d,23.3959675 +c653e520-df81-4a5f-b44b-bb1b4c1b7846,72.2100841 +59caed0d-53d6-473c-a88c-3726c7693f05,68.6029434 +b441f6a0-ce7f-4ad9-b96b-b41d73a94ae7,72.7861425 +26491840-bfd4-4496-9ca9-ad9220a2de47,35.3662223 +b4f233ce-a494-4bb6-9cf2-73bb6854e58a,89.1524680 +11c979b7-2661-4375-8143-7c9b54b90627,19.5755431 +a8057f73-312e-48e6-b344-f4a510a2c4a8,22.9237047 diff --git a/integration_tests/data/schema_tests/data_test_equality_floats_columns.csv b/integration_tests/data/schema_tests/data_test_equality_floats_columns.csv new file mode 100644 index 00000000..77beeae9 --- /dev/null +++ b/integration_tests/data/schema_tests/data_test_equality_floats_columns.csv @@ -0,0 +1,11 @@ +id,float_number,to_ignore +05ac09c4-f947-45a8-8c14-88f430f8b294,62.3888186,a +cfae9054-940b-42a1-84d4-052daae6194f,81.2511656,a +6029501d-c274-49f2-a69d-4c75a3d9931d,23.3959675,a +c653e520-df81-4a5f-b44b-bb1b4c1b7846,72.2100841,a +59caed0d-53d6-473c-a88c-3726c7693f05,68.6029434,a +b441f6a0-ce7f-4ad9-b96b-b41d73a94ae7,72.7861425,a +26491840-bfd4-4496-9ca9-ad9220a2de47,35.3662223,a +b4f233ce-a494-4bb6-9cf2-73bb6854e58a,89.1524680,a +11c979b7-2661-4375-8143-7c9b54b90627,19.5755431,a +a8057f73-312e-48e6-b344-f4a510a2c4a8,22.9237047,a diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index a9531e78..8ac1320c 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -32,7 +32,7 @@ seeds: sql: data_events_20180103: +schema: events - + data_get_column_values_dropped: # this.incorporate() to hardcode the node's type as otherwise dbt doesn't know it yet +post-hook: "{% do adapter.drop_relation(this.incorporate(type='table')) %}" @@ -53,3 +53,11 @@ seeds: data_test_sequential_timestamps: +column_types: my_timestamp: timestamp + + data_test_equality_floats: + +column_types: + float_number: float + + data_test_equality_floats_columns: + +column_types: + float_number: float diff --git a/integration_tests/models/generic_tests/schema.yml b/integration_tests/models/generic_tests/schema.yml index a7658d3f..d96cc6c3 100644 --- a/integration_tests/models/generic_tests/schema.yml +++ b/integration_tests/models/generic_tests/schema.yml @@ -210,6 +210,40 @@ models: - last_name - email + - name: test_equality_floats + tests: + # test precision only + - dbt_utils.equality: + compare_model: ref('data_test_equality_floats') + precision: 4 + - dbt_utils.equality: + compare_model: ref('data_test_equality_floats') + precision: 8 + error_if: "<1" #sneaky way to ensure that the test is returning failing rows + warn_if: "<0" + + - name: test_equality_floats_columns + tests: + # Positive assertion tests + - dbt_utils.equality: + compare_model: ref('data_test_equality_floats_columns') + compare_columns: + - id + - float_number + precision: 4 + - dbt_utils.equality: + compare_model: ref('data_test_equality_floats_columns') + ignore_columns: + - to_ignore + precision: 4 + # all columns should fail even with rounding + - dbt_utils.equality: + compare_model: ref('data_test_equality_floats_columns') + precision: 4 + error_if: "<1" #sneaky way to ensure that the test is returning failing rows + warn_if: "<0" + + - name: test_fewer_rows_than tests: - dbt_utils.fewer_rows_than: diff --git a/integration_tests/models/generic_tests/test_equality_floats.sql b/integration_tests/models/generic_tests/test_equality_floats.sql new file mode 100644 index 00000000..9c84bb5c --- /dev/null +++ b/integration_tests/models/generic_tests/test_equality_floats.sql @@ -0,0 +1,9 @@ +with data as ( + + select * from {{ ref('data_test_equality_floats') }} + +) + +select + id, float_number + 0.0000001 as float_number +from data diff --git a/integration_tests/models/generic_tests/test_equality_floats_columns.sql b/integration_tests/models/generic_tests/test_equality_floats_columns.sql new file mode 100644 index 00000000..bda9b248 --- /dev/null +++ b/integration_tests/models/generic_tests/test_equality_floats_columns.sql @@ -0,0 +1,9 @@ +with data as ( + + select * from {{ ref('data_test_equality_floats') }} + +) + +select + id, float_number + 0.0000001 as float_number, 'b' as to_ignore +from data diff --git a/macros/generic_tests/equality.sql b/macros/generic_tests/equality.sql index 3ef74c10..e844a24a 100644 --- a/macros/generic_tests/equality.sql +++ b/macros/generic_tests/equality.sql @@ -1,14 +1,14 @@ -{% test equality(model, compare_model, compare_columns=None, ignore_columns=None) %} - {{ return(adapter.dispatch('test_equality', 'dbt_utils')(model, compare_model, compare_columns, ignore_columns)) }} +{% test equality(model, compare_model, compare_columns=None, ignore_columns=None, precision = None) %} + {{ return(adapter.dispatch('test_equality', 'dbt_utils')(model, compare_model, compare_columns, ignore_columns, precision)) }} {% endtest %} -{% macro default__test_equality(model, compare_model, compare_columns=None, ignore_columns=None) %} +{% macro default__test_equality(model, compare_model, compare_columns=None, ignore_columns=None, precision = None) %} {%- if compare_columns and ignore_columns -%} {{ exceptions.raise_compiler_error("Both a compare and an ignore list were provided to the `equality` macro. Only one is allowed") }} {%- endif -%} -{% set set_diff %} +{% set set_diff %} count(*) + coalesce(abs( sum(case when which_diff = 'a_minus_b' then 1 else 0 end) - sum(case when which_diff = 'b_minus_a' then 1 else 0 end) @@ -26,32 +26,62 @@ -- setup {%- do dbt_utils._is_relation(model, 'test_equality') -%} -{#- -If the compare_cols arg is provided, we can run this test without querying the -information schema — this allows the model to be an ephemeral model --#} - -{%- if not compare_columns -%} +{%- if not precision -%} + {#- + If the compare_cols arg is provided, we can run this test without querying the + information schema — this allows the model to be an ephemeral model + -#} + {%- if not compare_columns -%} + {%- do dbt_utils._is_ephemeral(model, 'test_equality') -%} + {%- set compare_columns = adapter.get_columns_in_relation(model)-%} + + + {%- if ignore_columns -%} + {#-- Lower case ignore columns for easier comparison --#} + {%- set ignore_columns = ignore_columns | map("lower") | list %} + + {# Filter out the excluded columns #} + {%- set include_columns = [] %} + {%- for column in compare_columns -%} + {%- if column.name | lower not in ignore_columns -%} + {% do include_columns.append(column) %} + {%- endif %} + {%- endfor %} + + {%- set compare_columns = include_columns | map(attribute='quoted') %} + {%- else -%} + {%- set compare_columns = compare_columns | map(attribute='quoted') %} + {%- endif -%} + {%- endif -%} + + {% set compare_cols_csv = compare_columns | join(', ') %} + +{% else %} + {#- + If rounding is required, we need to get the types, so it can't be ephermeral + -#} {%- do dbt_utils._is_ephemeral(model, 'test_equality') -%} - {%- set compare_columns = adapter.get_columns_in_relation(model) | map(attribute='name') -%} -{%- endif -%} - -{%- if ignore_columns -%} - {#-- Lower case ignore columns for easier comparison --#} - {%- set ignore_columns = ignore_columns | map("lower") | list %} - - {%- set include_columns = [] %} - {%- for column in compare_columns -%} - {%- if column | lower not in ignore_columns -%} - {% do include_columns.append(column) %} - {%- endif %} - {%- endfor %} + {%- set columns = adapter.get_columns_in_relation(model) -%} + + {% set columns_list = [] %} + {%- for col in columns -%} + {%- if ( + (col.name|lower in compare_columns|map('lower') or not compare_columns) and + (col.name|lower not in ignore_columns|map('lower') or not ignore_columns) + ) -%} + {# Databricks double type is not picked up by any number type checks in dbt #} + {%- if col.is_float() or col.is_numeric() or col.data_type == 'double' -%} + {# Cast is required due to postgres not having round for a double precision number #} + {%- do columns_list.append('round(cast(' ~ col.quoted ~ ' as ' ~ dbt.type_numeric() ~ '),' ~ precision ~ ') as ' ~ col.quoted) -%} + {%- else -%} + {%- do columns_list.append(col.quoted) -%} + {%- endif -%} + {% endif %} + {%- endfor -%} + + {% set compare_cols_csv = columns_list | join(', ') %} - {%- set compare_columns = include_columns %} - -{%- endif -%} - -{% set compare_cols_csv = compare_columns | join(', ') %} +{% endif %} with a as ( From 24105b450d4d32c60b99a76b934b33ef24cc9c3d Mon Sep 17 00:00:00 2001 From: Ryan Hill Date: Tue, 7 Feb 2023 13:07:44 +0000 Subject: [PATCH 03/17] CI fix? --- integration_tests/dbt_project.yml | 1 + .../models/generic_tests/test_equality_floats_columns.sql | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 8ac1320c..7fe825f1 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -61,3 +61,4 @@ seeds: data_test_equality_floats_columns: +column_types: float_number: float + to_ignore: varchar(1) diff --git a/integration_tests/models/generic_tests/test_equality_floats_columns.sql b/integration_tests/models/generic_tests/test_equality_floats_columns.sql index bda9b248..b8087567 100644 --- a/integration_tests/models/generic_tests/test_equality_floats_columns.sql +++ b/integration_tests/models/generic_tests/test_equality_floats_columns.sql @@ -5,5 +5,5 @@ with data as ( ) select - id, float_number + 0.0000001 as float_number, 'b' as to_ignore + id, float_number + 0.0000001 as float_number, cast('b' as varchar(1)) as to_ignore from data From 12e3750185f4d7c36462d91b502244ce646ebbc8 Mon Sep 17 00:00:00 2001 From: Ryan Hill Date: Tue, 7 Feb 2023 13:17:20 +0000 Subject: [PATCH 04/17] CI fix 2.0 --- ...ts.csv => data_test_equality_floats_a.csv} | 0 .../data_test_equality_floats_b.csv | 11 +++ ...> data_test_equality_floats_columns_a.csv} | 0 .../data_test_equality_floats_columns_b.csv | 11 +++ integration_tests/dbt_project.yml | 13 +++- .../models/generic_tests/schema.yml | 67 +++++++++---------- .../generic_tests/test_equality_floats.sql | 9 --- .../test_equality_floats_columns.sql | 9 --- 8 files changed, 65 insertions(+), 55 deletions(-) rename integration_tests/data/schema_tests/{data_test_equality_floats.csv => data_test_equality_floats_a.csv} (100%) create mode 100644 integration_tests/data/schema_tests/data_test_equality_floats_b.csv rename integration_tests/data/schema_tests/{data_test_equality_floats_columns.csv => data_test_equality_floats_columns_a.csv} (100%) create mode 100644 integration_tests/data/schema_tests/data_test_equality_floats_columns_b.csv delete mode 100644 integration_tests/models/generic_tests/test_equality_floats.sql delete mode 100644 integration_tests/models/generic_tests/test_equality_floats_columns.sql diff --git a/integration_tests/data/schema_tests/data_test_equality_floats.csv b/integration_tests/data/schema_tests/data_test_equality_floats_a.csv similarity index 100% rename from integration_tests/data/schema_tests/data_test_equality_floats.csv rename to integration_tests/data/schema_tests/data_test_equality_floats_a.csv diff --git a/integration_tests/data/schema_tests/data_test_equality_floats_b.csv b/integration_tests/data/schema_tests/data_test_equality_floats_b.csv new file mode 100644 index 00000000..0306a9aa --- /dev/null +++ b/integration_tests/data/schema_tests/data_test_equality_floats_b.csv @@ -0,0 +1,11 @@ +id,float_number +05ac09c4-f947-45a8-8c14-88f430f8b294,62.3888187 +cfae9054-940b-42a1-84d4-052daae6194f,81.2511657 +6029501d-c274-49f2-a69d-4c75a3d9931d,23.3959676 +c653e520-df81-4a5f-b44b-bb1b4c1b7846,72.2100842 +59caed0d-53d6-473c-a88c-3726c7693f05,68.6029435 +b441f6a0-ce7f-4ad9-b96b-b41d73a94ae7,72.7861426 +26491840-bfd4-4496-9ca9-ad9220a2de47,35.3662224 +b4f233ce-a494-4bb6-9cf2-73bb6854e58a,89.1524681 +11c979b7-2661-4375-8143-7c9b54b90627,19.5755432 +a8057f73-312e-48e6-b344-f4a510a2c4a8,22.9237048 diff --git a/integration_tests/data/schema_tests/data_test_equality_floats_columns.csv b/integration_tests/data/schema_tests/data_test_equality_floats_columns_a.csv similarity index 100% rename from integration_tests/data/schema_tests/data_test_equality_floats_columns.csv rename to integration_tests/data/schema_tests/data_test_equality_floats_columns_a.csv diff --git a/integration_tests/data/schema_tests/data_test_equality_floats_columns_b.csv b/integration_tests/data/schema_tests/data_test_equality_floats_columns_b.csv new file mode 100644 index 00000000..e26305de --- /dev/null +++ b/integration_tests/data/schema_tests/data_test_equality_floats_columns_b.csv @@ -0,0 +1,11 @@ +id,float_number,to_ignore +05ac09c4-f947-45a8-8c14-88f430f8b294,62.3888186,b +cfae9054-940b-42a1-84d4-052daae6194f,81.2511656,b +6029501d-c274-49f2-a69d-4c75a3d9931d,23.3959675,b +c653e520-df81-4a5f-b44b-bb1b4c1b7846,72.2100841,b +59caed0d-53d6-473c-a88c-3726c7693f05,68.6029434,b +b441f6a0-ce7f-4ad9-b96b-b41d73a94ae7,72.7861425,b +26491840-bfd4-4496-9ca9-ad9220a2de47,35.3662223,b +b4f233ce-a494-4bb6-9cf2-73bb6854e58a,89.1524680,b +11c979b7-2661-4375-8143-7c9b54b90627,19.5755431,b +a8057f73-312e-48e6-b344-f4a510a2c4a8,22.9237047,b diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 7fe825f1..830643ec 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -54,11 +54,18 @@ seeds: +column_types: my_timestamp: timestamp - data_test_equality_floats: + data_test_equality_floats_a: +column_types: float_number: float - data_test_equality_floats_columns: + data_test_equality_floats_columns_a: + +column_types: + float_number: float + + data_test_equality_floats_b: + +column_types: + float_number: float + + data_test_equality_floats_columns_b: +column_types: float_number: float - to_ignore: varchar(1) diff --git a/integration_tests/models/generic_tests/schema.yml b/integration_tests/models/generic_tests/schema.yml index d96cc6c3..461f391e 100644 --- a/integration_tests/models/generic_tests/schema.yml +++ b/integration_tests/models/generic_tests/schema.yml @@ -160,6 +160,39 @@ seeds: ignore_columns: - col_c + - name: data_test_equality_floats_a + tests: + # test precision only + - dbt_utils.equality: + compare_model: ref('data_test_equality_floats_b') + precision: 4 + - dbt_utils.equality: + compare_model: ref('data_test_equality_floats_b') + precision: 8 + error_if: "<1" #sneaky way to ensure that the test is returning failing rows + warn_if: "<0" + + - name: data_test_equality_floats_columns_a + tests: + # Positive assertion tests + - dbt_utils.equality: + compare_model: ref('data_test_equality_floats_columns_b') + compare_columns: + - id + - float_number + precision: 4 + - dbt_utils.equality: + compare_model: ref('data_test_equality_floats_columns_b') + ignore_columns: + - to_ignore + precision: 4 + # all columns should fail even with rounding + - dbt_utils.equality: + compare_model: ref('data_test_equality_floats_columns_b') + precision: 4 + error_if: "<1" #sneaky way to ensure that the test is returning failing rows + warn_if: "<0" + models: - name: recency_time_included tests: @@ -210,40 +243,6 @@ models: - last_name - email - - name: test_equality_floats - tests: - # test precision only - - dbt_utils.equality: - compare_model: ref('data_test_equality_floats') - precision: 4 - - dbt_utils.equality: - compare_model: ref('data_test_equality_floats') - precision: 8 - error_if: "<1" #sneaky way to ensure that the test is returning failing rows - warn_if: "<0" - - - name: test_equality_floats_columns - tests: - # Positive assertion tests - - dbt_utils.equality: - compare_model: ref('data_test_equality_floats_columns') - compare_columns: - - id - - float_number - precision: 4 - - dbt_utils.equality: - compare_model: ref('data_test_equality_floats_columns') - ignore_columns: - - to_ignore - precision: 4 - # all columns should fail even with rounding - - dbt_utils.equality: - compare_model: ref('data_test_equality_floats_columns') - precision: 4 - error_if: "<1" #sneaky way to ensure that the test is returning failing rows - warn_if: "<0" - - - name: test_fewer_rows_than tests: - dbt_utils.fewer_rows_than: diff --git a/integration_tests/models/generic_tests/test_equality_floats.sql b/integration_tests/models/generic_tests/test_equality_floats.sql deleted file mode 100644 index 9c84bb5c..00000000 --- a/integration_tests/models/generic_tests/test_equality_floats.sql +++ /dev/null @@ -1,9 +0,0 @@ -with data as ( - - select * from {{ ref('data_test_equality_floats') }} - -) - -select - id, float_number + 0.0000001 as float_number -from data diff --git a/integration_tests/models/generic_tests/test_equality_floats_columns.sql b/integration_tests/models/generic_tests/test_equality_floats_columns.sql deleted file mode 100644 index b8087567..00000000 --- a/integration_tests/models/generic_tests/test_equality_floats_columns.sql +++ /dev/null @@ -1,9 +0,0 @@ -with data as ( - - select * from {{ ref('data_test_equality_floats') }} - -) - -select - id, float_number + 0.0000001 as float_number, cast('b' as varchar(1)) as to_ignore -from data From aa25bd2e03c626bf2b357e0b02ffb09e9ec379f2 Mon Sep 17 00:00:00 2001 From: Joel Labes Date: Tue, 2 May 2023 11:32:52 +1200 Subject: [PATCH 05/17] Update CHANGELOG.md --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7de08e52..438ea0f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,9 @@ * ZZZ by @YYY in https://github.com/dbt-labs/dbt-utils/pull/XXX ## Fixes * Fix legacy links in README by @dbeatty10 in https://github.com/dbt-labs/dbt-utils/pull/796 +## Quality of life +## Under the hood +## Contributors: # dbt utils v1.1.0 ## What's Changed From c8a39564b5c11924bc8768ea8a0bc651e8e6bd98 Mon Sep 17 00:00:00 2001 From: Ryan Hill Date: Tue, 2 May 2023 10:28:52 +0100 Subject: [PATCH 06/17] Check for subset of columns (Close #785) --- .../models/generic_tests/equality_less_columns.sql | 9 +++++++++ integration_tests/models/generic_tests/schema.yml | 7 +++++++ integration_tests/models/sql/test_union.sql | 4 ++-- macros/generic_tests/equality.sql | 13 +++++++++++++ 4 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 integration_tests/models/generic_tests/equality_less_columns.sql diff --git a/integration_tests/models/generic_tests/equality_less_columns.sql b/integration_tests/models/generic_tests/equality_less_columns.sql new file mode 100644 index 00000000..415bf949 --- /dev/null +++ b/integration_tests/models/generic_tests/equality_less_columns.sql @@ -0,0 +1,9 @@ +with data as ( + + select * from {{ ref('data_test_equality_b') }} + +) + +select + col_a, col_b +from data diff --git a/integration_tests/models/generic_tests/schema.yml b/integration_tests/models/generic_tests/schema.yml index 461f391e..4cd626a6 100644 --- a/integration_tests/models/generic_tests/schema.yml +++ b/integration_tests/models/generic_tests/schema.yml @@ -250,3 +250,10 @@ models: - dbt_utils.fewer_rows_than: compare_model: ref('data_test_fewer_rows_than_table_2') group_by_columns: ['col_a'] + + - name: equality_less_columns + tests: + - dbt_utils.equality: + compare_model: ref('data_test_equality_a') + error_if: "<1" #sneaky way to ensure that the test is returning failing rows + warn_if: "<0" diff --git a/integration_tests/models/sql/test_union.sql b/integration_tests/models/sql/test_union.sql index 69836833..8d675ede 100644 --- a/integration_tests/models/sql/test_union.sql +++ b/integration_tests/models/sql/test_union.sql @@ -2,7 +2,7 @@ select id, name, - favorite_color + favorite_color, + favorite_number from {{ ref('test_union_base') }} - diff --git a/macros/generic_tests/equality.sql b/macros/generic_tests/equality.sql index e844a24a..e6f812b7 100644 --- a/macros/generic_tests/equality.sql +++ b/macros/generic_tests/equality.sql @@ -23,9 +23,22 @@ {{ return('') }} {% endif %} + + -- setup {%- do dbt_utils._is_relation(model, 'test_equality') -%} +{# Ensure there are no extra columns in the compare_model vs model #} +{%- if not compare_columns and not ignore_columns -%} + {%- do dbt_utils._is_ephemeral(model, 'test_equality') -%} + {%- set compare_columns_set = set(adapter.get_columns_in_relation(model) | map(attribute='quoted')) -%} + {%- do dbt_utils._is_ephemeral(model, 'test_equality') -%} + {%- set compare_model_columns_set = set(adapter.get_columns_in_relation(compare_model) | map(attribute='quoted')) -%} + {% if compare_columns_set != compare_model_columns_set %} + {{ return("select 1, 'b_minus_a' as which_diff from " ~ compare_model) }} + {% endif %} +{% endif %} + {%- if not precision -%} {#- If the compare_cols arg is provided, we can run this test without querying the From b722128e16f9b5f1439ec2427c99ea99ecb7726c Mon Sep 17 00:00:00 2001 From: Ryan Hill Date: Tue, 2 May 2023 10:43:07 +0100 Subject: [PATCH 07/17] cast type --- macros/generic_tests/equality.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/generic_tests/equality.sql b/macros/generic_tests/equality.sql index e6f812b7..5c6143cb 100644 --- a/macros/generic_tests/equality.sql +++ b/macros/generic_tests/equality.sql @@ -35,7 +35,7 @@ {%- do dbt_utils._is_ephemeral(model, 'test_equality') -%} {%- set compare_model_columns_set = set(adapter.get_columns_in_relation(compare_model) | map(attribute='quoted')) -%} {% if compare_columns_set != compare_model_columns_set %} - {{ return("select 1, 'b_minus_a' as which_diff from " ~ compare_model) }} + {{ return("select 'b_minus_a'::text as which_diff from " ~ compare_model) }} {% endif %} {% endif %} From f786fda8df69ae02770cc257bd6ccf3c58536c98 Mon Sep 17 00:00:00 2001 From: Ryan Hill Date: Tue, 2 May 2023 10:47:06 +0100 Subject: [PATCH 08/17] cast type across warehouses --- macros/generic_tests/equality.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/generic_tests/equality.sql b/macros/generic_tests/equality.sql index 5c6143cb..4167e5ed 100644 --- a/macros/generic_tests/equality.sql +++ b/macros/generic_tests/equality.sql @@ -35,7 +35,7 @@ {%- do dbt_utils._is_ephemeral(model, 'test_equality') -%} {%- set compare_model_columns_set = set(adapter.get_columns_in_relation(compare_model) | map(attribute='quoted')) -%} {% if compare_columns_set != compare_model_columns_set %} - {{ return("select 'b_minus_a'::text as which_diff from " ~ compare_model) }} + {{ return("select cast('b_minus_a' as " ~ dbt.type_string() ~") as which_diff from " ~ compare_model) }} {% endif %} {% endif %} From 405d8ec8e2c2524211d4d5fc574aaed5675c0c83 Mon Sep 17 00:00:00 2001 From: Ryan Hill Date: Thu, 18 May 2023 12:00:41 +0100 Subject: [PATCH 09/17] swap to copiler error, account for ignore columns --- .../models/generic_tests/schema.yml | 4 +- macros/generic_tests/equality.sql | 40 ++++++++++++++++--- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/integration_tests/models/generic_tests/schema.yml b/integration_tests/models/generic_tests/schema.yml index 4cd626a6..0a1dd78b 100644 --- a/integration_tests/models/generic_tests/schema.yml +++ b/integration_tests/models/generic_tests/schema.yml @@ -255,5 +255,5 @@ models: tests: - dbt_utils.equality: compare_model: ref('data_test_equality_a') - error_if: "<1" #sneaky way to ensure that the test is returning failing rows - warn_if: "<0" + ignore_columns: + - col_c diff --git a/macros/generic_tests/equality.sql b/macros/generic_tests/equality.sql index 4167e5ed..eda54e63 100644 --- a/macros/generic_tests/equality.sql +++ b/macros/generic_tests/equality.sql @@ -29,14 +29,44 @@ {%- do dbt_utils._is_relation(model, 'test_equality') -%} {# Ensure there are no extra columns in the compare_model vs model #} -{%- if not compare_columns and not ignore_columns -%} +{%- if not compare_columns -%} {%- do dbt_utils._is_ephemeral(model, 'test_equality') -%} - {%- set compare_columns_set = set(adapter.get_columns_in_relation(model) | map(attribute='quoted')) -%} - {%- do dbt_utils._is_ephemeral(model, 'test_equality') -%} - {%- set compare_model_columns_set = set(adapter.get_columns_in_relation(compare_model) | map(attribute='quoted')) -%} + {%- do dbt_utils._is_ephemeral(compare_model, 'test_equality') -%} + + {%- set model_columns = adapter.get_columns_in_relation(model) -%} + {%- set compare_model_columns = adapter.get_columns_in_relation(compare_model) -%} + + + {%- if ignore_columns -%} + {#-- Lower case ignore columns for easier comparison --#} + {%- set ignore_columns = ignore_columns | map("lower") | list %} + + {# Filter out the excluded columns #} + {%- set include_columns = [] %} + {%- set include_model_columns = [] %} + {%- for column in model_columns -%} + {%- if column.name | lower not in ignore_columns -%} + {% do include_columns.append(column) %} + {%- endif %} + {%- endfor %} + {%- for column in compare_model_columns -%} + {%- if column.name | lower not in ignore_columns -%} + {% do include_model_columns.append(column) %} + {%- endif %} + {%- endfor %} + + {%- set compare_columns_set = set(include_columns | map(attribute='quoted')) %} + {%- set compare_model_columns_set = set(include_model_columns | map(attribute='quoted')) %} + {%- else -%} + {%- set compare_columns_set = set(model_columns | map(attribute='quoted')) %} + {%- set compare_model_columns_set = set(compare_model_columns | map(attribute='quoted')) %} + {%- endif -%} + {% if compare_columns_set != compare_model_columns_set %} - {{ return("select cast('b_minus_a' as " ~ dbt.type_string() ~") as which_diff from " ~ compare_model) }} + {{ exceptions.raise_compiler_error(compare_model ~" has less columns than " ~ model ~ ", please ensure they have the same columns or use the `compare_columns` or `ignore_columns` arguments to subset them.") }} {% endif %} + + {% endif %} {%- if not precision -%} From a0364364b1be90c44b29b9f459a70ef31f6751fb Mon Sep 17 00:00:00 2001 From: Ryan Hill Date: Mon, 22 May 2023 09:23:26 +0100 Subject: [PATCH 10/17] Update CL --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 438ea0f3..0e5ab8e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,7 +33,10 @@ ## New features * ZZZ by @YYY in https://github.com/dbt-labs/dbt-utils/pull/XXX ## Fixes -* Fix legacy links in README by @dbeatty10 in https://github.com/dbt-labs/dbt-utils/pull/796 +- Equality test will now raise an error when the second model has less columns than the first ([#785](https://github.com/dbt-labs/dbt-utils/issues/785)) +## Contributors: +- [@rlh1994](https://github.com/rlh1994) + ## Quality of life ## Under the hood ## Contributors: From d4caa527b2440022bb3b98a7d61e1ac860a4f94b Mon Sep 17 00:00:00 2001 From: Ryan Hill Date: Thu, 8 Jun 2023 10:29:12 +0100 Subject: [PATCH 11/17] allow for different cased names --- macros/generic_tests/equality.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/macros/generic_tests/equality.sql b/macros/generic_tests/equality.sql index eda54e63..bceb77d3 100644 --- a/macros/generic_tests/equality.sql +++ b/macros/generic_tests/equality.sql @@ -55,11 +55,11 @@ {%- endif %} {%- endfor %} - {%- set compare_columns_set = set(include_columns | map(attribute='quoted')) %} - {%- set compare_model_columns_set = set(include_model_columns | map(attribute='quoted')) %} + {%- set compare_columns_set = set(include_columns | map(attribute='quoted') | map("lower")) %} + {%- set compare_model_columns_set = set(include_model_columns | map(attribute='quoted') | map("lower")) %} {%- else -%} - {%- set compare_columns_set = set(model_columns | map(attribute='quoted')) %} - {%- set compare_model_columns_set = set(compare_model_columns | map(attribute='quoted')) %} + {%- set compare_columns_set = set(model_columns | map(attribute='quoted') | map("lower")) %} + {%- set compare_model_columns_set = set(compare_model_columns | map(attribute='quoted') | map("lower")) %} {%- endif -%} {% if compare_columns_set != compare_model_columns_set %} From 4ef3c06ab9658edddf1b21397cd1d060de9e8a25 Mon Sep 17 00:00:00 2001 From: Ryan Hill Date: Mon, 25 Sep 2023 16:02:06 +0100 Subject: [PATCH 12/17] fix CL --- CHANGELOG.md | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e5ab8e4..690f5a77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,35 +11,24 @@ # Unreleased ## New features - The `equality` test now accepts an additional argument, `precision` to aide in comparing floating point numbers ([#757](https://github.com/dbt-labs/dbt-utils/issues/757), [#765](https://github.com/dbt-labs/dbt-utils/pull/765)) -## Contributors: -- [@rlh1994](https://github.com/rlh1994) - -# Unreleased -## New features - Add option to ignore columns in equality test ([#734](https://github.com/dbt-labs/dbt-utils/issues/734), [#737](https://github.com/dbt-labs/dbt-utils/pull/737)) -## Contributors: -- [@brunocostalopes](https://github.com/brunocostalopes) - -# Unreleased ## Fixes - deduplicate macro for Databricks now uses the QUALIFY clause, which fixes NULL columns issues from the default natural join logic - deduplicate macro for Redshift now uses the QUALIFY clause, which fixes NULL columns issues from the default natural join logic +- Equality test will now raise an error when the second model has less columns than the first ([#785](https://github.com/dbt-labs/dbt-utils/issues/785)) ## Contributors: -[@graciegoheen](https://github.com/graciegoheen) -[@yauhen-sobaleu](https://github.com/yauhen-sobaleu) +- [@graciegoheen](https://github.com/graciegoheen) +- [@yauhen-sobaleu](https://github.com/yauhen-sobaleu) +- [@rlh1994](https://github.com/rlh1994) +- [@brunocostalopes](https://github.com/brunocostalopes) + # dbt utils v1.1.1 ## New features -* ZZZ by @YYY in https://github.com/dbt-labs/dbt-utils/pull/XXX +* Improve the performance of the `at_least_one` test by pruning early. This is especially helpful when running against external tables. By @joshuahuntley in https://github.com/dbt-labs/dbt-utils/pull/775 ## Fixes -- Equality test will now raise an error when the second model has less columns than the first ([#785](https://github.com/dbt-labs/dbt-utils/issues/785)) -## Contributors: -- [@rlh1994](https://github.com/rlh1994) - -## Quality of life -## Under the hood -## Contributors: +* Fix legacy links in README by @dbeatty10 in https://github.com/dbt-labs/dbt-utils/pull/796 # dbt utils v1.1.0 ## What's Changed From a481ad12ab37096d2e5c7d9915ff6efa6c327b17 Mon Sep 17 00:00:00 2001 From: Ryan Hill Date: Mon, 25 Sep 2023 16:03:48 +0100 Subject: [PATCH 13/17] linting --- macros/generic_tests/equality.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/generic_tests/equality.sql b/macros/generic_tests/equality.sql index bceb77d3..37ccc11c 100644 --- a/macros/generic_tests/equality.sql +++ b/macros/generic_tests/equality.sql @@ -101,7 +101,7 @@ {% else %} {#- - If rounding is required, we need to get the types, so it can't be ephermeral + If rounding is required, we need to get the types, so it cannot be ephermeral -#} {%- do dbt_utils._is_ephemeral(model, 'test_equality') -%} {%- set columns = adapter.get_columns_in_relation(model) -%} From e22fc2ca858ff94b33b0e89adb0ccf356db174be Mon Sep 17 00:00:00 2001 From: Ryan Hill Date: Tue, 24 Oct 2023 19:07:44 +0100 Subject: [PATCH 14/17] Rename to exclude_columns --- README.md | 6 ++--- .../models/generic_tests/schema.yml | 6 ++--- macros/generic_tests/equality.sql | 26 +++++++++---------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index a13b282d..55c926c3 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ This test supports the `group_by_columns` parameter; see [Grouping in tests](#gr ### equality ([source](macros/generic_tests/equality.sql)) -Asserts the equality of two relations. Optionally specify a subset of columns to compare or ignore, and a precision to compare numeric columns on. +Asserts the equality of two relations. Optionally specify a subset of columns to compare or exclude, and a precision to compare numeric columns on. **Usage:** @@ -139,11 +139,11 @@ models: precision: 4 # compare all columns except the ones on the ignore list - - name: model_name_ignore_columns + - name: model_name_exclude_columns tests: - dbt_utils.equality: compare_model: ref('other_table_name') - ignore_columns: + exclude_columns: - third_column ``` diff --git a/integration_tests/models/generic_tests/schema.yml b/integration_tests/models/generic_tests/schema.yml index 0a1dd78b..022b0a59 100644 --- a/integration_tests/models/generic_tests/schema.yml +++ b/integration_tests/models/generic_tests/schema.yml @@ -157,7 +157,7 @@ seeds: - col_b - dbt_utils.equality: compare_model: ref('data_test_equality_b') - ignore_columns: + exclude_columns: - col_c - name: data_test_equality_floats_a @@ -183,7 +183,7 @@ seeds: precision: 4 - dbt_utils.equality: compare_model: ref('data_test_equality_floats_columns_b') - ignore_columns: + exclude_columns: - to_ignore precision: 4 # all columns should fail even with rounding @@ -255,5 +255,5 @@ models: tests: - dbt_utils.equality: compare_model: ref('data_test_equality_a') - ignore_columns: + exclude_columns: - col_c diff --git a/macros/generic_tests/equality.sql b/macros/generic_tests/equality.sql index 37ccc11c..12d3e2c4 100644 --- a/macros/generic_tests/equality.sql +++ b/macros/generic_tests/equality.sql @@ -1,10 +1,10 @@ -{% test equality(model, compare_model, compare_columns=None, ignore_columns=None, precision = None) %} - {{ return(adapter.dispatch('test_equality', 'dbt_utils')(model, compare_model, compare_columns, ignore_columns, precision)) }} +{% test equality(model, compare_model, compare_columns=None, exclude_columns=None, precision = None) %} + {{ return(adapter.dispatch('test_equality', 'dbt_utils')(model, compare_model, compare_columns, exclude_columns, precision)) }} {% endtest %} -{% macro default__test_equality(model, compare_model, compare_columns=None, ignore_columns=None, precision = None) %} +{% macro default__test_equality(model, compare_model, compare_columns=None, exclude_columns=None, precision = None) %} -{%- if compare_columns and ignore_columns -%} +{%- if compare_columns and exclude_columns -%} {{ exceptions.raise_compiler_error("Both a compare and an ignore list were provided to the `equality` macro. Only one is allowed") }} {%- endif -%} @@ -37,20 +37,20 @@ {%- set compare_model_columns = adapter.get_columns_in_relation(compare_model) -%} - {%- if ignore_columns -%} + {%- if exclude_columns -%} {#-- Lower case ignore columns for easier comparison --#} - {%- set ignore_columns = ignore_columns | map("lower") | list %} + {%- set exclude_columns = exclude_columns | map("lower") | list %} {# Filter out the excluded columns #} {%- set include_columns = [] %} {%- set include_model_columns = [] %} {%- for column in model_columns -%} - {%- if column.name | lower not in ignore_columns -%} + {%- if column.name | lower not in exclude_columns -%} {% do include_columns.append(column) %} {%- endif %} {%- endfor %} {%- for column in compare_model_columns -%} - {%- if column.name | lower not in ignore_columns -%} + {%- if column.name | lower not in exclude_columns -%} {% do include_model_columns.append(column) %} {%- endif %} {%- endfor %} @@ -63,7 +63,7 @@ {%- endif -%} {% if compare_columns_set != compare_model_columns_set %} - {{ exceptions.raise_compiler_error(compare_model ~" has less columns than " ~ model ~ ", please ensure they have the same columns or use the `compare_columns` or `ignore_columns` arguments to subset them.") }} + {{ exceptions.raise_compiler_error(compare_model ~" has less columns than " ~ model ~ ", please ensure they have the same columns or use the `compare_columns` or `exclude_columns` arguments to subset them.") }} {% endif %} @@ -79,14 +79,14 @@ {%- set compare_columns = adapter.get_columns_in_relation(model)-%} - {%- if ignore_columns -%} + {%- if exclude_columns -%} {#-- Lower case ignore columns for easier comparison --#} - {%- set ignore_columns = ignore_columns | map("lower") | list %} + {%- set exclude_columns = exclude_columns | map("lower") | list %} {# Filter out the excluded columns #} {%- set include_columns = [] %} {%- for column in compare_columns -%} - {%- if column.name | lower not in ignore_columns -%} + {%- if column.name | lower not in exclude_columns -%} {% do include_columns.append(column) %} {%- endif %} {%- endfor %} @@ -110,7 +110,7 @@ {%- for col in columns -%} {%- if ( (col.name|lower in compare_columns|map('lower') or not compare_columns) and - (col.name|lower not in ignore_columns|map('lower') or not ignore_columns) + (col.name|lower not in exclude_columns|map('lower') or not exclude_columns) ) -%} {# Databricks double type is not picked up by any number type checks in dbt #} {%- if col.is_float() or col.is_numeric() or col.data_type == 'double' -%} From 311d37e6dfc45270bb572e49a95740fb3c00068c Mon Sep 17 00:00:00 2001 From: Ryan Hill Date: Tue, 24 Oct 2023 19:08:39 +0100 Subject: [PATCH 15/17] Fix typo --- macros/generic_tests/equality.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/generic_tests/equality.sql b/macros/generic_tests/equality.sql index 12d3e2c4..329ef8f0 100644 --- a/macros/generic_tests/equality.sql +++ b/macros/generic_tests/equality.sql @@ -101,7 +101,7 @@ {% else %} {#- - If rounding is required, we need to get the types, so it cannot be ephermeral + If rounding is required, we need to get the types, so it cannot be ephemeral -#} {%- do dbt_utils._is_ephemeral(model, 'test_equality') -%} {%- set columns = adapter.get_columns_in_relation(model) -%} From ae9558538d19fd6ed5bc86289d00796bdc550688 Mon Sep 17 00:00:00 2001 From: gwen windflower Date: Fri, 1 Mar 2024 14:00:39 -0600 Subject: [PATCH 16/17] Add package-lock.yaml to .gitignore --- integration_tests/.gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/.gitignore b/integration_tests/.gitignore index 638c8c44..b4500d79 100644 --- a/integration_tests/.gitignore +++ b/integration_tests/.gitignore @@ -1,6 +1,6 @@ - target/ dbt_modules/ logs/ .env/ profiles.yml +package-lock.yml From cd7bd1c235ccdb02a8b3d14819ed90d3cc5eebdc Mon Sep 17 00:00:00 2001 From: Ryan Hill Date: Mon, 4 Mar 2024 09:35:23 +0000 Subject: [PATCH 17/17] Update comments --- macros/generic_tests/equality.sql | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/macros/generic_tests/equality.sql b/macros/generic_tests/equality.sql index 329ef8f0..d7d7197c 100644 --- a/macros/generic_tests/equality.sql +++ b/macros/generic_tests/equality.sql @@ -70,15 +70,14 @@ {% endif %} {%- if not precision -%} - {#- - If the compare_cols arg is provided, we can run this test without querying the - information schema — this allows the model to be an ephemeral model - -#} {%- if not compare_columns -%} + {# + You cannot get the columns in an ephemeral model (due to not existing in the information schema), + so if the user does not provide an explicit list of columns we must error in the case it is ephemeral + #} {%- do dbt_utils._is_ephemeral(model, 'test_equality') -%} {%- set compare_columns = adapter.get_columns_in_relation(model)-%} - {%- if exclude_columns -%} {#-- Lower case ignore columns for easier comparison --#} {%- set exclude_columns = exclude_columns | map("lower") | list %} @@ -92,16 +91,16 @@ {%- endfor %} {%- set compare_columns = include_columns | map(attribute='quoted') %} - {%- else -%} + {%- else -%} {# Compare columns provided #} {%- set compare_columns = compare_columns | map(attribute='quoted') %} {%- endif -%} {%- endif -%} {% set compare_cols_csv = compare_columns | join(', ') %} -{% else %} +{% else %} {# Precision required #} {#- - If rounding is required, we need to get the types, so it cannot be ephemeral + If rounding is required, we need to get the types, so it cannot be ephemeral even if they provide column names -#} {%- do dbt_utils._is_ephemeral(model, 'test_equality') -%} {%- set columns = adapter.get_columns_in_relation(model) -%} @@ -116,7 +115,7 @@ {%- if col.is_float() or col.is_numeric() or col.data_type == 'double' -%} {# Cast is required due to postgres not having round for a double precision number #} {%- do columns_list.append('round(cast(' ~ col.quoted ~ ' as ' ~ dbt.type_numeric() ~ '),' ~ precision ~ ') as ' ~ col.quoted) -%} - {%- else -%} + {%- else -%} {# Non-numeric type #} {%- do columns_list.append(col.quoted) -%} {%- endif -%} {% endif %}