From d4cb42b313a1812ab7494aa9ba5c3bef1faf89b5 Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Mon, 12 Jun 2023 11:00:00 -0700 Subject: [PATCH 1/2] Updates to support a broader class of incremental functionality --- dbt/include/duckdb/macros/adapters.sql | 8 ------- dbt/include/duckdb/macros/columns.sql | 23 +++++++++++++++++++ .../macros/materializations/incremental.sql | 4 ++-- tests/functional/adapter/test_incremental.py | 16 ++++++++++++- 4 files changed, 40 insertions(+), 11 deletions(-) create mode 100644 dbt/include/duckdb/macros/columns.sql diff --git a/dbt/include/duckdb/macros/adapters.sql b/dbt/include/duckdb/macros/adapters.sql index 984bc5cd..bc055855 100644 --- a/dbt/include/duckdb/macros/adapters.sql +++ b/dbt/include/duckdb/macros/adapters.sql @@ -195,14 +195,6 @@ def materialize(df, con): {% do return(get_incremental_delete_insert_sql(arg_dict)) %} {% endmacro %} -{% macro duckdb__get_incremental_delete_insert_sql(arg_dict) %} - {% do return(get_delete_insert_merge_sql(arg_dict["target_relation"].include(database=adapter.use_database()), arg_dict["temp_relation"], arg_dict["unique_key"], arg_dict["dest_columns"])) %} -{% endmacro %} - -{% macro duckdb__get_incremental_append_sql(arg_dict) %} - {% do return(get_insert_into_sql(arg_dict["target_relation"].include(database=adapter.use_database()), arg_dict["temp_relation"], arg_dict["dest_columns"])) %} -{% endmacro %} - {% macro location_exists(location) -%} {% do return(adapter.location_exists(location)) %} {% endmacro %} diff --git a/dbt/include/duckdb/macros/columns.sql b/dbt/include/duckdb/macros/columns.sql new file mode 100644 index 00000000..b97f2f8b --- /dev/null +++ b/dbt/include/duckdb/macros/columns.sql @@ -0,0 +1,23 @@ +{% macro duckdb__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %} + + {% if add_columns %} + {% for column in add_columns %} + {% set sql -%} + alter {{ relation.type }} {{ relation }} add column + {{ column.name }} {{ column.data_type }} + {%- endset -%} + {% do run_query(sql) %} + {% endfor %} + {% endif %} + + {% if remove_columns %} + {% for column in remove_columns %} + {% set sql -%} + alter {{ relation.type }} {{ relation }} drop column + {{ column.name }} + {%- endset -%} + {% do run_query(sql) %} + {% endfor %} + {% endif %} + +{% endmacro %} \ No newline at end of file diff --git a/dbt/include/duckdb/macros/materializations/incremental.sql b/dbt/include/duckdb/macros/materializations/incremental.sql index 5ba2e342..3f1992ec 100644 --- a/dbt/include/duckdb/macros/materializations/incremental.sql +++ b/dbt/include/duckdb/macros/materializations/incremental.sql @@ -58,9 +58,9 @@ {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#} {% set incremental_strategy = config.get('incremental_strategy') or 'default' %} - {% set incremental_predicates = config.get('incremental_predicates', none) %} + {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %} {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %} - {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'predicates': incremental_predicates }) %} + {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %} {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %} {% set language = "sql" %} diff --git a/tests/functional/adapter/test_incremental.py b/tests/functional/adapter/test_incremental.py index 3235d2f3..c34d7626 100644 --- a/tests/functional/adapter/test_incremental.py +++ b/tests/functional/adapter/test_incremental.py @@ -1,7 +1,21 @@ from dbt.tests.adapter.incremental.test_incremental_unique_id import ( BaseIncrementalUniqueKey, ) +from dbt.tests.adapter.incremental.test_incremental_predicates import ( + BaseIncrementalPredicates, +) +from dbt.tests.adapter.incremental.test_incremental_on_schema_change import ( + BaseIncrementalOnSchemaChange, +) + + +class TestIncrementalUniqueKey(BaseIncrementalUniqueKey): + pass + + +class TestIncrementalPredicates(BaseIncrementalPredicates): + pass -class TestBaseIncrementalUniqueKey(BaseIncrementalUniqueKey): +class TestIncrementalOnSchemaChange(BaseIncrementalOnSchemaChange): pass From b1d8fc41ec4816f3afef33cb1e5522a56d4fde0e Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Mon, 12 Jun 2023 11:21:45 -0700 Subject: [PATCH 2/2] add missing newline --- dbt/include/duckdb/macros/columns.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/duckdb/macros/columns.sql b/dbt/include/duckdb/macros/columns.sql index b97f2f8b..f0d5a19e 100644 --- a/dbt/include/duckdb/macros/columns.sql +++ b/dbt/include/duckdb/macros/columns.sql @@ -20,4 +20,4 @@ {% endfor %} {% endif %} -{% endmacro %} \ No newline at end of file +{% endmacro %}