Snowflake create or replace

core/dbt/include/global_project/macros/adapters/common.sql

@@ -60,7 +60,6 @@
   );
 {% endmacro %}
 
-
 {% macro create_view_as(relation, sql) -%}
   {{ adapter_macro('create_view_as', relation, sql) }}
 {%- endmacro %}

plugins/snowflake/dbt/include/snowflake/macros/adapters.sql

@@ -5,7 +5,7 @@
 
   {%- set transient = config.get('transient', default=true) -%}
 
-  create {% if temporary -%}
+  create or replace {% if temporary -%}
     temporary
   {%- elif transient -%}
     transient

plugins/snowflake/dbt/include/snowflake/macros/materializations/incremental.sql

@@ -0,0 +1,75 @@
+
+{% materialization incremental, adapter='snowflake' -%}
+
+  {%- set unique_key = config.get('unique_key') -%}
+  {%- set sql_where = config.get('sql_where') -%}
+  {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%}
+  {%- set full_refresh_mode = (flags.FULL_REFRESH == True) -%}
+  {%- set identifier = model['alias'] -%}
+
+  {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}
+  {%- set target_relation = api.Relation.create(database=database, identifier=identifier, schema=schema, type='table') -%}
+  {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}
+  {%- set exists_not_as_table = (old_relation is not none and not old_relation.is_table) -%}
+  {%- set force_create = full_refresh_mode -%}
+
+  -- setup
+
+  {% set source_sql -%}
+     -- wrap sql in parens to make it a subquery --
+     (
+        select * from (
+            {{ sql }}
+        )
+    )
+  {%- endset -%}
+
+  {{ run_hooks(pre_hooks, inside_transaction=False) }}
+
+  -- `BEGIN` happens here:
+  {{ run_hooks(pre_hooks, inside_transaction=True) }}
+
+  -- build model
+  {% if force_create or old_relation is none -%}
+    {%- call statement('main') -%}
+
+      {# -- create or replace logic because we're in a full refresh or table is non existant. #}
+      {% if old_relation is not none and old_relation.type == 'view' %}
+        {# -- I'm preserving one of the old checks here for a view, and to make sure Snowflake doesn't
+        -- complain that we're running a replace table on a view. #}
+          {{ log("Dropping relation " ~ old_relation ~ " because it is a view and this model is a table.") }}
+          {{ adapter.drop_relation(old_relation) }}
+      {% endif %}
+      {# -- now create or replace the table because we're in full-refresh #}
+      {{create_table_as(false, target_relation, source_sql)}}
+    {%- endcall -%}
+
+  {%- else -%}
+    {# -- here is the incremental part #}
+    {% set dest_columns = adapter.get_columns_in_relation(target_relation) %}
+    {% set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') %}
+    {%- call statement('main') -%}
+
+      {%- if unique_key is none -%}
+        {# -- if no unique_key is provided run regular insert as Snowflake may complain #}
+        insert into {{ target_relation }} ({{ dest_cols_csv }})
+        (
+          select {{ dest_cols_csv }}
+          from {{ source_sql }}
+        );
+      {%- else -%}
+        {# -- use merge if a unique key is provided #}
+        {{ get_merge_sql(target_relation, source_sql, unique_key, dest_columns) }}
+      {%- endif -%}
+    {% endcall %}
+
+  {%- endif %}
+
+  {{ run_hooks(post_hooks, inside_transaction=True) }}
+
+  -- `COMMIT` happens here
+  {{ adapter.commit() }}
+
+  {{ run_hooks(post_hooks, inside_transaction=False) }}
+
+{%- endmaterialization %}

plugins/snowflake/dbt/include/snowflake/macros/materializations/table.sql

@@ -1,44 +1,34 @@
 {% materialization table, adapter='snowflake' %}
   {%- set identifier = model['alias'] -%}
-  {%- set tmp_identifier = model['name'] + '__dbt_tmp' -%}
-  {%- set backup_identifier = model['name'] + '__dbt_backup' -%}
   {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%}
 
   {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}
   {%- set target_relation = api.Relation.create(identifier=identifier,
                                                 schema=schema,
                                                 database=database, type='table') -%}
-  {%- set intermediate_relation = api.Relation.create(identifier=tmp_identifier,
-                                                      schema=schema,
-                                                      database=database, type='table') -%}
 
-  /*
+  /* --TODO: Is this still up to date?
       See ../view/view.sql for more information about this relation.
   */
 
-  -- drop the backup relation if it exists, then make a new one that uses the old relation's type
-  {%- set backup_relation = adapter.get_relation(database=database, schema=schema, identifier=backup_identifier) -%}
-  {% if backup_relation is not none -%}
-    {{ adapter.drop_relation(backup_relation) }}
-  {%- endif %}
-  {%- set backup_relation = api.Relation.create(identifier=backup_identifier,
-                                                schema=schema,
-                                                database=database,
-                                                type=(old_relation.type or 'table')) -%}
-
   {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}
   {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}
   {%- set create_as_temporary = (exists_as_table and non_destructive_mode) -%}
 
 
   -- drop the temp relations if they exists for some reason
-  {{ adapter.drop_relation(intermediate_relation) }}
+  {# {{ adapter.drop_relation(intermediate_relation) }} #}
 
+  --- FIXME: Do we want to put this block all together? I think it serves no purpose, but need to check
   -- setup: if the target relation already exists, truncate or drop it (if it's a view)
+  {# TODO: Would like to check this. New materialsiation makes these tests a bit moot. We should
+  be able to deprecate non-destructive flag all together here. #}
   {% if non_destructive_mode -%}
     {% if exists_as_table -%}
-      {{ adapter.truncate_relation(old_relation) }}
+      --noop we can do away with this step all together since the table can be replaced in Snowflake.
+      {# {{ adapter.truncate_relation(old_relation) }} #}
     {% elif exists_as_view -%}
+      --noop. I think we should also be able to do away with this and call a replace.
       {{ adapter.drop_relation(old_relation) }}
       {%- set old_relation = none -%}
     {%- endif %}
@@ -49,52 +39,22 @@
   -- `BEGIN` happens here:
   {{ run_hooks(pre_hooks, inside_transaction=True) }}
 
-  -- build model
+  --build model
   {% call statement('main') -%}
-    {%- if non_destructive_mode -%}
-      {%- if old_relation is not none -%}
-        {{ create_table_as(create_as_temporary, intermediate_relation, sql) }}
-
-        {% set dest_columns = adapter.get_columns_in_relation(old_relation) %}
-        {% set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') %}
-
-        insert into {{ target_relation }} ({{ dest_cols_csv }}) (
-          select {{ dest_cols_csv }}
-          from {{ intermediate_relation.include(database=(not create_as_temporary), schema=(not create_as_temporary)) }}
-        );
-      {%- else -%}
-        {{ create_table_as(create_as_temporary, target_relation, sql) }}
-      {%- endif -%}
-    {%- else -%}
-      {{ create_table_as(create_as_temporary, intermediate_relation, sql) }}
-    {%- endif -%}
-  {%- endcall %}
-
-  -- cleanup
-  {% if non_destructive_mode -%}
-    -- noop
-  {%- else -%}
-    {% if old_relation is not none %}
-      {% if old_relation.type == 'view' %}
-        {#-- This is the primary difference between Snowflake and Redshift. Renaming this view
-          -- would cause an error if the view has become invalid due to upstream schema changes #}
-        {{ log("Dropping relation " ~ old_relation ~ " because it is a view and this model is a table.") }}
-        {{ drop_relation_if_exists(old_relation) }}
-      {% else %}
-        {{ adapter.rename_relation(target_relation, backup_relation) }}
-      {% endif %}
+     {# Drop the relation if it was a view to essencially "convert" it in a table. This does lead to 
+        downtime but I think it makes sense and should happen. Impact will be minimal I suspect. #}
+    {% if old_relation is not none and old_relation.type == 'view' %}
+      {{ log("Dropping relation " ~ old_relation ~ " because it is a view and this model is a table.") }}
+      {{ drop_relation_if_exists(old_relation) }}
     {% endif %}
 
-    {{ adapter.rename_relation(intermediate_relation, target_relation) }}
-  {%- endif %}
+    {{ create_table_as(create_as_temporary, target_relation, sql) }}
+  {%- endcall %}
 
   {{ run_hooks(post_hooks, inside_transaction=True) }}
 
   -- `COMMIT` happens here
   {{ adapter.commit() }}
 
-  -- finally, drop the existing/backup relation after the commit
-  {{ drop_relation_if_exists(backup_relation) }}
-
   {{ run_hooks(post_hooks, inside_transaction=False) }}
 {% endmaterialization %}