Merge pull request #1888 from fishtown-analytics/fix/column-quoting

Fix quoting on columns for seeds/incremental models (#1847)
dbt-labs · Nov 5, 2019 · d7eee61 · d7eee61
2 parents 2d0f938 + e0d4e93
commit d7eee61
Show file tree

Hide file tree

Showing 25 changed files with 621 additions and 74 deletions.
diff --git a/core/dbt/include/global_project/macros/adapters/common.sql b/core/dbt/include/global_project/macros/adapters/common.sql
@@ -133,10 +133,10 @@
   {%- set tmp_column = column_name + "__dbt_alter" -%}
 
   {% call statement('alter_column_type') %}
-    alter table {{ relation }} add column {{ tmp_column }} {{ new_column_type }};
-    update {{ relation }} set {{ tmp_column }} = {{ column_name }};
-    alter table {{ relation }} drop column {{ column_name }} cascade;
-    alter table {{ relation }} rename column {{ tmp_column }} to {{ column_name }}
+    alter table {{ relation }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }};
+    update {{ relation }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }};
+    alter table {{ relation }} drop column {{ adapter.quote(column_name) }} cascade;
+    alter table {{ relation }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }}
   {% endcall %}
 
 {% endmacro %}

diff --git a/core/dbt/include/global_project/macros/materializations/common/merge.sql b/core/dbt/include/global_project/macros/materializations/common/merge.sql
@@ -10,7 +10,7 @@
 
 
 {% macro common_get_merge_sql(target, source, unique_key, dest_columns) -%}
-    {%- set dest_cols_csv = dest_columns | map(attribute="name") | join(', ') -%}
+    {%- set dest_cols_csv =  get_quoted_csv(dest_columns | map(attribute="name")) -%}
 
     merge into {{ target }} as DBT_INTERNAL_DEST
     using {{ source }} as DBT_INTERNAL_SOURCE
@@ -24,7 +24,7 @@
     {% if unique_key %}
     when matched then update set
         {% for column in dest_columns -%}
-            {{ column.name }} = DBT_INTERNAL_SOURCE.{{ column.name }}
+            {{ adapter.quote(column.name) }} = DBT_INTERNAL_SOURCE.{{ adapter.quote(column.name) }}
             {%- if not loop.last %}, {%- endif %}
         {%- endfor %}
     {% endif %}
@@ -48,7 +48,8 @@
 
 
 {% macro common_get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%}
-    {%- set dest_cols_csv = dest_columns | map(attribute="name") | join(', ') -%}
+
+    {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%}
 
     {% if unique_key is not none %}
     delete from {{ target }}

diff --git a/core/dbt/include/global_project/macros/materializations/seed/seed.sql b/core/dbt/include/global_project/macros/materializations/seed/seed.sql
@@ -19,7 +19,7 @@
         {%- for col_name in agate_table.column_names -%}
             {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}
             {%- set type = column_override.get(col_name, inferred_type) -%}
-            {{ col_name | string }} {{ type }} {%- if not loop.last -%}, {%- endif -%}
+            {{ adapter.quote(col_name | string) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}
         {%- endfor -%}
     )
   {% endset %}
@@ -46,8 +46,19 @@
 {% endmacro %}
 
 
+{% macro get_quoted_csv(column_names) %}
+    {% set quoted = [] %}
+    {% for col in column_names -%}
+        {%- do quoted.append(adapter.quote(col)) -%}
+    {%- endfor %}
+
+    {%- set dest_cols_csv = quoted | join(', ') -%}
+    {{ return(dest_cols_csv) }}
+{% endmacro %}
+
+
 {% macro basic_load_csv_rows(model, batch_size, agate_table) %}
-    {% set cols_sql = ", ".join(agate_table.column_names) %}
+    {% set cols_sql = get_quoted_csv(agate_table.column_names) %}
     {% set bindings = [] %}
 
     {% set statements = [] %}
@@ -56,7 +67,7 @@
         {% set bindings = [] %}
 
         {% for row in chunk %}
-            {% set _ = bindings.extend(row) %}
+            {% do bindings.extend(row) %}
         {% endfor %}
 
         {% set sql %}
@@ -70,10 +81,10 @@
             {%- endfor %}
         {% endset %}
 
-        {% set _ = adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}
+        {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}
 
         {% if loop.index0 == 0 %}
-            {% set _ = statements.append(sql) %}
+            {% do statements.append(sql) %}
         {% endif %}
     {% endfor %}
 

diff --git a/plugins/snowflake/dbt/include/snowflake/macros/adapters.sql b/plugins/snowflake/dbt/include/snowflake/macros/adapters.sql
@@ -122,6 +122,6 @@
 
 {% macro snowflake__alter_column_type(relation, column_name, new_column_type) -%}
   {% call statement('alter_column_type') %}
-    alter table {{ relation }} alter {{ column_name }} set data type {{ new_column_type }};
+    alter table {{ relation }} alter {{ adapter.quote(column_name) }} set data type {{ new_column_type }};
   {% endcall %}
 {% endmacro %}
diff --git a/plugins/snowflake/dbt/include/snowflake/macros/materializations/merge.sql b/plugins/snowflake/dbt/include/snowflake/macros/materializations/merge.sql
@@ -6,7 +6,7 @@
        is provided, then this macro will do a proper merge instead.
     #}
 
-    {%- set dest_cols_csv = dest_columns | map(attribute="name") | join(', ') -%}
+    {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute='name')) -%}
 
     {%- if unique_key is none -%}
 

diff --git a/test/integration/001_simple_copy_test/snowflake-seed-initial/seed.csv b/test/integration/001_simple_copy_test/snowflake-seed-initial/seed.csv
@@ -0,0 +1,101 @@
+ID,FIRST_NAME,LAST_NAME,EMAIL,GENDER,IP_ADDRESS
+1,Jack,Hunter,[email protected],Male,59.80.20.168
+2,Kathryn,Walker,[email protected],Female,194.121.179.35
+3,Gerald,Ryan,[email protected],Male,11.3.212.243
+4,Bonnie,Spencer,[email protected],Female,216.32.196.175
+5,Harold,Taylor,[email protected],Male,253.10.246.136
+6,Jacqueline,Griffin,[email protected],Female,16.13.192.220
+7,Wanda,Arnold,[email protected],Female,232.116.150.64
+8,Craig,Ortiz,[email protected],Male,199.126.106.13
+9,Gary,Day,[email protected],Male,35.81.68.186
+10,Rose,Wright,[email protected],Female,236.82.178.100
+11,Raymond,Kelley,[email protected],Male,213.65.166.67
+12,Gerald,Robinson,[email protected],Male,72.232.194.193
+13,Mildred,Martinez,[email protected],Female,198.29.112.5
+14,Dennis,Arnold,[email protected],Male,86.96.3.250
+15,Judy,Gray,[email protected],Female,79.218.162.245
+16,Theresa,Garza,[email protected],Female,21.59.100.54
+17,Gerald,Robertson,[email protected],Male,131.134.82.96
+18,Philip,Hernandez,[email protected],Male,254.196.137.72
+19,Julia,Gonzalez,[email protected],Female,84.240.227.174
+20,Andrew,Davis,[email protected],Male,9.255.67.25
+21,Kimberly,Harper,[email protected],Female,198.208.120.253
+22,Mark,Martin,[email protected],Male,233.138.182.153
+23,Cynthia,Ruiz,[email protected],Female,18.178.187.201
+24,Samuel,Carroll,[email protected],Male,128.113.96.122
+25,Jennifer,Larson,[email protected],Female,98.234.85.95
+26,Ashley,Perry,[email protected],Female,247.173.114.52
+27,Howard,Rodriguez,[email protected],Male,231.188.95.26
+28,Amy,Brooks,[email protected],Female,141.199.174.118
+29,Louise,Warren,[email protected],Female,96.105.158.28
+30,Tina,Watson,[email protected],Female,251.142.118.177
+31,Janice,Kelley,[email protected],Female,239.167.34.233
+32,Terry,Mccoy,[email protected],Male,117.201.183.203
+33,Jeffrey,Morgan,[email protected],Male,78.101.78.149
+34,Louis,Harvey,[email protected],Male,51.50.0.167
+35,Philip,Miller,[email protected],Male,103.255.222.110
+36,Willie,Marshall,[email protected],Male,149.219.91.68
+37,Patrick,Lopez,[email protected],Male,250.136.229.89
+38,Adam,Jenkins,[email protected],Male,7.36.112.81
+39,Benjamin,Cruz,[email protected],Male,32.38.98.15
+40,Ruby,Hawkins,[email protected],Female,135.171.129.255
+41,Carlos,Barnes,[email protected],Male,240.197.85.140
+42,Ruby,Griffin,[email protected],Female,19.29.135.24
+43,Sean,Mason,[email protected],Male,159.219.155.249
+44,Anthony,Payne,[email protected],Male,235.168.199.218
+45,Steve,Cruz,[email protected],Male,238.201.81.198
+46,Anthony,Garcia,[email protected],Male,25.85.10.18
+47,Doris,Lopez,[email protected],Female,245.218.51.238
+48,Susan,Nichols,[email protected],Female,199.99.9.61
+49,Wanda,Ferguson,[email protected],Female,236.241.135.21
+50,Andrea,Pierce,[email protected],Female,132.40.10.209
+51,Lawrence,Phillips,[email protected],Male,72.226.82.87
+52,Judy,Gilbert,[email protected],Female,196.250.15.142
+53,Eric,Williams,[email protected],Male,222.202.73.126
+54,Ralph,Romero,[email protected],Male,123.184.125.212
+55,Jean,Wilson,[email protected],Female,176.106.32.194
+56,Lori,Reynolds,[email protected],Female,114.181.203.22
+57,Donald,Moreno,[email protected],Male,233.249.97.60
+58,Steven,Berry,[email protected],Male,186.193.50.50
+59,Theresa,Shaw,[email protected],Female,120.37.71.222
+60,John,Stephens,[email protected],Male,191.87.127.115
+61,Richard,Jacobs,[email protected],Male,66.210.83.155
+62,Andrew,Lawson,[email protected],Male,54.98.36.94
+63,Peter,Morgan,[email protected],Male,14.77.29.106
+64,Nicole,Garrett,[email protected],Female,21.127.74.68
+65,Joshua,Kim,[email protected],Male,57.255.207.41
+66,Ralph,Roberts,[email protected],Male,222.143.131.109
+67,George,Montgomery,[email protected],Male,76.75.111.77
+68,Gerald,Alvarez,[email protected],Male,58.157.186.194
+69,Donald,Olson,[email protected],Male,69.65.74.135
+70,Carlos,Morgan,[email protected],Male,96.20.140.87
+71,Aaron,Stanley,[email protected],Male,163.119.217.44
+72,Virginia,Long,[email protected],Female,204.150.194.182
+73,Robert,Berry,[email protected],Male,104.19.48.241
+74,Antonio,Brooks,[email protected],Male,210.31.7.24
+75,Ruby,Garcia,[email protected],Female,233.218.162.214
+76,Jack,Hanson,[email protected],Male,31.55.46.199
+77,Kathryn,Nelson,[email protected],Female,14.189.146.41
+78,Jason,Reed,[email protected],Male,141.189.89.255
+79,George,Coleman,[email protected],Male,81.189.221.144
+80,Rose,King,[email protected],Female,212.123.168.231
+81,Johnny,Holmes,[email protected],Male,177.3.93.188
+82,Katherine,Gilbert,[email protected],Female,199.215.169.61
+83,Joshua,Thomas,[email protected],Male,0.8.205.30
+84,Julie,Perry,[email protected],Female,60.116.114.192
+85,Richard,Perry,[email protected],Male,181.125.70.232
+86,Kenneth,Ruiz,[email protected],Male,189.105.137.109
+87,Jose,Morgan,[email protected],Male,101.134.215.156
+88,Donald,Campbell,[email protected],Male,102.120.215.84
+89,Debra,Collins,[email protected],Female,90.13.153.235
+90,Jesse,Johnson,[email protected],Male,225.178.125.53
+91,Elizabeth,Stone,[email protected],Female,123.184.126.221
+92,Angela,Rogers,[email protected],Female,98.104.132.187
+93,Emily,Dixon,[email protected],Female,39.190.75.57
+94,Albert,Scott,[email protected],Male,40.209.13.189
+95,Barbara,Peterson,[email protected],Female,75.249.136.180
+96,Adam,Greene,[email protected],Male,184.173.109.144
+97,Earl,Sanders,[email protected],Male,247.34.90.117
+98,Angela,Brooks,[email protected],Female,10.63.249.126
+99,Harold,Foster,[email protected],Male,139.214.40.244
+100,Carl,Meyer,[email protected],Male,204.117.7.88