Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add support for late binding views (Redshift) #614

Merged
merged 6 commits into from
Jan 2, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions dbt/adapters/bigquery.py
Original file line number Diff line number Diff line change
Expand Up @@ -341,6 +341,12 @@ def get_existing_schemas(cls, profile, model_name=None):
all_datasets = client.list_datasets()
return [ds.name for ds in all_datasets]

@classmethod
def get_columns_in_table(cls, profile, schema_name, table_name,
model_name=None):
raise dbt.exceptions.NotImplementedException(
'`get_columns_in_table` is not implemented for this adapter!')

@classmethod
def check_schema_exists(cls, profile, schema, model_name=None):
conn = cls.get_connection(profile, model_name)
Expand Down
10 changes: 8 additions & 2 deletions dbt/adapters/default.py
Original file line number Diff line number Diff line change
Expand Up @@ -174,8 +174,7 @@ def get_missing_columns(cls, profile,
if col_name in missing_columns]

@classmethod
def get_columns_in_table(cls, profile, schema_name, table_name,
model_name=None):
def _get_columns_in_table_sql(cls, schema_name, table_name):
sql = """
select column_name, data_type, character_maximum_length
from information_schema.columns
Expand All @@ -186,6 +185,13 @@ def get_columns_in_table(cls, profile, schema_name, table_name,
sql += (" AND table_schema = '{schema_name}'"
.format(schema_name=schema_name))

return sql

@classmethod
def get_columns_in_table(cls, profile, schema_name, table_name,
model_name=None):

sql = cls._get_columns_in_table_sql(schema_name, table_name)
connection, cursor = cls.add_query(
profile, sql, model_name)

Expand Down
51 changes: 51 additions & 0 deletions dbt/adapters/redshift.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,57 @@ def type(cls):
def date_function(cls):
return 'getdate()'

@classmethod
def _get_columns_in_table_sql(cls, schema_name, table_name):
# TODO : how do we make this a macro?
if schema_name is None:
table_schema_filter = '1=1'
else:
table_schema_filter = "table_schema = '{schema_name}'".format(
schema_name=schema_name)

sql = """
with bound_views as (
select
table_schema,
column_name,
data_type,
character_maximum_length

from information_schema.columns
where table_name = '{table_name}'
),

unbound_views as (
select
view_schema,
col_name,
col_type,
case
when col_type like 'character%'
then REGEXP_SUBSTR(col_type, '[0-9]+')::int
else null
end as character_maximum_length

from pg_get_late_binding_view_cols()
cols(view_schema name, view_name name, col_name name,
col_type varchar, col_num int)
where view_name = '{table_name}'
),

unioned as (
select * from bound_views
union all
select * from unbound_views
)

select column_name, data_type, character_maximum_length
from unioned
where {table_schema_filter}
""".format(table_name=table_name,
table_schema_filter=table_schema_filter).strip()
return sql

@classmethod
def drop(cls, profile, schema, relation, relation_type, model_name=None):
global drop_lock
Expand Down
10 changes: 10 additions & 0 deletions dbt/include/global_project/macros/adapters/redshift.sql
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,16 @@
{%- endmacro %}


{% macro redshift__create_view_as(identifier, sql) -%}

{% set bind_qualifier = '' if config.get('bind', default=True) else 'with no schema binding' %}

create view "{{ schema }}"."{{ identifier }}" as (
{{ sql }}
) {{ bind_qualifier }};
{% endmacro %}


{% macro redshift__create_archive_table(schema, identifier, columns) -%}
create table if not exists "{{ schema }}"."{{ identifier }}" (
{{ column_list_for_create_table(columns) }}
Expand Down
3 changes: 2 additions & 1 deletion dbt/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ class SourceConfig(object):
'sort',
'sql_where',
'unique_key',
'sort_type'
'sort_type',
'bind'
]

def __init__(self, active_project, own_project, fqn):
Expand Down
3 changes: 2 additions & 1 deletion dbt/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@
'sort_type',
'pre-hook',
'post-hook',
'vars'
'vars',
'bind',
]


Expand Down