diff --git a/Makefile b/Makefile index c6d937a9ba9..3990df87e57 100644 --- a/Makefile +++ b/Makefile @@ -17,8 +17,7 @@ test-integration: @echo "Integration test run starting..." @time docker-compose run test tox -e integration-postgres-py27,integration-postgres-py36,integration-snowflake-py27,integration-snowflake-py36,integration-bigquery-py27,integration-bigquery-py36 -test-new: - @echo "Test run starting..." - @echo "Changed test files:" - @echo "${changed_tests}" - @docker-compose run test /usr/src/app/test/runner.sh ${changed_tests} + +test-quick: + @echo "Integration test run starting..." + @time docker-compose run test tox -e integration-postgres-py36 -- -x diff --git a/dbt/adapters/bigquery/__init__.py b/dbt/adapters/bigquery/__init__.py new file mode 100644 index 00000000000..f5c01e93ffc --- /dev/null +++ b/dbt/adapters/bigquery/__init__.py @@ -0,0 +1,7 @@ +from dbt.adapters.bigquery.impl import BigQueryAdapter +from dbt.adapters.bigquery.relation import BigQueryRelation + +__all__ = [ + BigQueryAdapter, + BigQueryRelation, +] diff --git a/dbt/adapters/bigquery.py b/dbt/adapters/bigquery/impl.py similarity index 76% rename from dbt/adapters/bigquery.py rename to dbt/adapters/bigquery/impl.py index 92aec09ac6b..e4d0dd39239 100644 --- a/dbt/adapters/bigquery.py +++ b/dbt/adapters/bigquery/impl.py @@ -3,6 +3,7 @@ from contextlib import contextmanager import dbt.compat +import dbt.deprecations import dbt.exceptions import dbt.schema import dbt.flags as flags @@ -10,6 +11,7 @@ import dbt.clients.agate_helper from dbt.adapters.postgres import PostgresAdapter +from dbt.adapters.bigquery.relation import BigQueryRelation from dbt.contracts.connection import validate_connection from dbt.logger import GLOBAL_LOGGER as logger @@ -25,6 +27,7 @@ class BigQueryAdapter(PostgresAdapter): context_functions = [ + # deprecated -- use versions that take relations instead "query_for_existing", "execute_model", "drop", @@ -35,17 +38,24 @@ class BigQueryAdapter(PostgresAdapter): "expand_target_column_types", "load_dataframe", + # versions of adapter functions that take / return Relations + "list_relations", + "get_relation", + "drop_relation", + "rename_relation", + "get_columns_in_table" ] + Relation = BigQueryRelation + Column = dbt.schema.BigQueryColumn + SCOPE = ('https://www.googleapis.com/auth/bigquery', 'https://www.googleapis.com/auth/cloud-platform', 'https://www.googleapis.com/auth/drive') QUERY_TIMEOUT = 300 - Column = dbt.schema.BigQueryColumn - @classmethod def handle_error(cls, error, message, sql): logger.debug(message.format(sql=sql)) @@ -165,17 +175,14 @@ def close(cls, connection): return connection @classmethod - def query_for_existing(cls, profile, schemas, model_name=None): - if not isinstance(schemas, (list, tuple)): - schemas = [schemas] - - conn = cls.get_connection(profile, model_name) - client = conn.get('handle') + def list_relations(cls, profile, project_cfg, schema, model_name=None): + connection = cls.get_connection(profile, model_name) + credentials = connection.get('credentials', {}) + client = connection.get('handle') - all_tables = [] - for schema in schemas: - dataset = cls.get_dataset(profile, schema, model_name) - all_tables.extend(client.list_tables(dataset)) + bigquery_dataset = cls.get_dataset( + profile, project_cfg, schema, model_name) + all_tables = client.list_tables(bigquery_dataset) relation_types = { 'TABLE': 'table', @@ -183,42 +190,50 @@ def query_for_existing(cls, profile, schemas, model_name=None): 'EXTERNAL': 'external' } - existing = [(table.table_id, relation_types.get(table.table_type)) - for table in all_tables] - - return dict(existing) + return [cls.Relation.create( + project=credentials.get('project'), + schema=schema, + identifier=table.table_id, + quote_policy={ + 'schema': True, + 'identifier': True + }, + type=relation_types.get(table.table_type)) + for table in all_tables] @classmethod - def table_exists(cls, profile, schema, table, model_name=None): - tables = cls.query_for_existing(profile, schema, model_name) - exists = tables.get(table) is not None - return exists - - @classmethod - def drop(cls, profile, schema, relation, relation_type, model_name=None): + def drop_relation(cls, profile, project_cfg, relation, model_name=None): conn = cls.get_connection(profile, model_name) client = conn.get('handle') - dataset = cls.get_dataset(profile, schema, model_name) - relation_object = dataset.table(relation) + dataset = cls.get_dataset( + profile, project_cfg, relation.schema, model_name) + relation_object = dataset.table(relation.identifier) client.delete_table(relation_object) @classmethod - def rename(cls, profile, schema, from_name, to_name, model_name=None): + def rename(cls, profile, project_cfg, schema, + from_name, to_name, model_name=None): raise dbt.exceptions.NotImplementedException( '`rename` is not implemented for this adapter!') + @classmethod + def rename_relation(cls, profile, project_cfg, from_relation, to_relation, + model_name=None): + raise dbt.exceptions.NotImplementedException( + '`rename_relation` is not implemented for this adapter!') + @classmethod def get_timeout(cls, conn): credentials = conn['credentials'] return credentials.get('timeout_seconds', cls.QUERY_TIMEOUT) @classmethod - def materialize_as_view(cls, profile, dataset, model): + def materialize_as_view(cls, profile, project_cfg, dataset, model): model_name = model.get('name') model_sql = model.get('injected_sql') - conn = cls.get_connection(profile, model_name) + conn = cls.get_connection(profile, project_cfg, model_name) client = conn.get('handle') view_ref = dataset.table(model_name) @@ -249,12 +264,13 @@ def poll_until_job_completes(cls, job, timeout): raise job.exception() @classmethod - def make_date_partitioned_table(cls, profile, dataset_name, identifier, - model_name=None): + def make_date_partitioned_table(cls, profile, project_cfg, dataset_name, + identifier, model_name=None): conn = cls.get_connection(profile, model_name) client = conn.get('handle') - dataset = cls.get_dataset(profile, dataset_name, identifier) + dataset = cls.get_dataset(profile, project_cfg, + dataset_name, identifier) table_ref = dataset.table(identifier) table = google.cloud.bigquery.Table(table_ref) table.partitioning_type = 'DAY' @@ -262,8 +278,8 @@ def make_date_partitioned_table(cls, profile, dataset_name, identifier, return client.create_table(table) @classmethod - def materialize_as_table(cls, profile, dataset, model, model_sql, - decorator=None): + def materialize_as_table(cls, profile, project_cfg, dataset, + model, model_sql, decorator=None): model_name = model.get('name') conn = cls.get_connection(profile, model_name) @@ -289,7 +305,8 @@ def materialize_as_table(cls, profile, dataset, model, model_sql, return "CREATE TABLE" @classmethod - def execute_model(cls, profile, model, materialization, sql_override=None, + def execute_model(cls, profile, project_cfg, model, + materialization, sql_override=None, decorator=None, model_name=None): if sql_override is None: @@ -302,13 +319,15 @@ def execute_model(cls, profile, model, materialization, sql_override=None, model_name = model.get('name') model_schema = model.get('schema') - dataset = cls.get_dataset(profile, model_schema, model_name) + dataset = cls.get_dataset(profile, project_cfg, + model_schema, model_name) if materialization == 'view': - res = cls.materialize_as_view(profile, dataset, model) + res = cls.materialize_as_view(profile, project_cfg, dataset, model) elif materialization == 'table': - res = cls.materialize_as_table(profile, dataset, model, - sql_override, decorator) + res = cls.materialize_as_table( + profile, project_cfg, dataset, model, + sql_override, decorator) else: msg = "Invalid relation type: '{}'".format(materialization) raise dbt.exceptions.RuntimeException(msg, model) @@ -356,18 +375,18 @@ def add_begin_query(cls, profile, name): '`add_begin_query` is not implemented for this adapter!') @classmethod - def create_schema(cls, profile, schema, model_name=None): + def create_schema(cls, profile, project_cfg, schema, model_name=None): logger.debug('Creating schema "%s".', schema) conn = cls.get_connection(profile, model_name) client = conn.get('handle') - dataset = cls.get_dataset(profile, schema, model_name) + dataset = cls.get_dataset(profile, project_cfg, schema, model_name) with cls.exception_handler(profile, 'create dataset', model_name): client.create_dataset(dataset) @classmethod - def drop_tables_in_schema(cls, profile, dataset): + def drop_tables_in_schema(cls, profile, project_cfg, dataset): conn = cls.get_connection(profile) client = conn.get('handle') @@ -375,22 +394,23 @@ def drop_tables_in_schema(cls, profile, dataset): client.delete_table(table.reference) @classmethod - def drop_schema(cls, profile, schema, model_name=None): + def drop_schema(cls, profile, project_cfg, schema, model_name=None): logger.debug('Dropping schema "%s".', schema) - if not cls.check_schema_exists(profile, schema, model_name): + if not cls.check_schema_exists(profile, project_cfg, + schema, model_name): return conn = cls.get_connection(profile) client = conn.get('handle') - dataset = cls.get_dataset(profile, schema, model_name) + dataset = cls.get_dataset(profile, project_cfg, schema, model_name) with cls.exception_handler(profile, 'drop dataset', model_name): - cls.drop_tables_in_schema(profile, dataset) + cls.drop_tables_in_schema(profile, project_cfg, dataset) client.delete_dataset(dataset) @classmethod - def get_existing_schemas(cls, profile, model_name=None): + def get_existing_schemas(cls, profile, project_cfg, model_name=None): conn = cls.get_connection(profile, model_name) client = conn.get('handle') @@ -399,7 +419,8 @@ def get_existing_schemas(cls, profile, model_name=None): return [ds.dataset_id for ds in all_datasets] @classmethod - def get_columns_in_table(cls, profile, schema_name, table_name, + def get_columns_in_table(cls, profile, project_cfg, + schema_name, table_name, database=None, model_name=None): # BigQuery does not have databases -- the database parameter is here @@ -419,16 +440,15 @@ def get_columns_in_table(cls, profile, schema_name, table_name, columns = [] for col in table_schema: - name = col.name - data_type = col.field_type - - column = cls.Column(col.name, col.field_type, col.fields, col.mode) + column = cls.Column( + col.name, col.field_type, col.fields, col.mode) columns.append(column) return columns @classmethod - def check_schema_exists(cls, profile, schema, model_name=None): + def check_schema_exists(cls, profile, project_cfg, + schema, model_name=None): conn = cls.get_connection(profile, model_name) client = conn.get('handle') @@ -437,7 +457,7 @@ def check_schema_exists(cls, profile, schema, model_name=None): return any([ds.dataset_id == schema for ds in all_datasets]) @classmethod - def get_dataset(cls, profile, dataset_name, model_name=None): + def get_dataset(cls, profile, project_cfg, dataset_name, model_name=None): conn = cls.get_connection(profile, model_name) client = conn.get('handle') @@ -468,13 +488,18 @@ def quote(cls, identifier): return '`{}`'.format(identifier) @classmethod - def quote_schema_and_table(cls, profile, schema, table, model_name=None): + def quote_schema_and_table(cls, profile, project_cfg, schema, + table, model_name=None): + return cls.render_relation(profile, project_cfg, + cls.quote(schema), + cls.quote(table)) + + @classmethod + def render_relation(cls, profile, project_cfg, schema, table): connection = cls.get_connection(profile) credentials = connection.get('credentials', {}) project = credentials.get('project') - return '{}.{}.{}'.format(cls.quote(project), - cls.quote(schema), - cls.quote(table)) + return '{}.{}.{}'.format(cls.quote(project), schema, table) @classmethod def convert_text_type(cls, agate_table, col_idx): @@ -504,10 +529,11 @@ def _agate_to_schema(cls, agate_table, column_override): return bq_schema @classmethod - def load_dataframe(cls, profile, schema, table_name, agate_table, + def load_dataframe(cls, profile, project_cfg, schema, + table_name, agate_table, column_override, model_name=None): bq_schema = cls._agate_to_schema(agate_table, column_override) - dataset = cls.get_dataset(profile, schema, None) + dataset = cls.get_dataset(profile, project_cfg, schema, None) table = dataset.table(table_name) conn = cls.get_connection(profile, None) client = conn.get('handle') @@ -524,7 +550,7 @@ def load_dataframe(cls, profile, schema, table_name, agate_table, cls.poll_until_job_completes(job, cls.get_timeout(conn)) @classmethod - def expand_target_column_types(cls, profile, temp_table, to_schema, - to_table, model_name=None): + def expand_target_column_types(cls, profile, project_cfg, temp_table, + to_schema, to_table, model_name=None): # This is a no-op on BigQuery pass diff --git a/dbt/adapters/bigquery/relation.py b/dbt/adapters/bigquery/relation.py new file mode 100644 index 00000000000..5c8f130c71e --- /dev/null +++ b/dbt/adapters/bigquery/relation.py @@ -0,0 +1,143 @@ +from dbt.adapters.default.relation import DefaultRelation +from dbt.utils import filter_null_values + + +class BigQueryRelation(DefaultRelation): + + DEFAULTS = { + 'metadata': { + 'type': 'BigQueryRelation' + }, + 'quote_character': '`', + 'quote_policy': { + 'project': True, + 'schema': True, + 'identifier': True + }, + 'include_policy': { + 'project': True, + 'schema': True, + 'identifier': True + } + } + + PATH_SCHEMA = { + 'type': 'object', + 'properties': { + 'project': {'type': ['string', 'null']}, + 'schema': {'type': ['string', 'null']}, + 'identifier': {'type': 'string'}, + }, + 'required': ['project', 'schema', 'identifier'], + } + + POLICY_SCHEMA = { + 'type': 'object', + 'properties': { + 'project': {'type': 'boolean'}, + 'schema': {'type': 'boolean'}, + 'identifier': {'type': 'boolean'}, + }, + 'required': ['project', 'schema', 'identifier'], + } + + SCHEMA = { + 'type': 'object', + 'properties': { + 'metadata': { + 'type': 'object', + 'properties': { + 'type': { + 'type': 'string', + 'const': 'BigQueryRelation', + }, + }, + }, + 'type': { + 'enum': DefaultRelation.RelationTypes + [None], + }, + 'path': PATH_SCHEMA, + 'include_policy': POLICY_SCHEMA, + 'quote_policy': POLICY_SCHEMA, + 'quote_character': {'type': 'string'}, + }, + 'required': ['metadata', 'type', 'path', 'include_policy', + 'quote_policy', 'quote_character'] + } + + PATH_ELEMENTS = ['project', 'schema', 'identifier'] + + def matches(self, project=None, schema=None, identifier=None): + search = filter_null_values({ + 'project': project, + 'schema': schema, + 'identifier': identifier + }) + + if not search: + # nothing was passed in + pass + + for k, v in search.items(): + if self.get_path_part(k) != v: + return False + + return True + + @classmethod + def create_from_node(cls, profile, node, **kwargs): + return cls.create( + project=profile.get('project'), + schema=node.get('schema'), + identifier=node.get('name'), + **kwargs) + + @classmethod + def create(cls, project=None, schema=None, + identifier=None, table_name=None, + type=None, **kwargs): + if table_name is None: + table_name = identifier + + return cls(type=type, + path={ + 'project': project, + 'schema': schema, + 'identifier': identifier + }, + table_name=table_name, + **kwargs) + + def quote(self, project=None, schema=None, identifier=None): + policy = filter_null_values({ + 'project': project, + 'schema': schema, + 'identifier': identifier + }) + + return self.incorporate(quote_policy=policy) + + def include(self, project=None, schema=None, identifier=None): + policy = filter_null_values({ + 'project': project, + 'schema': schema, + 'identifier': identifier + }) + + return self.incorporate(include_policy=policy) + + @property + def project(self): + return self.path.get('project') + + @property + def schema(self): + return self.path.get('schema') + + @property + def dataset(self): + return self.path.get('schema') + + @property + def identifier(self): + return self.path.get('identifier') diff --git a/dbt/adapters/default/__init__.py b/dbt/adapters/default/__init__.py new file mode 100644 index 00000000000..c56e8b54380 --- /dev/null +++ b/dbt/adapters/default/__init__.py @@ -0,0 +1,7 @@ +from dbt.adapters.default.impl import DefaultAdapter +from dbt.adapters.default.relation import DefaultRelation + +__all__ = [ + DefaultAdapter, + DefaultRelation, +] diff --git a/dbt/adapters/default.py b/dbt/adapters/default/impl.py similarity index 74% rename from dbt/adapters/default.py rename to dbt/adapters/default/impl.py index 2980a401ba9..cfe474434f2 100644 --- a/dbt/adapters/default.py +++ b/dbt/adapters/default/impl.py @@ -1,5 +1,4 @@ import copy -import itertools import multiprocessing import time import agate @@ -13,7 +12,10 @@ from dbt.contracts.connection import validate_connection from dbt.logger import GLOBAL_LOGGER as logger +from dbt.schema import Column +from dbt.utils import filter_null_values +from dbt.adapters.default.relation import DefaultRelation lock = multiprocessing.Lock() connections_in_use = {} @@ -25,27 +27,42 @@ class DefaultAdapter(object): requires = {} context_functions = [ - "already_exists", "get_columns_in_table", "get_missing_columns", + "expand_target_column_types", + + # deprecated -- use versions that take relations instead + "already_exists", "query_for_existing", "rename", "drop", "truncate", - "add_query", - "expand_target_column_types", + + # just deprecated. going away in a future release "quote_schema_and_table", + + # versions of adapter functions that take / return Relations + "list_relations", + "get_relation", + "drop_relation", + "rename_relation", + "truncate_relation", + ] + + profile_functions = [ "execute", - "convert_type" + "add_query", ] raw_functions = [ "get_status", "get_result_from_cursor", "quote", + "convert_type" ] - Column = dbt.schema.Column + Relation = DefaultRelation + Column = Column ### # ADAPTER-SPECIFIC FUNCTIONS -- each of these must be overridden in @@ -74,28 +91,38 @@ def get_status(cls, cursor): '`get_status` is not implemented for this adapter!') @classmethod - def alter_column_type(cls, profile, schema, table, column_name, - new_column_type, model_name=None): + def alter_column_type(cls, profile, project_cfg, schema, table, + column_name, new_column_type, model_name=None): raise dbt.exceptions.NotImplementedException( '`alter_column_type` is not implemented for this adapter!') @classmethod - def query_for_existing(cls, profile, schemas, model_name=None): - raise dbt.exceptions.NotImplementedException( - '`query_for_existing` is not implemented for this adapter!') + def query_for_existing(cls, profile, project_cfg, schemas, + model_name=None): + if not isinstance(schemas, (list, tuple)): + schemas = [schemas] + + all_relations = [] + + for schema in schemas: + all_relations.extend( + cls.list_relations(profile, project_cfg, schema, model_name)) + + return {relation.identifier: relation.type + for relation in all_relations} @classmethod - def get_existing_schemas(cls, profile, model_name=None): + def get_existing_schemas(cls, profile, project_cfg, model_name=None): raise dbt.exceptions.NotImplementedException( '`get_existing_schemas` is not implemented for this adapter!') @classmethod - def check_schema_exists(cls, profile, schema): + def check_schema_exists(cls, profile, project_cfg, schema): raise dbt.exceptions.NotImplementedException( '`check_schema_exists` is not implemented for this adapter!') @classmethod - def cancel_connection(cls, project, connection): + def cancel_connection(cls, project_cfg, connection): raise dbt.exceptions.NotImplementedException( '`cancel_connection` is not implemented for this adapter!') @@ -115,43 +142,60 @@ def get_result_from_cursor(cls, cursor): return dbt.clients.agate_helper.table_from_data(data) @classmethod - def drop(cls, profile, schema, relation, relation_type, model_name=None): - if relation_type == 'view': - return cls.drop_view(profile, schema, relation, model_name) - elif relation_type == 'table': - return cls.drop_table(profile, schema, relation, model_name) - else: - raise RuntimeError( - "Invalid relation_type '{}'" - .format(relation_type)) + def drop(cls, profile, project_cfg, schema, + relation, relation_type, model_name=None): + identifier = relation + relation = cls.Relation.create( + schema=schema, + identifier=identifier, + type=relation_type) + + return cls.drop_relation(profile, project_cfg, relation, model_name) @classmethod - def drop_relation(cls, profile, schema, rel_name, rel_type, model_name): - relation = cls.quote_schema_and_table(profile, schema, rel_name) - sql = 'drop {} if exists {} cascade'.format(rel_type, relation) + def drop_relation(cls, profile, project_cfg, relation, model_name=None): + if relation.type is None: + dbt.exceptions.raise_compiler_error( + 'Tried to drop relation {}, but its type is null.' + .format(relation)) + + sql = 'drop {} if exists {} cascade'.format(relation.type, relation) connection, cursor = cls.add_query(profile, sql, model_name) @classmethod - def drop_view(cls, profile, schema, view, model_name): - cls.drop_relation(profile, schema, view, 'view', model_name) + def truncate(cls, profile, project_cfg, schema, table, model_name=None): + relation = cls.Relation.create( + schema=schema, + identifier=table, + type='table') - @classmethod - def drop_table(cls, profile, schema, table, model_name): - cls.drop_relation(profile, schema, table, 'table', model_name) + return cls.truncate_relation(profile, project_cfg, + relation, model_name) @classmethod - def truncate(cls, profile, schema, table, model_name=None): - relation = cls.quote_schema_and_table(profile, schema, table) + def truncate_relation(cls, profile, project_cfg, + relation, model_name=None): sql = 'truncate table {}'.format(relation) connection, cursor = cls.add_query(profile, sql, model_name) @classmethod - def rename(cls, profile, schema, from_name, to_name, model_name=None): - from_relation = cls.quote_schema_and_table(profile, schema, from_name) - to_relation = cls.quote(to_name) - sql = 'alter table {} rename to {}'.format(from_relation, to_relation) + def rename(cls, profile, project_cfg, schema, + from_name, to_name, model_name=None): + return cls.rename_relation( + profile, project_cfg, + from_relation=cls.Relation.create( + schema=schema, identifier=from_name), + to_relation=cls.Relation.create( + identifier=to_name), + model_name=model_name) + + @classmethod + def rename_relation(cls, profile, project_cfg, from_relation, + to_relation, model_name=None): + sql = 'alter table {} rename to {}'.format( + from_relation, to_relation.include(schema=False)) connection, cursor = cls.add_query(profile, sql, model_name) @@ -160,7 +204,7 @@ def is_cancelable(cls): return True @classmethod - def get_missing_columns(cls, profile, + def get_missing_columns(cls, profile, project_cfg, from_schema, from_table, to_schema, to_table, model_name=None): @@ -168,11 +212,11 @@ def get_missing_columns(cls, profile, missing from to_table""" from_columns = {col.name: col for col in cls.get_columns_in_table( - profile, from_schema, from_table, + profile, project_cfg, from_schema, from_table, model_name=model_name)} to_columns = {col.name: col for col in cls.get_columns_in_table( - profile, to_schema, to_table, + profile, project_cfg, to_schema, to_table, model_name=model_name)} missing_columns = set(from_columns.keys()) - set(to_columns.keys()) @@ -182,7 +226,6 @@ def get_missing_columns(cls, profile, @classmethod def _get_columns_in_table_sql(cls, schema_name, table_name, database): - schema_filter = '1=1' if schema_name is not None: schema_filter = "table_schema = '{}'".format(schema_name) @@ -207,9 +250,8 @@ def _get_columns_in_table_sql(cls, schema_name, table_name, database): return sql @classmethod - def get_columns_in_table(cls, profile, schema_name, table_name, - database=None, model_name=None): - + def get_columns_in_table(cls, profile, project_cfg, schema_name, + table_name, database=None, model_name=None): sql = cls._get_columns_in_table_sql(schema_name, table_name, database) connection, cursor = cls.add_query( profile, sql, model_name) @@ -229,18 +271,19 @@ def _table_columns_to_dict(cls, columns): return {col.name: col for col in columns} @classmethod - def expand_target_column_types(cls, profile, + def expand_target_column_types(cls, profile, project_cfg, temp_table, to_schema, to_table, model_name=None): reference_columns = cls._table_columns_to_dict( cls.get_columns_in_table( - profile, None, temp_table, model_name=model_name)) + profile, project_cfg, None, temp_table, model_name=model_name)) target_columns = cls._table_columns_to_dict( cls.get_columns_in_table( - profile, to_schema, to_table, model_name=model_name)) + profile, project_cfg, to_schema, to_table, + model_name=model_name)) for column_name, reference_column in reference_columns.items(): target_column = target_columns.get(column_name) @@ -255,20 +298,77 @@ def expand_target_column_types(cls, profile, to_schema, to_table) - cls.alter_column_type(profile, to_schema, to_table, - column_name, new_type, model_name) + cls.alter_column_type(profile, project_cfg, to_schema, + to_table, column_name, new_type, + model_name) + + ### + # RELATIONS + ### + @classmethod + def list_relations(cls, profile, project_cfg, schema, model_name=None): + raise dbt.exceptions.NotImplementedException( + '`list_relations` is not implemented for this adapter!') + + @classmethod + def _make_match_kwargs(cls, project_cfg, schema, identifier): + if identifier is not None and \ + project_cfg.get('quoting', {}).get('identifier') is False: + identifier = identifier.lower() + + if schema is not None and \ + project_cfg.get('quoting', {}).get('schema') is False: + schema = schema.lower() + + return filter_null_values({'identifier': identifier, + 'schema': schema}) + + @classmethod + def get_relation(cls, profile, project_cfg, schema=None, identifier=None, + relations_list=None, model_name=None): + if schema is None and relations_list is None: + raise dbt.exceptions.RuntimeException( + 'get_relation needs either a schema to query, or a list ' + 'of relations to use') + + if relations_list is None: + relations_list = cls.list_relations( + profile, project_cfg, schema, model_name) + + matches = [] + + search = cls._make_match_kwargs(project_cfg, schema, identifier) + + for relation in relations_list: + if relation.matches(**search): + matches.append(relation) + + if len(matches) > 1: + dbt.exceptions.get_relation_returned_multiple_results( + {'identifier': identifier, 'schema': schema}, matches) + + elif matches: + return matches[0] + + return None ### # SANE ANSI SQL DEFAULTS ### @classmethod - def get_create_schema_sql(cls, schema): - return ('create schema if not exists "{schema}"' + def get_create_schema_sql(cls, project_cfg, schema): + if project_cfg.get('quoting', {}).get('schema', True): + schema = cls.quote(schema) + + return ('create schema if not exists {schema}' .format(schema=schema)) @classmethod - def get_drop_schema_sql(cls, schema): - return ('drop schema if exists "{schema} cascade"' + def get_drop_schema_sql(cls, project_cfg, schema): + if project_cfg.get('quoting', {}).get('schema', True): + schema = cls.quote(schema) + + return ('drop schema if exists {schema} cascade' .format(schema=schema)) ### @@ -276,7 +376,7 @@ def get_drop_schema_sql(cls, schema): # although some adapters may override them ### @classmethod - def get_default_schema(cls, profile): + def get_default_schema(cls, profile, project_cfg): return profile.get('schema') @classmethod @@ -593,9 +693,9 @@ def execute_all(cls, profile, sqls, model_name=None): return connection @classmethod - def create_schema(cls, profile, schema, model_name=None): + def create_schema(cls, profile, project_cfg, schema, model_name=None): logger.debug('Creating schema "%s".', schema) - sql = cls.get_create_schema_sql(schema) + sql = cls.get_create_schema_sql(project_cfg, schema) res = cls.add_query(profile, sql, model_name) cls.commit_if_has_connection(profile, model_name) @@ -603,30 +703,25 @@ def create_schema(cls, profile, schema, model_name=None): return res @classmethod - def drop_schema(cls, profile, schema, model_name=None): + def drop_schema(cls, profile, project_cfg, schema, model_name=None): logger.debug('Dropping schema "%s".', schema) - sql = cls.get_drop_schema_sql(schema) + sql = cls.get_drop_schema_sql(project_cfg, schema) return cls.add_query(profile, sql, model_name) @classmethod - def table_exists(cls, profile, schema, table, model_name=None): - tables = cls.query_for_existing(profile, schema, model_name) - exists = tables.get(table) is not None - return exists - - @classmethod - def already_exists(cls, profile, schema, table, model_name=None): - """ - Alias for `table_exists`. - """ - return cls.table_exists(profile, schema, table, model_name) + def already_exists(cls, profile, project_cfg, + schema, table, model_name=None): + relation = cls.get_relation( + profile, project_cfg, schema=schema, identifier=table) + return relation is not None @classmethod def quote(cls, identifier): return '"{}"'.format(identifier.replace('"', '""')) @classmethod - def quote_schema_and_table(cls, profile, schema, table, model_name=None): + def quote_schema_and_table(cls, profile, project_cfg, + schema, table, model_name=None): return '{}.{}'.format(cls.quote(schema), cls.quote(table)) @@ -661,7 +756,7 @@ def convert_time_type(cls, agate_table, col_idx): '`convert_time_type` is not implemented for this adapter!') @classmethod - def convert_type(cls, profiel, agate_table, col_idx, model_name): + def convert_type(cls, agate_table, col_idx): return cls.convert_agate_type(agate_table, col_idx) @classmethod diff --git a/dbt/adapters/default/relation.py b/dbt/adapters/default/relation.py new file mode 100644 index 00000000000..869a022a856 --- /dev/null +++ b/dbt/adapters/default/relation.py @@ -0,0 +1,245 @@ +from dbt.api import APIObject +from dbt.utils import filter_null_values + +import dbt.exceptions + + +class DefaultRelation(APIObject): + + Table = "table" + View = "view" + CTE = "cte" + + RelationTypes = [ + Table, + View, + CTE + ] + + DEFAULTS = { + 'metadata': { + 'type': 'DefaultRelation' + }, + 'quote_character': '"', + 'quote_policy': { + 'database': True, + 'schema': True, + 'identifier': True + }, + 'include_policy': { + 'database': False, + 'schema': True, + 'identifier': True + } + } + + PATH_SCHEMA = { + 'type': 'object', + 'properties': { + 'database': {'type': ['string', 'null']}, + 'schema': {'type': ['string', 'null']}, + 'identifier': {'type': 'string'}, + }, + 'required': ['database', 'schema', 'identifier'], + } + + POLICY_SCHEMA = { + 'type': 'object', + 'properties': { + 'database': {'type': 'boolean'}, + 'schema': {'type': 'boolean'}, + 'identifier': {'type': 'boolean'}, + }, + 'required': ['database', 'schema', 'identifier'], + } + + SCHEMA = { + 'type': 'object', + 'properties': { + 'metadata': { + 'type': 'object', + 'properties': { + 'type': { + 'type': 'string', + 'const': 'DefaultRelation', + }, + }, + }, + 'type': { + 'enum': RelationTypes + [None], + }, + 'path': PATH_SCHEMA, + 'include_policy': POLICY_SCHEMA, + 'quote_policy': POLICY_SCHEMA, + 'quote_character': {'type': 'string'}, + }, + 'required': ['metadata', 'type', 'path', 'include_policy', + 'quote_policy', 'quote_character'] + } + + PATH_ELEMENTS = ['database', 'schema', 'identifier'] + + def matches(self, database=None, schema=None, identifier=None): + search = filter_null_values({ + 'database': database, + 'schema': schema, + 'identifier': identifier + }) + + if not search: + # nothing was passed in + raise dbt.exceptions.RuntimeException( + "Tried to match relation, but no search path was passed!") + + exact_match = True + approximate_match = True + + for k, v in search.items(): + if self.get_path_part(k) != v: + exact_match = False + + if self.get_path_part(k).lower() != v.lower(): + approximate_match = False + + if approximate_match and not exact_match: + target = self.create( + database=database, schema=schema, identifier=identifier) + dbt.exceptions.approximate_relation_match(target, self) + + return exact_match + + def get_path_part(self, part): + return self.path.get(part) + + def should_quote(self, part): + return self.quote_policy.get(part) + + def should_include(self, part): + return self.include_policy.get(part) + + def quote(self, database=None, schema=None, identifier=None): + policy = filter_null_values({ + 'database': database, + 'schema': schema, + 'identifier': identifier + }) + + return self.incorporate(quote_policy=policy) + + def include(self, database=None, schema=None, identifier=None): + policy = filter_null_values({ + 'database': database, + 'schema': schema, + 'identifier': identifier + }) + + return self.incorporate(include_policy=policy) + + def render(self, use_table_name=True): + parts = [] + + for k in self.PATH_ELEMENTS: + if self.should_include(k): + path_part = self.get_path_part(k) + + if path_part is None: + continue + elif k == 'identifier': + if use_table_name: + path_part = self.table + else: + path_part = self.identifier + + parts.append( + self.quote_if( + path_part, + self.should_quote(k))) + + if len(parts) == 0: + raise dbt.exceptions.RuntimeException( + "No path parts are included! Nothing to render.") + + return '.'.join(parts) + + def quote_if(self, identifier, should_quote): + if should_quote: + return self.quoted(identifier) + + return identifier + + def quoted(self, identifier): + return '{quote_char}{identifier}{quote_char}'.format( + quote_char=self.quote_character, + identifier=identifier) + + @classmethod + def create_from_node(cls, profile, node, table_name=None, **kwargs): + return cls.create( + database=profile.get('dbname'), + schema=node.get('schema'), + identifier=node.get('name'), + table_name=table_name, + **kwargs) + + @classmethod + def create(cls, database=None, schema=None, + identifier=None, table_name=None, + type=None, **kwargs): + if table_name is None: + table_name = identifier + + return cls(type=type, + path={ + 'database': database, + 'schema': schema, + 'identifier': identifier + }, + table_name=table_name, + **kwargs) + + def __repr__(self): + return "<{} {}>".format(self.__class__.__name__, self.render()) + + def __hash__(self): + return hash(self.render()) + + def __str__(self): + return self.render() + + @property + def path(self): + return self.get('path', {}) + + @property + def database(self): + return self.path.get('database') + + @property + def schema(self): + return self.path.get('schema') + + @property + def identifier(self): + return self.path.get('identifier') + + # Here for compatibility with old Relation interface + @property + def name(self): + return self.identifier + + # Here for compatibility with old Relation interface + @property + def table(self): + return self.table_name + + @property + def is_table(self): + return self.type == self.Table + + @property + def is_cte(self): + return self.type == self.CTE + + @property + def is_view(self): + return self.type == self.View diff --git a/dbt/adapters/postgres/__init__.py b/dbt/adapters/postgres/__init__.py new file mode 100644 index 00000000000..82bb6ed5419 --- /dev/null +++ b/dbt/adapters/postgres/__init__.py @@ -0,0 +1,5 @@ +from dbt.adapters.postgres.impl import PostgresAdapter + +__all__ = [ + PostgresAdapter, +] diff --git a/dbt/adapters/postgres.py b/dbt/adapters/postgres/impl.py similarity index 77% rename from dbt/adapters/postgres.py rename to dbt/adapters/postgres/impl.py index 14374cc088c..17797dc9fe3 100644 --- a/dbt/adapters/postgres.py +++ b/dbt/adapters/postgres/impl.py @@ -7,7 +7,6 @@ import dbt.exceptions import agate -from dbt.utils import chunks from dbt.logger import GLOBAL_LOGGER as logger @@ -17,8 +16,6 @@ class PostgresAdapter(dbt.adapters.default.DefaultAdapter): @contextmanager def exception_handler(cls, profile, sql, model_name=None, connection_name=None): - connection = cls.get_connection(profile, connection_name) - try: yield @@ -86,7 +83,24 @@ def open_connection(cls, connection): return result @classmethod - def alter_column_type(cls, profile, schema, table, column_name, + def cancel_connection(cls, profile, connection): + connection_name = connection.get('name') + pid = connection.get('handle').get_backend_pid() + + sql = "select pg_terminate_backend({})".format(pid) + + logger.debug("Cancelling query '{}' ({})".format(connection_name, pid)) + + _, cursor = cls.add_query(profile, sql, 'master') + res = cursor.fetchone() + + logger.debug("Cancel query '{}': {}".format(connection_name, res)) + + # DATABASE INSPECTION FUNCTIONS + # These require the profile AND project, as they need to know + # database-specific configs at the project level. + @classmethod + def alter_column_type(cls, profile, project, schema, table, column_name, new_column_type, model_name=None): """ 1. Create a new column (w/ temp name and correct type) @@ -95,19 +109,20 @@ def alter_column_type(cls, profile, schema, table, column_name, 4. Rename the new column to existing column """ + relation = cls.Relation.create(schema=schema, identifier=table) + opts = { - "schema": schema, - "table": table, + "relation": relation, "old_column": column_name, "tmp_column": "{}__dbt_alter".format(column_name), "dtype": new_column_type } sql = """ - alter table "{schema}"."{table}" add column "{tmp_column}" {dtype}; - update "{schema}"."{table}" set "{tmp_column}" = "{old_column}"; - alter table "{schema}"."{table}" drop column "{old_column}" cascade; - alter table "{schema}"."{table}" rename column "{tmp_column}" to "{old_column}"; + alter table {relation} add column "{tmp_column}" {dtype}; + update {relation} set "{tmp_column}" = "{old_column}"; + alter table {relation} drop column "{old_column}" cascade; + alter table {relation} rename column "{tmp_column}" to "{old_column}"; """.format(**opts).strip() # noqa connection, cursor = cls.add_query(profile, sql, model_name) @@ -115,31 +130,33 @@ def alter_column_type(cls, profile, schema, table, column_name, return connection, cursor @classmethod - def query_for_existing(cls, profile, schemas, model_name=None): - if not isinstance(schemas, (list, tuple)): - schemas = [schemas] - - schema_list = ",".join(["'{}'".format(schema) for schema in schemas]) - + def list_relations(cls, profile, project, schema, model_name=None): sql = """ - select tablename as name, 'table' as type from pg_tables - where schemaname in ({schema_list}) + select tablename as name, schemaname as schema, 'table' as type from pg_tables + where schemaname ilike '{schema}' union all - select viewname as name, 'view' as type from pg_views - where schemaname in ({schema_list}) - """.format(schema_list=schema_list).strip() # noqa + select viewname as name, schemaname as schema, 'view' as type from pg_views + where schemaname ilike '{schema}' + """.format(schema=schema).strip() # noqa connection, cursor = cls.add_query(profile, sql, model_name, auto_begin=False) results = cursor.fetchall() - existing = [(name, relation_type) for (name, relation_type) in results] - - return dict(existing) + return [cls.Relation.create( + database=profile.get('dbname'), + schema=_schema, + identifier=name, + quote_policy={ + 'schema': True, + 'identifier': True + }, + type=type) + for (name, _schema, type) in results] @classmethod - def get_existing_schemas(cls, profile, model_name=None): + def get_existing_schemas(cls, profile, project, model_name=None): sql = "select distinct nspname from pg_namespace" connection, cursor = cls.add_query(profile, sql, model_name, @@ -149,7 +166,7 @@ def get_existing_schemas(cls, profile, model_name=None): return [row[0] for row in results] @classmethod - def check_schema_exists(cls, profile, schema, model_name=None): + def check_schema_exists(cls, profile, project, schema, model_name=None): sql = """ select count(*) from pg_namespace where nspname = '{schema}' """.format(schema=schema).strip() # noqa @@ -160,20 +177,6 @@ def check_schema_exists(cls, profile, schema, model_name=None): return results[0] > 0 - @classmethod - def cancel_connection(cls, profile, connection): - connection_name = connection.get('name') - pid = connection.get('handle').get_backend_pid() - - sql = "select pg_terminate_backend({})".format(pid) - - logger.debug("Cancelling query '{}' ({})".format(connection_name, pid)) - - _, cursor = cls.add_query(profile, sql, 'master') - res = cursor.fetchone() - - logger.debug("Cancel query '{}': {}".format(connection_name, res)) - @classmethod def convert_text_type(cls, agate_table, col_idx): return "text" diff --git a/dbt/adapters/redshift/__init__.py b/dbt/adapters/redshift/__init__.py new file mode 100644 index 00000000000..a92a6acbd80 --- /dev/null +++ b/dbt/adapters/redshift/__init__.py @@ -0,0 +1,5 @@ +from dbt.adapters.redshift.impl import RedshiftAdapter + +__all__ = [ + RedshiftAdapter, +] diff --git a/dbt/adapters/redshift.py b/dbt/adapters/redshift/impl.py similarity index 83% rename from dbt/adapters/redshift.py rename to dbt/adapters/redshift/impl.py index bab408e7628..311e09d3923 100644 --- a/dbt/adapters/redshift.py +++ b/dbt/adapters/redshift/impl.py @@ -91,7 +91,21 @@ def _get_columns_in_table_sql(cls, schema_name, table_name, database): return sql @classmethod - def drop(cls, profile, schema, relation, relation_type, model_name=None): + def drop_relation(cls, profile, project, relation, model_name=None): + """ + In Redshift, DROP TABLE ... CASCADE should not be used + inside a transaction. Redshift doesn't prevent the CASCADE + part from conflicting with concurrent transactions. If we do + attempt to drop two tables with CASCADE at once, we'll often + get the dreaded: + + table was dropped by a concurrent transaction + + So, we need to lock around calls to the underlying + drop_relation() function. + + https://docs.aws.amazon.com/redshift/latest/dg/r_DROP_TABLE.html + """ global drop_lock to_return = None @@ -106,8 +120,8 @@ def drop(cls, profile, schema, relation, relation_type, model_name=None): cls.begin(profile, connection.get('name')) - to_return = super(PostgresAdapter, cls).drop( - profile, schema, relation, relation_type, model_name) + to_return = super(PostgresAdapter, cls).drop_relation( + profile, project, relation, model_name) cls.commit(profile, connection) cls.begin(profile, connection.get('name')) diff --git a/dbt/adapters/snowflake/__init__.py b/dbt/adapters/snowflake/__init__.py new file mode 100644 index 00000000000..8ff756fdf4e --- /dev/null +++ b/dbt/adapters/snowflake/__init__.py @@ -0,0 +1,7 @@ +from dbt.adapters.snowflake.impl import SnowflakeAdapter +from dbt.adapters.snowflake.relation import SnowflakeRelation + +__all__ = [ + SnowflakeAdapter, + SnowflakeRelation, +] diff --git a/dbt/adapters/snowflake.py b/dbt/adapters/snowflake/impl.py similarity index 70% rename from dbt/adapters/snowflake.py rename to dbt/adapters/snowflake/impl.py index dc5169e216d..72836cf2518 100644 --- a/dbt/adapters/snowflake.py +++ b/dbt/adapters/snowflake/impl.py @@ -10,15 +10,17 @@ import dbt.compat import dbt.exceptions -import dbt.flags as flags from dbt.adapters.postgres import PostgresAdapter -from dbt.contracts.connection import validate_connection +from dbt.adapters.snowflake.relation import SnowflakeRelation from dbt.logger import GLOBAL_LOGGER as logger +from dbt.utils import filter_null_values class SnowflakeAdapter(PostgresAdapter): + Relation = SnowflakeRelation + @classmethod @contextmanager def exception_handler(cls, profile, sql, model_name=None, @@ -104,38 +106,40 @@ def open_connection(cls, connection): return result @classmethod - def query_for_existing(cls, profile, schemas, model_name=None): - if not isinstance(schemas, (list, tuple)): - schemas = [schemas] - - schema_list = ",".join(["'{}'".format(schema) for schema in schemas]) - + def list_relations(cls, profile, project_cfg, schema, model_name=None): sql = """ - select TABLE_NAME as name, TABLE_TYPE as type - from INFORMATION_SCHEMA.TABLES - where TABLE_SCHEMA in ({schema_list}) - """.format(schema_list=schema_list).strip() # noqa + select + table_name as name, table_schema as schema, table_type as type + from information_schema.tables + where table_schema ilike '{schema}' + """.format(schema=schema).strip() # noqa + + _, cursor = cls.add_query( + profile, sql, model_name, auto_begin=False) - _, cursor = cls.add_query(profile, sql, model_name, auto_begin=False) results = cursor.fetchall() relation_type_lookup = { 'BASE TABLE': 'table', 'VIEW': 'view' - } - existing = [(name, relation_type_lookup.get(relation_type)) - for (name, relation_type) in results] - - return dict(existing) + } + return [cls.Relation.create( + database=profile.get('database'), + schema=_schema, + identifier=name, + quote_policy={ + 'schema': True, + 'identifier': True + }, + type=relation_type_lookup.get(type)) + for (name, _schema, type) in results] @classmethod - def rename(cls, profile, schema, from_name, to_name, model_name=None): - sql = (('alter table "{schema}"."{from_name}" ' - 'rename to "{schema}"."{to_name}"') - .format(schema=schema, - from_name=from_name, - to_name=to_name)) + def rename_relation(cls, profile, project_cfg, from_relation, + to_relation, model_name=None): + sql = 'alter table {} rename to {}'.format( + from_relation, to_relation) connection, cursor = cls.add_query(profile, sql, model_name) @@ -144,18 +148,8 @@ def add_begin_query(cls, profile, name): return cls.add_query(profile, 'BEGIN', name, auto_begin=False) @classmethod - def create_schema(cls, profile, schema, model_name=None): - logger.debug('Creating schema "%s".', schema) - sql = cls.get_create_schema_sql(schema) - res = cls.add_query(profile, sql, model_name) - - cls.commit_if_has_connection(profile, model_name) - - return res - - @classmethod - def get_existing_schemas(cls, profile, model_name=None): - sql = "select distinct SCHEMA_NAME from INFORMATION_SCHEMA.SCHEMATA" + def get_existing_schemas(cls, profile, project_cfg, model_name=None): + sql = "select distinct schema_name from information_schema.schemata" connection, cursor = cls.add_query(profile, sql, model_name, auto_begin=False) @@ -164,11 +158,12 @@ def get_existing_schemas(cls, profile, model_name=None): return [row[0] for row in results] @classmethod - def check_schema_exists(cls, profile, schema, model_name=None): + def check_schema_exists(cls, profile, project_cfg, + schema, model_name=None): sql = """ select count(*) - from INFORMATION_SCHEMA.SCHEMATA - where SCHEMA_NAME = '{schema}' + from information_schema.schemata + where upper(schema_name) = upper('{schema}') """.format(schema=schema).strip() # noqa connection, cursor = cls.add_query(profile, sql, model_name, @@ -224,6 +219,19 @@ def add_query(cls, profile, sql, model_name=None, auto_begin=True, return connection, cursor + @classmethod + def _make_match_kwargs(cls, project_cfg, schema, identifier): + if identifier is not None and \ + project_cfg.get('quoting', {}).get('identifier', True) is False: + identifier = identifier.upper() + + if schema is not None and \ + project_cfg.get('quoting', {}).get('schema', True) is False: + schema = schema.upper() + + return filter_null_values({'identifier': identifier, + 'schema': schema}) + @classmethod def cancel_connection(cls, profile, connection): handle = connection['handle'] @@ -239,3 +247,28 @@ def cancel_connection(cls, profile, connection): res = cursor.fetchone() logger.debug("Cancel query '{}': {}".format(connection_name, res)) + + @classmethod + def _get_columns_in_table_sql(cls, schema_name, table_name, database): + schema_filter = '1=1' + if schema_name is not None: + schema_filter = "table_schema ilike '{}'".format(schema_name) + + db_prefix = '' if database is None else '{}.'.format(database) + + sql = """ + select + column_name, + data_type, + character_maximum_length, + numeric_precision || ',' || numeric_scale as numeric_size + + from {db_prefix}information_schema.columns + where table_name ilike '{table_name}' + and {schema_filter} + order by ordinal_position + """.format(db_prefix=db_prefix, + table_name=table_name, + schema_filter=schema_filter).strip() + + return sql diff --git a/dbt/adapters/snowflake/relation.py b/dbt/adapters/snowflake/relation.py new file mode 100644 index 00000000000..d8765c54775 --- /dev/null +++ b/dbt/adapters/snowflake/relation.py @@ -0,0 +1,52 @@ +from dbt.adapters.default.relation import DefaultRelation + + +class SnowflakeRelation(DefaultRelation): + DEFAULTS = { + 'metadata': { + 'type': 'SnowflakeRelation' + }, + 'quote_character': '"', + 'quote_policy': { + 'database': True, + 'schema': True, + 'identifier': True, + }, + 'include_policy': { + 'database': False, + 'schema': True, + 'identifier': True, + } + } + + SCHEMA = { + 'type': 'object', + 'properties': { + 'metadata': { + 'type': 'object', + 'properties': { + 'type': { + 'type': 'string', + 'const': 'SnowflakeRelation', + }, + }, + }, + 'type': { + 'enum': DefaultRelation.RelationTypes + [None], + }, + 'path': DefaultRelation.PATH_SCHEMA, + 'include_policy': DefaultRelation.POLICY_SCHEMA, + 'quote_policy': DefaultRelation.POLICY_SCHEMA, + 'quote_character': {'type': 'string'}, + }, + 'required': ['metadata', 'type', 'path', 'include_policy', + 'quote_policy', 'quote_character'] + } + + @classmethod + def create_from_node(cls, profile, node, **kwargs): + return cls.create( + database=profile.get('database'), + schema=node.get('schema'), + identifier=node.get('name'), + **kwargs) diff --git a/dbt/api/__init__.py b/dbt/api/__init__.py new file mode 100644 index 00000000000..c51e0a83548 --- /dev/null +++ b/dbt/api/__init__.py @@ -0,0 +1,5 @@ +from dbt.api.object import APIObject + +__all__ = [ + APIObject +] diff --git a/dbt/api/object.py b/dbt/api/object.py new file mode 100644 index 00000000000..d4c1089f0c9 --- /dev/null +++ b/dbt/api/object.py @@ -0,0 +1,82 @@ +import copy +from jsonschema import Draft4Validator + +from dbt.exceptions import ValidationException +from dbt.utils import deep_merge + + +class APIObject(dict): + """ + A serializable / deserializable object intended for + use in a future dbt API. + + To create a new object, you'll want to extend this + class, and then implement the SCHEMA property (a + valid JSON schema), the DEFAULTS property (default + settings for this object), and a static method that + calls this constructor. + """ + + SCHEMA = { + 'type': 'object', + 'properties': {} + } + + DEFAULTS = {} + + def __init__(self, *args, **kwargs): + """ + Create and validate an instance. Note that it's + not a good idea to override this. + """ + defaults = copy.deepcopy(self.DEFAULTS) + settings = copy.deepcopy(kwargs) + + d = deep_merge(defaults, settings) + super(APIObject, self).__init__(*args, **d) + self.__dict__ = self + self.validate() + + def incorporate(self, **kwargs): + """ + Given a list of kwargs, incorporate these arguments + into a new copy of this instance, and return the new + instance after validating. + """ + existing = copy.deepcopy(dict(self)) + updates = copy.deepcopy(kwargs) + return type(self)(**deep_merge(existing, updates)) + + def serialize(self): + """ + Return a dict representation of this object. + """ + return dict(self) + + @classmethod + def deserialize(cls, settings): + """ + Convert a dict representation of this object into + an actual object for internal use. + """ + return cls(**settings) + + def validate(self): + """ + Using the SCHEMA property, validate the attributes + of this instance. If any attributes are missing or + invalid, raise a ValidationException. + """ + validator = Draft4Validator(self.SCHEMA) + + errors = [] + + for error in validator.iter_errors(self.serialize()): + errors.append('property "{}", {}'.format( + ".".join(error.path), error.message)) + + if errors: + raise ValidationException( + 'Invalid arguments passed to "{}" instance: {}' + .format(type(self).__name__, + ", ".join(errors))) diff --git a/dbt/compilation.py b/dbt/compilation.py index 1a6f385b954..ab89bd54343 100644 --- a/dbt/compilation.py +++ b/dbt/compilation.py @@ -166,7 +166,7 @@ def compile_node(self, node, flat_graph): }) context = dbt.context.runtime.generate( - compiled_node, self.project.cfg, flat_graph) + compiled_node, self.project, flat_graph) compiled_node['compiled_sql'] = dbt.clients.jinja.get_rendered( node.get('raw_sql'), diff --git a/dbt/context/common.py b/dbt/context/common.py index 703657f7ac2..32a6b000d52 100644 --- a/dbt/context/common.py +++ b/dbt/context/common.py @@ -30,10 +30,12 @@ class DatabaseWrapper(object): functions. """ - def __init__(self, model, adapter, profile): + def __init__(self, model, adapter, profile, project): self.model = model self.adapter = adapter self.profile = profile + self.project = project + self.Relation = adapter.Relation # Fun with metaprogramming # Most adapter functions take `profile` as the first argument, and @@ -42,16 +44,21 @@ def __init__(self, model, adapter, profile): for context_function in self.adapter.context_functions: setattr(self, context_function, - self.wrap_with_profile_and_model_name(context_function)) + self.wrap(context_function, (self.profile, self.project,))) + + for profile_function in self.adapter.profile_functions: + setattr(self, + profile_function, + self.wrap(profile_function, (self.profile,))) for raw_function in self.adapter.raw_functions: setattr(self, raw_function, getattr(self.adapter, raw_function)) - def wrap_with_profile_and_model_name(self, fn): + def wrap(self, fn, arg_prefix): def wrapped(*args, **kwargs): - args = (self.profile,) + args + args = arg_prefix + args kwargs['model_name'] = self.model.get('name') return getattr(self.adapter, fn)(*args, **kwargs) @@ -278,7 +285,43 @@ def _return(value): raise dbt.exceptions.MacroReturn(value) -def generate(model, project, flat_graph, provider=None): +def get_this_relation(db_wrapper, project_cfg, profile, model): + table_name = dbt.utils.model_immediate_name( + model, dbt.flags.NON_DESTRUCTIVE) + + return db_wrapper.adapter.Relation.create_from_node( + profile, model, table_name=table_name) + + +def create_relation(relation_type, quoting_config): + + class RelationWithContext(relation_type): + @classmethod + def create(cls, *args, **kwargs): + quote_policy = quoting_config + + if 'quote_policy' in kwargs: + quote_policy = dbt.utils.merge( + quote_policy, + kwargs.pop('quote_policy')) + + return relation_type.create(*args, + quote_policy=quote_policy, + **kwargs) + + return RelationWithContext + + +def create_adapter(adapter_type, relation_type): + + class AdapterWithContext(adapter_type): + + Relation = relation_type + + return AdapterWithContext + + +def generate(model, project_cfg, flat_graph, provider=None): """ Not meant to be called directly. Call with either: dbt.context.parser.generate @@ -289,8 +332,8 @@ def generate(model, project, flat_graph, provider=None): raise dbt.exceptions.InternalException( "Invalid provider given to context: {}".format(provider)) - target_name = project.get('target') - profile = project.get('outputs').get(target_name) + target_name = project_cfg.get('target') + profile = project_cfg.get('outputs').get(target_name) target = profile.copy() target.pop('pass', None) target['name'] = target_name @@ -302,11 +345,22 @@ def generate(model, project, flat_graph, provider=None): pre_hooks = model.get('config', {}).get('pre-hook') post_hooks = model.get('config', {}).get('post-hook') - db_wrapper = DatabaseWrapper(model, adapter, profile) - cli_var_overrides = project.get('cli_vars', {}) + relation_type = create_relation(adapter.Relation, + project_cfg.get('quoting')) + + db_wrapper = DatabaseWrapper(model, + create_adapter(adapter, relation_type), + profile, + project_cfg) + + cli_var_overrides = project_cfg.get('cli_vars', {}) context = dbt.utils.merge(context, { "adapter": db_wrapper, + "api": { + "Relation": relation_type, + "Column": adapter.Column, + }, "column": adapter.Column, "config": provider.Config(model), "env_var": _env_var, @@ -322,7 +376,8 @@ def generate(model, project, flat_graph, provider=None): }, "post_hooks": post_hooks, "pre_hooks": pre_hooks, - "ref": provider.ref(model, project, profile, flat_graph), + "ref": provider.ref(db_wrapper, model, project_cfg, + profile, flat_graph), "return": _return, "schema": model.get('schema', schema), "sql": model.get('injected_sql'), @@ -330,7 +385,7 @@ def generate(model, project, flat_graph, provider=None): "fromjson": fromjson, "tojson": tojson, "target": target, - "this": dbt.utils.Relation(profile, adapter, model, use_temp=True), + "this": get_this_relation(db_wrapper, project_cfg, profile, model), "try_or_compiler_error": try_or_compiler_error(model) }) @@ -342,7 +397,7 @@ def generate(model, project, flat_graph, provider=None): context = _add_macros(context, model, flat_graph) - context["write"] = write(model, project.get('target-path'), 'run') + context["write"] = write(model, project_cfg.get('target-path'), 'run') context["render"] = render(context, model) context["var"] = Var(model, context=context, overrides=cli_var_overrides) context['context'] = context diff --git a/dbt/context/parser.py b/dbt/context/parser.py index bf254bb3636..f43f55b6748 100644 --- a/dbt/context/parser.py +++ b/dbt/context/parser.py @@ -1,15 +1,12 @@ -import dbt.utils import dbt.exceptions import dbt.context.common -from dbt.adapters.factory import get_adapter - execute = False -def ref(model, project, profile, flat_graph): +def ref(db_wrapper, model, project_cfg, profile, flat_graph): def ref(*args): if len(args) == 1 or len(args) == 2: @@ -18,8 +15,7 @@ def ref(*args): else: dbt.exceptions.ref_invalid_args(model, args) - adapter = get_adapter(profile) - return dbt.utils.Relation(profile, adapter, model) + return db_wrapper.adapter.Relation.create_from_node(profile, model) return ref @@ -51,6 +47,6 @@ def get(self, name, validator=None, default=None): return '' -def generate(model, project, flat_graph): +def generate(model, project_cfg, flat_graph): return dbt.context.common.generate( - model, project, flat_graph, dbt.context.parser) + model, project_cfg, flat_graph, dbt.context.parser) diff --git a/dbt/context/runtime.py b/dbt/context/runtime.py index 1a67ab1f5b1..efe0c375f1f 100644 --- a/dbt/context/runtime.py +++ b/dbt/context/runtime.py @@ -1,12 +1,8 @@ -import json - -from dbt.adapters.factory import get_adapter -from dbt.compat import basestring +from dbt.utils import get_materialization, add_ephemeral_model_prefix import dbt.clients.jinja import dbt.context.common import dbt.flags -import dbt.utils from dbt.logger import GLOBAL_LOGGER as logger # noqa @@ -14,8 +10,9 @@ execute = True -def ref(model, project, profile, flat_graph): - current_project = project.get('name') +def ref(db_wrapper, model, project_cfg, profile, flat_graph): + current_project = project_cfg.get('name') + adapter = db_wrapper.adapter def do_ref(*args): target_model_name = None @@ -48,11 +45,16 @@ def do_ref(*args): target_model_name, target_model_package) - if dbt.utils.get_materialization(target_model) == 'ephemeral': - model['extra_ctes'][target_model_id] = None + is_ephemeral = (get_materialization(target_model) == 'ephemeral') - adapter = get_adapter(profile) - return dbt.utils.Relation(profile, adapter, target_model) + if is_ephemeral: + model['extra_ctes'][target_model_id] = None + return adapter.Relation.create( + type=adapter.Relation.CTE, + identifier=add_ephemeral_model_prefix( + target_model_name)).quote(identifier=False) + else: + return adapter.Relation.create_from_node(profile, target_model) return do_ref @@ -90,6 +92,6 @@ def get(self, name, validator=None, default=None): return to_return -def generate(model, project, flat_graph): +def generate(model, project_cfg, flat_graph): return dbt.context.common.generate( - model, project, flat_graph, dbt.context.runtime) + model, project_cfg, flat_graph, dbt.context.runtime) diff --git a/dbt/contracts/graph/parsed.py b/dbt/contracts/graph/parsed.py index afc96dc280e..8d44c8736bb 100644 --- a/dbt/contracts/graph/parsed.py +++ b/dbt/contracts/graph/parsed.py @@ -25,6 +25,7 @@ Required('post-hook'): [hook_contract], Required('pre-hook'): [hook_contract], Required('vars'): dict, + Required('quoting'): dict, Required('column_types'): dict, }, extra=ALLOW_EXTRA) diff --git a/dbt/exceptions.py b/dbt/exceptions.py index 9ce61b9f801..fa2fdb2d7fa 100644 --- a/dbt/exceptions.py +++ b/dbt/exceptions.py @@ -248,9 +248,21 @@ def missing_config(model, name): model) -def missing_relation(relation_name, model=None): +def missing_relation(relation, model=None): raise_compiler_error( - "Relation {} not found!".format(relation_name), + "Relation {} not found!".format(relation), + model) + + +def relation_wrong_type(relation, expected_type, model=None): + raise_compiler_error( + ('Trying to create {expected_type} {relation}, ' + 'but it currently exists as a {current_type}. Either ' + 'drop {relation} manually, or run dbt with ' + '`--full-refresh` and dbt will drop it for you.') + .format(relation=relation, + current_type=relation.type, + expected_type=expected_type), model) @@ -293,6 +305,28 @@ def raise_dep_not_found(node, node_description, required_pkg): '`dbt deps`.'.format(node_description, required_pkg), node=node) +def multiple_matching_relations(kwargs, matches): + raise_compiler_error( + 'get_relation returned more than one relation with the given args. ' + 'Please specify a database or schema to narrow down the result set.' + '\n{}\n\n{}' + .format(kwargs, matches)) + + +def get_relation_returned_multiple_results(kwargs, matches): + multiple_matching_relations(kwargs, matches) + + +def approximate_relation_match(target, relation): + raise_compiler_error( + 'When searching for a relation, dbt found an approximate match. ' + 'Instead of guessing \nwhich relation to use, dbt will move on. ' + 'Please delete {relation}, or rename it to be less ambiguous.' + '\nSearched for: {target}\nFound: {relation}' + .format(target=target, + relation=relation)) + + def raise_duplicate_resource_name(node_1, node_2): duped_name = node_1['name'] diff --git a/dbt/include/global_project/macros/adapters/bigquery.sql b/dbt/include/global_project/macros/adapters/bigquery.sql index be427b1d0d2..d1e4784fe69 100644 --- a/dbt/include/global_project/macros/adapters/bigquery.sql +++ b/dbt/include/global_project/macros/adapters/bigquery.sql @@ -10,18 +10,20 @@ {{ return(partition_by_clause) }} {%- endmacro -%} -{% macro bigquery__create_table_as(temporary, identifier, sql) -%} + +{% macro bigquery__create_table_as(temporary, relation, sql) -%} {%- set raw_partition_by = config.get('partition_by', none) -%} - create or replace table `{{ schema }}`.`{{ identifier }}` + create or replace table {{ relation }} {{ partition_by(raw_partition_by) }} as ( {{ sql }} ); {% endmacro %} -{% macro bigquery__create_view_as(identifier, sql) -%} - create or replace view `{{ schema }}`.`{{ identifier }}` as ( + +{% macro bigquery__create_view_as(relation, sql) -%} + create or replace view {{ relation }} as ( {{ sql }} ); {% endmacro %} diff --git a/dbt/include/global_project/macros/adapters/common.sql b/dbt/include/global_project/macros/adapters/common.sql index 9d02ca32315..7d6a42dba1d 100644 --- a/dbt/include/global_project/macros/adapters/common.sql +++ b/dbt/include/global_project/macros/adapters/common.sql @@ -29,37 +29,40 @@ {%- endmacro %} {% macro create_schema(schema_name) %} - create schema if not exists "{{ schema_name }}"; + create schema if not exists {{ schema_name }}; {% endmacro %} -{% macro create_table_as(temporary, identifier, sql) -%} - {{ adapter_macro('create_table_as', temporary, identifier, sql) }} +{% macro create_table_as(temporary, relation, sql) -%} + {{ adapter_macro('create_table_as', temporary, relation, sql) }} {%- endmacro %} -{% macro default__create_table_as(temporary, identifier, sql) -%} - create {% if temporary: -%}temporary{%- endif %} table "{{ schema }}"."{{ identifier }}" as ( +{% macro default__create_table_as(temporary, relation, sql) -%} + create {% if temporary: -%}temporary{%- endif %} table + {{ relation.include(schema=(not temporary)) }} + as ( {{ sql }} ); {% endmacro %} -{% macro create_view_as(identifier, sql) -%} - {{ adapter_macro('create_view_as', identifier, sql) }} + +{% macro create_view_as(relation, sql) -%} + {{ adapter_macro('create_view_as', relation, sql) }} {%- endmacro %} -{% macro default__create_view_as(identifier, sql) -%} - create view "{{ schema }}"."{{ identifier }}" as ( +{% macro default__create_view_as(relation, sql) -%} + create view {{ relation }} as ( {{ sql }} ); {% endmacro %} -{% macro create_archive_table(schema, identifier, columns) -%} - {{ adapter_macro('create_archive_table', schema, identifier, columns) }} +{% macro create_archive_table(relation, columns) -%} + {{ adapter_macro('create_archive_table', relation, columns) }} {%- endmacro %} -{% macro default__create_archive_table(schema, identifier, columns) -%} - create table if not exists "{{ schema }}"."{{ identifier }}" ( +{% macro default__create_archive_table(relation, columns) -%} + create table if not exists {{ relation }} ( {{ column_list_for_create_table(columns) }} ); {% endmacro %} diff --git a/dbt/include/global_project/macros/adapters/postgres.sql b/dbt/include/global_project/macros/adapters/postgres.sql deleted file mode 100644 index 2e945b17516..00000000000 --- a/dbt/include/global_project/macros/adapters/postgres.sql +++ /dev/null @@ -1,6 +0,0 @@ -{% macro postgres__create_table_as(temporary, identifier, sql) -%} - create {% if temporary: -%}temporary{%- endif %} table - {% if not temporary: -%}"{{ schema }}".{%- endif %}"{{ identifier }}" as ( - {{ sql }} - ); -{% endmacro %} diff --git a/dbt/include/global_project/macros/adapters/redshift.sql b/dbt/include/global_project/macros/adapters/redshift.sql index 6fc15111455..b6696ca22ff 100644 --- a/dbt/include/global_project/macros/adapters/redshift.sql +++ b/dbt/include/global_project/macros/adapters/redshift.sql @@ -5,7 +5,7 @@ {%- if dist in ['all', 'even'] -%} diststyle {{ dist }} {%- else -%} - diststyle key distkey ("{{ dist }}") + diststyle key distkey ({{ dist }}) {%- endif -%} {%- endif -%} @@ -19,7 +19,7 @@ {%- set sort = [sort] -%} {%- endif -%} {%- for item in sort -%} - "{{ item }}" + {{ item }} {%- if not loop.last -%},{%- endif -%} {%- endfor -%} ) @@ -27,7 +27,7 @@ {%- endmacro -%} -{% macro redshift__create_table_as(temporary, identifier, sql) -%} +{% macro redshift__create_table_as(temporary, relation, sql) -%} {%- set _dist = config.get('dist') -%} {%- set _sort_type = config.get( @@ -37,33 +37,28 @@ 'sort', validator=validation.any[list, basestring]) -%} - {% if temporary %} - {% set relation = adapter.quote(identifier) %} - {% else %} - {% set relation = adapter.quote(schema) ~ '.' ~ adapter.quote(identifier) %} - {% endif %} - - create {% if temporary -%}temporary{%- endif %} table {{ relation }} - {{ dist(_dist) }} - {{ sort(_sort_type, _sort) }} + create {% if temporary -%}temporary{%- endif %} table + {{ relation.include(schema=(not temporary)) }} + {{ dist(_dist) }} + {{ sort(_sort_type, _sort) }} as ( {{ sql }} ); {%- endmacro %} -{% macro redshift__create_view_as(identifier, sql) -%} +{% macro redshift__create_view_as(relation, sql) -%} {% set bind_qualifier = '' if config.get('bind', default=True) else 'with no schema binding' %} - create view "{{ schema }}"."{{ identifier }}" as ( + create view {{ relation }} as ( {{ sql }} ) {{ bind_qualifier }}; {% endmacro %} -{% macro redshift__create_archive_table(schema, identifier, columns) -%} - create table if not exists "{{ schema }}"."{{ identifier }}" ( +{% macro redshift__create_archive_table(relation, columns) -%} + create table if not exists {{ relation }} ( {{ column_list_for_create_table(columns) }} ) {{ dist('dbt_updated_at') }} diff --git a/dbt/include/global_project/macros/adapters/snowflake.sql b/dbt/include/global_project/macros/adapters/snowflake.sql index 28b876f6569..94c81e867fc 100644 --- a/dbt/include/global_project/macros/adapters/snowflake.sql +++ b/dbt/include/global_project/macros/adapters/snowflake.sql @@ -1,10 +1,7 @@ -{% macro snowflake__create_table_as(temporary, identifier, sql) -%} +{% macro snowflake__create_table_as(temporary, relation, sql) -%} {% if temporary %} - use schema "{{ schema }}"; + use schema {{ schema }}; {% endif %} - create {% if temporary: -%}temporary{%- endif %} table - {% if not temporary: -%}"{{ schema }}".{%- endif %}"{{ identifier }}" as ( - {{ sql }} - ); + {{ default__create_table_as(temporary, relation, sql) }} {% endmacro %} diff --git a/dbt/include/global_project/macros/materializations/archive/archive.sql b/dbt/include/global_project/macros/materializations/archive/archive.sql index f9ba53c431f..8ac99e9ae7c 100644 --- a/dbt/include/global_project/macros/materializations/archive/archive.sql +++ b/dbt/include/global_project/macros/materializations/archive/archive.sql @@ -1,87 +1,105 @@ -{% macro archive_select(source_schema, source_table, target_schema, target_table, unique_key, updated_at) %} +{% macro archive_select(source_relation, target_relation, unique_key, updated_at) %} - with "current_data" as ( + with current_data as ( select - {% for col in adapter.get_columns_in_table(source_schema, source_table) %} + {% for col in adapter.get_columns_in_table(source_relation.schema, source_relation.identifier) %} "{{ col.name }}" {% if not loop.last %},{% endif %} {% endfor %}, {{ updated_at }} as "dbt_updated_at", {{ unique_key }} as "dbt_pk", {{ updated_at }} as "valid_from", null::timestamp as "tmp_valid_to" - from "{{ source_schema }}"."{{ source_table }}" + from {{ source_relation }} ), - "archived_data" as ( + archived_data as ( select - {% for col in adapter.get_columns_in_table(source_schema, source_table) %} + {% for col in adapter.get_columns_in_table(source_relation.schema, source_relation.identifier) %} "{{ col.name }}" {% if not loop.last %},{% endif %} {% endfor %}, {{ updated_at }} as "dbt_updated_at", {{ unique_key }} as "dbt_pk", "valid_from", "valid_to" as "tmp_valid_to" - from "{{ target_schema }}"."{{ target_table }}" + from {{ target_relation }} ), - "insertions" as ( + insertions as ( select - "current_data".*, + current_data.*, null::timestamp as "valid_to" - from "current_data" - left outer join "archived_data" - on "archived_data"."dbt_pk" = "current_data"."dbt_pk" - where "archived_data"."dbt_pk" is null or ( - "archived_data"."dbt_pk" is not null and - "current_data"."dbt_updated_at" > "archived_data"."dbt_updated_at" and - "archived_data"."tmp_valid_to" is null + from current_data + left outer join archived_data + on archived_data."dbt_pk" = current_data."dbt_pk" + where archived_data."dbt_pk" is null or ( + archived_data."dbt_pk" is not null and + current_data."dbt_updated_at" > archived_data."dbt_updated_at" and + archived_data."tmp_valid_to" is null ) ), - "updates" as ( + updates as ( select - "archived_data".*, - "current_data"."dbt_updated_at" as "valid_to" - from "current_data" - left outer join "archived_data" - on "archived_data"."dbt_pk" = "current_data"."dbt_pk" - where "archived_data"."dbt_pk" is not null - and "archived_data"."dbt_updated_at" < "current_data"."dbt_updated_at" - and "archived_data"."tmp_valid_to" is null + archived_data.*, + current_data."dbt_updated_at" as "valid_to" + from current_data + left outer join archived_data + on archived_data."dbt_pk" = current_data."dbt_pk" + where archived_data."dbt_pk" is not null + and archived_data."dbt_updated_at" < current_data."dbt_updated_at" + and archived_data."tmp_valid_to" is null ), - "merged" as ( + merged as ( - select *, 'update' as "change_type" from "updates" + select *, 'update' as "change_type" from updates union all - select *, 'insert' as "change_type" from "insertions" + select *, 'insert' as "change_type" from insertions ) select *, md5("dbt_pk" || '|' || "dbt_updated_at") as "scd_id" - from "merged" + from merged {% endmacro %} {% materialization archive, default %} {%- set config = model['config'] -%} + + {%- set target_schema = config.get('target_schema') -%} + {%- set target_table = config.get('target_table') -%} + {%- set source_schema = config.get('source_schema') -%} {%- set source_table = config.get('source_table') -%} - {%- if not adapter.already_exists(source_schema, source_table) -%} - {{ exceptions.missing_relation(source_table) }} + {%- set source_relation = adapter.get_relation( + schema=source_schema, + identifier=source_table) -%} + + {%- set target_relation = adapter.get_relation( + schema=target_schema, + identifier=target_table) -%} + + {%- if source_relation is none -%} + {{ exceptions.missing_relation(source_relation) }} + {%- endif -%} + + {%- if target_relation is none -%} + {%- set target_relation = api.Relation.create( + schema=target_schema, + identifier=target_table) -%} + {%- elif not target_relation.is_table -%} + {{ exceptions.relation_wrong_type(target_relation, 'table') }} {%- endif -%} {%- set source_columns = adapter.get_columns_in_table(source_schema, source_table) -%} - {%- set target_schema = config.get('target_schema') -%} - {%- set target_table = config.get('target_table') -%} {%- set unique_key = config.get('unique_key') -%} {%- set updated_at = config.get('updated_at') -%} {%- set dest_columns = source_columns + [ @@ -96,7 +114,7 @@ {% endcall %} {% call statement() %} - {{ create_archive_table(target_schema, target_table, dest_columns) }} + {{ create_archive_table(target_relation, dest_columns) }} {% endcall %} {% set missing_columns = adapter.get_missing_columns(source_schema, source_table, target_schema, target_table) %} @@ -104,24 +122,26 @@ {% for col in missing_columns %} {% call statement() %} - alter table "{{ target_schema }}"."{{ target_table }}" add column "{{ col.name }}" {{ col.data_type }}; + alter table {{ target_relation }} + add column "{{ col.name }}" {{ col.data_type }}; {% endcall %} {% endfor %} {%- set identifier = model['name'] -%} {%- set tmp_identifier = model['name'] + '__dbt_archival_tmp' -%} + {%- set tmp_relation = api.Relation.create(identifier=tmp_identifier, type='table') -%} {% call statement() %} {% set tmp_table_sql -%} with dbt_archive_sbq as ( - {{ archive_select(source_schema, source_table, target_schema, target_table, unique_key, updated_at) }} + {{ archive_select(source_relation, target_relation, unique_key, updated_at) }} ) select * from dbt_archive_sbq {%- endset %} - {{ dbt.create_table_as(temporary=True, identifier=tmp_identifier, sql=tmp_table_sql) }} + {{ dbt.create_table_as(True, tmp_relation, tmp_table_sql) }} {% endcall %} @@ -130,15 +150,15 @@ to_table=target_table) }} {% call statement('main') -%} - update "{{ target_schema }}"."{{ identifier }}" set "valid_to" = "tmp"."valid_to" - from "{{ tmp_identifier }}" as "tmp" - where "tmp"."scd_id" = "{{ target_schema }}"."{{ identifier }}"."scd_id" + update {{ target_relation }} set "valid_to" = tmp."valid_to" + from {{ tmp_relation }} as tmp + where tmp."scd_id" = {{ target_relation }}."scd_id" and "change_type" = 'update'; - insert into "{{ target_schema }}"."{{ identifier }}" ( + insert into {{ target_relation }} ( {{ column_list(dest_columns) }} ) - select {{ column_list(dest_columns) }} from "{{ tmp_identifier }}" + select {{ column_list(dest_columns) }} from {{ tmp_relation }} where "change_type" = 'insert'; {% endcall %} diff --git a/dbt/include/global_project/macros/materializations/helpers.sql b/dbt/include/global_project/macros/materializations/helpers.sql index 1a20316c2f3..d816f46b5cc 100644 --- a/dbt/include/global_project/macros/materializations/helpers.sql +++ b/dbt/include/global_project/macros/materializations/helpers.sql @@ -45,10 +45,8 @@ {{ make_hook_config(sql, inside_transaction=False) }} {% endmacro %} - -{% macro drop_if_exists(existing, schema, name) %} - {% set existing_type = existing.get(name) %} - {% if existing_type is not none %} - {{ adapter.drop(schema, name, existing_type) }} +{% macro drop_relation_if_exists(relation) %} + {% if relation is not none %} + {{ adapter.drop_relation(relation) }} {% endif %} {% endmacro %} diff --git a/dbt/include/global_project/macros/materializations/incremental/incremental.sql b/dbt/include/global_project/macros/materializations/incremental/incremental.sql index e6f33710c1d..71945f29f08 100644 --- a/dbt/include/global_project/macros/materializations/incremental/incremental.sql +++ b/dbt/include/global_project/macros/materializations/incremental/incremental.sql @@ -1,13 +1,13 @@ -{% macro dbt__incremental_delete(schema, model) -%} +{% macro dbt__incremental_delete(target_relation, tmp_relation) -%} {%- set unique_key = config.require('unique_key') -%} {%- set identifier = model['name'] -%} delete - from "{{ schema }}"."{{ identifier }}" + from {{ target_relation }} where ({{ unique_key }}) in ( select ({{ unique_key }}) - from "{{ identifier }}__dbt_incremental_tmp" + from {{ tmp_relation.include(schema=False) }} ); {%- endmacro %} @@ -19,23 +19,31 @@ {%- set identifier = model['name'] -%} {%- set tmp_identifier = model['name'] + '__dbt_incremental_tmp' -%} + {%- set existing_relations = adapter.list_relations(schema=schema) -%} + {%- set old_relation = adapter.get_relation(relations_list=existing_relations, + schema=schema, identifier=identifier) -%} + {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, type='table') -%} + {%- set tmp_relation = api.Relation.create(identifier=tmp_identifier, + schema=schema, type='table') -%} + {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} {%- set full_refresh_mode = (flags.FULL_REFRESH == True) -%} - {%- set existing = adapter.query_for_existing(schema) -%} - {%- set existing_type = existing.get(identifier) -%} - {%- set exists_as_table = (existing_type == 'table') -%} + {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%} + {%- set exists_not_as_table = (old_relation is not none and not old_relation.is_table) -%} + {%- set should_truncate = (non_destructive_mode and full_refresh_mode and exists_as_table) -%} - {%- set should_drop = (not should_truncate and (full_refresh_mode or (existing_type not in (none, 'table')))) -%} + {%- set should_drop = (not should_truncate and (full_refresh_mode or exists_not_as_table)) -%} {%- set force_create = (flags.FULL_REFRESH and not flags.NON_DESTRUCTIVE) -%} -- setup - {% if existing_type is none -%} + {% if old_relation is none -%} -- noop {%- elif should_truncate -%} - {{ adapter.truncate(schema, identifier) }} + {{ adapter.truncate_relation(old_relation) }} {%- elif should_drop -%} - {{ adapter.drop(schema, identifier, existing_type) }} + {{ adapter.drop_relation(old_relation) }} + {%- set old_relation = none -%} {%- endif %} {{ run_hooks(pre_hooks, inside_transaction=False) }} @@ -44,9 +52,9 @@ {{ run_hooks(pre_hooks, inside_transaction=True) }} -- build model - {% if force_create or not adapter.already_exists(schema, identifier) -%} + {% if force_create or old_relation is none -%} {%- call statement('main') -%} - {{ create_table_as(False, identifier, sql) }} + {{ create_table_as(False, target_relation, sql) }} {%- endcall -%} {%- else -%} {%- call statement() -%} @@ -60,7 +68,7 @@ or ({{ sql_where }}) is null {%- endset %} - {{ dbt.create_table_as(temporary=True, identifier=tmp_identifier, sql=tmp_table_sql) }} + {{ dbt.create_table_as(True, tmp_relation, tmp_table_sql) }} {%- endcall -%} @@ -74,14 +82,14 @@ {% if unique_key is not none -%} - {{ dbt__incremental_delete(schema, model) }} + {{ dbt__incremental_delete(target_relation, tmp_relation) }} {%- endif %} - insert into "{{ schema }}"."{{ identifier }}" ({{ dest_cols_csv }}) + insert into {{ target_relation }} ({{ dest_cols_csv }}) ( select {{ dest_cols_csv }} - from "{{ identifier }}__dbt_incremental_tmp" + from {{ tmp_relation.include(schema=False) }} ); {% endcall %} {%- endif %} diff --git a/dbt/include/global_project/macros/materializations/seed/seed.sql b/dbt/include/global_project/macros/materializations/seed/seed.sql index 5aad5212c2a..f0a8ca86f95 100644 --- a/dbt/include/global_project/macros/materializations/seed/seed.sql +++ b/dbt/include/global_project/macros/materializations/seed/seed.sql @@ -3,8 +3,8 @@ {{ adapter_macro('create_csv_table', model) }} {%- endmacro %} -{% macro reset_csv_table(model, full_refresh, existing) -%} - {{ adapter_macro('reset_csv_table', model, full_refresh, existing) }} +{% macro reset_csv_table(model, full_refresh, old_relation) -%} + {{ adapter_macro('reset_csv_table', model, full_refresh, old_relation) }} {%- endmacro %} {% macro load_csv_rows(model) -%} @@ -16,12 +16,12 @@ {%- set column_override = model['config'].get('column_types', {}) -%} {% set sql %} - create table {{ adapter.quote(model['schema']) }}.{{ adapter.quote(model['name']) }} ( - {% for col_name in agate_table.column_names %} - {% set inferred_type = adapter.convert_type(agate_table, loop.index0) %} - {% set type = column_override.get(col_name, inferred_type) %} - {{ col_name | string }} {{ type }} {% if not loop.last %}, {% endif %} - {% endfor %} + create table {{ this.render(False) }} ( + {%- for col_name in agate_table.column_names -%} + {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%} + {%- set type = column_override.get(col_name, inferred_type) -%} + {{ col_name | string }} {{ type }} {%- if not loop.last -%}, {%- endif -%} + {%- endfor -%} ) {% endset %} @@ -33,14 +33,14 @@ {% endmacro %} -{% macro default__reset_csv_table(model, full_refresh, existing) %} +{% macro default__reset_csv_table(model, full_refresh, old_relation) %} {% set sql = "" %} {% if full_refresh %} - {{ drop_if_exists(existing, model['schema'], model['name']) }} + {{ adapter.drop_relation(old_relation) }} {% set sql = create_csv_table(model) %} {% else %} - {{ adapter.truncate(model['schema'], model['name']) }} - {% set sql = "truncate table " ~ adapter.quote(model['schema']) ~ "." ~ adapter.quote(model['name']) %} + {{ adapter.truncate_relation(old_relation) }} + {% set sql = "truncate table " ~ old_relation %} {% endif %} {{ return(sql) }} @@ -62,7 +62,7 @@ {% endfor %} {% set sql %} - insert into {{ adapter.quote(model['schema']) }}.{{ adapter.quote(model['name']) }} ({{ cols_sql }}) values + insert into {{ this.render(False) }} ({{ cols_sql }}) values {% for row in chunk -%} ({%- for column in agate_table.column_names -%} %s @@ -88,8 +88,14 @@ {%- set identifier = model['name'] -%} {%- set full_refresh_mode = (flags.FULL_REFRESH == True) -%} - {%- set existing = adapter.query_for_existing(schema) -%} - {%- set existing_type = existing.get(identifier) -%} + {%- set existing_relations = adapter.list_relations(schema=schema) -%} + + {%- set old_relation = adapter.get_relation(relations_list=existing_relations, + schema=schema, identifier=identifier) -%} + + {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%} + {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%} + {%- set csv_table = model["agate_table"] -%} {{ run_hooks(pre_hooks, inside_transaction=False) }} @@ -99,12 +105,12 @@ -- build model {% set create_table_sql = "" %} - {% if existing_type and existing_type != 'table' %} - {{ dbt.exceptions.raise_compiler_error("Cannot seed to '{}', it is a view".format(identifier)) }} - {% elif existing_type is none%} - {% set create_table_sql = create_csv_table(model) %} + {% if exists_as_view %} + {{ exceptions.raise_compiler_error("Cannot seed to '{}', it is a view".format(old_relation)) }} + {% elif exists_as_table %} + {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation) %} {% else %} - {% set create_table_sql = reset_csv_table(model, full_refresh_mode, existing) %} + {% set create_table_sql = create_csv_table(model) %} {% endif %} {% set status = 'CREATE' if full_refresh_mode else 'INSERT' %} diff --git a/dbt/include/global_project/macros/materializations/table/bigquery_table.sql b/dbt/include/global_project/macros/materializations/table/bigquery_table.sql index 25b19c29706..3c50969bbab 100644 --- a/dbt/include/global_project/macros/materializations/table/bigquery_table.sql +++ b/dbt/include/global_project/macros/materializations/table/bigquery_table.sql @@ -1,8 +1,7 @@ - -{% macro make_date_partitioned_table(model, dates, should_create, verbose=False) %} +{% macro make_date_partitioned_table(model, relation, dates, should_create, verbose=False) %} {% if should_create %} - {{ adapter.make_date_partitioned_table(model.schema, model.name) }} + {{ adapter.make_date_partitioned_table(relation.schema, relation.identifier) }} {% endif %} {% for date in dates %} @@ -31,8 +30,10 @@ {%- set identifier = model['name'] -%} {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} - {%- set existing = adapter.query_for_existing(schema) -%} - {%- set existing_type = existing.get(identifier) -%} + {%- set existing_relations = adapter.list_relations(schema=schema) -%} + {%- set old_relation = adapter.get_relation(relations_list=existing_relations, identifier=identifier) -%} + {%- set exists_not_as_table = (old_relation is not none and not old_relation.is_table) -%} + {%- set target_relation = api.Relation.create(schema=schema, identifier=identifier, type='table') -%} {%- set verbose = config.get('verbose', False) -%} {# partitions: iterate over each partition, running a separate query in a for-loop #} @@ -55,16 +56,18 @@ Since dbt uses WRITE_TRUNCATE mode for tables, we only need to drop this thing if it is not a table. If it _is_ already a table, then we can overwrite it without downtime #} - {%- if existing_type is not none and existing_type != 'table' -%} - {{ adapter.drop(schema, identifier, existing_type) }} + {%- if exists_not_as_table -%} + {{ adapter.drop_relation(old_relation) }} {%- endif -%} -- build model {% if partitions %} - {{ make_date_partitioned_table(model, partitions, (existing_type != 'table'), verbose) }} + {# Create the dp-table if 1. it does not exist or 2. it existed, but we just dropped it #} + {%- set should_create = (old_relation is none or exists_not_as_table) -%} + {{ make_date_partitioned_table(model, target_relation, partitions, should_create, verbose) }} {% else %} {% call statement('main') -%} - {{ create_table_as(False, identifier, sql) }} + {{ create_table_as(False, target_relation, sql) }} {% endcall -%} {% endif %} diff --git a/dbt/include/global_project/macros/materializations/table/table.sql b/dbt/include/global_project/macros/materializations/table/table.sql index 96691afdbec..f03d8b46d8d 100644 --- a/dbt/include/global_project/macros/materializations/table/table.sql +++ b/dbt/include/global_project/macros/materializations/table/table.sql @@ -2,17 +2,29 @@ {%- set identifier = model['name'] -%} {%- set tmp_identifier = identifier + '__dbt_tmp' -%} {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} - {%- set existing = adapter.query_for_existing(schema) -%} - {%- set existing_type = existing.get(identifier) -%} - {{ drop_if_exists(existing, schema, tmp_identifier) }} + {%- set existing_relations = adapter.list_relations(schema=schema) -%} + {%- set old_relation = adapter.get_relation(relations_list=existing_relations, + schema=schema, identifier=identifier) -%} + {%- set target_relation = api.Relation.create(identifier=identifier, + schema=schema, type='table') -%} + {%- set intermediate_relation = api.Relation.create(identifier=tmp_identifier, + schema=schema, type='table') -%} + {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%} + {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%} + {%- set create_as_temporary = (exists_as_table and non_destructive_mode) -%} - -- setup + + -- drop the temp relation if it exists for some reason + {{ adapter.drop_relation(intermediate_relation) }} + + -- setup: if the target relation already exists, truncate or drop it {% if non_destructive_mode -%} - {% if existing_type == 'table' -%} - {{ adapter.truncate(schema, identifier) }} - {% elif existing_type == 'view' -%} - {{ adapter.drop(schema, identifier, existing_type) }} + {% if exists_as_table -%} + {{ adapter.truncate_relation(old_relation) }} + {% elif exists_as_view -%} + {{ adapter.drop_relation(old_relation) }} + {%- set old_relation = none -%} {%- endif %} {%- endif %} @@ -24,21 +36,21 @@ -- build model {% call statement('main') -%} {%- if non_destructive_mode -%} - {%- if adapter.already_exists(schema, identifier) -%} - {{ create_table_as(True, tmp_identifier, sql) }} + {%- if old_relation is not none -%} + {{ create_table_as(create_as_temporary, intermediate_relation, sql) }} {% set dest_columns = adapter.get_columns_in_table(schema, identifier) %} {% set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') %} - insert into {{ schema }}.{{ identifier }} ({{ dest_cols_csv }}) ( + insert into {{ target_relation }} ({{ dest_cols_csv }}) ( select {{ dest_cols_csv }} - from "{{ tmp_identifier }}" + from {{ intermediate_relation.include(schema=(not create_as_temporary)) }} ); {%- else -%} - {{ create_table_as(False, identifier, sql) }} + {{ create_table_as(create_as_temporary, target_relation, sql) }} {%- endif -%} {%- else -%} - {{ create_table_as(False, tmp_identifier, sql) }} + {{ create_table_as(create_as_temporary, intermediate_relation, sql) }} {%- endif -%} {%- endcall %} @@ -48,8 +60,8 @@ {% if non_destructive_mode -%} -- noop {%- else -%} - {{ drop_if_exists(existing, schema, identifier) }} - {{ adapter.rename(schema, tmp_identifier, identifier) }} + {{ drop_relation_if_exists(old_relation) }} + {{ adapter.rename_relation(intermediate_relation, target_relation) }} {%- endif %} -- `COMMIT` happens here diff --git a/dbt/include/global_project/macros/materializations/view/bigquery_view.sql b/dbt/include/global_project/macros/materializations/view/bigquery_view.sql index 4b21ee5e44f..1ab47ea9c68 100644 --- a/dbt/include/global_project/macros/materializations/view/bigquery_view.sql +++ b/dbt/include/global_project/macros/materializations/view/bigquery_view.sql @@ -1,33 +1,38 @@ - {% materialization view, adapter='bigquery' -%} {%- set identifier = model['name'] -%} {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} - {%- set existing = adapter.query_for_existing(schema) -%} - {%- set existing_type = existing.get(identifier) -%} - - {%- if existing_type is not none -%} - {%- if existing_type == 'table' and not flags.FULL_REFRESH -%} - {# this is only intended for date partitioned tables, but we cant see that field in the context #} - {% set error_message -%} - Trying to create model '{{ identifier }}' as a view, but it already exists as a table. - Either drop the '{{ schema }}.{{ identifier }}' table manually, or use --full-refresh - {%- endset %} - {{ exceptions.raise_compiler_error(error_message) }} + + {%- set existing_relations = adapter.list_relations(schema=schema) -%} + + {%- set old_relation = adapter.get_relation( + relations_list=existing_relations, + schema=schema, identifier=identifier) -%} + + {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%} + + {%- set target_relation = api.Relation.create( + identifier=identifier, schema=schema, + type='view') -%} + + -- drop if exists + {%- if old_relation is not none -%} + {%- if old_relation.is_table and not flags.FULL_REFRESH -%} + {{ exceptions.relation_wrong_type(old_relation, 'view') }} {%- endif -%} - {{ adapter.drop(schema, identifier, existing_type) }} + {{ adapter.drop_relation(old_relation) }} {%- endif -%} -- build model - {% if existing_type == 'view' and non_destructive_mode -%} + {% if exists_as_view and non_destructive_mode -%} {% call noop_statement('main', status="PASS", res=None) -%} -- Not running : non-destructive mode {{ sql }} {%- endcall %} {%- else -%} {% call statement('main') -%} - {{ create_view_as(identifier, sql) }} + {{ create_view_as(target_relation, sql) }} {%- endcall %} {%- endif %} diff --git a/dbt/include/global_project/macros/materializations/view/view.sql b/dbt/include/global_project/macros/materializations/view/view.sql index b5bba061142..1f0036116bc 100644 --- a/dbt/include/global_project/macros/materializations/view/view.sql +++ b/dbt/include/global_project/macros/materializations/view/view.sql @@ -3,14 +3,22 @@ {%- set identifier = model['name'] -%} {%- set tmp_identifier = identifier + '__dbt_tmp' -%} {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} - {%- set existing = adapter.query_for_existing(schema) -%} - {%- set existing_type = existing.get(identifier) -%} + + {%- set existing_relations = adapter.list_relations(schema=schema) -%} + {%- set old_relation = adapter.get_relation(relations_list=existing_relations, + schema=schema, identifier=identifier) -%} + {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, + type='view') -%} + {%- set intermediate_relation = api.Relation.create(identifier=tmp_identifier, + schema=schema, type='view') -%} + + {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%} {%- set has_transactional_hooks = (hooks | selectattr('transaction', 'equalto', True) | list | length) > 0 %} - {%- set should_ignore = non_destructive_mode and existing_type == 'view' %} + {%- set should_ignore = non_destructive_mode and exists_as_view %} {{ run_hooks(pre_hooks, inside_transaction=False) }} - {{ drop_if_exists(existing, schema, tmp_identifier) }} + {{ adapter.drop_relation(intermediate_relation) }} -- `BEGIN` happens here: {{ run_hooks(pre_hooks, inside_transaction=True) }} @@ -29,7 +37,7 @@ {%- endcall %} {%- else -%} {% call statement('main') -%} - {{ create_view_as(tmp_identifier, sql) }} + {{ create_view_as(intermediate_relation, sql) }} {%- endcall %} {%- endif %} @@ -37,8 +45,8 @@ -- cleanup {% if not should_ignore -%} - {{ drop_if_exists(existing, schema, identifier) }} - {{ adapter.rename(schema, tmp_identifier, identifier) }} + {{ drop_relation_if_exists(old_relation) }} + {{ adapter.rename_relation(intermediate_relation, target_relation) }} {%- endif %} {# diff --git a/dbt/links.py b/dbt/links.py new file mode 100644 index 00000000000..3be2c275d9e --- /dev/null +++ b/dbt/links.py @@ -0,0 +1,2 @@ + +SnowflakeQuotingDocs = 'https://docs.getdbt.com/v0.10/docs/configuring-quoting' diff --git a/dbt/logger.py b/dbt/logger.py index 301c3a72bc3..2ad662aabe8 100644 --- a/dbt/logger.py +++ b/dbt/logger.py @@ -33,6 +33,14 @@ logger.setLevel(logging.DEBUG) logging.getLogger().setLevel(logging.CRITICAL) +# Quiet these down in the logs +logging.getLogger('botocore').setLevel(logging.INFO) +logging.getLogger('requests').setLevel(logging.INFO) +logging.getLogger('urllib3').setLevel(logging.INFO) +logging.getLogger('google').setLevel(logging.INFO) +logging.getLogger('snowflake.connector').setLevel(logging.INFO) +logging.getLogger('parsedatetime').setLevel(logging.INFO) + # Redirect warnings through our logging setup # They will be logged to a file below logging.captureWarnings(True) diff --git a/dbt/main.py b/dbt/main.py index 3282090156f..29388240bab 100644 --- a/dbt/main.py +++ b/dbt/main.py @@ -232,6 +232,8 @@ def invoke_dbt(parsed): return None + proj.log_warnings() + flags.NON_DESTRUCTIVE = getattr(proj.args, 'non_destructive', False) arg_drop_existing = getattr(proj.args, 'drop_existing', False) diff --git a/dbt/model.py b/dbt/model.py index 24a9cfd8520..e169f25007f 100644 --- a/dbt/model.py +++ b/dbt/model.py @@ -12,7 +12,7 @@ class SourceConfig(object): ConfigKeys = DBTConfigKeys AppendListFields = ['pre-hook', 'post-hook'] - ExtendDictFields = ['vars', 'column_types'] + ExtendDictFields = ['vars', 'column_types', 'quoting'] ClobberFields = [ 'schema', 'enabled', @@ -77,7 +77,8 @@ def config(self): active_config = self.load_config_from_active_project() if self.active_project['name'] == self.own_project['name']: - cfg = self._merge(defaults, active_config, self.in_model_config) + cfg = self._merge(defaults, active_config, + self.in_model_config) else: own_config = self.load_config_from_own_project() diff --git a/dbt/node_runners.py b/dbt/node_runners.py index e4247c91b2a..7da8080c6ce 100644 --- a/dbt/node_runners.py +++ b/dbt/node_runners.py @@ -89,7 +89,7 @@ def is_ephemeral(cls, node): def is_ephemeral_model(cls, node): return cls.is_refable(node) and cls.is_ephemeral(node) - def safe_run(self, flat_graph, existing): + def safe_run(self, flat_graph): catchable_errors = (dbt.exceptions.CompilationException, dbt.exceptions.RuntimeException) @@ -104,7 +104,7 @@ def safe_run(self, flat_graph, existing): # for ephemeral nodes, we only want to compile, not run if not self.is_ephemeral_model(self.node): - result = self.run(compiled_node, existing, flat_graph) + result = self.run(compiled_node, flat_graph) except catchable_errors as e: if e.node is None: @@ -147,11 +147,11 @@ def safe_run(self, flat_graph, existing): def before_execute(self): raise NotImplementedException() - def execute(self, compiled_node, existing, flat_graph): + def execute(self, compiled_node, flat_graph): raise NotImplementedException() - def run(self, compiled_node, existing, flat_graph): - return self.execute(compiled_node, existing, flat_graph) + def run(self, compiled_node, flat_graph): + return self.execute(compiled_node, flat_graph) def after_execute(self, result): raise NotImplementedException() @@ -208,7 +208,7 @@ def before_execute(self): def after_execute(self, result): pass - def execute(self, compiled_node, existing, flat_graph): + def execute(self, compiled_node, flat_graph): return RunModelResult(compiled_node) def compile(self, flat_graph): @@ -250,33 +250,34 @@ def _node_context(cls, adapter, project, node): def call_get_columns_in_table(schema_name, table_name): return adapter.get_columns_in_table( - profile, schema_name, table_name, model_name=node.get('name')) + profile, project, schema_name, + table_name, model_name=node.get('name')) def call_get_missing_columns(from_schema, from_table, to_schema, to_table): return adapter.get_missing_columns( - profile, from_schema, from_table, + profile, project, from_schema, from_table, to_schema, to_table, node.get('name')) - def call_table_exists(schema, table): - return adapter.table_exists( - profile, schema, table, node.get('name')) + def call_already_exists(schema, table): + return adapter.already_exists( + profile, project, schema, table, node.get('name')) return { "run_started_at": dbt.tracking.active_user.run_started_at, "invocation_id": dbt.tracking.active_user.invocation_id, "get_columns_in_table": call_get_columns_in_table, "get_missing_columns": call_get_missing_columns, - "already_exists": call_table_exists, + "already_exists": call_already_exists, } @classmethod def create_schemas(cls, project, adapter, flat_graph): profile = project.run_environment() required_schemas = cls.get_model_schemas(flat_graph) - existing_schemas = set(adapter.get_existing_schemas(profile)) + existing_schemas = set(adapter.get_existing_schemas(profile, project)) for schema in (required_schemas - existing_schemas): - adapter.create_schema(profile, schema) + adapter.create_schema(profile, project, schema) class ModelRunner(CompileRunner): @@ -345,12 +346,12 @@ def create_schemas(cls, project, adapter, flat_graph): # is the one defined in the profile. Create this schema if it # does not exist, otherwise subsequent queries will fail. Generally, # dbt expects that this schema will exist anyway. - required_schemas.add(adapter.get_default_schema(profile)) + required_schemas.add(adapter.get_default_schema(profile, project)) - existing_schemas = set(adapter.get_existing_schemas(profile)) + existing_schemas = set(adapter.get_existing_schemas(profile, project)) for schema in (required_schemas - existing_schemas): - adapter.create_schema(profile, schema) + adapter.create_schema(profile, project, schema) @classmethod def before_run(cls, project, adapter, flat_graph): @@ -407,8 +408,9 @@ def after_execute(self, result): track_model_run(self.node_index, self.num_nodes, result) self.print_result_line(result) - def execute(self, model, existing, flat_graph): - context = dbt.context.runtime.generate(model, self.project, flat_graph) + def execute(self, model, flat_graph): + context = dbt.context.runtime.generate( + model, self.project.cfg, flat_graph) materialization_macro = dbt.utils.get_materialization_macro( flat_graph, @@ -467,7 +469,7 @@ def execute_test(self, test): def before_execute(self): self.print_start_line() - def execute(self, test, existing, flat_graph): + def execute(self, test, flat_graph): status = self.execute_test(test) return RunModelResult(test, status=status) diff --git a/dbt/project.py b/dbt/project.py index a59ab8189f6..cb541027f3d 100644 --- a/dbt/project.py +++ b/dbt/project.py @@ -12,6 +12,8 @@ import dbt.compat import dbt.context.common import dbt.clients.system +import dbt.ui.printer +import dbt.links from dbt.logger import GLOBAL_LOGGER as logger # noqa @@ -25,6 +27,7 @@ 'outputs': {'default': {}}, 'target': 'default', 'models': {}, + 'quoting': {}, 'profile': None, 'packages': [], 'modules-path': 'dbt_modules' @@ -91,6 +94,9 @@ def __init__(self, cfg, profiles, profiles_dir, profile_to_load=None, if self.cfg.get('models') is None: self.cfg['models'] = {} + if self.cfg.get('quoting') is None: + self.cfg['quoting'] = {} + if self.cfg['models'].get('vars') is None: self.cfg['models']['vars'] = {} @@ -220,6 +226,23 @@ def validate(self): "Expected project configuration '{}' was not supplied" .format('.'.join(e.path)), self) + def log_warnings(self): + target_cfg = self.run_environment() + db_type = target_cfg.get('type') + + if db_type == 'snowflake' and self.cfg \ + .get('quoting', {}) \ + .get('identifier') is None: + msg = dbt.ui.printer.yellow( + 'You are using Snowflake, but you did not specify a ' + 'quoting strategy for your identifiers.\nQuoting ' + 'behavior for Snowflake will change in a future release, ' + 'so it is recommended that you define this explicitly.\n\n' + 'For more information, see: {}\n' + ) + + logger.warn(msg.format(dbt.links.SnowflakeQuotingDocs)) + def hashed_name(self): if self.cfg.get("name", None) is None: return None diff --git a/dbt/runner.py b/dbt/runner.py index efb9ff7427e..923bb9857e4 100644 --- a/dbt/runner.py +++ b/dbt/runner.py @@ -69,7 +69,6 @@ def get_runners(self, Runner, adapter, node_dependency_list): def call_runner(self, data): runner = data['runner'] - existing = data['existing'] flat_graph = data['flat_graph'] if runner.skip: @@ -79,7 +78,7 @@ def call_runner(self, data): if not runner.is_ephemeral_model(runner.node): runner.before_execute() - result = runner.safe_run(flat_graph, existing) + result = runner.safe_run(flat_graph) if not runner.is_ephemeral_model(runner.node): runner.after_execute(result) @@ -110,10 +109,6 @@ def execute_nodes(self, linker, Runner, flat_graph, node_dependency_list): dbt.ui.printer.print_timestamped_line("") schemas = list(Runner.get_model_schemas(flat_graph)) - if len(schemas) > 0: - existing = adapter.query_for_existing(profile, schemas) - else: - existing = {} node_runners = self.get_runners(Runner, adapter, node_dependency_list) pool = ThreadPool(num_threads) @@ -124,7 +119,6 @@ def execute_nodes(self, linker, Runner, flat_graph, node_dependency_list): args_list = [] for runner in runners: args_list.append({ - 'existing': existing, 'flat_graph': flat_graph, 'runner': runner }) diff --git a/dbt/utils.py b/dbt/utils.py index da0b9036816..71a3f02af98 100644 --- a/dbt/utils.py +++ b/dbt/utils.py @@ -28,6 +28,7 @@ 'vars', 'column_types', 'bind', + 'quoting', ] @@ -37,52 +38,6 @@ class ExitCodes(object): UnhandledError = 2 -class Relation(object): - def __init__(self, profile, adapter, node, use_temp=False): - self.node = node - self.schema = node.get('schema') - self.name = node.get('name') - - if use_temp: - self.table = self._get_table_name(node) - else: - self.table = self.name - - self.materialized = get_materialization(node) - self.sql = node.get('injected_sql') - - self.do_quote = self._get_quote_function(profile, adapter) - - def _get_quote_function(self, profile, adapter): - - # make a closure so we don't need to store the profile - # on the `Relation` object. That shouldn't be accessible in user-land - def quote(schema, table): - return adapter.quote_schema_and_table( - profile=profile, - schema=schema, - table=table - ) - - return quote - - def _get_table_name(self, node): - return model_immediate_name(node, dbt.flags.NON_DESTRUCTIVE) - - def final_name(self): - if self.materialized == 'ephemeral': - msg = "final_name() was called on an ephemeral model" - dbt.exceptions.raise_compiler_error(msg, self.node) - else: - return self.do_quote(self.schema, self.name) - - def __repr__(self): - if self.materialized == 'ephemeral': - return '__dbt__CTE__{}'.format(self.name) - else: - return self.do_quote(self.schema, self.table) - - def coalesce(*args): for arg in args: if arg is not None: @@ -441,3 +396,12 @@ def parse_cli_vars(var_string): logger.error( "The YAML provided in the --vars argument is not valid.\n") raise + + +def filter_null_values(input): + return dict((k, v) for (k, v) in input.items() + if v is not None) + + +def add_ephemeral_model_prefix(s): + return '__dbt__CTE__{}'.format(s) diff --git a/requirements.txt b/requirements.txt index 57cd9ed3602..55c785bff56 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,3 +13,4 @@ colorama==0.3.9 google-cloud-bigquery==0.29.0 requests>=2.18.0 agate>=1.6,<2 +jsonschema==2.6.0 diff --git a/setup.py b/setup.py index 26c02753e17..a3cb877cdb6 100644 --- a/setup.py +++ b/setup.py @@ -45,5 +45,6 @@ 'colorama==0.3.9', 'google-cloud-bigquery==0.29.0', 'agate>=1.6,<2', + 'jsonschema==2.6.0', ] ) diff --git a/test/integration/001_simple_copy_test/models/advanced_incremental.sql b/test/integration/001_simple_copy_test/models/advanced_incremental.sql new file mode 100644 index 00000000000..0072463c1a0 --- /dev/null +++ b/test/integration/001_simple_copy_test/models/advanced_incremental.sql @@ -0,0 +1,17 @@ +{{ + config( + materialized = "incremental", + sql_where = "TRUE", + unique_key = "id" + ) +}} + + +select * +from {{ ref('seed') }} + +{% if adapter.already_exists(this.schema, this.table) and not flags.FULL_REFRESH %} + + where id > (select max(id) from {{this}}) + +{% endif %} diff --git a/test/integration/001_simple_copy_test/models/compound_sort.sql b/test/integration/001_simple_copy_test/models/compound_sort.sql index 236de4a08b9..64b41ca7ebd 100644 --- a/test/integration/001_simple_copy_test/models/compound_sort.sql +++ b/test/integration/001_simple_copy_test/models/compound_sort.sql @@ -6,4 +6,4 @@ ) }} -select * from "{{ target.schema }}"."seed" +select * from {{ ref('seed') }} diff --git a/test/integration/001_simple_copy_test/models/disabled.sql b/test/integration/001_simple_copy_test/models/disabled.sql index 44d93ff8b83..1d10a0c8d41 100644 --- a/test/integration/001_simple_copy_test/models/disabled.sql +++ b/test/integration/001_simple_copy_test/models/disabled.sql @@ -5,4 +5,4 @@ ) }} -select * from "{{ target.schema }}"."seed" +select * from {{ ref('seed') }} diff --git a/test/integration/001_simple_copy_test/models/incremental.sql b/test/integration/001_simple_copy_test/models/incremental.sql index 591b77faa5d..72e0c872e6b 100644 --- a/test/integration/001_simple_copy_test/models/incremental.sql +++ b/test/integration/001_simple_copy_test/models/incremental.sql @@ -5,4 +5,4 @@ ) }} -select * from "{{ target.schema }}"."seed" +select * from {{ ref('seed') }} diff --git a/test/integration/001_simple_copy_test/models/interleaved_sort.sql b/test/integration/001_simple_copy_test/models/interleaved_sort.sql index ff4eba3c19b..147370396e2 100644 --- a/test/integration/001_simple_copy_test/models/interleaved_sort.sql +++ b/test/integration/001_simple_copy_test/models/interleaved_sort.sql @@ -6,4 +6,4 @@ ) }} -select * from "{{ target.schema }}"."seed" +select * from {{ ref('seed') }} diff --git a/test/integration/001_simple_copy_test/models/materialized.sql b/test/integration/001_simple_copy_test/models/materialized.sql index 5130aa9e4db..d29676d672b 100644 --- a/test/integration/001_simple_copy_test/models/materialized.sql +++ b/test/integration/001_simple_copy_test/models/materialized.sql @@ -5,4 +5,4 @@ }} -- this is a unicode character: å -select * from "{{ target.schema }}"."seed" +select * from {{ ref('seed') }} diff --git a/test/integration/012_profile_config_tests/models/view.sql b/test/integration/001_simple_copy_test/models/view_model.sql similarity index 53% rename from test/integration/012_profile_config_tests/models/view.sql rename to test/integration/001_simple_copy_test/models/view_model.sql index 15f6caffe13..9838c5b839a 100644 --- a/test/integration/012_profile_config_tests/models/view.sql +++ b/test/integration/001_simple_copy_test/models/view_model.sql @@ -4,4 +4,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ ref('seed') }} diff --git a/test/integration/001_simple_copy_test/test_simple_copy.py b/test/integration/001_simple_copy_test/test_simple_copy.py index fcf720f7aac..8733126d786 100644 --- a/test/integration/001_simple_copy_test/test_simple_copy.py +++ b/test/integration/001_simple_copy_test/test_simple_copy.py @@ -21,13 +21,13 @@ def models(self): @attr(type="postgres") def test__postgres__simple_copy(self): - self.use_default_project({"data-paths": [self.dir("seed-initial")]}) self.use_profile("postgres") + self.use_default_project({"data-paths": [self.dir("seed-initial")]}) self.run_dbt(["seed"]) self.run_dbt() - self.assertTablesEqual("seed","view") + self.assertTablesEqual("seed","view_model") self.assertTablesEqual("seed","incremental") self.assertTablesEqual("seed","materialized") @@ -35,14 +35,14 @@ def test__postgres__simple_copy(self): self.run_dbt(["seed"]) self.run_dbt() - self.assertTablesEqual("seed","view") + self.assertTablesEqual("seed","view_model") self.assertTablesEqual("seed","incremental") self.assertTablesEqual("seed","materialized") @attr(type="postgres") def test__postgres__dbt_doesnt_run_empty_models(self): - self.use_default_project({"data-paths": [self.dir("seed-initial")]}) self.use_profile("postgres") + self.use_default_project({"data-paths": [self.dir("seed-initial")]}) self.run_dbt(["seed"]) self.run_dbt() @@ -54,20 +54,88 @@ def test__postgres__dbt_doesnt_run_empty_models(self): @attr(type="snowflake") def test__snowflake__simple_copy(self): - self.use_default_project({"data-paths": [self.dir("seed-initial")]}) self.use_profile("snowflake") + self.use_default_project({"data-paths": [self.dir("seed-initial")]}) self.run_dbt(["seed"]) self.run_dbt() - self.assertTablesEqual("seed","view") - self.assertTablesEqual("seed","incremental") - self.assertTablesEqual("seed","materialized") + self.assertTablesEqual("seed", "view_model") + self.assertTablesEqual("seed", "incremental") + self.assertTablesEqual("seed", "materialized") self.use_default_project({"data-paths": [self.dir("seed-update")]}) self.run_dbt(["seed"]) self.run_dbt() - self.assertTablesEqual("seed","view") - self.assertTablesEqual("seed","incremental") - self.assertTablesEqual("seed","materialized") + self.assertTablesEqual("seed", "view_model") + self.assertTablesEqual("seed", "incremental") + self.assertTablesEqual("seed", "materialized") + + @attr(type="snowflake") + def test__snowflake__simple_copy__quoting_on(self): + self.use_profile("snowflake") + self.use_default_project({ + "data-paths": [self.dir("seed-initial")], + "quoting": {"identifier": True}, + }) + + self.run_dbt(["seed"]) + self.run_dbt() + + self.assertTablesEqual("seed", "view_model") + self.assertTablesEqual("seed", "incremental") + self.assertTablesEqual("seed", "materialized") + + self.use_default_project({ + "data-paths": [self.dir("seed-update")], + "quoting": {"identifier": True}, + }) + self.run_dbt(["seed"]) + self.run_dbt() + + self.assertTablesEqual("seed", "view_model") + self.assertTablesEqual("seed", "incremental") + self.assertTablesEqual("seed", "materialized") + + @attr(type="snowflake") + def test__snowflake__simple_copy__quoting_off(self): + self.use_profile("snowflake") + self.use_default_project({ + "data-paths": [self.dir("seed-initial")], + "quoting": {"identifier": False}, + }) + + self.run_dbt(["seed"]) + self.run_dbt() + + self.assertTablesEqual("SEED", "VIEW_MODEL") + self.assertTablesEqual("SEED", "INCREMENTAL") + self.assertTablesEqual("SEED", "MATERIALIZED") + + self.use_default_project({ + "data-paths": [self.dir("seed-update")], + "quoting": {"identifier": False}, + }) + self.run_dbt(["seed"]) + self.run_dbt() + + self.assertTablesEqual("SEED", "VIEW_MODEL") + self.assertTablesEqual("SEED", "INCREMENTAL") + self.assertTablesEqual("SEED", "MATERIALIZED") + + @attr(type="snowflake") + def test__snowflake__seed__quoting_switch(self): + self.use_profile("snowflake") + self.use_default_project({ + "data-paths": [self.dir("seed-initial")], + "quoting": {"identifier": False}, + }) + + self.run_dbt(["seed"]) + + self.use_default_project({ + "data-paths": [self.dir("seed-update")], + "quoting": {"identifier": True}, + }) + self.run_dbt(["seed"], expect_pass=False) diff --git a/test/integration/002_varchar_widening_test/models/incremental.sql b/test/integration/002_varchar_widening_test/models/incremental.sql index 82992c64e1a..10141cd3f13 100644 --- a/test/integration/002_varchar_widening_test/models/incremental.sql +++ b/test/integration/002_varchar_widening_test/models/incremental.sql @@ -5,4 +5,10 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed + +{% if adapter.already_exists(this.schema, this.table) %} + + where id > (select max(id) from {{this}}) + +{% endif %} diff --git a/test/integration/002_varchar_widening_test/models/materialized.sql b/test/integration/002_varchar_widening_test/models/materialized.sql index 314afafac59..79dd477a36e 100644 --- a/test/integration/002_varchar_widening_test/models/materialized.sql +++ b/test/integration/002_varchar_widening_test/models/materialized.sql @@ -4,4 +4,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/002_varchar_widening_test/seed.sql b/test/integration/002_varchar_widening_test/seed.sql index 43d320f45a6..7406b18a067 100644 --- a/test/integration/002_varchar_widening_test/seed.sql +++ b/test/integration/002_varchar_widening_test/seed.sql @@ -1,4 +1,4 @@ -create table "{schema}"."seed" ( +create table {schema}.seed ( id BIGSERIAL PRIMARY KEY, first_name VARCHAR(50), last_name VARCHAR(50), @@ -7,7 +7,7 @@ create table "{schema}"."seed" ( ip_address VARCHAR(20) ); -insert into "{schema}"."seed" (first_name, last_name, email, gender, ip_address) values +insert into {schema}.seed (first_name, last_name, email, gender, ip_address) values ('Jack', 'Hunter', 'jhunter0@pbs.org', 'Male', '59.80.20.168'), ('Kathryn', 'Walker', 'kwalker1@ezinearticles.com', 'Female', '194.121.179.35'), ('Gerald', 'Ryan', 'gryan2@com.com', 'Male', '11.3.212.243'), diff --git a/test/integration/002_varchar_widening_test/test_varchar_widening.py b/test/integration/002_varchar_widening_test/test_varchar_widening.py index fcf9a6b7d83..e53a5d5ea0f 100644 --- a/test/integration/002_varchar_widening_test/test_varchar_widening.py +++ b/test/integration/002_varchar_widening_test/test_varchar_widening.py @@ -16,8 +16,8 @@ def models(self): @attr(type='postgres') def test__postgres__varchar_widening(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() self.run_sql_file("test/integration/002_varchar_widening_test/seed.sql") self.run_dbt() @@ -34,18 +34,18 @@ def test__postgres__varchar_widening(self): @attr(type='snowflake') def test__snowflake__varchar_widening(self): - self.use_default_project() self.use_profile('snowflake') + self.use_default_project() self.run_sql_file("test/integration/002_varchar_widening_test/seed.sql") self.run_dbt() - self.assertTablesEqual("seed","incremental") - self.assertTablesEqual("seed","materialized") + self.assertTablesEqual("SEED", "incremental") + self.assertTablesEqual("SEED", "materialized") self.run_sql_file("test/integration/002_varchar_widening_test/update.sql") self.run_dbt() - self.assertTablesEqual("seed","incremental") - self.assertTablesEqual("seed","materialized") + self.assertTablesEqual("SEED", "incremental") + self.assertTablesEqual("SEED", "materialized") diff --git a/test/integration/002_varchar_widening_test/update.sql b/test/integration/002_varchar_widening_test/update.sql index e216a3ac798..7398d436534 100644 --- a/test/integration/002_varchar_widening_test/update.sql +++ b/test/integration/002_varchar_widening_test/update.sql @@ -1,6 +1,6 @@ -ALTER TABLE "{schema}"."seed" ALTER COLUMN gender TYPE varchar(300); +ALTER TABLE {schema}.seed ALTER COLUMN gender TYPE varchar(300); -insert into "{schema}"."seed" (first_name, last_name, email, gender, ip_address) values +insert into {schema}.seed (first_name, last_name, email, gender, ip_address) values ('Annie', 'Reynolds', 'areynolds0@nifty.com', 'Amerisource Bergen', '133.30.242.211'), ('Doris', 'Wood', 'dwood1@skyrock.com', 'Bliss World, LLC', '128.229.89.207'), ('Andrea', 'Ray', 'aray2@google.co.jp', 'Nelco Laboratories, Inc.', '109.74.153.45'), diff --git a/test/integration/003_simple_reference_test/models/ephemeral_copy.sql b/test/integration/003_simple_reference_test/models/ephemeral_copy.sql index 66823f4030a..c6c2c53e900 100644 --- a/test/integration/003_simple_reference_test/models/ephemeral_copy.sql +++ b/test/integration/003_simple_reference_test/models/ephemeral_copy.sql @@ -4,4 +4,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/003_simple_reference_test/models/incremental_copy.sql b/test/integration/003_simple_reference_test/models/incremental_copy.sql index 82992c64e1a..d7e63d2a5d1 100644 --- a/test/integration/003_simple_reference_test/models/incremental_copy.sql +++ b/test/integration/003_simple_reference_test/models/incremental_copy.sql @@ -5,4 +5,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/003_simple_reference_test/models/materialized_copy.sql b/test/integration/003_simple_reference_test/models/materialized_copy.sql index 314afafac59..79dd477a36e 100644 --- a/test/integration/003_simple_reference_test/models/materialized_copy.sql +++ b/test/integration/003_simple_reference_test/models/materialized_copy.sql @@ -4,4 +4,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/003_simple_reference_test/models/view_copy.sql b/test/integration/003_simple_reference_test/models/view_copy.sql index 15f6caffe13..e0560b694c9 100644 --- a/test/integration/003_simple_reference_test/models/view_copy.sql +++ b/test/integration/003_simple_reference_test/models/view_copy.sql @@ -4,4 +4,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/003_simple_reference_test/seed.sql b/test/integration/003_simple_reference_test/seed.sql index 550964ea8a2..63f5384a442 100644 --- a/test/integration/003_simple_reference_test/seed.sql +++ b/test/integration/003_simple_reference_test/seed.sql @@ -1,13 +1,13 @@ -create table "{schema}"."summary_expected" ( +create table {schema}.summary_expected ( gender VARCHAR(10), ct BIGINT ); -insert into "{schema}"."summary_expected" (gender, ct) values +insert into {schema}.summary_expected (gender, ct) values ('Female', 40), ('Male', 60); -create table "{schema}"."seed" ( +create table {schema}.seed ( id BIGSERIAL PRIMARY KEY, first_name VARCHAR(50), last_name VARCHAR(50), @@ -16,7 +16,7 @@ create table "{schema}"."seed" ( ip_address VARCHAR(20) ); -insert into "{schema}"."seed" (first_name, last_name, email, gender, ip_address) values +insert into {schema}.seed (first_name, last_name, email, gender, ip_address) values ('Jack', 'Hunter', 'jhunter0@pbs.org', 'Male', '59.80.20.168'), ('Kathryn', 'Walker', 'kwalker1@ezinearticles.com', 'Female', '194.121.179.35'), ('Gerald', 'Ryan', 'gryan2@com.com', 'Male', '11.3.212.243'), diff --git a/test/integration/003_simple_reference_test/test_simple_reference.py b/test/integration/003_simple_reference_test/test_simple_reference.py index 91b9336b12d..2f0ab0a6916 100644 --- a/test/integration/003_simple_reference_test/test_simple_reference.py +++ b/test/integration/003_simple_reference_test/test_simple_reference.py @@ -16,9 +16,10 @@ def models(self): @attr(type='postgres') def test__postgres__simple_reference(self): - self.use_default_project() self.use_profile('postgres') - self.run_sql_file("test/integration/003_simple_reference_test/seed.sql") + self.use_default_project() + self.run_sql_file( + "test/integration/003_simple_reference_test/seed.sql") self.run_dbt() @@ -50,42 +51,43 @@ def test__postgres__simple_reference(self): @attr(type='snowflake') def test__snowflake__simple_reference(self): - self.use_default_project() self.use_profile('snowflake') + self.use_default_project() self.run_sql_file("test/integration/003_simple_reference_test/seed.sql") self.run_dbt() # Copies should match - self.assertTablesEqual("seed","incremental_copy") - self.assertTablesEqual("seed","materialized_copy") - self.assertTablesEqual("seed","view_copy") + self.assertTablesEqual("SEED", "incremental_copy") + self.assertTablesEqual("SEED", "materialized_copy") + self.assertTablesEqual("SEED", "view_copy") # Summaries should match - self.assertTablesEqual("summary_expected","incremental_summary") - self.assertTablesEqual("summary_expected","materialized_summary") - self.assertTablesEqual("summary_expected","view_summary") - self.assertTablesEqual("summary_expected","ephemeral_summary") + self.assertTablesEqual("SUMMARY_EXPECTED", "incremental_summary") + self.assertTablesEqual("SUMMARY_EXPECTED", "materialized_summary") + self.assertTablesEqual("SUMMARY_EXPECTED", "view_summary") + self.assertTablesEqual("SUMMARY_EXPECTED", "ephemeral_summary") - self.run_sql_file("test/integration/003_simple_reference_test/update.sql") + self.run_sql_file( + "test/integration/003_simple_reference_test/update.sql") self.run_dbt() # Copies should match - self.assertTablesEqual("seed","incremental_copy") - self.assertTablesEqual("seed","materialized_copy") - self.assertTablesEqual("seed","view_copy") + self.assertTablesEqual("SEED", "incremental_copy") + self.assertTablesEqual("SEED", "materialized_copy") + self.assertTablesEqual("SEED", "view_copy") # Summaries should match - self.assertTablesEqual("summary_expected","incremental_summary") - self.assertTablesEqual("summary_expected","materialized_summary") - self.assertTablesEqual("summary_expected","view_summary") - self.assertTablesEqual("summary_expected","ephemeral_summary") + self.assertTablesEqual("SUMMARY_EXPECTED", "incremental_summary") + self.assertTablesEqual("SUMMARY_EXPECTED", "materialized_summary") + self.assertTablesEqual("SUMMARY_EXPECTED", "view_summary") + self.assertTablesEqual("SUMMARY_EXPECTED", "ephemeral_summary") @attr(type='postgres') def test__postgres__simple_reference_with_models(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() self.run_sql_file("test/integration/003_simple_reference_test/seed.sql") # Run materialized_copy, ephemeral_copy, and their dependents @@ -100,8 +102,8 @@ def test__postgres__simple_reference_with_models(self): @attr(type='postgres') def test__postgres__simple_reference_with_models_and_children(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() self.run_sql_file("test/integration/003_simple_reference_test/seed.sql") # Run materialized_copy, ephemeral_copy, and their dependents @@ -136,8 +138,8 @@ def test__postgres__simple_reference_with_models_and_children(self): @attr(type='snowflake') def test__snowflake__simple_reference_with_models(self): - self.use_default_project() self.use_profile('snowflake') + self.use_default_project() self.run_sql_file("test/integration/003_simple_reference_test/seed.sql") # Run materialized_copy & ephemeral_copy @@ -145,15 +147,15 @@ def test__snowflake__simple_reference_with_models(self): self.run_dbt(['run', '--models', 'materialized_copy', 'ephemeral_copy']) # Copies should match - self.assertTablesEqual("seed","materialized_copy") + self.assertTablesEqual("SEED", "materialized_copy") created_models = self.get_models_in_schema() self.assertTrue('materialized_copy' in created_models) @attr(type='snowflake') def test__snowflake__simple_reference_with_models_and_children(self): - self.use_default_project() self.use_profile('snowflake') + self.use_default_project() self.run_sql_file("test/integration/003_simple_reference_test/seed.sql") # Run materialized_copy, ephemeral_copy, and their dependents @@ -162,11 +164,11 @@ def test__snowflake__simple_reference_with_models_and_children(self): self.run_dbt(['run', '--models', 'materialized_copy+', 'ephemeral_copy+']) # Copies should match - self.assertTablesEqual("seed","materialized_copy") + self.assertTablesEqual("SEED", "materialized_copy") # Summaries should match - self.assertTablesEqual("summary_expected","materialized_summary") - self.assertTablesEqual("summary_expected","ephemeral_summary") + self.assertTablesEqual("SUMMARY_EXPECTED", "materialized_summary") + self.assertTablesEqual("SUMMARY_EXPECTED", "ephemeral_summary") created_models = self.get_models_in_schema() diff --git a/test/integration/003_simple_reference_test/update.sql b/test/integration/003_simple_reference_test/update.sql index 816e4876078..abf3c0b340a 100644 --- a/test/integration/003_simple_reference_test/update.sql +++ b/test/integration/003_simple_reference_test/update.sql @@ -1,10 +1,10 @@ -truncate table "{schema}"."summary_expected"; -insert into "{schema}"."summary_expected" (gender, ct) values +truncate table {schema}.summary_expected; +insert into {schema}.summary_expected (gender, ct) values ('Female', 94), ('Male', 106); -insert into "{schema}"."seed" (first_name, last_name, email, gender, ip_address) values +insert into {schema}.seed (first_name, last_name, email, gender, ip_address) values ('Michael', 'Perez', 'mperez0@chronoengine.com', 'Male', '106.239.70.175'), ('Shawn', 'Mccoy', 'smccoy1@reddit.com', 'Male', '24.165.76.182'), ('Kathleen', 'Payne', 'kpayne2@cargocollective.com', 'Female', '113.207.168.106'), diff --git a/test/integration/004_simple_archive_test/invalidate_postgres.sql b/test/integration/004_simple_archive_test/invalidate_postgres.sql index 779f9b09804..30902625059 100644 --- a/test/integration/004_simple_archive_test/invalidate_postgres.sql +++ b/test/integration/004_simple_archive_test/invalidate_postgres.sql @@ -1,12 +1,12 @@ -- update records 11 - 21. Change email and updated_at field -update "{schema}"."seed" set +update {schema}.seed set "updated_at" = "updated_at" + interval '1 hour', "email" = 'new_' || "email" where "id" >= 10 and "id" <= 20; -- invalidate records 11 - 21 -update "{schema}"."archive_expected" set +update {schema}.archive_expected set "valid_to" = "updated_at" + interval '1 hour' where "id" >= 10 and "id" <= 20; diff --git a/test/integration/004_simple_archive_test/invalidate_snowflake.sql b/test/integration/004_simple_archive_test/invalidate_snowflake.sql index 276c3da6a4d..879d2966d05 100644 --- a/test/integration/004_simple_archive_test/invalidate_snowflake.sql +++ b/test/integration/004_simple_archive_test/invalidate_snowflake.sql @@ -1,12 +1,12 @@ -- update records 11 - 21. Change email and updated_at field -update "{schema}"."seed" set +update {schema}.seed set "updated_at" = DATEADD(hour, 1, "updated_at"), "email" = 'new_' || "email" where "id" >= 10 and "id" <= 20; -- invalidate records 11 - 21 -update "{schema}"."archive_expected" set +update {schema}.archive_expected set "valid_to" = DATEADD(hour, 1, "updated_at") where "id" >= 10 and "id" <= 20; diff --git a/test/integration/004_simple_archive_test/seed.sql b/test/integration/004_simple_archive_test/seed.sql index c3256b7d735..99b5bf4bbe5 100644 --- a/test/integration/004_simple_archive_test/seed.sql +++ b/test/integration/004_simple_archive_test/seed.sql @@ -1,4 +1,4 @@ -create table "{schema}"."seed" ( +create table {schema}.seed ( "id" INTEGER, "first_name" VARCHAR(50), "last_name" VARCHAR(50), @@ -8,7 +8,7 @@ create table "{schema}"."seed" ( "updated_at" TIMESTAMP WITHOUT TIME ZONE ); -create table "{schema}"."archive_expected" ( +create table {schema}.archive_expected ( "id" INTEGER, "first_name" VARCHAR(50), "last_name" VARCHAR(50), @@ -26,7 +26,7 @@ create table "{schema}"."archive_expected" ( -- seed inserts -insert into "{schema}"."seed" ("id", "first_name", "last_name", "email", "gender", "ip_address", "updated_at") values +insert into {schema}.seed ("id", "first_name", "last_name", "email", "gender", "ip_address", "updated_at") values (1, 'Judith', 'Kennedy', 'jkennedy0@phpbb.com', 'Female', '54.60.24.128', '2015-12-24 12:19:28'), (2, 'Arthur', 'Kelly', 'akelly1@eepurl.com', 'Male', '62.56.24.215', '2015-10-28 16:22:15'), (3, 'Rachel', 'Moreno', 'rmoreno2@msu.edu', 'Female', '31.222.249.23', '2016-04-05 02:05:30'), @@ -50,7 +50,7 @@ insert into "{schema}"."seed" ("id", "first_name", "last_name", "email", "gender -- populate archive table -insert into "{schema}"."archive_expected" ( +insert into {schema}.archive_expected ( "id", "first_name", "last_name", @@ -77,4 +77,4 @@ select null::timestamp as valid_to, "updated_at" as dbt_updated_at, md5("id" || '-' || "first_name" || '|' || "updated_at"::text) as scd_id -from "{schema}"."seed"; +from {schema}.seed; diff --git a/test/integration/004_simple_archive_test/test_simple_archive.py b/test/integration/004_simple_archive_test/test_simple_archive.py index 58bb9f4cc51..c2e7a3c3d23 100644 --- a/test/integration/004_simple_archive_test/test_simple_archive.py +++ b/test/integration/004_simple_archive_test/test_simple_archive.py @@ -16,6 +16,11 @@ def models(self): @property def project_config(self): + source_table = 'seed' + + if self.adapter_type == 'snowflake': + source_table = source_table.upper() + return { "archive": [ { @@ -23,7 +28,7 @@ def project_config(self): "target_schema": self.unique_schema(), "tables": [ { - "source_table": "seed", + "source_table": source_table, "target_table": "archive_actual", "updated_at": '"updated_at"', "unique_key": '''"id" || '-' || "first_name"''' @@ -35,8 +40,8 @@ def project_config(self): @attr(type='postgres') def test__postgres__simple_archive(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() self.run_sql_file("test/integration/004_simple_archive_test/seed.sql") self.run_dbt(["archive"]) @@ -52,17 +57,17 @@ def test__postgres__simple_archive(self): @attr(type='snowflake') def test__snowflake__simple_archive(self): - self.use_default_project() self.use_profile('snowflake') + self.use_default_project() self.run_sql_file("test/integration/004_simple_archive_test/seed.sql") self.run_dbt(["archive"]) - self.assertTablesEqual("archive_expected","archive_actual") + self.assertTablesEqual("ARCHIVE_EXPECTED", "archive_actual") self.run_sql_file("test/integration/004_simple_archive_test/invalidate_snowflake.sql") self.run_sql_file("test/integration/004_simple_archive_test/update.sql") self.run_dbt(["archive"]) - self.assertTablesEqual("archive_expected","archive_actual") + self.assertTablesEqual("ARCHIVE_EXPECTED", "archive_actual") diff --git a/test/integration/004_simple_archive_test/update.sql b/test/integration/004_simple_archive_test/update.sql index d574e254702..32b954d7f0c 100644 --- a/test/integration/004_simple_archive_test/update.sql +++ b/test/integration/004_simple_archive_test/update.sql @@ -1,6 +1,6 @@ -- insert v2 of the 11 - 21 records -insert into "{schema}"."archive_expected" ( +insert into {schema}.archive_expected ( "id", "first_name", "last_name", @@ -27,12 +27,12 @@ select null::timestamp as "valid_to", "updated_at" as "dbt_updated_at", md5("id" || '-' || "first_name" || '|' || "updated_at"::text) as "scd_id" -from "{schema}"."seed" +from {schema}.seed where "id" >= 10 and "id" <= 20; -- insert 10 new records -insert into "{schema}"."seed" ("id", "first_name", "last_name", "email", "gender", "ip_address", "updated_at") values +insert into {schema}.seed ("id", "first_name", "last_name", "email", "gender", "ip_address", "updated_at") values (21, 'Judy', 'Robinson', 'jrobinsonk@blogs.com', 'Female', '208.21.192.232', '2016-09-18 08:27:38'), (22, 'Kevin', 'Alvarez', 'kalvarezl@buzzfeed.com', 'Male', '228.106.146.9', '2016-07-29 03:07:37'), (23, 'Barbara', 'Carr', 'bcarrm@pen.io', 'Female', '106.165.140.17', '2015-09-24 13:27:23'), @@ -46,7 +46,7 @@ insert into "{schema}"."seed" ("id", "first_name", "last_name", "email", "gender -- add these new records to the archive table -insert into "{schema}"."archive_expected" ( +insert into {schema}.archive_expected ( "id", "first_name", "last_name", @@ -73,5 +73,5 @@ select null::timestamp as "valid_to", "updated_at" as "dbt_updated_at", md5("id" || '-' || "first_name" || '|' || "updated_at"::text) as "scd_id" -from "{schema}"."seed" +from {schema}.seed where "id" > 20; diff --git a/test/integration/005_simple_seed_test/test_seed_type_override.py b/test/integration/005_simple_seed_test/test_seed_type_override.py index 658cd9d2065..98f26598693 100644 --- a/test/integration/005_simple_seed_test/test_seed_type_override.py +++ b/test/integration/005_simple_seed_test/test_seed_type_override.py @@ -1,6 +1,7 @@ from nose.plugins.attrib import attr from test.integration.base import DBTIntegrationTest + class TestSimpleSeedColumnOverride(DBTIntegrationTest): @property @@ -23,6 +24,7 @@ def project_config(self): } } + class TestSimpleSeedColumnOverridePostgres(TestSimpleSeedColumnOverride): @property def models(self): @@ -83,5 +85,3 @@ def profile_config(self): def test_simple_seed_with_column_override_bq(self): self.run_dbt(["seed"]) self.run_dbt(["test"]) - - diff --git a/test/integration/007_graph_selection_tests/models/base_users.sql b/test/integration/007_graph_selection_tests/models/base_users.sql index dd664b5cf32..fbab29807f3 100644 --- a/test/integration/007_graph_selection_tests/models/base_users.sql +++ b/test/integration/007_graph_selection_tests/models/base_users.sql @@ -5,4 +5,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/007_graph_selection_tests/seed.sql b/test/integration/007_graph_selection_tests/seed.sql index 550964ea8a2..63f5384a442 100644 --- a/test/integration/007_graph_selection_tests/seed.sql +++ b/test/integration/007_graph_selection_tests/seed.sql @@ -1,13 +1,13 @@ -create table "{schema}"."summary_expected" ( +create table {schema}.summary_expected ( gender VARCHAR(10), ct BIGINT ); -insert into "{schema}"."summary_expected" (gender, ct) values +insert into {schema}.summary_expected (gender, ct) values ('Female', 40), ('Male', 60); -create table "{schema}"."seed" ( +create table {schema}.seed ( id BIGSERIAL PRIMARY KEY, first_name VARCHAR(50), last_name VARCHAR(50), @@ -16,7 +16,7 @@ create table "{schema}"."seed" ( ip_address VARCHAR(20) ); -insert into "{schema}"."seed" (first_name, last_name, email, gender, ip_address) values +insert into {schema}.seed (first_name, last_name, email, gender, ip_address) values ('Jack', 'Hunter', 'jhunter0@pbs.org', 'Male', '59.80.20.168'), ('Kathryn', 'Walker', 'kwalker1@ezinearticles.com', 'Female', '194.121.179.35'), ('Gerald', 'Ryan', 'gryan2@com.com', 'Male', '11.3.212.243'), diff --git a/test/integration/007_graph_selection_tests/test_graph_selection.py b/test/integration/007_graph_selection_tests/test_graph_selection.py index 86aef1abc87..6d15d462b56 100644 --- a/test/integration/007_graph_selection_tests/test_graph_selection.py +++ b/test/integration/007_graph_selection_tests/test_graph_selection.py @@ -13,8 +13,8 @@ def models(self): @attr(type='postgres') def test__postgres__specific_model(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() self.run_sql_file("test/integration/007_graph_selection_tests/seed.sql") self.run_dbt(['run', '--models', 'users']) @@ -27,13 +27,13 @@ def test__postgres__specific_model(self): @attr(type='snowflake') def test__snowflake__specific_model(self): - self.use_default_project() self.use_profile('snowflake') + self.use_default_project() self.run_sql_file("test/integration/007_graph_selection_tests/seed.sql") self.run_dbt(['run', '--models', 'users']) - self.assertTablesEqual("seed", "users") + self.assertTablesEqual("SEED", "users") created_models = self.get_models_in_schema() self.assertFalse('users_rollup' in created_models) self.assertFalse('base_users' in created_models) @@ -42,8 +42,8 @@ def test__snowflake__specific_model(self): @attr(type='postgres') def test__postgres__specific_model_and_children(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() self.run_sql_file("test/integration/007_graph_selection_tests/seed.sql") self.run_dbt(['run', '--models', 'users+']) @@ -56,14 +56,14 @@ def test__postgres__specific_model_and_children(self): @attr(type='snowflake') def test__snowflake__specific_model_and_children(self): - self.use_default_project() self.use_profile('snowflake') + self.use_default_project() self.run_sql_file("test/integration/007_graph_selection_tests/seed.sql") self.run_dbt(['run', '--models', 'users+']) - self.assertTablesEqual("seed", "users") - self.assertTablesEqual("summary_expected", "users_rollup") + self.assertTablesEqual("SEED", "users") + self.assertTablesEqual("SUMMARY_EXPECTED", "users_rollup") created_models = self.get_models_in_schema() self.assertFalse('base_users' in created_models) self.assertFalse('emails' in created_models) @@ -71,8 +71,8 @@ def test__snowflake__specific_model_and_children(self): @attr(type='postgres') def test__postgres__specific_model_and_parents(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() self.run_sql_file("test/integration/007_graph_selection_tests/seed.sql") self.run_dbt(['run', '--models', '+users_rollup']) @@ -85,14 +85,14 @@ def test__postgres__specific_model_and_parents(self): @attr(type='snowflake') def test__snowflake__specific_model_and_parents(self): - self.use_default_project() self.use_profile('snowflake') + self.use_default_project() self.run_sql_file("test/integration/007_graph_selection_tests/seed.sql") self.run_dbt(['run', '--models', '+users_rollup']) - self.assertTablesEqual("seed", "users") - self.assertTablesEqual("summary_expected", "users_rollup") + self.assertTablesEqual("SEED", "users") + self.assertTablesEqual("SUMMARY_EXPECTED", "users_rollup") created_models = self.get_models_in_schema() self.assertFalse('base_users' in created_models) self.assertFalse('emails' in created_models) @@ -100,8 +100,8 @@ def test__snowflake__specific_model_and_parents(self): @attr(type='postgres') def test__postgres__specific_model_with_exclusion(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() self.run_sql_file("test/integration/007_graph_selection_tests/seed.sql") self.run_dbt(['run', '--models', '+users_rollup', '--exclude', 'users_rollup']) @@ -114,13 +114,13 @@ def test__postgres__specific_model_with_exclusion(self): @attr(type='snowflake') def test__snowflake__specific_model_with_exclusion(self): - self.use_default_project() self.use_profile('snowflake') + self.use_default_project() self.run_sql_file("test/integration/007_graph_selection_tests/seed.sql") self.run_dbt(['run', '--models', '+users_rollup', '--exclude', 'users_rollup']) - self.assertTablesEqual("seed", "users") + self.assertTablesEqual("SEED", "users") created_models = self.get_models_in_schema() self.assertFalse('base_users' in created_models) self.assertFalse('users_rollup' in created_models) diff --git a/test/integration/007_graph_selection_tests/test_schema_test_graph_selection.py b/test/integration/007_graph_selection_tests/test_schema_test_graph_selection.py index d01e490123f..c12a856c188 100644 --- a/test/integration/007_graph_selection_tests/test_schema_test_graph_selection.py +++ b/test/integration/007_graph_selection_tests/test_schema_test_graph_selection.py @@ -23,14 +23,10 @@ def project_config(self): ] } - def setUp(self): - DBTIntegrationTest.setUp(self) - - self.use_default_project() - self.project = read_project('dbt_project.yml') - def run_schema_and_assert(self, include, exclude, expected_tests): self.use_profile('postgres') + self.use_default_project() + self.project = read_project('dbt_project.yml') self.run_sql_file("test/integration/007_graph_selection_tests/seed.sql") self.run_dbt(["deps"]) diff --git a/test/integration/009_data_tests_test/models/table_copy.sql b/test/integration/009_data_tests_test/models/table_copy.sql index b7d5d7b745e..56e90a6d93c 100644 --- a/test/integration/009_data_tests_test/models/table_copy.sql +++ b/test/integration/009_data_tests_test/models/table_copy.sql @@ -5,4 +5,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/009_data_tests_test/seed.sql b/test/integration/009_data_tests_test/seed.sql index caac3985e6b..445c087cd66 100644 --- a/test/integration/009_data_tests_test/seed.sql +++ b/test/integration/009_data_tests_test/seed.sql @@ -1,4 +1,4 @@ -create table "{schema}"."seed" ( +create table {schema}.seed ( favorite_color VARCHAR(10), id INTEGER, first_name VARCHAR(11), @@ -8,7 +8,7 @@ create table "{schema}"."seed" ( ); -INSERT INTO "{schema}"."seed" +INSERT INTO {schema}.seed (favorite_color, id, first_name, email, ip_address, updated_at) VALUES ('blue', 1,'Larry','lking0@miitbeian.gov.cn','69.135.206.194','2008-09-12 19:08:31'), diff --git a/test/integration/010_permission_tests/models/view.sql b/test/integration/010_permission_tests/models/view.sql deleted file mode 100644 index ce40d77a0c0..00000000000 --- a/test/integration/010_permission_tests/models/view.sql +++ /dev/null @@ -1,2 +0,0 @@ - -select * from "{{ this.schema }}"."seed" diff --git a/test/integration/010_permission_tests/models/view_model.sql b/test/integration/010_permission_tests/models/view_model.sql new file mode 100644 index 00000000000..31dc91a8d51 --- /dev/null +++ b/test/integration/010_permission_tests/models/view_model.sql @@ -0,0 +1,2 @@ + +select * from {{ this.schema }}.seed diff --git a/test/integration/011_invalid_model_tests/models-2/view.sql b/test/integration/011_invalid_model_tests/models-2/view_model.sql similarity index 51% rename from test/integration/011_invalid_model_tests/models-2/view.sql rename to test/integration/011_invalid_model_tests/models-2/view_model.sql index ae0485cbcf1..162396e7850 100644 --- a/test/integration/011_invalid_model_tests/models-2/view.sql +++ b/test/integration/011_invalid_model_tests/models-2/view_model.sql @@ -4,4 +4,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/011_invalid_model_tests/models-3/dependent.sql b/test/integration/011_invalid_model_tests/models-3/dependent.sql index acf363bb859..7e8723a868a 100644 --- a/test/integration/011_invalid_model_tests/models-3/dependent.sql +++ b/test/integration/011_invalid_model_tests/models-3/dependent.sql @@ -1,4 +1,4 @@ -- view is disabled (defined in-model) -select * from {{ ref('view') }} +select * from {{ ref('view_model') }} diff --git a/test/integration/011_invalid_model_tests/models-3/view.sql b/test/integration/011_invalid_model_tests/models-3/view_model.sql similarity index 50% rename from test/integration/011_invalid_model_tests/models-3/view.sql rename to test/integration/011_invalid_model_tests/models-3/view_model.sql index ae8f688759a..b615ec282d8 100644 --- a/test/integration/011_invalid_model_tests/models-3/view.sql +++ b/test/integration/011_invalid_model_tests/models-3/view_model.sql @@ -4,4 +4,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/021_concurrency_test/models/view.sql b/test/integration/012_profile_config_tests/models/view_model.sql similarity index 53% rename from test/integration/021_concurrency_test/models/view.sql rename to test/integration/012_profile_config_tests/models/view_model.sql index 15f6caffe13..e0560b694c9 100644 --- a/test/integration/021_concurrency_test/models/view.sql +++ b/test/integration/012_profile_config_tests/models/view_model.sql @@ -4,4 +4,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/012_profile_config_tests/test_profile_config.py b/test/integration/012_profile_config_tests/test_profile_config.py index fbfdaf67df5..c9969020f69 100644 --- a/test/integration/012_profile_config_tests/test_profile_config.py +++ b/test/integration/012_profile_config_tests/test_profile_config.py @@ -42,6 +42,6 @@ def profile_config(self): def test_deprecated_run_target_config(self): self.run_dbt() - self.assertTablesEqual("seed","view") + self.assertTablesEqual("seed","view_model") self.assertFalse('run-target' in dbt.deprecations.active_deprecations) diff --git a/test/integration/013_context_var_tests/test_context_vars.py b/test/integration/013_context_var_tests/test_context_vars.py index dc19d3583c8..4b92ea480ae 100644 --- a/test/integration/013_context_var_tests/test_context_vars.py +++ b/test/integration/013_context_var_tests/test_context_vars.py @@ -1,9 +1,9 @@ from nose.plugins.attrib import attr from test.integration.base import DBTIntegrationTest -import dbt.flags import os + class TestContextVars(DBTIntegrationTest): def setUp(self): @@ -89,7 +89,10 @@ def test_env_vars_dev(self): self.run_dbt(['run']) ctx = self.get_ctx_vars() - self.assertEqual(ctx['this'], '"{}"."context__dbt_tmp"'.format(self.unique_schema())) + self.assertEqual( + ctx['this'], + '"{}"."context__dbt_tmp"'.format(self.unique_schema())) + self.assertEqual(ctx['this.name'], 'context') self.assertEqual(ctx['this.schema'], self.unique_schema()) self.assertEqual(ctx['this.table'], 'context__dbt_tmp') @@ -111,7 +114,10 @@ def test_env_vars_prod(self): self.run_dbt(['run', '--target', 'prod']) ctx = self.get_ctx_vars() - self.assertEqual(ctx['this'], '"{}"."context__dbt_tmp"'.format(self.unique_schema())) + self.assertEqual( + ctx['this'], + '"{}"."context__dbt_tmp"'.format(self.unique_schema())) + self.assertEqual(ctx['this.name'], 'context') self.assertEqual(ctx['this.schema'], self.unique_schema()) self.assertEqual(ctx['this.table'], 'context__dbt_tmp') diff --git a/test/integration/017_runtime_materialization_tests/create_view__dbt_tmp.sql b/test/integration/017_runtime_materialization_tests/create_view__dbt_tmp.sql index eeb4d2bca2d..6d92f998671 100644 --- a/test/integration/017_runtime_materialization_tests/create_view__dbt_tmp.sql +++ b/test/integration/017_runtime_materialization_tests/create_view__dbt_tmp.sql @@ -1,4 +1,4 @@ -create view {schema}.view__dbt_tmp as ( +create view {schema}.view_model__dbt_tmp as ( select 1 as id ); diff --git a/test/integration/017_runtime_materialization_tests/models/incremental.sql b/test/integration/017_runtime_materialization_tests/models/incremental.sql index 82992c64e1a..d7e63d2a5d1 100644 --- a/test/integration/017_runtime_materialization_tests/models/incremental.sql +++ b/test/integration/017_runtime_materialization_tests/models/incremental.sql @@ -5,4 +5,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/017_runtime_materialization_tests/models/materialized.sql b/test/integration/017_runtime_materialization_tests/models/materialized.sql index 314afafac59..79dd477a36e 100644 --- a/test/integration/017_runtime_materialization_tests/models/materialized.sql +++ b/test/integration/017_runtime_materialization_tests/models/materialized.sql @@ -4,4 +4,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/017_runtime_materialization_tests/models/view.sql b/test/integration/017_runtime_materialization_tests/models/view.sql index 15f6caffe13..e0560b694c9 100644 --- a/test/integration/017_runtime_materialization_tests/models/view.sql +++ b/test/integration/017_runtime_materialization_tests/models/view.sql @@ -4,4 +4,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/017_runtime_materialization_tests/test_runtime_materialization.py b/test/integration/017_runtime_materialization_tests/test_runtime_materialization.py index a2acbad9b8e..95f5f136219 100644 --- a/test/integration/017_runtime_materialization_tests/test_runtime_materialization.py +++ b/test/integration/017_runtime_materialization_tests/test_runtime_materialization.py @@ -82,5 +82,5 @@ def test_delete__dbt_tmp_relation(self): self.run_sql_file("test/integration/017_runtime_materialization_tests/create_view__dbt_tmp.sql") self.run_dbt(['run', '--model', 'view']) - self.assertTableDoesNotExist('view__dbt_tmp') + self.assertTableDoesNotExist('view__model_dbt_tmp') self.assertTablesEqual("seed","view") diff --git a/test/integration/018_adapter_ddl_tests/models/materialized.sql b/test/integration/018_adapter_ddl_tests/models/materialized.sql index a950c6bdf09..edd9c8e04bf 100644 --- a/test/integration/018_adapter_ddl_tests/models/materialized.sql +++ b/test/integration/018_adapter_ddl_tests/models/materialized.sql @@ -6,4 +6,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/020_ephemeral_test/models/base/base.sql b/test/integration/020_ephemeral_test/models/base/base.sql index edd3e7c6f46..5d9b6de149b 100644 --- a/test/integration/020_ephemeral_test/models/base/base.sql +++ b/test/integration/020_ephemeral_test/models/base/base.sql @@ -1,3 +1,3 @@ {{ config(materialized='ephemeral') }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/020_ephemeral_test/seed.sql b/test/integration/020_ephemeral_test/seed.sql index 28309b400f8..3c618ccab3a 100644 --- a/test/integration/020_ephemeral_test/seed.sql +++ b/test/integration/020_ephemeral_test/seed.sql @@ -1,4 +1,4 @@ -create table "{schema}"."seed" ( +create table {schema}.seed ( id BIGSERIAL PRIMARY KEY, first_name VARCHAR(50), last_name VARCHAR(50), @@ -8,7 +8,7 @@ create table "{schema}"."seed" ( ); -insert into "{schema}"."seed" (first_name, last_name, email, gender, ip_address) values +insert into {schema}.seed (first_name, last_name, email, gender, ip_address) values ('Jack', 'Hunter', 'jhunter0@pbs.org', 'Male', '59.80.20.168'), ('Kathryn', 'Walker', 'kwalker1@ezinearticles.com', 'Female', '194.121.179.35'), ('Gerald', 'Ryan', 'gryan2@com.com', 'Male', '11.3.212.243'), diff --git a/test/integration/020_ephemeral_test/test_ephemeral.py b/test/integration/020_ephemeral_test/test_ephemeral.py index 06f4f5c5894..4b56543bb45 100644 --- a/test/integration/020_ephemeral_test/test_ephemeral.py +++ b/test/integration/020_ephemeral_test/test_ephemeral.py @@ -1,6 +1,7 @@ from nose.plugins.attrib import attr from test.integration.base import DBTIntegrationTest + class TestEphemeral(DBTIntegrationTest): def setUp(self): @@ -16,22 +17,22 @@ def models(self): @attr(type='postgres') def test__postgres(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() self.run_sql_file("test/integration/020_ephemeral_test/seed.sql") - result = self.run_dbt() + self.run_dbt() self.assertTablesEqual("seed", "dependent") self.assertTablesEqual("seed", "double_dependent") @attr(type='snowflake') def test__snowflake(self): - self.use_default_project() self.use_profile('snowflake') + self.use_default_project() self.run_sql_file("test/integration/020_ephemeral_test/seed.sql") self.run_dbt() - self.assertTablesEqual("seed", "dependent") - self.assertTablesEqual("seed", "double_dependent") + self.assertTablesEqual("SEED", "dependent") + self.assertTablesEqual("SEED", "double_dependent") diff --git a/test/integration/021_concurrency_test/models/dep.sql b/test/integration/021_concurrency_test/models/dep.sql index 046bab975be..746be81aa64 100644 --- a/test/integration/021_concurrency_test/models/dep.sql +++ b/test/integration/021_concurrency_test/models/dep.sql @@ -4,4 +4,4 @@ ) }} -select * from {{ref('view')}} +select * from {{ref('view_model')}} diff --git a/test/integration/021_concurrency_test/models/invalid.sql b/test/integration/021_concurrency_test/models/invalid.sql index 4ea041f6c75..fa7046f9ebc 100644 --- a/test/integration/021_concurrency_test/models/invalid.sql +++ b/test/integration/021_concurrency_test/models/invalid.sql @@ -4,4 +4,4 @@ ) }} -select a_field_that_does_not_exist from "{{ this.schema }}"."seed" +select a_field_that_does_not_exist from {{ this.schema }}.seed diff --git a/test/integration/021_concurrency_test/models/table_a.sql b/test/integration/021_concurrency_test/models/table_a.sql index 314afafac59..79dd477a36e 100644 --- a/test/integration/021_concurrency_test/models/table_a.sql +++ b/test/integration/021_concurrency_test/models/table_a.sql @@ -4,4 +4,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/021_concurrency_test/models/table_b.sql b/test/integration/021_concurrency_test/models/table_b.sql index 314afafac59..79dd477a36e 100644 --- a/test/integration/021_concurrency_test/models/table_b.sql +++ b/test/integration/021_concurrency_test/models/table_b.sql @@ -4,4 +4,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/001_simple_copy_test/models/view.sql b/test/integration/021_concurrency_test/models/view_model.sql similarity index 52% rename from test/integration/001_simple_copy_test/models/view.sql rename to test/integration/021_concurrency_test/models/view_model.sql index 014818a472a..e0560b694c9 100644 --- a/test/integration/001_simple_copy_test/models/view.sql +++ b/test/integration/021_concurrency_test/models/view_model.sql @@ -4,4 +4,4 @@ ) }} -select * from "{{ target.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/021_concurrency_test/seed.sql b/test/integration/021_concurrency_test/seed.sql index 28309b400f8..3c618ccab3a 100644 --- a/test/integration/021_concurrency_test/seed.sql +++ b/test/integration/021_concurrency_test/seed.sql @@ -1,4 +1,4 @@ -create table "{schema}"."seed" ( +create table {schema}.seed ( id BIGSERIAL PRIMARY KEY, first_name VARCHAR(50), last_name VARCHAR(50), @@ -8,7 +8,7 @@ create table "{schema}"."seed" ( ); -insert into "{schema}"."seed" (first_name, last_name, email, gender, ip_address) values +insert into {schema}.seed (first_name, last_name, email, gender, ip_address) values ('Jack', 'Hunter', 'jhunter0@pbs.org', 'Male', '59.80.20.168'), ('Kathryn', 'Walker', 'kwalker1@ezinearticles.com', 'Female', '194.121.179.35'), ('Gerald', 'Ryan', 'gryan2@com.com', 'Male', '11.3.212.243'), diff --git a/test/integration/021_concurrency_test/test_concurrency.py b/test/integration/021_concurrency_test/test_concurrency.py index d55db512396..6a5449178d1 100644 --- a/test/integration/021_concurrency_test/test_concurrency.py +++ b/test/integration/021_concurrency_test/test_concurrency.py @@ -17,13 +17,13 @@ def models(self): @attr(type='postgres') def test__postgres__concurrency(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() self.run_sql_file("test/integration/021_concurrency_test/seed.sql") self.run_dbt(expect_pass=False) - self.assertTablesEqual("seed", "view") + self.assertTablesEqual("seed", "view_model") self.assertTablesEqual("seed", "dep") self.assertTablesEqual("seed", "table_a") self.assertTablesEqual("seed", "table_b") @@ -34,7 +34,7 @@ def test__postgres__concurrency(self): self.run_dbt(expect_pass=False) - self.assertTablesEqual("seed", "view") + self.assertTablesEqual("seed", "view_model") self.assertTablesEqual("seed", "dep") self.assertTablesEqual("seed", "table_a") self.assertTablesEqual("seed", "table_b") @@ -43,22 +43,22 @@ def test__postgres__concurrency(self): @attr(type='snowflake') def test__snowflake__concurrency(self): - self.use_default_project() self.use_profile('snowflake') + self.use_default_project() self.run_sql_file("test/integration/021_concurrency_test/seed.sql") self.run_dbt(expect_pass=False) - self.assertTablesEqual("seed", "view") - self.assertTablesEqual("seed", "dep") - self.assertTablesEqual("seed", "table_a") - self.assertTablesEqual("seed", "table_b") + self.assertTablesEqual("SEED", "view_model") + self.assertTablesEqual("SEED", "dep") + self.assertTablesEqual("SEED", "table_a") + self.assertTablesEqual("SEED", "table_b") self.run_sql_file("test/integration/021_concurrency_test/update.sql") self.run_dbt(expect_pass=False) - self.assertTablesEqual("seed", "view") - self.assertTablesEqual("seed", "dep") - self.assertTablesEqual("seed", "table_a") - self.assertTablesEqual("seed", "table_b") + self.assertTablesEqual("SEED", "view_model") + self.assertTablesEqual("SEED", "dep") + self.assertTablesEqual("SEED", "table_a") + self.assertTablesEqual("SEED", "table_b") diff --git a/test/integration/021_concurrency_test/update.sql b/test/integration/021_concurrency_test/update.sql index e78d5d7d6df..8a8d2ac1a8b 100644 --- a/test/integration/021_concurrency_test/update.sql +++ b/test/integration/021_concurrency_test/update.sql @@ -1,4 +1,4 @@ -insert into "{schema}"."seed" (first_name, last_name, email, gender, ip_address) values +insert into {schema}.seed (first_name, last_name, email, gender, ip_address) values ('Michael', 'Perez', 'mperez0@chronoengine.com', 'Male', '106.239.70.175'), ('Shawn', 'Mccoy', 'smccoy1@reddit.com', 'Male', '24.165.76.182'), ('Kathleen', 'Payne', 'kpayne2@cargocollective.com', 'Female', '113.207.168.106'), diff --git a/test/integration/022_bigquery_test/adapter-models/test_flattened_get_columns_in_table.sql b/test/integration/022_bigquery_test/adapter-models/test_flattened_get_columns_in_table.sql index 1a741dbd102..330e0d67111 100644 --- a/test/integration/022_bigquery_test/adapter-models/test_flattened_get_columns_in_table.sql +++ b/test/integration/022_bigquery_test/adapter-models/test_flattened_get_columns_in_table.sql @@ -1,8 +1,9 @@ - - {% set source = ref('source') %} {% set cols = adapter.get_columns_in_table(source.schema, source.name) %} +{{ log('source') }} +{{ log(source) }} + {% set flattened = [] %} {% for col in cols %} {% if col.mode == 'REPEATED' %} diff --git a/test/integration/022_bigquery_test/adapter-models/test_get_columns_in_table.sql b/test/integration/022_bigquery_test/adapter-models/test_get_columns_in_table.sql index 3653fee869c..58503c93b92 100644 --- a/test/integration/022_bigquery_test/adapter-models/test_get_columns_in_table.sql +++ b/test/integration/022_bigquery_test/adapter-models/test_get_columns_in_table.sql @@ -1,5 +1,3 @@ - - {% set source = ref('source') %} {% set cols = adapter.get_columns_in_table(source.schema, source.name) %} diff --git a/test/integration/022_bigquery_test/models/schema.yml b/test/integration/022_bigquery_test/models/schema.yml index 63aa552c6b9..88114ee08a4 100644 --- a/test/integration/022_bigquery_test/models/schema.yml +++ b/test/integration/022_bigquery_test/models/schema.yml @@ -1,5 +1,5 @@ -view: +view_model: constraints: not_null: - id @@ -10,7 +10,7 @@ view: - dupe # fails was_materialized: - - {name: view, type: view} + - {name: view_model, type: view} table_model: constraints: diff --git a/test/integration/022_bigquery_test/models/table_model.sql b/test/integration/022_bigquery_test/models/table_model.sql index 86b2e2f1deb..5922162c756 100644 --- a/test/integration/022_bigquery_test/models/table_model.sql +++ b/test/integration/022_bigquery_test/models/table_model.sql @@ -1,4 +1,4 @@ {{ config(materialized = "table") }} -select * from {{ ref('view') }} +select * from {{ ref('view_model') }} diff --git a/test/integration/022_bigquery_test/models/view.sql b/test/integration/022_bigquery_test/models/view_model.sql similarity index 100% rename from test/integration/022_bigquery_test/models/view.sql rename to test/integration/022_bigquery_test/models/view_model.sql diff --git a/test/integration/022_bigquery_test/test_bigquery_date_partitioning.py b/test/integration/022_bigquery_test/test_bigquery_date_partitioning.py index 3277c70f490..c4c946fca6e 100644 --- a/test/integration/022_bigquery_test/test_bigquery_date_partitioning.py +++ b/test/integration/022_bigquery_test/test_bigquery_date_partitioning.py @@ -30,4 +30,3 @@ def test__bigquery_date_partitioning(self): self.assertFalse(result.skipped) # status = # of failing rows self.assertEqual(result.status, 0) - diff --git a/test/integration/023_exit_codes_test/test_exit_codes.py b/test/integration/023_exit_codes_test/test_exit_codes.py index 01af65ce2ec..3831c3298a8 100644 --- a/test/integration/023_exit_codes_test/test_exit_codes.py +++ b/test/integration/023_exit_codes_test/test_exit_codes.py @@ -35,24 +35,24 @@ def project_config(self): @attr(type='postgres') def test_exit_code_run_succeed(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() _, success = self.run_dbt_and_check(['run', '--model', 'good']) self.assertTrue(success) self.assertTableDoesExist('good') @attr(type='postgres') def test__exit_code_run_fail(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() _, success = self.run_dbt_and_check(['run', '--model', 'bad']) self.assertFalse(success) self.assertTableDoesNotExist('bad') @attr(type='postgres') def test___schema_test_pass(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() _, success = self.run_dbt_and_check(['run', '--model', 'good']) self.assertTrue(success) _, success = self.run_dbt_and_check(['test', '--model', 'good']) @@ -60,8 +60,8 @@ def test___schema_test_pass(self): @attr(type='postgres') def test___schema_test_fail(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() _, success = self.run_dbt_and_check(['run', '--model', 'dupe']) self.assertTrue(success) _, success = self.run_dbt_and_check(['test', '--model', 'dupe']) @@ -69,15 +69,15 @@ def test___schema_test_fail(self): @attr(type='postgres') def test___compile(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() _, success = self.run_dbt_and_check(['compile']) self.assertTrue(success) @attr(type='postgres') def test___archive_pass(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() self.run_dbt_and_check(['run', '--model', 'good']) _, success = self.run_dbt_and_check(['archive']) @@ -115,8 +115,8 @@ def project_config(self): @attr(type='postgres') def test___archive_fail(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() _, success = self.run_dbt_and_check(['run', '--model', 'good']) self.assertTrue(success) diff --git a/test/integration/024_custom_schema_test/models/view_1.sql b/test/integration/024_custom_schema_test/models/view_1.sql index a42ec4a2916..501c773e8f0 100644 --- a/test/integration/024_custom_schema_test/models/view_1.sql +++ b/test/integration/024_custom_schema_test/models/view_1.sql @@ -1,3 +1,3 @@ -select * from "{{ target.schema }}"."seed" +select * from {{ target.schema }}.seed diff --git a/test/integration/024_custom_schema_test/seed.sql b/test/integration/024_custom_schema_test/seed.sql index b041208db00..607759b0781 100644 --- a/test/integration/024_custom_schema_test/seed.sql +++ b/test/integration/024_custom_schema_test/seed.sql @@ -1,6 +1,6 @@ -drop table if exists "{schema}"."seed" cascade; -create table "{schema}"."seed" ( +drop table if exists {schema}.seed cascade; +create table {schema}.seed ( id BIGSERIAL PRIMARY KEY, first_name VARCHAR(50), last_name VARCHAR(50), @@ -9,17 +9,17 @@ create table "{schema}"."seed" ( ip_address VARCHAR(20) ); -drop table if exists "{schema}"."agg" cascade; -create table "{schema}"."agg" ( +drop table if exists {schema}.agg cascade; +create table {schema}.agg ( last_name VARCHAR(50), count BIGINT ); -insert into "{schema}"."seed" (first_name, last_name, email, gender, ip_address) values +insert into {schema}.seed (first_name, last_name, email, gender, ip_address) values ('Jack', 'Hunter', 'jhunter0@pbs.org', 'Male', '59.80.20.168'), ('Kathryn', 'Walker', 'kwalker1@ezinearticles.com', 'Female', '194.121.179.35'), ('Gerald', 'Ryan', 'gryan2@com.com', 'Male', '11.3.212.243'); -insert into "{schema}"."agg" (last_name, count) values +insert into {schema}.agg (last_name, count) values ('Hunter', 2), ('Walker', 2), ('Ryan', 2); diff --git a/test/integration/028_cli_vars/test_cli_vars.py b/test/integration/028_cli_vars/test_cli_vars.py index 9e6cb06e383..166dcb7f4c5 100644 --- a/test/integration/028_cli_vars/test_cli_vars.py +++ b/test/integration/028_cli_vars/test_cli_vars.py @@ -14,8 +14,8 @@ def models(self): @attr(type='postgres') def test__cli_vars_longform(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() cli_vars = { "variable_1": "abc", @@ -39,16 +39,16 @@ def models(self): @attr(type='postgres') def test__cli_vars_shorthand(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() self.run_dbt(["run", "--vars", "simple: abc"]) self.run_dbt(["test", "--vars", "simple: abc"]) @attr(type='postgres') def test__cli_vars_longer(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() self.run_dbt(["run", "--vars", "{simple: abc, unused: def}"]) self.run_dbt(["test", "--vars", "{simple: abc, unused: def}"]) diff --git a/test/integration/base.py b/test/integration/base.py index 0e37bfb81fd..32982d56f46 100644 --- a/test/integration/base.py +++ b/test/integration/base.py @@ -6,6 +6,7 @@ import json from dbt.adapters.factory import get_adapter +from dbt.project import Project from dbt.logger import GLOBAL_LOGGER as logger @@ -120,8 +121,14 @@ def bigquery_profile(self): } def unique_schema(self): - schema = self.schema - return "{}_{}".format(self.prefix, schema) + schema = self.schema + + to_return = "{}_{}".format(self.prefix, schema) + + if self.adapter_type == 'snowflake': + return to_return.upper() + + return to_return.lower() def get_profile(self, adapter_type): if adapter_type == 'postgres': @@ -132,6 +139,8 @@ def get_profile(self, adapter_type): return self.bigquery_profile() def setUp(self): + self.adapter_type = 'postgres' + # create a dbt_project.yml base_project_config = { @@ -169,6 +178,8 @@ def setUp(self): profile = profile_config.get('test').get('outputs').get(target) + project = Project(project_config, profile_config, DBT_CONFIG_DIR) + adapter = get_adapter(profile) # it's important to use a different connection handle here so @@ -177,15 +188,19 @@ def setUp(self): connection = adapter.acquire_connection(profile, '__test') self.handle = connection.get('handle') self.adapter_type = profile.get('type') - self.profile = profile + self.adapter = adapter + self._profile = profile + self._profile_config = profile_config + self.project = project if self.adapter_type == 'bigquery': schema_name = self.unique_schema() - adapter.drop_schema(profile, schema_name, '__test') - adapter.create_schema(profile, schema_name, '__test') + adapter.drop_schema(profile, project, schema_name, '__test') + adapter.create_schema(profile, project, schema_name, '__test') else: - self.run_sql('DROP SCHEMA IF EXISTS "{}" CASCADE'.format(self.unique_schema())) - self.run_sql('CREATE SCHEMA "{}"'.format(self.unique_schema())) + schema = self.adapter.quote(self.unique_schema()) + self.run_sql('DROP SCHEMA IF EXISTS {} CASCADE'.format(schema)) + self.run_sql('CREATE SCHEMA {}'.format(schema)) def use_default_project(self, overrides=None): # create a dbt_project.yml @@ -202,10 +217,15 @@ def use_default_project(self, overrides=None): project_config.update(self.project_config) project_config.update(overrides or {}) + project = Project(project_config, self._profile_config, DBT_CONFIG_DIR) + self.project = project + with open("dbt_project.yml", 'w') as f: yaml.safe_dump(project_config, f, default_flow_style=True) def use_profile(self, adapter_type): + self.adapter_type = adapter_type + profile_config = {} default_profile_config = self.get_profile(adapter_type) @@ -221,19 +241,25 @@ def use_profile(self, adapter_type): profile = profile_config.get('test').get('outputs').get('default2') adapter = get_adapter(profile) + self.adapter = adapter + # it's important to use a different connection handle here so # we don't look into an incomplete transaction connection = adapter.acquire_connection(profile, '__test') self.handle = connection.get('handle') self.adapter_type = profile.get('type') - self.profile = profile + self._profile_config = profile_config + self._profile = profile if self.adapter_type == 'bigquery': - adapter.drop_schema(profile, self.unique_schema(), '__test') - adapter.create_schema(profile, self.unique_schema(), '__test') + adapter.drop_schema(profile, self.project, + self.unique_schema(), '__test') + adapter.create_schema(profile, self.project, + self.unique_schema(), '__test') else: - self.run_sql('DROP SCHEMA IF EXISTS "{}" CASCADE'.format(self.unique_schema())) - self.run_sql('CREATE SCHEMA "{}"'.format(self.unique_schema())) + schema = self.adapter.quote(self.unique_schema()) + self.run_sql('DROP SCHEMA IF EXISTS {} CASCADE'.format(schema)) + self.run_sql('CREATE SCHEMA {}'.format(schema)) def tearDown(self): os.remove(DBT_PROFILES) @@ -246,13 +272,14 @@ def tearDown(self): except: os.rename("dbt_modules", "dbt_modules-{}".format(time.time())) - adapter = get_adapter(self.profile) + adapter = get_adapter(self._profile) if self.adapter_type == 'bigquery': - adapter.drop_schema(self.profile, self.unique_schema(), '__test') + adapter.drop_schema(self._profile, self.project, + self.unique_schema(), '__test') else: - self.run_sql('DROP SCHEMA IF EXISTS "{}" CASCADE' - .format(self.unique_schema())) + schema = self.adapter.quote(self.unique_schema()) + self.run_sql('DROP SCHEMA IF EXISTS {} CASCADE'.format(schema)) self.handle.close() # hack for BQ -- TODO @@ -276,8 +303,10 @@ def run_dbt(self, args=None, expect_pass=True): args = ["--strict"] + args logger.info("Invoking dbt with {}".format(args)) - res, success = dbt.handle_and_check(args) - self.assertEqual(success, expect_pass, "dbt exit state did not match expected") + res, success = dbt.handle_and_check(args) + self.assertEqual( + success, expect_pass, + "dbt exit state did not match expected") return res @@ -331,11 +360,12 @@ def get_table_columns(self, table, schema=None): sql = """ select column_name, data_type, character_maximum_length from information_schema.columns - where table_name = '{}' - and table_schema = '{}' + where table_name ilike '{}' + and table_schema ilike '{}' order by column_name asc""" - result = self.run_sql(sql.format(table, schema), fetch='all') + result = self.run_sql(sql.format(table.replace('"', ''), schema), + fetch='all') return result @@ -348,7 +378,7 @@ def get_models_in_schema(self, schema=None): else table_type end as materialization from information_schema.tables - where table_schema = '{}' + where table_schema ilike '{}' order by table_name """ @@ -356,33 +386,46 @@ def get_models_in_schema(self, schema=None): return {model_name: materialization for (model_name, materialization) in result} - def assertTablesEqual(self, table_a, table_b, table_a_schema=None, table_b_schema=None): - table_a_schema = self.unique_schema() if table_a_schema is None else table_a_schema - table_b_schema = self.unique_schema() if table_b_schema is None else table_b_schema - - self.assertTableColumnsEqual(table_a, table_b, table_a_schema, table_b_schema) - self.assertTableRowCountsEqual(table_a, table_b, table_a_schema, table_b_schema) - + def _assertTablesEqualSql(self, table_a_schema, table_a, table_b_schema, table_b): columns = self.get_table_columns(table_a, table_a_schema) - columns_csv = ", ".join(['"{}"'.format(record[0]) - for record in columns]) - table_sql = "SELECT {} FROM {}" + + if self.adapter_type == 'snowflake': + columns_csv = ", ".join(['"{}"'.format(record[0]) for record in columns]) + else: + columns_csv = ", ".join(['{}'.format(record[0]) for record in columns]) sql = """ SELECT COUNT(*) FROM ( - (SELECT {columns} FROM "{table_a_schema}"."{table_a}" EXCEPT - SELECT {columns} FROM "{table_b_schema}"."{table_b}") + (SELECT {columns} FROM {table_a_schema}.{table_a} EXCEPT + SELECT {columns} FROM {table_b_schema}.{table_b}) UNION ALL - (SELECT {columns} FROM "{table_b_schema}"."{table_b}" EXCEPT - SELECT {columns} FROM "{table_a_schema}"."{table_a}") + (SELECT {columns} FROM {table_b_schema}.{table_b} EXCEPT + SELECT {columns} FROM {table_a_schema}.{table_a}) ) AS a""".format( columns=columns_csv, - table_a_schema=table_a_schema, - table_b_schema=table_b_schema, - table_a=table_a, - table_b=table_b + table_a_schema=self.adapter.quote(table_a_schema), + table_b_schema=self.adapter.quote(table_b_schema), + table_a=self.adapter.quote(table_a), + table_b=self.adapter.quote(table_b) ) + return sql + + def assertTablesEqual(self, table_a, table_b, + table_a_schema=None, table_b_schema=None): + table_a_schema = self.unique_schema() \ + if table_a_schema is None else table_a_schema + + table_b_schema = self.unique_schema() \ + if table_b_schema is None else table_b_schema + + self.assertTableColumnsEqual(table_a, table_b, + table_a_schema, table_b_schema) + self.assertTableRowCountsEqual(table_a, table_b, + table_a_schema, table_b_schema) + + sql = self._assertTablesEqualSql(table_a_schema, table_a, + table_b_schema, table_b) result = self.run_sql(sql, fetch='one') self.assertEquals( @@ -391,12 +434,22 @@ def assertTablesEqual(self, table_a, table_b, table_a_schema=None, table_b_schem sql ) - def assertTableRowCountsEqual(self, table_a, table_b, table_a_schema=None, table_b_schema=None): - table_a_schema = self.unique_schema() if table_a_schema is None else table_a_schema - table_b_schema = self.unique_schema() if table_b_schema is None else table_b_schema - - table_a_result = self.run_sql('SELECT COUNT(*) FROM "{}"."{}"'.format(table_a_schema, table_a), fetch='one') - table_b_result = self.run_sql('SELECT COUNT(*) FROM "{}"."{}"'.format(table_b_schema, table_b), fetch='one') + def assertTableRowCountsEqual(self, table_a, table_b, + table_a_schema=None, table_b_schema=None): + table_a_schema = self.unique_schema() \ + if table_a_schema is None else table_a_schema + + table_b_schema = self.unique_schema() \ + if table_b_schema is None else table_b_schema + + table_a_result = self.run_sql( + 'SELECT COUNT(*) FROM {}.{}' + .format(self.adapter.quote(table_a_schema), + self.adapter.quote(table_a)), fetch='one') + table_b_result = self.run_sql( + 'SELECT COUNT(*) FROM {}.{}' + .format(self.adapter.quote(table_b_schema), + self.adapter.quote(table_b)), fetch='one') self.assertEquals( table_a_result[0], diff --git a/test/unit/test_compiler.py b/test/unit/test_compiler.py index c47b2d699f3..9e2fe88eca1 100644 --- a/test/unit/test_compiler.py +++ b/test/unit/test_compiler.py @@ -38,6 +38,7 @@ def setUp(self): 'post-hook': [], 'pre-hook': [], 'vars': {}, + 'quoting': {}, 'column_types': {}, } diff --git a/test/unit/test_parser.py b/test/unit/test_parser.py index 249cd840368..570c48c4ba0 100644 --- a/test/unit/test_parser.py +++ b/test/unit/test_parser.py @@ -28,6 +28,7 @@ def setUp(self): 'profile': 'test', 'project-root': os.path.abspath('.'), 'target': 'test', + 'quoting': {}, 'outputs': { 'test': { 'type': 'postgres', @@ -42,6 +43,7 @@ def setUp(self): 'version': '0.1', 'project-root': os.path.abspath('./dbt_modules/snowplow'), 'target': 'test', + 'quoting': {}, 'outputs': { 'test': { 'type': 'postgres', @@ -57,6 +59,7 @@ def setUp(self): 'post-hook': [], 'pre-hook': [], 'vars': {}, + 'quoting': {}, 'column_types': {}, } @@ -66,6 +69,7 @@ def setUp(self): 'post-hook': [], 'pre-hook': [], 'vars': {}, + 'quoting': {}, 'column_types': {}, }