Feature/alias (#800)

Implement model aliasing Co-authored-by: Brian Abelson <[email protected]> Co-authored-by: Jonathan Kaczynski <[email protected]>
dbt-labs · Jul 4, 2018 · 540631f · 540631f
1 parent 145a82b
commit 540631f
Show file tree

Hide file tree

Showing 41 changed files with 342 additions and 33 deletions.
diff --git a/Makefile b/Makefile
@@ -17,7 +17,20 @@ test-integration:
 	@echo "Integration test run starting..."
 	@time docker-compose run test tox -e integration-postgres-py27,integration-postgres-py36,integration-snowflake-py27,integration-snowflake-py36,integration-bigquery-py27,integration-bigquery-py36
 
-
 test-quick:
 	@echo "Integration test run starting..."
 	@time docker-compose run test tox -e integration-postgres-py36 -- -x
+
+clean:
+	rm -f .coverage
+	rm -rf .eggs/
+	rm -rf .tox/
+	rm -rf build/
+	rm -rf dbt.egg-info/
+	rm -f dbt_project.yml
+	rm -rf dist/
+	rm -f htmlcov/*.{css,html,js,json,png}
+	rm -rf logs/
+	rm -rf target/
+	find . -type f -name '*.pyc' -delete
+	find . -type d -name '__pycache__' -depth -delete
diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py
@@ -231,12 +231,13 @@ def get_timeout(cls, conn):
     @classmethod
     def materialize_as_view(cls, profile, project_cfg, dataset, model):
         model_name = model.get('name')
+        model_alias = model.get('alias')
         model_sql = model.get('injected_sql')
 
         conn = cls.get_connection(profile, project_cfg, model_name)
         client = conn.get('handle')
 
-        view_ref = dataset.table(model_name)
+        view_ref = dataset.table(model_alias)
         view = google.cloud.bigquery.Table(view_ref)
         view.view_query = model_sql
         view.view_use_legacy_sql = False
@@ -281,14 +282,15 @@ def make_date_partitioned_table(cls, profile, project_cfg, dataset_name,
     def materialize_as_table(cls, profile, project_cfg, dataset,
                              model, model_sql, decorator=None):
         model_name = model.get('name')
+        model_alias = model.get('alias')
 
         conn = cls.get_connection(profile, model_name)
         client = conn.get('handle')
 
         if decorator is None:
-            table_name = model_name
+            table_name = model_alias
         else:
-            table_name = "{}${}".format(model_name, decorator)
+            table_name = "{}${}".format(model_alias, decorator)
 
         table_ref = dataset.table(table_name)
         job_config = google.cloud.bigquery.QueryJobConfig()
@@ -299,7 +301,8 @@ def materialize_as_table(cls, profile, project_cfg, dataset,
         query_job = client.query(model_sql, job_config=job_config)
 
         # this waits for the job to complete
-        with cls.exception_handler(profile, model_sql, model_name, model_name):
+        with cls.exception_handler(profile, model_sql, model_alias,
+                                   model_name):
             query_job.result(timeout=cls.get_timeout(conn))
 
         return "CREATE TABLE"

diff --git a/dbt/adapters/bigquery/relation.py b/dbt/adapters/bigquery/relation.py
@@ -89,7 +89,7 @@ def create_from_node(cls, profile, node, **kwargs):
         return cls.create(
             project=profile.get('project'),
             schema=node.get('schema'),
-            identifier=node.get('name'),
+            identifier=node.get('alias'),
             **kwargs)
 
     @classmethod

diff --git a/dbt/adapters/default/relation.py b/dbt/adapters/default/relation.py
@@ -177,7 +177,7 @@ def create_from_node(cls, profile, node, table_name=None, **kwargs):
         return cls.create(
             database=profile.get('dbname'),
             schema=node.get('schema'),
-            identifier=node.get('name'),
+            identifier=node.get('alias'),
             table_name=table_name,
             **kwargs)
 

diff --git a/dbt/adapters/snowflake/relation.py b/dbt/adapters/snowflake/relation.py
@@ -48,5 +48,5 @@ def create_from_node(cls, profile, node, **kwargs):
         return cls.create(
             database=profile.get('database'),
             schema=node.get('schema'),
-            identifier=node.get('name'),
+            identifier=node.get('alias'),
             **kwargs)
diff --git a/dbt/compilation.py b/dbt/compilation.py
@@ -274,18 +274,27 @@ def get_all_projects(self):
     def _check_resource_uniqueness(cls, flat_graph):
         nodes = flat_graph['nodes']
         names_resources = {}
+        alias_resources = {}
 
         for resource, node in nodes.items():
             if node.get('resource_type') not in NodeType.refable():
                 continue
 
             name = node['name']
+            alias = "{}.{}".format(node['schema'], node['alias'])
+
             existing_node = names_resources.get(name)
             if existing_node is not None:
                 dbt.exceptions.raise_duplicate_resource_name(
                         existing_node, node)
 
+            existing_alias = alias_resources.get(alias)
+            if existing_alias is not None:
+                dbt.exceptions.raise_ambiguous_alias(
+                        existing_alias, node)
+
             names_resources[name] = node
+            alias_resources[alias] = node
 
     def compile(self):
         linker = Linker()

diff --git a/dbt/contracts/graph/parsed.py b/dbt/contracts/graph/parsed.py
@@ -87,6 +87,12 @@
                     'The actual database string that this will build into.'
                 )
             },
+            'alias': {
+                'type': 'string',
+                'description': (
+                    'The name of the relation that this will build into'
+                )
+            },
             'refs': {
                 'type': 'array',
                 'items': {
@@ -148,7 +154,7 @@
         },
         'required': UNPARSED_NODE_CONTRACT['required'] + [
             'unique_id', 'fqn', 'schema', 'refs', 'depends_on', 'empty',
-            'config', 'tags',
+            'config', 'tags', 'alias',
         ]
     }
 )

diff --git a/dbt/exceptions.py b/dbt/exceptions.py
@@ -164,6 +164,10 @@ def ref_bad_context(model, target_model_name, target_model_package):
 To fix this, add the following hint to the top of the model "{model_name}":
 
 -- depends_on: {ref_string}"""
+    # This explicitly references model['name'], instead of model['alias'], for
+    # better error messages. Ex. If models foo_users and bar_users are aliased
+    # to 'users', in their respective schemas, then you would want to see
+    # 'bar_users' in your error messge instead of just 'users'.
     error_msg = base_error_msg.format(
         model_name=model['name'],
         model_path=model['path'],
@@ -339,3 +343,16 @@ def raise_duplicate_resource_name(node_1, node_2):
             duped_name,
             node_1['unique_id'], node_1['original_file_path'],
             node_2['unique_id'], node_2['original_file_path']))
+
+
+def raise_ambiguous_alias(node_1, node_2):
+    duped_name = "{}.{}".format(node_1['schema'], node_1['alias'])
+
+    raise_compiler_error(
+        'dbt found two resources with the database representation "{}".\ndbt '
+        'cannot create two resources with identical database representations. '
+        'To fix this,\nchange the "schema" or "alias" configuration of one of '
+        'these resources:\n- {} ({})\n- {} ({})'.format(
+            duped_name,
+            node_1['unique_id'], node_1['original_file_path'],
+            node_2['unique_id'], node_2['original_file_path']))
diff --git a/dbt/include/global_project/macros/materializations/archive/archive.sql b/dbt/include/global_project/macros/materializations/archive/archive.sql
@@ -127,8 +127,8 @@
     {% endcall %}
   {% endfor %}
 
-  {%- set identifier = model['name'] -%}
-  {%- set tmp_identifier = model['name'] + '__dbt_archival_tmp' -%}
+  {%- set identifier = model['alias'] -%}
+  {%- set tmp_identifier = identifier + '__dbt_archival_tmp' -%}
   {%- set tmp_relation = api.Relation.create(identifier=tmp_identifier, type='table') -%}
 
   {% call statement() %}

diff --git a/dbt/include/global_project/macros/materializations/incremental/incremental.sql b/dbt/include/global_project/macros/materializations/incremental/incremental.sql
@@ -1,7 +1,6 @@
 {% macro dbt__incremental_delete(target_relation, tmp_relation) -%}
 
   {%- set unique_key = config.require('unique_key') -%}
-  {%- set identifier = model['name'] -%}
 
   delete
   from {{ target_relation }}
@@ -16,8 +15,8 @@
   {%- set sql_where = config.require('sql_where') -%}
   {%- set unique_key = config.get('unique_key') -%}
 
-  {%- set identifier = model['name'] -%}
-  {%- set tmp_identifier = model['name'] + '__dbt_incremental_tmp' -%}
+  {%- set identifier = model['alias'] -%}
+  {%- set tmp_identifier = identifier + '__dbt_incremental_tmp' -%}
 
   {%- set existing_relations = adapter.list_relations(schema=schema) -%}
   {%- set old_relation = adapter.get_relation(relations_list=existing_relations,

diff --git a/dbt/include/global_project/macros/materializations/seed/bigquery.sql b/dbt/include/global_project/macros/materializations/seed/bigquery.sql
@@ -10,6 +10,6 @@
 {% macro bigquery__load_csv_rows(model) %}
 
   {%- set column_override = model['config'].get('column_types', {}) -%}
-  {{ adapter.load_dataframe(model['schema'], model['name'], model['agate_table'], column_override) }}
+  {{ adapter.load_dataframe(model['schema'], model['alias'], model['agate_table'], column_override) }}
 
 {% endmacro %}
diff --git a/dbt/include/global_project/macros/materializations/seed/seed.sql b/dbt/include/global_project/macros/materializations/seed/seed.sql
@@ -86,7 +86,7 @@
 
 {% materialization seed, default %}
 
-  {%- set identifier = model['name'] -%}
+  {%- set identifier = model['alias'] -%}
   {%- set full_refresh_mode = (flags.FULL_REFRESH == True) -%}
   {%- set existing_relations = adapter.list_relations(schema=schema) -%}
 

diff --git a/dbt/include/global_project/macros/materializations/table/bigquery_table.sql b/dbt/include/global_project/macros/materializations/table/bigquery_table.sql
@@ -28,7 +28,7 @@
 
 {% materialization table, adapter='bigquery' -%}
 
-  {%- set identifier = model['name'] -%}
+  {%- set identifier = model['alias'] -%}
   {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%}
   {%- set existing_relations = adapter.list_relations(schema=schema) -%}
   {%- set old_relation = adapter.get_relation(relations_list=existing_relations, identifier=identifier) -%}

diff --git a/dbt/include/global_project/macros/materializations/table/table.sql b/dbt/include/global_project/macros/materializations/table/table.sql
@@ -1,5 +1,5 @@
 {% materialization table, default %}
-  {%- set identifier = model['name'] -%}
+  {%- set identifier = model['alias'] -%}
   {%- set tmp_identifier = identifier + '__dbt_tmp' -%}
   {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%}
 

diff --git a/dbt/include/global_project/macros/materializations/view/bigquery_view.sql b/dbt/include/global_project/macros/materializations/view/bigquery_view.sql
@@ -1,6 +1,6 @@
 {% materialization view, adapter='bigquery' -%}
 
-  {%- set identifier = model['name'] -%}
+  {%- set identifier = model['alias'] -%}
   {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%}
 
   {%- set existing_relations = adapter.list_relations(schema=schema) -%}

diff --git a/dbt/include/global_project/macros/materializations/view/view.sql b/dbt/include/global_project/macros/materializations/view/view.sql
@@ -1,6 +1,6 @@
 {%- materialization view, default -%}
 
-  {%- set identifier = model['name'] -%}
+  {%- set identifier = model['alias'] -%}
   {%- set tmp_identifier = identifier + '__dbt_tmp' -%}
   {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%}
 

diff --git a/dbt/model.py b/dbt/model.py
@@ -14,6 +14,7 @@ class SourceConfig(object):
     AppendListFields = ['pre-hook', 'post-hook']
     ExtendDictFields = ['vars', 'column_types', 'quoting']
     ClobberFields = [
+        'alias',
         'schema',
         'enabled',
         'materialized',

diff --git a/dbt/node_runners.py b/dbt/node_runners.py
@@ -260,17 +260,17 @@ def _node_context(cls, adapter, project, node):
         def call_get_columns_in_table(schema_name, table_name):
             return adapter.get_columns_in_table(
                 profile, project, schema_name,
-                table_name, model_name=node.get('name'))
+                table_name, model_name=node.get('alias'))
 
         def call_get_missing_columns(from_schema, from_table,
                                      to_schema, to_table):
             return adapter.get_missing_columns(
                 profile, project, from_schema, from_table,
-                to_schema, to_table, node.get('name'))
+                to_schema, to_table, node.get('alias'))
 
         def call_already_exists(schema, table):
             return adapter.already_exists(
-                profile, project, schema, table, node.get('name'))
+                profile, project, schema, table, node.get('alias'))
 
         return {
             "run_started_at": dbt.tracking.active_user.run_started_at,
@@ -388,8 +388,7 @@ def after_hooks(cls, project, adapter, results, flat_graph, elapsed):
     def describe_node(self):
         materialization = dbt.utils.get_materialization(self.node)
         schema_name = self.node.get('schema')
-        node_name = self.node.get('name')
-
+        node_name = self.node.get('alias')
         return "{} model {}.{}".format(materialization, schema_name, node_name)
 
     def print_start_line(self):
@@ -499,7 +498,7 @@ class SeedRunner(ModelRunner):
 
     def describe_node(self):
         schema_name = self.node.get('schema')
-        return "seed file {}.{}".format(schema_name, self.node["name"])
+        return "seed file {}.{}".format(schema_name, self.node['alias'])
 
     @classmethod
     def before_run(cls, project, adapter, flat_graph):

diff --git a/dbt/parser/base.py b/dbt/parser/base.py
@@ -90,10 +90,12 @@ def parse_node(cls, node, node_path, root_project_config,
         config_dict.update(config.config)
         node['config'] = config_dict
 
-        # Set this temporarily so get_rendered() below has access to a schema
+        # Set this temporarily so get_rendered() has access to a schema & alias
         profile = dbt.utils.get_profile_from_project(root_project_config)
         default_schema = profile.get('schema', 'public')
         node['schema'] = default_schema
+        default_alias = node.get('name')
+        node['alias'] = default_alias
 
         context = dbt.context.parser.generate(node, root_project_config,
                                               {"macros": macros})
@@ -113,6 +115,7 @@ def parse_node(cls, node, node_path, root_project_config,
         get_schema = context.get('generate_schema_name',
                                  lambda x: default_schema)
         node['schema'] = get_schema(schema_override)
+        node['alias'] = config.config.get('alias', default_alias)
 
         # Overwrite node config
         config_dict = node.get('config', {})

diff --git a/dbt/task/seed.py b/dbt/task/seed.py
@@ -32,9 +32,9 @@ def show_table(self, result):
         rand_table = table.order_by(lambda x: random.random())
 
         schema = result.node['schema']
-        name = result.node['name']
+        alias = result.node['alias']
 
-        header = "Random sample of table: {}.{}".format(schema, name)
+        header = "Random sample of table: {}.{}".format(schema, alias)
         logger.info("")
         logger.info(header)
         logger.info("-" * len(header))

diff --git a/dbt/ui/printer.py b/dbt/ui/printer.py
@@ -156,7 +156,7 @@ def print_model_result_line(result, schema_name, index, total):
             info=info,
             model_type=get_materialization(model),
             schema=schema_name,
-            relation=model.get('name')),
+            relation=model.get('alias')),
         status,
         index,
         total,
@@ -187,7 +187,7 @@ def print_seed_result_line(result, schema_name, index, total):
         "{info} seed file {schema}.{relation}".format(
             info=info,
             schema=schema_name,
-            relation=model.get('name')),
+            relation=model.get('alias')),
         status,
         index,
         total,

diff --git a/dbt/utils.py b/dbt/utils.py
@@ -16,6 +16,7 @@
 
 
 DBTConfigKeys = [
+    'alias',
     'schema',
     'enabled',
     'materialized',
@@ -65,7 +66,7 @@ def get_model_name_or_none(model):
     elif isinstance(model, basestring):
         name = model
     elif isinstance(model, dict):
-        name = model.get('name')
+        name = model['alias']
     else:
         name = model.nice_name
     return name
@@ -87,7 +88,7 @@ def model_immediate_name(model, non_destructive):
     seeds.
     """
 
-    model_name = model.get('name')
+    model_name = model['alias']
     is_incremental = (get_materialization(model) == 'incremental')
     is_seed = is_type(model, 'seed')
 

diff --git a/test/integration/026_aliases_test/macros/cast.sql b/test/integration/026_aliases_test/macros/cast.sql
@@ -0,0 +1,13 @@
+
+
+{% macro string_literal(s) -%}
+  {{ adapter_macro('test.string_literal', s) }}
+{%- endmacro %}
+
+{% macro default__string_literal(s) %}
+    '{{ s }}'::text
+{% endmacro %}
+
+{% macro bigquery__string_literal(s) %}
+    cast('{{ s }}' as string)
+{% endmacro %}
diff --git a/test/integration/026_aliases_test/macros/expect_value.sql b/test/integration/026_aliases_test/macros/expect_value.sql
@@ -0,0 +1,10 @@
+
+-- cross-db compatible test, similar to accepted_values
+
+{% macro test_expect_value(model, field, value) %}
+
+select count(*)
+from {{ model }}
+where {{ field }} != '{{ value }}'
+
+{% endmacro %}