From 67b56488d300513dc3e227fc6033ba909e55c331 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Sun, 3 Mar 2019 16:36:14 -0500 Subject: [PATCH 1/4] 0.13.0 fixes around database quoting and rendering - do not quote snowflake database identifiers by default - do not find relations in source schemas in list_relations - do not render database names in stdout if a custom database is not specified --- core/dbt/adapters/base/impl.py | 9 ++++++++- core/dbt/contracts/graph/manifest.py | 5 +++-- core/dbt/node_runners.py | 16 +++++++++++----- .../snowflake/dbt/adapters/snowflake/relation.py | 2 +- 4 files changed, 23 insertions(+), 9 deletions(-) diff --git a/core/dbt/adapters/base/impl.py b/core/dbt/adapters/base/impl.py index a77094c2301..7a7d372f37e 100644 --- a/core/dbt/adapters/base/impl.py +++ b/core/dbt/adapters/base/impl.py @@ -229,7 +229,14 @@ def _relations_cache_for_schemas(self, manifest): if not dbt.flags.USE_CACHE: return - schemas = manifest.get_used_schemas() + # We only really need to cache relations for resources that + # dbt will try to build. Even the executable() list is probably + # more expansive than necessary. Really, we just want to avoid + # caching Sources here, as there could be _many_ different schemas + # in the list, and dbt largely doesn't need to know if those sources + # exist or not. + resource_types = NodeType.executable() + schemas = manifest.get_used_schemas(resource_types) relations = [] # add all relations diff --git a/core/dbt/contracts/graph/manifest.py b/core/dbt/contracts/graph/manifest.py index 99866cd8ddb..7b9a8508aeb 100644 --- a/core/dbt/contracts/graph/manifest.py +++ b/core/dbt/contracts/graph/manifest.py @@ -4,7 +4,7 @@ PARSED_MACRO_CONTRACT, PARSED_DOCUMENTATION_CONTRACT, \ PARSED_SOURCE_DEFINITION_CONTRACT from dbt.contracts.graph.compiled import COMPILED_NODE_CONTRACT, CompiledNode -from dbt.exceptions import ValidationException +from dbt.exceptions import ValidationException, raise_duplicate_resource_name from dbt.node_types import NodeType from dbt.logger import GLOBAL_LOGGER as logger from dbt import tracking @@ -401,10 +401,11 @@ def __getattr__(self, name): type(self).__name__, name) ) - def get_used_schemas(self): + def get_used_schemas(self, resource_types=None): return frozenset({ (node.database, node.schema) for node in self.nodes.values() + if resource_types and node.resource_type in resource_types }) def get_used_databases(self): diff --git a/core/dbt/node_runners.py b/core/dbt/node_runners.py index b949f0221f5..4c988bb0165 100644 --- a/core/dbt/node_runners.py +++ b/core/dbt/node_runners.py @@ -286,11 +286,17 @@ def compile(self, manifest): class ModelRunner(CompileRunner): + def get_node_representation(self): + if self.config.credentials.database == self.node.database: + template = "{0.schema}.{0.alias}" + else: + template = "{0.database}.{0.schema}.{0.alias}" + + return template.format(self.node) + def describe_node(self): - materialization = dbt.utils.get_materialization(self.node) - return "{0} model {1.database}.{1.schema}.{1.alias}".format( - materialization, self.node - ) + return "{} model {}".format(self.node.get_materialization(), + self.get_node_representation()) def print_start_line(self): description = self.describe_node() @@ -477,7 +483,7 @@ def print_result_line(self, result): class SeedRunner(ModelRunner): def describe_node(self): - return "seed file {0.database}.{0.schema}.{0.alias}".format(self.node) + return "seed file {}".format(self.get_node_representation()) def before_execute(self): description = self.describe_node() diff --git a/plugins/snowflake/dbt/adapters/snowflake/relation.py b/plugins/snowflake/dbt/adapters/snowflake/relation.py index a494fb89363..5563f904c1d 100644 --- a/plugins/snowflake/dbt/adapters/snowflake/relation.py +++ b/plugins/snowflake/dbt/adapters/snowflake/relation.py @@ -9,7 +9,7 @@ class SnowflakeRelation(BaseRelation): }, 'quote_character': '"', 'quote_policy': { - 'database': True, + 'database': False, 'schema': False, 'identifier': False, }, From 328ce82bae5471f5bda57049234a47860c31c13a Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Mon, 4 Mar 2019 20:43:37 -0500 Subject: [PATCH 2/4] fix unit tests --- test/unit/test_snowflake_adapter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/unit/test_snowflake_adapter.py b/test/unit/test_snowflake_adapter.py index d97ee2b4c7e..db5394c3fe3 100644 --- a/test/unit/test_snowflake_adapter.py +++ b/test/unit/test_snowflake_adapter.py @@ -69,7 +69,7 @@ def test_quoting_on_drop_schema(self): ) self.mock_execute.assert_has_calls([ - mock.call('drop schema if exists "test_database"."test_schema" cascade', None) + mock.call('drop schema if exists test_database."test_schema" cascade', None) ]) def test_quoting_on_drop(self): @@ -84,7 +84,7 @@ def test_quoting_on_drop(self): self.mock_execute.assert_has_calls([ mock.call( - 'drop table if exists "test_database"."test_schema".test_table cascade', + 'drop table if exists test_database."test_schema".test_table cascade', None ) ]) @@ -100,7 +100,7 @@ def test_quoting_on_truncate(self): self.adapter.truncate_relation(relation) self.mock_execute.assert_has_calls([ - mock.call('truncate table "test_database"."test_schema".test_table', None) + mock.call('truncate table test_database."test_schema".test_table', None) ]) def test_quoting_on_rename(self): @@ -125,7 +125,7 @@ def test_quoting_on_rename(self): ) self.mock_execute.assert_has_calls([ mock.call( - 'alter table "test_database"."test_schema".table_a rename to "test_database"."test_schema".table_b', + 'alter table test_database."test_schema".table_a rename to test_database."test_schema".table_b', None ) ]) From 22a2887df29194d10fb9254f2bd6062b0fef0620 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Mon, 4 Mar 2019 22:50:23 -0500 Subject: [PATCH 3/4] add missing import --- core/dbt/adapters/base/impl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/core/dbt/adapters/base/impl.py b/core/dbt/adapters/base/impl.py index 7a7d372f37e..5f18f8d7367 100644 --- a/core/dbt/adapters/base/impl.py +++ b/core/dbt/adapters/base/impl.py @@ -18,6 +18,7 @@ from dbt.logger import GLOBAL_LOGGER as logger from dbt.schema import Column from dbt.utils import filter_null_values, translate_aliases +from dbt.node_types import NodeType from dbt.adapters.base.meta import AdapterMeta, available, available_raw, \ available_deprecated From 7eb033e71ae7d3335e5282ba1fe4d2d18cfa078f Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Tue, 5 Mar 2019 17:54:29 -0500 Subject: [PATCH 4/4] fix incorrect schema filtering logic --- core/dbt/contracts/graph/manifest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/contracts/graph/manifest.py b/core/dbt/contracts/graph/manifest.py index 7b9a8508aeb..06f8a9270a8 100644 --- a/core/dbt/contracts/graph/manifest.py +++ b/core/dbt/contracts/graph/manifest.py @@ -405,7 +405,7 @@ def get_used_schemas(self, resource_types=None): return frozenset({ (node.database, node.schema) for node in self.nodes.values() - if resource_types and node.resource_type in resource_types + if not resource_types or node.resource_type in resource_types }) def get_used_databases(self):