From e4f37ab057abfb1556e0d42a84b552a52cb756ae Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Fri, 12 Jul 2019 15:59:24 -0600 Subject: [PATCH] Split Parsed and Compiled nodes into subtypes A few test changes to account for removed fields based on types --- core/dbt/compilation.py | 19 ++-- core/dbt/contracts/graph/compiled.py | 97 ++++++++++++++++--- core/dbt/contracts/graph/parsed.py | 47 ++++++++- core/dbt/node_types.py | 30 ++++-- core/dbt/parser/analysis.py | 17 +++- core/dbt/parser/base.py | 12 ++- core/dbt/parser/data_test.py | 6 ++ core/dbt/parser/docs.py | 14 ++- core/dbt/parser/hooks.py | 7 +- core/dbt/parser/macros.py | 4 + core/dbt/parser/models.py | 6 ++ core/dbt/parser/schemas.py | 10 +- core/dbt/parser/seeds.py | 6 ++ core/dbt/parser/snapshots.py | 2 +- .../test_docs_generate.py | 16 --- test/unit/test_compiler.py | 20 ++-- test/unit/test_context.py | 7 +- test/unit/test_contracts_graph_compiled.py | 6 +- test/unit/test_contracts_graph_parsed.py | 10 +- test/unit/test_manifest.py | 33 ++++--- test/unit/test_parser.py | 82 +++++++++------- 21 files changed, 312 insertions(+), 139 deletions(-) diff --git a/core/dbt/compilation.py b/core/dbt/compilation.py index 3ee95f94b92..d595ca69629 100644 --- a/core/dbt/compilation.py +++ b/core/dbt/compilation.py @@ -15,19 +15,21 @@ import dbt.flags import dbt.loader import dbt.config -from dbt.contracts.graph.compiled import InjectedCTE, CompiledNode, \ - CompiledTestNode +from dbt.contracts.graph.compiled import InjectedCTE, COMPILED_TYPES +from dbt.contracts.graph.parsed import ParsedNode from dbt.logger import GLOBAL_LOGGER as logger graph_file_name = 'graph.gpickle' -def _compiled_type_for(model): - if model.resource_type == NodeType.Test: - return CompiledTestNode - else: - return CompiledNode +def _compiled_type_for(model: ParsedNode): + if model.resource_type not in COMPILED_TYPES: + raise dbt.exceptions.InternalException( + 'Asked to compile {} node, but it has no compiled form' + .format(model.resource_type) + ) + return COMPILED_TYPES[model.resource_type] def print_compile_stats(stats): @@ -75,7 +77,8 @@ def recursively_prepend_ctes(model, manifest): return (model, model.extra_ctes, manifest) if dbt.flags.STRICT_MODE: - assert isinstance(model, (CompiledNode, CompiledTestNode)) + assert isinstance(model, tuple(COMPILED_TYPES.values())), \ + 'Bad model type: {}'.format(type(model)) prepended_ctes = [] diff --git a/core/dbt/contracts/graph/compiled.py b/core/dbt/contracts/graph/compiled.py index a0db5b00d07..a6a4eb19e5d 100644 --- a/core/dbt/contracts/graph/compiled.py +++ b/core/dbt/contracts/graph/compiled.py @@ -1,8 +1,27 @@ from dbt.contracts.graph.parsed import ( - ParsedNodeMixins, ParsedNode, ParsedSourceDefinition, - ParsedNodeDefaults, TestType, ParsedTestNode, TestConfig + ParsedNode, + ParsedAnalysisNode, + ParsedDocumentation, + ParsedMacro, + ParsedModelNode, + ParsedHookNode, + ParsedRPCNode, + ParsedSeedNode, + ParsedSnapshotNode, + ParsedSourceDefinition, + ParsedTestNode, + TestConfig, +) +from dbt.node_types import ( + NodeType, + AnalysisType, + ModelType, + OperationType, + RPCCallType, + SeedType, + SnapshotType, + TestType, ) - from dbt.contracts.util import Replaceable from hologram import JsonSchemaMixin @@ -23,7 +42,7 @@ class InjectedCTE(JsonSchemaMixin, Replaceable): @dataclass -class CompiledNodeDefaults(ParsedNodeDefaults, ParsedNodeMixins): +class CompiledNode(ParsedNode): compiled: bool = False compiled_sql: Optional[str] = None extra_ctes_injected: bool = False @@ -53,18 +72,38 @@ def set_cte(self, cte_id: str, sql: str): @dataclass -class CompiledNode(CompiledNodeDefaults): +class CompiledAnalysisNode(CompiledNode): + resource_type: AnalysisType + + +@dataclass +class CompiledHookNode(CompiledNode): + resource_type: OperationType index: Optional[int] = None - @classmethod - def from_parsed_node(cls, parsed, **kwargs): - dct = parsed.to_dict() - dct.update(kwargs) - return cls.from_dict(dct) + +@dataclass +class CompiledModelNode(CompiledNode): + resource_type: ModelType + + +@dataclass +class CompiledRPCNode(CompiledNode): + resource_type: RPCCallType + + +@dataclass +class CompiledSeedNode(CompiledNode): + resource_type: SeedType + + +@dataclass +class CompiledSnapshotNode(CompiledNode): + resource_type: SnapshotType @dataclass -class CompiledTestNode(CompiledNodeDefaults): +class CompiledTestNode(CompiledNode): resource_type: TestType column_name: Optional[str] = None config: TestConfig = field(default_factory=TestConfig) @@ -130,11 +169,43 @@ def _inject_ctes_into_sql(sql: str, ctes: List[InjectedCTE]) -> str: return str(parsed) +COMPILED_TYPES = { + NodeType.Analysis: CompiledAnalysisNode, + NodeType.Model: CompiledModelNode, + NodeType.Operation: CompiledHookNode, + NodeType.RPCCall: CompiledRPCNode, + NodeType.Seed: CompiledSeedNode, + NodeType.Snapshot: CompiledSnapshotNode, + NodeType.Test: CompiledTestNode, +} + + +def compiled_type_for(parsed: ParsedNode): + if parsed.resource_type in COMPILED_TYPES: + return COMPILED_TYPES[parsed.resource_type] + else: + return type(parsed) + + # We allow either parsed or compiled nodes, or parsed sources, as some # 'compile()' calls in the runner actually just return the original parsed # node they were given. CompileResultNode = Union[ - CompiledNode, ParsedNode, - CompiledTestNode, ParsedTestNode, + CompiledAnalysisNode, + CompiledModelNode, + CompiledHookNode, + CompiledRPCNode, + CompiledSeedNode, + CompiledSnapshotNode, + CompiledTestNode, + ParsedAnalysisNode, + ParsedDocumentation, + ParsedMacro, + ParsedModelNode, + ParsedHookNode, + ParsedRPCNode, + ParsedSeedNode, + ParsedSnapshotNode, ParsedSourceDefinition, + ParsedTestNode, ] diff --git a/core/dbt/contracts/graph/parsed.py b/core/dbt/contracts/graph/parsed.py index 2bd63569556..f4e64dc15e4 100644 --- a/core/dbt/contracts/graph/parsed.py +++ b/core/dbt/contracts/graph/parsed.py @@ -12,7 +12,8 @@ from dbt.contracts.util import Replaceable from dbt.logger import GLOBAL_LOGGER as logger # noqa from dbt.node_types import ( - NodeType, SourceType, SnapshotType, MacroType, TestType + NodeType, SourceType, SnapshotType, MacroType, TestType, OperationType, + SeedType, ModelType, AnalysisType, RPCCallType ) @@ -190,20 +191,44 @@ class ParsedNodeDefaults(ParsedNodeMandatory): build_path: Optional[str] = None -# TODO(jeb): hooks should get their own parsed type instead of including -# index everywhere! @dataclass class ParsedNode(ParsedNodeDefaults, ParsedNodeMixins): + pass + + +@dataclass +class ParsedAnalysisNode(ParsedNode): + resource_type: AnalysisType + + +@dataclass +class ParsedHookNode(ParsedNode): + resource_type: OperationType index: Optional[int] = None +@dataclass +class ParsedModelNode(ParsedNode): + resource_type: ModelType + + +@dataclass +class ParsedRPCNode(ParsedNode): + resource_type: RPCCallType + + +@dataclass +class ParsedSeedNode(ParsedNode): + resource_type: SeedType + + @dataclass class TestConfig(NodeConfig): severity: Severity = 'error' @dataclass -class ParsedTestNode(ParsedNodeDefaults, ParsedNodeMixins): +class ParsedTestNode(ParsedNode): resource_type: TestType column_name: Optional[str] = None config: TestConfig = field(default_factory=TestConfig) @@ -415,3 +440,17 @@ def tags(self): @property def has_freshness(self): return bool(self.freshness) and self.loaded_at_field is not None + + +PARSED_TYPES = { + NodeType.Analysis: ParsedAnalysisNode, + NodeType.Documentation: ParsedDocumentation, + NodeType.Macro: ParsedMacro, + NodeType.Model: ParsedModelNode, + NodeType.Operation: ParsedHookNode, + NodeType.RPCCall: ParsedRPCNode, + NodeType.Seed: ParsedSeedNode, + NodeType.Snapshot: ParsedSnapshotNode, + NodeType.Source: ParsedSourceDefinition, + NodeType.Test: ParsedTestNode, +} diff --git a/core/dbt/node_types.py b/core/dbt/node_types.py index ffbc1eb9af8..296134f699e 100644 --- a/core/dbt/node_types.py +++ b/core/dbt/node_types.py @@ -45,25 +45,41 @@ class UnparsedNodeType(StrEnum): RPCCall = str(NodeType.RPCCall) +class RunHookType(StrEnum): + Start = 'on-run-start' + End = 'on-run-end' + + +class AnalysisType(StrEnum): + Analysis = str(NodeType.Analysis) + + class DocumentationType(StrEnum): Documentation = str(NodeType.Documentation) -class RunHookType(StrEnum): - Start = 'on-run-start' - End = 'on-run-end' +class MacroType(StrEnum): + Macro = str(NodeType.Macro) + + +class ModelType(StrEnum): + Model = str(NodeType.Model) class OperationType(StrEnum): Operation = str(NodeType.Operation) -class SnapshotType(StrEnum): - Snapshot = str(NodeType.Snapshot) +class RPCCallType(StrEnum): + RPCCall = str(NodeType.RPCCall) -class MacroType(StrEnum): - Macro = str(NodeType.Macro) +class SeedType(StrEnum): + Seed = str(NodeType.Seed) + + +class SnapshotType(StrEnum): + Snapshot = str(NodeType.Snapshot) class SourceType(StrEnum): diff --git a/core/dbt/parser/analysis.py b/core/dbt/parser/analysis.py index c466ead1cfe..b18f85cae1a 100644 --- a/core/dbt/parser/analysis.py +++ b/core/dbt/parser/analysis.py @@ -1,6 +1,8 @@ +import os +from typing import Dict, Any +from dbt.contracts.graph.parsed import ParsedAnalysisNode, ParsedRPCNode from dbt.parser.base_sql import BaseSqlParser -import os class AnalysisParser(BaseSqlParser): @@ -8,7 +10,18 @@ class AnalysisParser(BaseSqlParser): def get_compiled_path(cls, name, relative_path): return os.path.join('analysis', relative_path) + def parse_from_dict( + self, + parsed_dict: Dict[str, Any] + ) -> ParsedAnalysisNode: + """Given a dictionary, return the parsed entity for this parser""" + return ParsedAnalysisNode.from_dict(parsed_dict) -class RPCCallParser(AnalysisParser): + +class RPCCallParser(BaseSqlParser): def get_compiled_path(cls, name, relative_path): return os.path.join('rpc', relative_path) + + def parse_from_dict(self, parsed_dict: Dict[str, Any]) -> ParsedRPCNode: + """Given a dictionary, return the parsed entity for this parser""" + return ParsedRPCNode.from_dict(parsed_dict) diff --git a/core/dbt/parser/base.py b/core/dbt/parser/base.py index 646288e426e..5ea1cc65d8b 100644 --- a/core/dbt/parser/base.py +++ b/core/dbt/parser/base.py @@ -1,4 +1,6 @@ +import abc import os +from typing import Dict, Any import dbt.exceptions import dbt.flags @@ -11,14 +13,13 @@ from dbt.include.global_project import PROJECT_NAME as GLOBAL_PROJECT_NAME from dbt.utils import coalesce from dbt.logger import GLOBAL_LOGGER as logger -from dbt.contracts.graph.parsed import ParsedNode from dbt.contracts.project import ProjectList from dbt.parser.source_config import SourceConfig from dbt import deprecations from dbt import hooks -class BaseParser: +class BaseParser(metaclass=abc.ABCMeta): def __init__(self, root_project_config, all_projects: ProjectList): self.root_project_config = root_project_config self.all_projects = all_projects @@ -251,8 +252,9 @@ def _update_parsed_node_info(self, parsed_node, config): self._mangle_hooks(config_dict) parsed_node.config = parsed_node.config.from_dict(config_dict) - def _parse_from_dict(self, parsed_dict): - return ParsedNode.from_dict(parsed_dict) + @abc.abstractmethod + def parse_from_dict(self, parsed_dict: Dict[str, Any]) -> Any: + """Given a dictionary, return the parsed entity for this parser""" def parse_node(self, node, node_path, package_project_config, tags=None, fqn_extra=None, fqn=None, snapshot_config=None, @@ -281,7 +283,7 @@ def parse_node(self, node, node_path, package_project_config, tags=None, config, node.to_dict(), node_path, config, tags, fqn, snapshot_config, column_name ) - parsed_node = self._parse_from_dict(parsed_dict) + parsed_node = self.parse_from_dict(parsed_dict) self._render_with_context(parsed_node, config) self._update_parsed_node_info(parsed_node, config) diff --git a/core/dbt/parser/data_test.py b/core/dbt/parser/data_test.py index 5169ba91e13..faab88f7cd1 100644 --- a/core/dbt/parser/data_test.py +++ b/core/dbt/parser/data_test.py @@ -1,4 +1,6 @@ +from typing import Dict, Any +from dbt.contracts.graph.parsed import ParsedTestNode from dbt.parser.base_sql import BaseSqlParser import dbt.utils @@ -7,3 +9,7 @@ class DataTestParser(BaseSqlParser): @classmethod def get_compiled_path(cls, name, relative_path): return dbt.utils.get_pseudo_test_path(name, relative_path, 'data_test') + + def parse_from_dict(self, parsed_dict: Dict[str, Any]) -> ParsedTestNode: + """Given a dictionary, return the parsed entity for this parser""" + return ParsedTestNode.from_dict(parsed_dict) diff --git a/core/dbt/parser/docs.py b/core/dbt/parser/docs.py index 18db1ab0754..456850f8301 100644 --- a/core/dbt/parser/docs.py +++ b/core/dbt/parser/docs.py @@ -1,3 +1,8 @@ +import os +from typing import Dict, Any + +import jinja2.runtime + import dbt.exceptions from dbt.parser.base import BaseParser from dbt.contracts.graph.unparsed import UnparsedDocumentationFile @@ -5,9 +10,6 @@ from dbt.clients.jinja import extract_toplevel_blocks, get_template from dbt.clients import system -import jinja2.runtime -import os - class DocumentationParser(BaseParser): @classmethod @@ -94,3 +96,9 @@ def load_and_parse(self, package_name, root_dir, relative_dirs): ) to_return[parsed.unique_id] = parsed return to_return + + def parse_from_dict( + self, parsed_dict: Dict[str, Any] + ) -> ParsedDocumentation: + """Given a dictionary, return the parsed entity for this parser""" + return ParsedDocumentation.from_dict(parsed_dict) diff --git a/core/dbt/parser/hooks.py b/core/dbt/parser/hooks.py index 4d5d38a0652..6f21750208e 100644 --- a/core/dbt/parser/hooks.py +++ b/core/dbt/parser/hooks.py @@ -1,10 +1,11 @@ - import collections +from typing import Dict, Any import dbt.flags import dbt.contracts.project import dbt.utils +from dbt.contracts.graph.parsed import ParsedHookNode from dbt.contracts.graph.unparsed import UnparsedRunHook from dbt.parser.base_sql import BaseSqlParser from dbt.node_types import NodeType, RunHookType @@ -74,3 +75,7 @@ def load_and_parse(self): hook_nodes.update(project_hooks) return hook_nodes + + def parse_from_dict(self, parsed_dict: Dict[str, Any]) -> ParsedHookNode: + """Given a dictionary, return the parsed entity for this parser""" + return ParsedHookNode.from_dict(parsed_dict) diff --git a/core/dbt/parser/macros.py b/core/dbt/parser/macros.py index e3976ce0d67..ab20d23ff18 100644 --- a/core/dbt/parser/macros.py +++ b/core/dbt/parser/macros.py @@ -1,4 +1,5 @@ import os +from typing import Dict, Any import jinja2.runtime @@ -71,6 +72,9 @@ def parse_macro_file(self, macro_file_path, macro_file_contents, root_path, return to_return + def parse_from_dict(self, parsed_dict: Dict[str, Any]) -> ParsedMacro: + return ParsedMacro.from_dict(parsed_dict) + def load_and_parse(self, package_name, root_dir, relative_dirs, resource_type, tags=None): extension = "[!.#~]*.sql" diff --git a/core/dbt/parser/models.py b/core/dbt/parser/models.py index 1b1c9eaa7d0..1f8f4e6a2a1 100644 --- a/core/dbt/parser/models.py +++ b/core/dbt/parser/models.py @@ -1,4 +1,6 @@ +from typing import Dict, Any +from dbt.contracts.graph.parsed import ParsedModelNode from dbt.parser.base_sql import BaseSqlParser @@ -6,3 +8,7 @@ class ModelParser(BaseSqlParser): @classmethod def get_compiled_path(cls, name, relative_path): return relative_path + + def parse_from_dict(self, parsed_dict: Dict[str, Any]) -> ParsedModelNode: + """Given a dictionary, return the parsed entity for this parser""" + return ParsedModelNode.from_dict(parsed_dict) diff --git a/core/dbt/parser/schemas.py b/core/dbt/parser/schemas.py index cbd908138c0..970f702c39d 100644 --- a/core/dbt/parser/schemas.py +++ b/core/dbt/parser/schemas.py @@ -23,7 +23,7 @@ from dbt.contracts.graph.unparsed import UnparsedNode, UnparsedNodeUpdate, \ UnparsedSourceDefinition from dbt.contracts.graph.parsed import ParsedNodePatch, ParsedTestNode, \ - ParsedSourceDefinition + ParsedSourceDefinition, ParsedNode from dbt.parser.base import MacrosKnownParser from dbt.config.renderer import ConfigRenderer from dbt.exceptions import JSONValidationException, validator_error_message @@ -286,7 +286,7 @@ def _parse_column(self, target, column, package_name, root_dir, path, ) continue - def _parse_from_dict(self, parsed_dict): + def parse_from_dict(self, parsed_dict): return ParsedTestNode.from_dict(parsed_dict) def build_test_node(self, test_target, package_name, test, root_dir, path, @@ -612,9 +612,9 @@ def _parse_format_version(self, path, test_yml): return version def load_and_parse(self, package_name, root_dir, relative_dirs): - new_tests = {} # test unique ID -> ParsedNode - node_patches = {} # model name -> dict - new_sources = {} # source unique ID -> ParsedSourceDefinition + new_tests: Dict[str, ParsedNode] = {} + node_patches: Dict[str, dict] = {} + new_sources: Dict[str, ParsedSourceDefinition] = {} iterator = self.find_schema_yml(package_name, root_dir, relative_dirs) diff --git a/core/dbt/parser/seeds.py b/core/dbt/parser/seeds.py index f53f39cfa0f..7da8ace3b46 100644 --- a/core/dbt/parser/seeds.py +++ b/core/dbt/parser/seeds.py @@ -1,4 +1,5 @@ import os +from typing import Dict, Any import dbt.flags import dbt.clients.agate_helper @@ -9,6 +10,7 @@ from dbt.node_types import NodeType from dbt.logger import GLOBAL_LOGGER as logger +from dbt.contracts.graph.parsed import ParsedSeedNode from dbt.contracts.graph.unparsed import UnparsedNode from dbt.parser.base import MacrosKnownParser @@ -57,3 +59,7 @@ def load_and_parse(self, package_name, root_dir, relative_dirs, tags=None): result[node_path] = parsed return result + + def parse_from_dict(self, parsed_dict: Dict[str, Any]) -> ParsedSeedNode: + """Given a dictionary, return the parsed entity for this parser""" + return ParsedSeedNode.from_dict(parsed_dict) diff --git a/core/dbt/parser/snapshots.py b/core/dbt/parser/snapshots.py index a80d63a1208..d09516dc691 100644 --- a/core/dbt/parser/snapshots.py +++ b/core/dbt/parser/snapshots.py @@ -56,7 +56,7 @@ def get_fqn(cls, node, package_project_config, extra=[]): return fqn - def _parse_from_dict(self, parsed_dict): + def parse_from_dict(self, parsed_dict): return IntermediateSnapshotNode.from_dict(parsed_dict) @staticmethod diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index 68deb74ae84..ad608123905 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -847,7 +847,6 @@ def expected_seeded_manifest(self, model_database=None): 'depends_on': {'nodes': ['seed.test.seed'], 'macros': []}, 'unique_id': 'model.test.model', 'fqn': ['test', 'model'], - 'index': None, 'tags': [], 'config': model_config, 'schema': my_schema_name, @@ -892,7 +891,6 @@ def expected_seeded_manifest(self, model_database=None): 'quoting': {}, 'tags': [], }, - 'index': None, 'patch_path': None, 'path': 'seed.csv', 'name': 'seed', @@ -1085,7 +1083,6 @@ def expected_postgres_references_manifest(self, model_database=None): 'description': '', 'docrefs': [], 'fqn': ['test', 'ephemeral_copy'], - 'index': None, 'name': 'ephemeral_copy', 'original_file_path': self.dir('ref_models/ephemeral_copy.sql'), 'package_name': 'test', @@ -1151,7 +1148,6 @@ def expected_postgres_references_manifest(self, model_database=None): } ], 'fqn': ['test', 'ephemeral_summary'], - 'index': None, 'name': 'ephemeral_summary', 'original_file_path': self.dir('ref_models/ephemeral_summary.sql'), 'package_name': 'test', @@ -1218,7 +1214,6 @@ def expected_postgres_references_manifest(self, model_database=None): } ], 'fqn': ['test', 'view_summary'], - 'index': None, 'name': 'view_summary', 'original_file_path': self.dir('ref_models/view_summary.sql'), 'package_name': 'test', @@ -1257,7 +1252,6 @@ def expected_postgres_references_manifest(self, model_database=None): 'description': '', 'docrefs': [], 'fqn': ['test', 'seed'], - 'index': None, 'name': 'seed', 'original_file_path': self.dir('seed/seed.csv'), 'package_name': 'test', @@ -1432,7 +1426,6 @@ def expected_bigquery_complex_manifest(self): 'sources': [], 'depends_on': {'macros': [], 'nodes': ['seed.test.seed']}, 'fqn': ['test', 'clustered'], - 'index': None, 'build_path': None, 'name': 'clustered', 'original_file_path': clustered_sql_path, @@ -1491,7 +1484,6 @@ def expected_bigquery_complex_manifest(self): 'sources': [], 'depends_on': {'macros': [], 'nodes': ['seed.test.seed']}, 'fqn': ['test', 'multi_clustered'], - 'index': None, 'name': 'multi_clustered', 'original_file_path': multi_clustered_sql_path, 'package_name': 'test', @@ -1550,7 +1542,6 @@ def expected_bigquery_complex_manifest(self): 'nodes': ['model.test.nested_table'] }, 'fqn': ['test', 'nested_view'], - 'index': None, 'name': 'nested_view', 'original_file_path': nested_view_sql_path, 'package_name': 'test', @@ -1609,7 +1600,6 @@ def expected_bigquery_complex_manifest(self): 'nodes': [] }, 'fqn': ['test', 'nested_table'], - 'index': None, 'name': 'nested_table', 'original_file_path': nested_table_sql_path, 'package_name': 'test', @@ -1629,7 +1619,6 @@ def expected_bigquery_complex_manifest(self): }, 'seed.test.seed': { 'build_path': None, - 'index': None, 'patch_path': None, 'path': 'seed.csv', 'name': 'seed', @@ -1917,7 +1906,6 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'extra_ctes': [], 'extra_ctes_injected': True, 'fqn': ['test', 'model'], - 'index': None, 'injected_sql': compiled_sql, 'name': 'model', 'original_file_path': model_sql_path, @@ -1969,7 +1957,6 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'extra_ctes': [], 'extra_ctes_injected': True, 'fqn': ['test', 'seed'], - 'index': None, 'injected_sql': '-- csv --', 'name': 'seed', 'original_file_path': self.dir('seed/seed.csv'), @@ -2233,7 +2220,6 @@ def expected_postgres_references_run_results(self): ], 'extra_ctes_injected': True, 'fqn': ['test', 'ephemeral_summary'], - 'index': None, 'injected_sql': ephemeral_injected_sql, 'name': 'ephemeral_summary', 'original_file_path': self.dir('ref_models/ephemeral_summary.sql'), @@ -2322,7 +2308,6 @@ def expected_postgres_references_run_results(self): 'extra_ctes': [], 'extra_ctes_injected': True, 'fqn': ['test', 'view_summary'], - 'index': None, 'injected_sql': view_compiled_sql, 'name': 'view_summary', 'original_file_path': self.dir('ref_models/view_summary.sql'), @@ -2378,7 +2363,6 @@ def expected_postgres_references_run_results(self): 'extra_ctes': [], 'extra_ctes_injected': True, 'fqn': ['test', 'seed'], - 'index': None, 'injected_sql': '-- csv --', 'name': 'seed', 'original_file_path': self.dir('seed/seed.csv'), diff --git a/test/unit/test_compiler.py b/test/unit/test_compiler.py index 96184b82eb5..eb3a27b3944 100644 --- a/test/unit/test_compiler.py +++ b/test/unit/test_compiler.py @@ -6,7 +6,7 @@ import dbt.compilation from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.parsed import NodeConfig, DependsOn -from dbt.contracts.graph.compiled import CompiledNode, InjectedCTE +from dbt.contracts.graph.compiled import CompiledModelNode, InjectedCTE from dbt.node_types import NodeType from datetime import datetime @@ -54,7 +54,7 @@ def test__prepend_ctes__already_has_cte(self): input_graph = Manifest( macros={}, nodes={ - 'model.root.view': CompiledNode( + 'model.root.view': CompiledModelNode( name='view', database='dbt', schema='analytics', @@ -80,7 +80,7 @@ def test__prepend_ctes__already_has_cte(self): 'with cte as (select * from something_else) ' 'select * from __dbt__CTE__ephemeral') ), - 'model.root.ephemeral': CompiledNode( + 'model.root.ephemeral': CompiledModelNode( name='ephemeral', database='dbt', schema='analytics', @@ -132,7 +132,7 @@ def test__prepend_ctes__no_ctes(self): input_graph = Manifest( macros={}, nodes={ - 'model.root.view': CompiledNode( + 'model.root.view': CompiledModelNode( name='view', database='dbt', schema='analytics', @@ -158,7 +158,7 @@ def test__prepend_ctes__no_ctes(self): compiled_sql=('with cte as (select * from something_else) ' 'select * from source_table') ), - 'model.root.view_no_cte': CompiledNode( + 'model.root.view_no_cte': CompiledModelNode( name='view_no_cte', database='dbt', schema='analytics', @@ -218,7 +218,7 @@ def test__prepend_ctes(self): input_graph = Manifest( macros={}, nodes={ - 'model.root.view': CompiledNode( + 'model.root.view': CompiledModelNode( name='view', database='dbt', schema='analytics', @@ -242,7 +242,7 @@ def test__prepend_ctes(self): injected_sql='', compiled_sql='select * from __dbt__CTE__ephemeral' ), - 'model.root.ephemeral': CompiledNode( + 'model.root.ephemeral': CompiledModelNode( name='ephemeral', database='dbt', schema='analytics', @@ -295,7 +295,7 @@ def test__prepend_ctes__multiple_levels(self): input_graph = Manifest( macros={}, nodes={ - 'model.root.view': CompiledNode( + 'model.root.view': CompiledModelNode( name='view', database='dbt', schema='analytics', @@ -319,7 +319,7 @@ def test__prepend_ctes__multiple_levels(self): injected_sql='', compiled_sql='select * from __dbt__CTE__ephemeral' ), - 'model.root.ephemeral': CompiledNode( + 'model.root.ephemeral': CompiledModelNode( name='ephemeral', database='dbt', schema='analytics', @@ -343,7 +343,7 @@ def test__prepend_ctes__multiple_levels(self): injected_sql='', compiled_sql='select * from __dbt__CTE__ephemeral_level_two' # noqa ), - 'model.root.ephemeral_level_two': CompiledNode( + 'model.root.ephemeral_level_two': CompiledModelNode( name='ephemeral_level_two', database='dbt', schema='analytics', diff --git a/test/unit/test_context.py b/test/unit/test_context.py index 2032d25cc4a..0b96adfd1a9 100644 --- a/test/unit/test_context.py +++ b/test/unit/test_context.py @@ -1,20 +1,21 @@ import unittest from unittest import mock -from dbt.contracts.graph.parsed import ParsedNode, NodeConfig, DependsOn +from dbt.contracts.graph.parsed import ParsedModelNode, NodeConfig, DependsOn from dbt.context import parser, runtime +from dbt.node_types import NodeType import dbt.exceptions from .mock_adapter import adapter_factory class TestVar(unittest.TestCase): def setUp(self): - self.model = ParsedNode( + self.model = ParsedModelNode( alias='model_one', name='model_one', database='dbt', schema='analytics', - resource_type='model', + resource_type=NodeType.Model, unique_id='model.root.model_one', fqn=['root', 'model_one'], package_name='root', diff --git a/test/unit/test_contracts_graph_compiled.py b/test/unit/test_contracts_graph_compiled.py index c403d826553..9c626e82871 100644 --- a/test/unit/test_contracts_graph_compiled.py +++ b/test/unit/test_contracts_graph_compiled.py @@ -1,5 +1,5 @@ from dbt.contracts.graph.compiled import ( - CompiledNode, InjectedCTE, CompiledTestNode + CompiledModelNode, InjectedCTE, CompiledTestNode ) from dbt.contracts.graph.parsed import ( DependsOn, NodeConfig, TestConfig @@ -9,8 +9,8 @@ from .utils import ContractTestCase -class TestCompiledNode(ContractTestCase): - ContractType = CompiledNode +class TestCompiledModelNode(ContractTestCase): + ContractType = CompiledModelNode def test_basic_uncompiled(self): node_dict = { diff --git a/test/unit/test_contracts_graph_parsed.py b/test/unit/test_contracts_graph_parsed.py index dd48e9add13..0bd2f113655 100644 --- a/test/unit/test_contracts_graph_parsed.py +++ b/test/unit/test_contracts_graph_parsed.py @@ -1,10 +1,10 @@ from dbt.node_types import NodeType from dbt.contracts.graph.parsed import ( - ParsedNode, DependsOn, NodeConfig, ColumnInfo, Hook, ParsedTestNode, + ParsedModelNode, DependsOn, NodeConfig, ColumnInfo, Hook, ParsedTestNode, TestConfig, ParsedSnapshotNode, TimestampSnapshotConfig, All, Docref, GenericSnapshotConfig, CheckSnapshotConfig, TimestampStrategy, CheckStrategy, IntermediateSnapshotNode, ParsedNodePatch, ParsedMacro, - MacroDependsOn, ParsedSourceDefinition, ParsedDocumentation, + MacroDependsOn, ParsedSourceDefinition, ParsedDocumentation, ParsedHookNode ) from dbt.contracts.graph.unparsed import Quoting, FreshnessThreshold @@ -53,8 +53,8 @@ def test_populated(self): self.assert_symmetric(cfg, cfg_dict) -class TestParsedNode(ContractTestCase): - ContractType = ParsedNode +class TestParsedModelNode(ContractTestCase): + ContractType = ParsedModelNode def test_ok(self): node_dict = { @@ -404,7 +404,7 @@ def patch_invalid(self): class TestParsedHookNode(ContractTestCase): - ContractType = ParsedNode + ContractType = ParsedHookNode def test_ok(self): node_dict = { diff --git a/test/unit/test_manifest.py b/test/unit/test_manifest.py index ed47a5f990f..2ab8f275d25 100644 --- a/test/unit/test_manifest.py +++ b/test/unit/test_manifest.py @@ -7,8 +7,10 @@ import dbt.flags from dbt import tracking from dbt.contracts.graph.manifest import Manifest, ManifestMetadata -from dbt.contracts.graph.parsed import ParsedNode, DependsOn, NodeConfig -from dbt.contracts.graph.compiled import CompiledNode +from dbt.contracts.graph.parsed import ( + ParsedModelNode, DependsOn, NodeConfig, ParsedSeedNode +) +from dbt.contracts.graph.compiled import CompiledModelNode from dbt.node_types import NodeType import freezegun @@ -18,7 +20,6 @@ 'depends_on', 'database', 'schema', 'name', 'resource_type', 'package_name', 'root_path', 'path', 'original_file_path', 'raw_sql', 'docrefs', 'description', 'columns', 'fqn', 'build_path', 'patch_path', - 'index', }) REQUIRED_COMPILED_NODE_KEYS = frozenset(REQUIRED_PARSED_NODE_KEYS | { @@ -46,7 +47,7 @@ def setUp(self): }) self.nested_nodes = { - 'model.snowplow.events': ParsedNode( + 'model.snowplow.events': ParsedModelNode( name='events', database='dbt', schema='analytics', @@ -65,7 +66,7 @@ def setUp(self): root_path='', raw_sql='does not matter' ), - 'model.root.events': ParsedNode( + 'model.root.events': ParsedModelNode( name='events', database='dbt', schema='analytics', @@ -84,7 +85,7 @@ def setUp(self): root_path='', raw_sql='does not matter' ), - 'model.root.dep': ParsedNode( + 'model.root.dep': ParsedModelNode( name='dep', database='dbt', schema='analytics', @@ -103,7 +104,7 @@ def setUp(self): root_path='', raw_sql='does not matter' ), - 'model.root.nested': ParsedNode( + 'model.root.nested': ParsedModelNode( name='nested', database='dbt', schema='analytics', @@ -122,7 +123,7 @@ def setUp(self): root_path='', raw_sql='does not matter' ), - 'model.root.sibling': ParsedNode( + 'model.root.sibling': ParsedModelNode( name='sibling', database='dbt', schema='analytics', @@ -141,7 +142,7 @@ def setUp(self): root_path='', raw_sql='does not matter' ), - 'model.root.multi': ParsedNode( + 'model.root.multi': ParsedModelNode( name='multi', database='dbt', schema='analytics', @@ -316,7 +317,7 @@ def test_get_resource_fqns_empty(self): def test_get_resource_fqns(self): nodes = copy.copy(self.nested_nodes) - nodes['seed.root.seed'] = ParsedNode( + nodes['seed.root.seed'] = ParsedSeedNode( name='seed', database='dbt', schema='analytics', @@ -371,7 +372,7 @@ def setUp(self): }) self.nested_nodes = { - 'model.snowplow.events': CompiledNode( + 'model.snowplow.events': CompiledModelNode( name='events', database='dbt', schema='analytics', @@ -395,7 +396,7 @@ def setUp(self): injected_sql=None, extra_ctes=[] ), - 'model.root.events': CompiledNode( + 'model.root.events': CompiledModelNode( name='events', database='dbt', schema='analytics', @@ -419,7 +420,7 @@ def setUp(self): injected_sql='and this also does not matter', extra_ctes=[] ), - 'model.root.dep': ParsedNode( + 'model.root.dep': ParsedModelNode( name='dep', database='dbt', schema='analytics', @@ -438,7 +439,7 @@ def setUp(self): root_path='', raw_sql='does not matter' ), - 'model.root.nested': ParsedNode( + 'model.root.nested': ParsedModelNode( name='nested', database='dbt', schema='analytics', @@ -457,7 +458,7 @@ def setUp(self): root_path='', raw_sql='does not matter' ), - 'model.root.sibling': ParsedNode( + 'model.root.sibling': ParsedModelNode( name='sibling', database='dbt', schema='analytics', @@ -476,7 +477,7 @@ def setUp(self): root_path='', raw_sql='does not matter' ), - 'model.root.multi': ParsedNode( + 'model.root.multi': ParsedModelNode( name='multi', database='dbt', schema='analytics', diff --git a/test/unit/test_parser.py b/test/unit/test_parser.py index 8640a4a4bd1..4b07ac20187 100644 --- a/test/unit/test_parser.py +++ b/test/unit/test_parser.py @@ -13,7 +13,7 @@ from dbt.node_types import NodeType from dbt.contracts.graph.manifest import Manifest -from dbt.contracts.graph.parsed import ParsedNode, ParsedMacro, \ +from dbt.contracts.graph.parsed import ParsedModelNode, ParsedMacro, \ ParsedNodePatch, ParsedSourceDefinition, NodeConfig, DependsOn, \ ColumnInfo, ParsedTestNode, TestConfig from dbt.contracts.graph.unparsed import FreshnessThreshold, Quoting, Time, \ @@ -950,7 +950,19 @@ def setUp(self): 'column_types': {}, 'tags': [], }) - self.test_config = self.model_config.replace(severity='ERROR') + + self.test_config = TestConfig.from_dict({ + 'enabled': True, + 'materialized': 'view', + 'persist_docs': {}, + 'post-hook': [], + 'pre-hook': [], + 'vars': {}, + 'quoting': {}, + 'column_types': {}, + 'tags': [], + 'severity': 'ERROR', + }) self.disabled_config = NodeConfig.from_dict({ 'enabled': False, @@ -983,7 +995,7 @@ def test__single_model(self): self._assert_parsed_sql_nodes( parser.parse_sql_nodes(models), { - 'model.root.model_one': ParsedNode( + 'model.root.model_one': ParsedModelNode( alias='model_one', name='model_one', database='test', @@ -1041,7 +1053,7 @@ def test__single_model__nested_configuration(self): self._assert_parsed_sql_nodes( parser.parse_sql_nodes(models), { - 'model.root.model_one': ParsedNode( + 'model.root.model_one': ParsedModelNode( alias='model_one', name='model_one', database='test', @@ -1088,7 +1100,7 @@ def test__empty_model(self): self._assert_parsed_sql_nodes( parser.parse_sql_nodes(models), { - 'model.root.model_one': ParsedNode( + 'model.root.model_one': ParsedModelNode( alias='model_one', name='model_one', database='test', @@ -1142,7 +1154,7 @@ def test__simple_dependency(self): self._assert_parsed_sql_nodes( parser.parse_sql_nodes(models), { - 'model.root.base': ParsedNode( + 'model.root.base': ParsedModelNode( alias='base', name='base', database='test', @@ -1164,7 +1176,7 @@ def test__simple_dependency(self): columns={} ), - 'model.root.events_tx': ParsedNode( + 'model.root.events_tx': ParsedModelNode( alias='events_tx', name='events_tx', database='test', @@ -1245,7 +1257,7 @@ def test__multiple_dependencies(self): self._assert_parsed_sql_nodes( parser.parse_sql_nodes(models), { - 'model.root.events': ParsedNode( + 'model.root.events': ParsedModelNode( alias='events', name='events', database='test', @@ -1267,7 +1279,7 @@ def test__multiple_dependencies(self): description='', columns={} ), - 'model.root.sessions': ParsedNode( + 'model.root.sessions': ParsedModelNode( alias='sessions', name='sessions', database='test', @@ -1289,7 +1301,7 @@ def test__multiple_dependencies(self): description='', columns={}, ), - 'model.root.events_tx': ParsedNode( + 'model.root.events_tx': ParsedModelNode( alias='events_tx', name='events_tx', database='test', @@ -1311,7 +1323,7 @@ def test__multiple_dependencies(self): description='', columns={} ), - 'model.root.sessions_tx': ParsedNode( + 'model.root.sessions_tx': ParsedModelNode( alias='sessions_tx', name='sessions_tx', database='test', @@ -1333,7 +1345,7 @@ def test__multiple_dependencies(self): description='', columns={} ), - 'model.root.multi': ParsedNode( + 'model.root.multi': ParsedModelNode( alias='multi', name='multi', database='test', @@ -1417,7 +1429,7 @@ def test__multiple_dependencies__packages(self): self._assert_parsed_sql_nodes( parser.parse_sql_nodes(models), { - 'model.snowplow.events': ParsedNode( + 'model.snowplow.events': ParsedModelNode( alias='events', name='events', database='test', @@ -1439,7 +1451,7 @@ def test__multiple_dependencies__packages(self): description='', columns={} ), - 'model.snowplow.sessions': ParsedNode( + 'model.snowplow.sessions': ParsedModelNode( alias='sessions', name='sessions', database='test', @@ -1461,7 +1473,7 @@ def test__multiple_dependencies__packages(self): description='', columns={} ), - 'model.snowplow.events_tx': ParsedNode( + 'model.snowplow.events_tx': ParsedModelNode( alias='events_tx', name='events_tx', database='test', @@ -1483,7 +1495,7 @@ def test__multiple_dependencies__packages(self): description='', columns={} ), - 'model.snowplow.sessions_tx': ParsedNode( + 'model.snowplow.sessions_tx': ParsedModelNode( alias='sessions_tx', name='sessions_tx', database='test', @@ -1505,7 +1517,7 @@ def test__multiple_dependencies__packages(self): description='', columns={} ), - 'model.root.multi': ParsedNode( + 'model.root.multi': ParsedModelNode( alias='multi', name='multi', database='test', @@ -1534,7 +1546,7 @@ def test__multiple_dependencies__packages(self): def test__process_refs__packages(self): nodes = { - 'model.snowplow.events': ParsedNode( + 'model.snowplow.events': ParsedModelNode( name='events', alias='events', database='test', @@ -1553,7 +1565,7 @@ def test__process_refs__packages(self): root_path=get_os_path('/usr/src/app'), raw_sql='does not matter', ), - 'model.root.events': ParsedNode( + 'model.root.events': ParsedModelNode( name='events', alias='events', database='test', @@ -1572,7 +1584,7 @@ def test__process_refs__packages(self): root_path=get_os_path('/usr/src/app'), raw_sql='does not matter', ), - 'model.root.dep': ParsedNode( + 'model.root.dep': ParsedModelNode( name='dep', alias='dep', database='test', @@ -1632,7 +1644,6 @@ def test__process_refs__packages(self): 'columns': {}, 'description': '', 'build_path': None, - 'index': None, 'patch_path': None, }, 'model.root.events': { @@ -1660,7 +1671,6 @@ def test__process_refs__packages(self): 'columns': {}, 'description': '', 'build_path': None, - 'index': None, 'patch_path': None, }, 'model.root.dep': { @@ -1688,7 +1698,6 @@ def test__process_refs__packages(self): 'columns': {}, 'description': '', 'build_path': None, - 'index': None, 'patch_path': None, } } @@ -1718,7 +1727,7 @@ def test__in_model_config(self): self._assert_parsed_sql_nodes( parser.parse_sql_nodes(models), { - 'model.root.model_one': ParsedNode( + 'model.root.model_one': ParsedModelNode( alias='model_one', name='model_one', database='test', @@ -1794,7 +1803,7 @@ def test__root_project_config(self): self._assert_parsed_sql_nodes( parser.parse_sql_nodes(models), { - 'model.root.table': ParsedNode( + 'model.root.table': ParsedModelNode( alias='table', name='table', database='test', @@ -1816,7 +1825,7 @@ def test__root_project_config(self): description='', columns={} ), - 'model.root.ephemeral': ParsedNode( + 'model.root.ephemeral': ParsedModelNode( alias='ephemeral', name='ephemeral', database='test', @@ -1838,7 +1847,7 @@ def test__root_project_config(self): description='', columns={} ), - 'model.root.view': ParsedNode( + 'model.root.view': ParsedModelNode( alias='view', name='view', database='test', @@ -1979,7 +1988,7 @@ def test__other_project_config(self): self._assert_parsed_sql_nodes( parser.parse_sql_nodes(models), parsed={ - 'model.root.table': ParsedNode( + 'model.root.table': ParsedModelNode( alias='table', name='table', database='test', @@ -2001,7 +2010,7 @@ def test__other_project_config(self): description='', columns={} ), - 'model.root.ephemeral': ParsedNode( + 'model.root.ephemeral': ParsedModelNode( alias='ephemeral', name='ephemeral', database='test', @@ -2023,7 +2032,7 @@ def test__other_project_config(self): description='', columns={} ), - 'model.root.view': ParsedNode( + 'model.root.view': ParsedModelNode( alias='view', name='view', database='test', @@ -2045,7 +2054,7 @@ def test__other_project_config(self): description='', columns={} ), - 'model.snowplow.multi_sort': ParsedNode( + 'model.snowplow.multi_sort': ParsedModelNode( alias='multi_sort', name='multi_sort', database='test', @@ -2069,7 +2078,7 @@ def test__other_project_config(self): ), }, disabled=[ - ParsedNode( + ParsedModelNode( name='disabled', resource_type=NodeType.Model, package_name='snowplow', @@ -2089,7 +2098,7 @@ def test__other_project_config(self): fqn=['snowplow', 'disabled'], columns={} ), - ParsedNode( + ParsedModelNode( name='package', resource_type=NodeType.Model, package_name='snowplow', @@ -2132,7 +2141,7 @@ def test__simple_data_test(self): self._assert_parsed_sql_nodes( parser.parse_sql_nodes(tests), { - 'test.root.no_events': ParsedNode( + 'test.root.no_events': ParsedTestNode( alias='no_events', name='no_events', database='test', @@ -2174,7 +2183,6 @@ def test__simple_macro(self): self.assertTrue(callable(result['macro.root.simple'].generator)) - self.assertEqual( result, { @@ -2250,7 +2258,7 @@ def test__simple_macro_used_in_model(self): self._assert_parsed_sql_nodes( parser.parse_sql_nodes(models), { - 'model.root.model_one': ParsedNode( + 'model.root.model_one': ParsedModelNode( alias='model_one', name='model_one', database='test', @@ -2296,7 +2304,7 @@ def test__macro_no_explicit_project_used_in_model(self): self._assert_parsed_sql_nodes( parser.parse_sql_nodes(models), { - 'model.root.model_one': ParsedNode( + 'model.root.model_one': ParsedModelNode( alias='model_one', name='model_one', database='test',