dbt Contexts

Rewrite/reorganize contexts
 - now they are objects that have a to_dict()
 - special decorators on properties/methods to indicate membership
 - reorganize contexts/ to remove many many import cycles

Context behavior changes:
 - 'dbt_version' is alwways available
 - 'target' is available as long as the profile has been parsed
 - 'project_name' is available in: query headers, hooks, models, and macro execution

Profiles/projects now render at load time
 - reading a profile or project file requires a ConfigRenderer
 - projects get an extra-fiddly special load for use during initial parsing
    - it returns the profile name and a function that, given a renderer, can return the rest of the Project
    - idea is: use the profile name to load the Profile, use that to build a ConfigRenderer that has a TargetContext, render the project with that
 - profiles.yml is rendered with the 'base' context
 - dbt_project.yml/schema.yml/packages.yml are rendered with the 'target' context: 'base' context + 'target'
 - docs are rendered with the docs context: 'target' context + the 'doc' function
 - query headers are rendered with the query header context: 'target' context + macros + 'project_name'
 - executed macros/models should have the same context as previously (query headers + adapter/model/etc related functions)

Moved actual ref/source searching into the manifest
Moved the rest of the parse utils into parser/manifest.py
Made the ref/source resolvers defined in the provider context a bit more sane/well-defined
Picked consistent-but-not-great names for all the various context generation functions
Moved write_node into ParsedNode

core/dbt/adapters/base/connections.py

@@ -14,8 +14,9 @@
 from dbt.contracts.connection import (
     Connection, Identifier, ConnectionState, AdapterRequiredConfig, LazyHandle
 )
+from dbt.contracts.graph.manifest import Manifest
 from dbt.adapters.base.query_headers import (
-    QueryStringSetter, MacroQueryStringSetter,
+    MacroQueryStringSetter,
 )
 from dbt.logger import GLOBAL_LOGGER as logger
 
@@ -39,13 +40,10 @@ def __init__(self, profile: AdapterRequiredConfig):
         self.profile = profile
         self.thread_connections: Dict[Hashable, Connection] = {}
         self.lock: RLock = dbt.flags.MP_CONTEXT.RLock()
-        self.query_header = QueryStringSetter(self.profile)
+        self.query_header: Optional[MacroQueryStringSetter] = None
 
-    def set_query_header(self, manifest=None) -> None:
-        if manifest is not None:
-            self.query_header = MacroQueryStringSetter(self.profile, manifest)
-        else:
-            self.query_header = QueryStringSetter(self.profile)
+    def set_query_header(self, manifest: Manifest) -> None:
+        self.query_header = MacroQueryStringSetter(self.profile, manifest)
 
     @staticmethod
     def get_thread_identifier() -> Hashable:
@@ -285,6 +283,8 @@ def commit_if_has_connection(self) -> None:
             self.commit()
 
     def _add_query_comment(self, sql: str) -> str:
+        if self.query_header is None:
+            return sql
         return self.query_header.add(sql)
 
     @abc.abstractmethod

core/dbt/adapters/base/impl.py

@@ -248,12 +248,14 @@ def connection_named(
         self, name: str, node: Optional[CompileResultNode] = None
     ) -> Iterator[None]:
         try:
-            self.connections.query_header.set(name, node)
+            if self.connections.query_header is not None:
+                self.connections.query_header.set(name, node)
             self.acquire_connection(name)
             yield
         finally:
             self.release_connection()
-            self.connections.query_header.reset()
+            if self.connections.query_header is not None:
+                self.connections.query_header.reset()
 
     @contextmanager
     def connection_for(
@@ -983,12 +985,12 @@ def execute_macro(
                 'dbt could not find a macro with the name "{}" in {}'
                 .format(macro_name, package_name)
             )
-        # This causes a reference cycle, as dbt.context.runtime.generate()
+        # This causes a reference cycle, as generate_runtime_macro()
         # ends up calling get_adapter, so the import has to be here.
-        import dbt.context.operation
-        macro_context = dbt.context.operation.generate(
-            model=macro,
-            runtime_config=self.config,
+        from dbt.context.providers import generate_runtime_macro
+        macro_context = generate_runtime_macro(
+            macro=macro,
+            config=self.config,
             manifest=manifest,
             package_name=project
         )

core/dbt/adapters/base/plugin.py

@@ -1,6 +1,7 @@
 from typing import List, Optional, Type
 
 from dbt.adapters.base import BaseAdapter, Credentials
+from dbt.exceptions import CompilationException
 
 
 class AdapterPlugin:
@@ -23,8 +24,12 @@ def __init__(
         self.adapter: Type[BaseAdapter] = adapter
         self.credentials: Type[Credentials] = credentials
         self.include_path: str = include_path
-        project = Project.from_project_root(include_path, {})
-        self.project_name: str = project.project_name
+        partial = Project.partial_load(include_path)
+        if partial.project_name is None:
+            raise CompilationException(
+                f'Invalid project at {include_path}: name not set!'
+            )
+        self.project_name: str = partial.project_name
         self.dependencies: List[str]
         if dependencies is None:
             self.dependencies = []

core/dbt/adapters/base/query_headers.py

@@ -3,8 +3,7 @@
 
 from dbt.clients.jinja import QueryStringGenerator
 
-# this generates an import cycle, as usual
-from dbt.context.base import QueryHeaderContext
+from dbt.context.configured import generate_query_header_context
 from dbt.contracts.connection import AdapterRequiredConfig
 from dbt.contracts.graph.compiled import CompileResultNode
 from dbt.contracts.graph.manifest import Manifest
@@ -68,9 +67,9 @@ def set(self, comment: Optional[str]):
 QueryStringFunc = Callable[[str, Optional[NodeWrapper]], str]
 
 
-class QueryStringSetter:
-    """The base query string setter. This is only used once."""
-    def __init__(self, config: AdapterRequiredConfig):
+class MacroQueryStringSetter:
+    def __init__(self, config: AdapterRequiredConfig, manifest: Manifest):
+        self.manifest = manifest
         self.config = config
 
         comment_macro = self._get_comment_macro()
@@ -88,17 +87,22 @@ def __init__(self, config: AdapterRequiredConfig):
         self.comment = _QueryComment(None)
         self.reset()
 
-    def _get_context(self):
-        return QueryHeaderContext(self.config).to_dict()
-
-    def _get_comment_macro(self) -> Optional[str]:
+    def _get_comment_macro(self):
+        if (
+            self.config.query_comment != NoValue() and
+            self.config.query_comment
+        ):
+            return self.config.query_comment
         # if the query comment is null/empty string, there is no comment at all
-        if not self.config.query_comment:
+        elif not self.config.query_comment:
             return None
         else:
             # else, the default
             return DEFAULT_QUERY_COMMENT
 
+    def _get_context(self) -> Dict[str, Any]:
+        return generate_query_header_context(self.config, self.manifest)
+
     def add(self, sql: str) -> str:
         return self.comment.add(sql)
 
@@ -111,21 +115,3 @@ def set(self, name: str, node: Optional[CompileResultNode]):
             wrapped = NodeWrapper(node)
         comment_str = self.generator(name, wrapped)
         self.comment.set(comment_str)
-
-
-class MacroQueryStringSetter(QueryStringSetter):
-    def __init__(self, config: AdapterRequiredConfig, manifest: Manifest):
-        self.manifest = manifest
-        super().__init__(config)
-
-    def _get_comment_macro(self):
-        if (
-            self.config.query_comment != NoValue() and
-            self.config.query_comment
-        ):
-            return self.config.query_comment
-        else:
-            return super()._get_comment_macro()
-
-    def _get_context(self) -> Dict[str, Any]:
-        return QueryHeaderContext(self.config).to_dict(self.manifest.macros)

core/dbt/clients/jinja.py

@@ -4,7 +4,7 @@
 import tempfile
 from contextlib import contextmanager
 from typing import (
-    List, Union, Set, Optional, Dict, Any, Callable, Iterator, Type
+    List, Union, Set, Optional, Dict, Any, Callable, Iterator, Type, NoReturn
 )
 
 import jinja2
@@ -374,7 +374,7 @@ def get_rendered(string, ctx, node=None,
     return render_template(template, ctx, node)
 
 
-def undefined_error(msg):
+def undefined_error(msg) -> NoReturn:
     raise jinja2.exceptions.UndefinedError(msg)
 
 

core/dbt/compilation.py

@@ -10,7 +10,7 @@
 from dbt.node_types import NodeType
 from dbt.linker import Linker
 
-import dbt.context.runtime
+from dbt.context.providers import generate_runtime_model
 import dbt.contracts.project
 import dbt.exceptions
 import dbt.flags
@@ -146,8 +146,9 @@ def compile_node(self, node, manifest, extra_context=None):
         })
         compiled_node = _compiled_type_for(node).from_dict(data)
 
-        context = dbt.context.runtime.generate(
-            compiled_node, self.config, manifest)
+        context = generate_runtime_model(
+            compiled_node, self.config, manifest
+        )
         context.update(extra_context)
 
         compiled_node.compiled_sql = dbt.clients.jinja.get_rendered(
@@ -253,13 +254,11 @@ def compile_node(adapter, config, node, manifest, extra_context, write=True):
         logger.debug('Writing injected SQL for node "{}"'.format(
             node.unique_id))
 
-        written_path = dbt.writer.write_node(
-            node,
+        node.build_path = node.write_node(
             config.target_path,
             'compiled',
-            node.injected_sql)
-
-        node.build_path = written_path
+            node.injected_sql
+        )
 
     return node
 

core/dbt/config/profile.py

@@ -6,15 +6,15 @@
 
 from dbt.clients.system import load_file_contents
 from dbt.clients.yaml_helper import load_yaml_text
-from dbt.contracts.connection import Credentials
+from dbt.contracts.connection import Credentials, HasCredentials
 from dbt.contracts.project import ProfileConfig, UserConfig
 from dbt.exceptions import DbtProfileError
 from dbt.exceptions import DbtProjectError
 from dbt.exceptions import ValidationException
 from dbt.exceptions import RuntimeException
 from dbt.exceptions import validator_error_message
 from dbt.logger import GLOBAL_LOGGER as logger
-from dbt.utils import parse_cli_vars, coerce_dict_str
+from dbt.utils import coerce_dict_str
 
 from .renderer import ConfigRenderer
 
@@ -73,7 +73,7 @@ def read_user_config(directory: str) -> UserConfig:
 
 
 @dataclass
-class Profile:
+class Profile(HasCredentials):
     profile_name: str
     target_name: str
     config: UserConfig
@@ -217,13 +217,11 @@ def render_profile(
         raw_profile: Dict[str, Any],
         profile_name: str,
         target_override: Optional[str],
-        cli_vars: Dict[str, Any],
+        renderer: ConfigRenderer,
     ) -> Tuple[str, Dict[str, Any]]:
         """This is a containment zone for the hateful way we're rendering
         profiles.
         """
-        renderer = ConfigRenderer(cli_vars=cli_vars)
-
         # rendering profiles is a bit complex. Two constraints cause trouble:
         # 1) users should be able to use environment/cli variables to specify
         #    the target in their profile.
@@ -255,7 +253,7 @@ def from_raw_profile_info(
         cls,
         raw_profile: Dict[str, Any],
         profile_name: str,
-        cli_vars: Dict[str, Any],
+        renderer: ConfigRenderer,
         user_cfg: Optional[Dict[str, Any]] = None,
         target_override: Optional[str] = None,
         threads_override: Optional[int] = None,
@@ -267,8 +265,7 @@ def from_raw_profile_info(
         :param raw_profile: The profile data for a single profile, from
             disk as yaml and its values rendered with jinja.
         :param profile_name: The profile name used.
-        :param cli_vars: The command-line variables passed as arguments,
-            as a dict.
+        :param renderer: The config renderer.
         :param user_cfg: The global config for the user, if it
             was present.
         :param target_override: The target to use, if provided on
@@ -285,7 +282,7 @@ def from_raw_profile_info(
 
         # TODO: should it be, and the values coerced to bool?
         target_name, profile_data = cls.render_profile(
-            raw_profile, profile_name, target_override, cli_vars
+            raw_profile, profile_name, target_override, renderer
         )
 
         # valid connections never include the number of threads, but it's
@@ -311,15 +308,14 @@ def from_raw_profiles(
         cls,
         raw_profiles: Dict[str, Any],
         profile_name: str,
-        cli_vars: Dict[str, Any],
+        renderer: ConfigRenderer,
         target_override: Optional[str] = None,
         threads_override: Optional[int] = None,
     ) -> 'Profile':
         """
         :param raw_profiles: The profile data, from disk as yaml.
         :param profile_name: The profile name to use.
-        :param cli_vars: The command-line variables passed as arguments, as a
-            dict.
+        :param renderer: The config renderer.
         :param target_override: The target to use, if provided on the command
             line.
         :param threads_override: The thread count to use, if provided on the
@@ -344,17 +340,18 @@ def from_raw_profiles(
         return cls.from_raw_profile_info(
             raw_profile=raw_profile,
             profile_name=profile_name,
-            cli_vars=cli_vars,
+            renderer=renderer,
             user_cfg=user_cfg,
             target_override=target_override,
             threads_override=threads_override,
         )
 
     @classmethod
-    def from_args(
+    def render_from_args(
         cls,
         args: Any,
-        project_profile_name: Optional[str] = None,
+        renderer: ConfigRenderer,
+        project_profile_name: Optional[str],
     ) -> 'Profile':
         """Given the raw profiles as read from disk and the name of the desired
         profile if specified, return the profile component of the runtime
@@ -370,7 +367,6 @@ def from_args(
             target could not be found.
         :returns Profile: The new Profile object.
         """
-        cli_vars = parse_cli_vars(getattr(args, 'vars', '{}'))
         threads_override = getattr(args, 'threads', None)
         target_override = getattr(args, 'target', None)
         raw_profiles = read_profile(args.profiles_dir)
@@ -380,7 +376,7 @@ def from_args(
         return cls.from_raw_profiles(
             raw_profiles=raw_profiles,
             profile_name=profile_name,
-            cli_vars=cli_vars,
+            renderer=renderer,
             target_override=target_override,
             threads_override=threads_override
         )