Use yaml renderer (with target context) for rendering selectors (dbt-…

…labs#5136)

* Use yaml renderer (with target context) for rendering selectors

* Changie

* Convert cli_vars tests

* Add test for var in profiles

* Add test for cli vars in packages

* Add test for vars in selectors

.changes/unreleased/Fixes-20220422-135645.yaml

@@ -0,0 +1,7 @@
+kind: Fixes
+body: Use yaml renderer (with target context) for rendering selectors
+time: 2022-04-22T13:56:45.147893-04:00
+custom:
+  Author: gshank
+  Issue: "5131"
+  PR: "5136"

core/dbt/config/renderer.py

@@ -114,12 +114,10 @@ def __init__(
     def name(self):
         "Project config"
 
+    # Uses SecretRenderer
     def get_package_renderer(self) -> BaseRenderer:
         return PackageRenderer(self.ctx_obj.cli_vars)
 
-    def get_selector_renderer(self) -> BaseRenderer:
-        return SelectorRenderer(self.ctx_obj.cli_vars)
-
     def render_project(
         self,
         project: Dict[str, Any],
@@ -136,8 +134,7 @@ def render_packages(self, packages: Dict[str, Any]):
         return package_renderer.render_data(packages)
 
     def render_selectors(self, selectors: Dict[str, Any]):
-        selector_renderer = self.get_selector_renderer()
-        return selector_renderer.render_data(selectors)
+        return self.render_data(selectors)
 
     def render_entry(self, value: Any, keypath: Keypath) -> Any:
         result = super().render_entry(value, keypath)
@@ -165,18 +162,10 @@ def should_render_keypath(self, keypath: Keypath) -> bool:
         return True
 
 
-class SelectorRenderer(BaseRenderer):
-    @property
-    def name(self):
-        return "Selector config"
-
-
 class SecretRenderer(BaseRenderer):
-    def __init__(self, cli_vars: Optional[Dict[str, Any]] = None) -> None:
+    def __init__(self, cli_vars: Dict[str, Any] = {}) -> None:
         # Generate contexts here because we want to save the context
         # object in order to retrieve the env_vars.
-        if cli_vars is None:
-            cli_vars = {}
         self.ctx_obj = SecretContext(cli_vars)
         context = self.ctx_obj.to_dict()
         super().__init__(context)

core/dbt/config/selectors.py

@@ -3,7 +3,7 @@
 from dbt.clients.yaml_helper import yaml, Loader, Dumper, load_yaml_text  # noqa: F401
 from dbt.dataclass_schema import ValidationError
 
-from .renderer import SelectorRenderer
+from .renderer import BaseRenderer
 
 from dbt.clients.system import (
     load_file_contents,
@@ -57,7 +57,7 @@ def selectors_from_dict(cls, data: Dict[str, Any]) -> "SelectorConfig":
     def render_from_dict(
         cls,
         data: Dict[str, Any],
-        renderer: SelectorRenderer,
+        renderer: BaseRenderer,
     ) -> "SelectorConfig":
         try:
             rendered = renderer.render_data(data)
@@ -72,7 +72,7 @@ def render_from_dict(
     def from_path(
         cls,
         path: Path,
-        renderer: SelectorRenderer,
+        renderer: BaseRenderer,
     ) -> "SelectorConfig":
         try:
             data = load_yaml_text(load_file_contents(str(path)))

core/dbt/config/utils.py

@@ -62,6 +62,8 @@ def get_project_config(
             user_config = read_user_config(flags.PROFILES_DIR)
         # Update flags
         flags.set_from_args(args, user_config)
+        if cli_vars is None:
+            cli_vars = {}
         profile = Profile.render_from_args(args, ProfileRenderer(cli_vars), profile_name)
     # Generate a project
     project = Project.from_project_root(

core/dbt/tests/fixtures/project.py

@@ -107,6 +107,9 @@ def test_data_dir(request):
 
 # This contains the profile target information, for simplicity in setting
 # up different profiles, particularly in the adapter repos.
+# Note: because we load the profile to create the adapter, this
+# fixture can't be used to test vars and env_vars or errors. The
+# profile must be written out after the test starts.
 @pytest.fixture(scope="class")
 def dbt_profile_target():
     return {

core/dbt/tests/util.py

@@ -24,6 +24,7 @@
 #   read_file
 #   get_artifact
 #   update_config_file
+#   write_config_file
 #   get_unique_ids_in_results
 #   check_result_nodes_by_name
 #   check_result_nodes_by_unique_id
@@ -149,6 +150,13 @@ def update_config_file(updates, *paths):
     write_file(new_yaml, *paths)
 
 
+# Write new config file
+def write_config_file(data, *paths):
+    if type(data) is dict:
+        data = yaml.safe_dump(data)
+    write_file(data, *paths)
+
+
 # Get the unique_ids in dbt command results
 def get_unique_ids_in_results(results):
     unique_ids = []

tests/functional/cli_vars/test_cli_var_override.py

@@ -0,0 +1,68 @@
+import pytest
+
+from dbt.tests.util import run_dbt
+
+
+models_override__schema_yml = """
+version: 2
+models:
+- name: test_vars
+  columns:
+  - name: field
+    tests:
+    - accepted_values:
+        values:
+        - override
+"""
+
+models_override__test_vars_sql = """
+select '{{ var("required") }}'::varchar as field
+"""
+
+
+# Tests that cli vars override vars set in the project config
+class TestCLIVarOverride:
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "schema.yml": models_override__schema_yml,
+            "test_vars.sql": models_override__test_vars_sql,
+        }
+
+    @pytest.fixture(scope="class")
+    def project_config_update(self):
+        return {
+            "vars": {
+                "required": "present",
+            },
+        }
+
+    def test__override_vars_global(self, project):
+        run_dbt(["run", "--vars", "{required: override}"])
+        run_dbt(["test"])
+
+
+# This one switches to setting a var in 'test'
+class TestCLIVarOverridePorject:
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "schema.yml": models_override__schema_yml,
+            "test_vars.sql": models_override__test_vars_sql,
+        }
+
+    @pytest.fixture(scope="class")
+    def project_config_update(self):
+        return {
+            "vars": {
+                "test": {
+                    "required": "present",
+                },
+            },
+        }
+
+    def test__override_vars_project_level(self, project):
+
+        # This should be "override"
+        run_dbt(["run", "--vars", "{required: override}"])
+        run_dbt(["test"])