Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add modules and tracking info to the configuration parsing context (#1320) #1441

Merged
merged 1 commit into from
May 7, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 2 additions & 4 deletions core/dbt/config/renderer.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
from dbt import compat
from dbt.clients.jinja import get_rendered
from dbt.context.common import env_var
from dbt.context.common import Var
from dbt.context.common import generate_config_context
from dbt.exceptions import DbtProfileError
from dbt.exceptions import DbtProjectError
from dbt.exceptions import RecursionException
Expand All @@ -13,8 +12,7 @@ class ConfigRenderer(object):
variables and a render type.
"""
def __init__(self, cli_vars):
self.context = {'env_var': env_var}
self.context['var'] = Var(None, self.context, cli_vars)
self.context = generate_config_context(cli_vars)

@staticmethod
def _is_hook_or_model_vars_path(keypath):
Expand Down
21 changes: 17 additions & 4 deletions core/dbt/context/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -329,6 +329,22 @@ def get_datetime_module_context():
}


def get_context_modules():
return {
'pytz': get_pytz_module_context(),
'datetime': get_datetime_module_context(),
}


def generate_config_context(cli_vars):
context = {
'env_var': env_var,
'modules': get_context_modules(),
}
context['var'] = Var(None, context, cli_vars)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is probably one of those things we'll want to consider when describing (and overhauling) the compilation context -- it's kind of wild that we need to provide a context to build the context... Obviously ok for now though

return _add_tracking(context)


def generate_base(model, model_dict, config, manifest, source_config,
provider, adapter=None):
"""Generate the common aspects of the config dict."""
Expand Down Expand Up @@ -370,10 +386,7 @@ def generate_base(model, model_dict, config, manifest, source_config,
"graph": manifest.to_flat_graph(),
"log": log,
"model": model_dict,
"modules": {
"pytz": get_pytz_module_context(),
"datetime": get_datetime_module_context(),
},
"modules": get_context_modules(),
"post_hooks": post_hooks,
"pre_hooks": pre_hooks,
"ref": provider.ref(db_wrapper, model, config, manifest),
Expand Down
44 changes: 44 additions & 0 deletions test/integration/039_config_test/test_configs.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
from datetime import datetime
import os
import shutil

from test.integration.base import DBTIntegrationTest, use_profile

Expand Down Expand Up @@ -39,3 +42,44 @@ def test_postgres_config_layering(self):
# make sure we overwrote the materialization properly
models = self.get_models_in_schema()
self.assertEqual(models['model'], 'table')


class TestTargetConfigs(DBTIntegrationTest):
@property
def schema(self):
return "config_039"

def unique_schema(self):
return super(TestTargetConfigs, self).unique_schema().upper()

@property
def models(self):
return "test/integration/039_config_test/models"

def setUp(self):
super(TestTargetConfigs, self).setUp()
self.init_targets = [d for d in os.listdir('.') if os.path.isdir(d) and d.startswith('target_')]

def tearDown(self):
super(TestTargetConfigs, self).tearDown()
for d in self.new_dirs():
shutil.rmtree(d)

def new_dirs(self):
for d in os.listdir('.'):
if os.path.isdir(d) and d not in self.init_targets and d.startswith('target_'):
yield d

@property
def project_config(self):
return {
'data-paths': ['test/integration/039_config_test/data'],
'target-path': "target_{{ modules.datetime.datetime.utcnow().strftime('%Y%m%dT%H%M%S') }}"
}

@use_profile('postgres')
def test_alternative_target_paths(self):
self.run_dbt(['seed'])
dirs = list(self.new_dirs())
self.assertEqual(len(dirs), 1)
self.assertTrue(os.path.exists(os.path.join(dirs[0], 'manifest.json')))