From abcbacaf6918ba2e4a82464ef433a1577c4cd839 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Mon, 29 Apr 2019 19:54:19 -0600 Subject: [PATCH] Implement test failure severity levels A small refactor to make test parsing easier to modify add concept of test modifier kwargs, pass them through to config plug the severity setting into test result handling Update existing tests Add integration tests severity settings for data tests, too --- core/dbt/contracts/graph/parsed.py | 5 +- core/dbt/exceptions.py | 10 + core/dbt/parser/schemas.py | 264 ++++++++++-------- core/dbt/parser/source_config.py | 4 + core/dbt/ui/printer.py | 11 +- .../test_docs_generate.py | 18 +- .../data/null_seed.csv | 21 ++ .../045_test_severity_tests/models/model.sql | 1 + .../045_test_severity_tests/models/schema.yml | 19 ++ .../045_test_severity_tests/test_severity.py | 43 +++ test/unit/test_parser.py | 107 ++++--- 11 files changed, 322 insertions(+), 181 deletions(-) create mode 100644 test/integration/045_test_severity_tests/data/null_seed.csv create mode 100644 test/integration/045_test_severity_tests/models/model.sql create mode 100644 test/integration/045_test_severity_tests/models/schema.yml create mode 100644 test/integration/045_test_severity_tests/test_severity.py diff --git a/core/dbt/contracts/graph/parsed.py b/core/dbt/contracts/graph/parsed.py index d0e77c20ab5..8647d83741b 100644 --- a/core/dbt/contracts/graph/parsed.py +++ b/core/dbt/contracts/graph/parsed.py @@ -80,11 +80,14 @@ } ] }, + 'severity': { + 'enum': ['ERROR', 'WARN'], + }, }, 'required': [ 'enabled', 'materialized', 'post-hook', 'pre-hook', 'vars', 'quoting', 'column_types', 'tags' - ] + ], } diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index 2319e748cc2..a2034ac6f7e 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -661,6 +661,16 @@ def warn_or_error(msg, node=None, log_fmt=None): logger.warning(msg) +def warn_or_raise(exc, log_fmt=None): + if dbt.flags.WARN_ERROR: + raise exc + else: + msg = str(exc) + if log_fmt is not None: + msg = log_fmt.format(msg) + logger.warning(msg) + + # Update this when a new function should be added to the # dbt context's `exceptions` key! CONTEXT_EXPORTS = { diff --git a/core/dbt/parser/schemas.py b/core/dbt/parser/schemas.py index a96961d4fa6..e7497b2d80e 100644 --- a/core/dbt/parser/schemas.py +++ b/core/dbt/parser/schemas.py @@ -67,81 +67,55 @@ def as_kwarg(key, value): return "{key}={value}".format(key=key, value=formatted_value) -def build_test_raw_sql(test_namespace, model, test_type, test_args): - """Build the raw SQL from a test definition. - - :param test_namespace: The test's namespace, if one exists - :param model: The model under test - :param test_type: The type of the test (unique_id, etc) - :param test_args: The arguments passed to the test as a list of `key=value` - strings - :return: A string of raw sql for the test node. - """ - # sort the dict so the keys are rendered deterministically (for tests) - kwargs = [as_kwarg(key, test_args[key]) for key in sorted(test_args)] - - if test_namespace is None: - macro_name = "test_{}".format(test_type) - else: - macro_name = "{}.test_{}".format(test_namespace, test_type) - - raw_sql = "{{{{ {macro}(model=ref('{model}'), {kwargs}) }}}}".format( - **{ - 'model': model['name'], - 'macro': macro_name, - 'kwargs': ", ".join(kwargs) - } - ) - return raw_sql +class TestBuilder(object): + """An object to hold assorted test settings and perform basic parsing + Test names have the following pattern: + - the test name itself may be namespaced (package.test) + - or it may not be namespaced (test) + - the test may have arguments embedded in the name (, severity=WARN) + - or it may not have arguments. -def build_source_test_raw_sql(test_namespace, source, table, test_type, - test_args): - """Build the raw SQL from a source test definition. - - :param test_namespace: The test's namespace, if one exists - :param source: The source under test. - :param table: The table under test - :param test_type: The type of the test (unique_id, etc) - :param test_args: The arguments passed to the test as a list of `key=value` - strings - :return: A string of raw sql for the test node. """ - # sort the dict so the keys are rendered deterministically (for tests) - kwargs = [as_kwarg(key, test_args[key]) for key in sorted(test_args)] - - if test_namespace is None: - macro_name = "test_{}".format(test_type) - else: - macro_name = "{}.test_{}".format(test_namespace, test_type) - - raw_sql = ( - "{{{{ {macro}(model=source('{source}', '{table}'), {kwargs}) }}}}" - .format( - source=source['name'], - table=table['name'], - macro=macro_name, - kwargs=", ".join(kwargs)) + TEST_NAME_PATTERN = re.compile( + r'((?P([a-zA-Z_][0-9a-zA-Z_]*))\.)?' + r'(?P([a-zA-Z_][0-9a-zA-Z_]*))' ) - return raw_sql + # map magic keys to default values + MODIFIER_ARGS = {'severity': 'ERROR'} + def __init__(self, test, target, column_name, package_name): + test_name, test_args = self.extract_test_args(test, column_name) + self.args = test_args + self.package_name = package_name + self.target = target + + match = self.TEST_NAME_PATTERN.match(test_name) + if match is None: + dbt.exceptions.raise_compiler_error( + 'Test name string did not match expected pattern: {}' + .format(test_name) + ) -def calculate_test_namespace(test_type, package_name): - test_namespace = None - split = test_type.split('.') - if len(split) > 1: - test_type = split[1] - package_name = split[0] - test_namespace = package_name + groups = match.groupdict() + self.name = groups['test_name'] + self.namespace = groups['test_namespace'] + self.modifiers = {} + for key, default in self.MODIFIER_ARGS.items(): + self.modifiers[key] = self.args.pop(key, default) - return test_namespace, test_type, package_name + if self.namespace is not None: + self.package_name = self.namespace + @staticmethod + def extract_test_args(test, name=None): + if not isinstance(test, dict): + dbt.exceptions.raise_compiler_error( + 'test must be dict or str, got {} (value {})'.format( + type(test), test + ) + ) -def _build_test_args(test, name): - if isinstance(test, basestring): - test_name = test - test_args = {} - elif isinstance(test, dict): test = list(test.items()) if len(test) != 1: dbt.exceptions.raise_compiler_error( @@ -149,27 +123,88 @@ def _build_test_args(test, name): ' {} instead ({} keys)'.format(test, len(test)) ) test_name, test_args = test[0] - else: - dbt.exceptions.raise_compiler_error( - 'test must be dict or str, got {} (value {})'.format( - type(test), test + + if not isinstance(test_args, dict): + dbt.exceptions.raise_compiler_error( + 'test arguments must be dict, got {} (value {})'.format( + type(test_args), test_args + ) ) - ) - if not isinstance(test_args, dict): - dbt.exceptions.raise_compiler_error( - 'test arguments must be dict, got {} (value {})'.format( - type(test_args), test_args + if not isinstance(test_name, basestring): + dbt.exceptions.raise_compiler_error( + 'test name must be a str, got {} (value {})'.format( + type(test_name), test_name + ) ) + if name is not None: + test_args['column_name'] = name + return test_name, test_args + + def severity(self): + return self.modifiers.get('severity', 'ERROR').upper() + + def test_kwargs_str(self): + # sort the dict so the keys are rendered deterministically (for tests) + return ', '.join(( + as_kwarg(key, self.args[key]) + for key in sorted(self.args) + )) + + def macro_name(self): + macro_name = 'test_{}'.format(self.name) + if self.namespace is not None: + macro_name = "{}.{}".format(self.namespace, macro_name) + return macro_name + + def build_model_str(self): + raise NotImplementedError('build_model_str not implemented!') + + def get_test_name(self): + raise NotImplementedError('get_test_name not implemented!') + + def build_raw_sql(self): + return ( + "{{{{ config(severity='{severity}') }}}}" + "{{{{ {macro}(model={model}, {kwargs}) }}}}" + ).format( + model=self.build_model_str(), + macro=self.macro_name(), + kwargs=self.test_kwargs_str(), + severity=self.severity() ) - if not isinstance(test_name, basestring): - dbt.exceptions.raise_compiler_error( - 'test name must be a str, got {} (value {})'.format( - type(test_name), test_name - ) + + +class RefTestBuilder(TestBuilder): + def build_model_str(self): + return "ref('{}')".format(self.target['name']) + + def get_test_name(self): + return get_nice_schema_test_name(self.name, + self.target['name'], + self.args) + + def describe_test_target(self): + return 'model "{}"'.format(self.target) + + +class SourceTestBuilder(TestBuilder): + def build_model_str(self): + return "source('{}', '{}')".format( + self.target['source']['name'], + self.target['table']['name'] ) - if name is not None: - test_args['column_name'] = name - return test_name, test_args + + def get_test_name(self): + target_name = '{}_{}'.format(self.target['source']['name'], + self.target['table']['name']) + return get_nice_schema_test_name( + 'source_' + self.name, + target_name, + self.args + ) + + def describe_test_target(self): + return 'source "{0[source]}.{0[table]}"'.format(self.target) def warn_invalid(filepath, key, value, explain): @@ -212,6 +247,8 @@ def add(self, column_name, description): class SchemaBaseTestParser(MacrosKnownParser): + Builder = TestBuilder + def _parse_column(self, target, column, package_name, root_dir, path, refs): # this should yield ParsedNodes where resource_type == NodeType.Test @@ -237,53 +274,38 @@ def _parse_column(self, target, column, package_name, root_dir, path, ) continue - def _build_raw_sql(self, test_namespace, target, test_type, test_args): - raise NotImplementedError - - def _generate_test_name(self, target, test_type, test_args): - """Returns a hashed_name, full_name pair.""" - raise NotImplementedError - - @staticmethod - def _describe_test_target(test_target): - raise NotImplementedError - def build_test_node(self, test_target, package_name, test, root_dir, path, column_name=None): """Build a test node against the given target (a model or a source). :param test_target: An unparsed form of the target. """ - test_type, test_args = _build_test_args(test, column_name) + if isinstance(test, basestring): + test = {test: {}} - test_namespace, test_type, package_name = calculate_test_namespace( - test_type, package_name - ) + test_info = self.Builder(test, test_target, column_name, package_name) - source_package = self.all_projects.get(package_name) + source_package = self.all_projects.get(test_info.package_name) if source_package is None: desc = '"{}" test on {}'.format( - test_type, self._describe_test_target(test_target) + test_info.name, test_info.describe_test_target() ) - dbt.exceptions.raise_dep_not_found(None, desc, test_namespace) + dbt.exceptions.raise_dep_not_found(None, desc, test_info.namespace) test_path = os.path.basename(path) - hashed_name, full_name = self._generate_test_name(test_target, - test_type, - test_args) + hashed_name, full_name = test_info.get_test_name() hashed_path = get_pseudo_test_path(hashed_name, test_path, 'schema_test') full_path = get_pseudo_test_path(full_name, test_path, 'schema_test') - raw_sql = self._build_raw_sql(test_namespace, test_target, test_type, - test_args) + raw_sql = test_info.build_raw_sql() unparsed = UnparsedNode( name=full_name, resource_type=NodeType.Test, - package_name=package_name, + package_name=test_info.package_name, root_path=root_dir, path=hashed_path, original_file_path=path, @@ -318,15 +340,7 @@ def build_test_node(self, test_target, package_name, test, root_dir, path, class SchemaModelParser(SchemaBaseTestParser): - def _build_raw_sql(self, test_namespace, target, test_type, test_args): - return build_test_raw_sql(test_namespace, target, test_type, test_args) - - def _generate_test_name(self, target, test_type, test_args): - return get_nice_schema_test_name(test_type, target['name'], test_args) - - @staticmethod - def _describe_test_target(test_target): - return 'model "{}"'.format(test_target) + Builder = RefTestBuilder def parse_models_entry(self, model_dict, path, package_name, root_dir): model_name = model_dict['name'] @@ -381,6 +395,8 @@ def parse_all(self, models, path, package_name, root_dir): class SchemaSourceParser(SchemaBaseTestParser): + Builder = SourceTestBuilder + def __init__(self, root_project_config, all_projects, macro_manifest): super(SchemaSourceParser, self).__init__( root_project_config=root_project_config, @@ -389,16 +405,16 @@ def __init__(self, root_project_config, all_projects, macro_manifest): ) self._renderer = ConfigRenderer(self.root_project_config.cli_vars) - def _build_raw_sql(self, test_namespace, target, test_type, test_args): - return build_source_test_raw_sql(test_namespace, target['source'], - target['table'], test_type, - test_args) + def _build_raw_sql(self, test_info): + return test_info.build_source_test_raw_sql() - def _generate_test_name(self, target, test_type, test_args): + def _generate_test_name(self, test_info): + target_name = '{}_{}'.format(test_info.target['source']['name'], + test_info.target['table']['name']) return get_nice_schema_test_name( - 'source_' + test_type, - '{}_{}'.format(target['source']['name'], target['table']['name']), - test_args + 'source_' + test_info.name, + target_name, + test_info.args ) @staticmethod diff --git a/core/dbt/parser/source_config.py b/core/dbt/parser/source_config.py index 65f3e6b93c3..119ccc99948 100644 --- a/core/dbt/parser/source_config.py +++ b/core/dbt/parser/source_config.py @@ -15,6 +15,7 @@ class SourceConfig(object): 'materialized', 'unique_key', 'database', + 'severity', } ConfigKeys = AppendListFields | ExtendDictFields | ClobberFields @@ -68,6 +69,9 @@ def config(self): elif self.node_type == NodeType.Archive: defaults['materialized'] = 'archive' + if self.node_type == NodeType.Test: + defaults['severity'] = 'ERROR' + active_config = self.load_config_from_active_project() if self.active_project.project_name == self.own_project.project_name: diff --git a/core/dbt/ui/printer.py b/core/dbt/ui/printer.py index e92816d5b97..cc02bd50d7a 100644 --- a/core/dbt/ui/printer.py +++ b/core/dbt/ui/printer.py @@ -127,10 +127,13 @@ def print_test_result_line(result, schema_name, index, total): color = red elif result.status > 0: - info = 'FAIL {}'.format(result.status) - color = red - - result.fail = True + if result.node.config['severity'] == 'ERROR' or dbt.flags.WARN_ERROR: + info = 'FAIL {}'.format(result.status) + color = red + result.fail = True + else: + info = 'WARN {}'.format(result.status) + color = yellow elif result.status == 0: info = 'PASS' color = green diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index 26339897ad5..312ef1c8299 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -911,6 +911,7 @@ def expected_seeded_manifest(self, model_database=None): 'quoting': {}, 'vars': config_vars, 'tags': [], + 'severity': 'ERROR', }, 'sources': [], 'depends_on': {'macros': [], 'nodes': ['model.test.model']}, @@ -921,7 +922,7 @@ def expected_seeded_manifest(self, model_database=None): 'original_file_path': schema_yml_path, 'package_name': 'test', 'path': _normalize('schema_test/not_null_model_id.sql'), - 'raw_sql': "{{ test_not_null(model=ref('model'), column_name='id') }}", + 'raw_sql': "{{ config(severity='ERROR') }}{{ test_not_null(model=ref('model'), column_name='id') }}", 'refs': [['model']], 'resource_type': 'test', 'root_path': os.getcwd(), @@ -942,6 +943,7 @@ def expected_seeded_manifest(self, model_database=None): 'quoting': {}, 'vars': config_vars, 'tags': [], + 'severity': 'ERROR', }, 'sources': [], 'depends_on': {'macros': [], 'nodes': ['model.test.model']}, @@ -952,7 +954,7 @@ def expected_seeded_manifest(self, model_database=None): 'original_file_path': schema_yml_path, 'package_name': 'test', 'path': _normalize('schema_test/nothing_model_.sql'), - 'raw_sql': "{{ test.test_nothing(model=ref('model'), ) }}", + 'raw_sql': "{{ config(severity='ERROR') }}{{ test.test_nothing(model=ref('model'), ) }}", 'refs': [['model']], 'resource_type': 'test', 'root_path': os.getcwd(), @@ -974,6 +976,7 @@ def expected_seeded_manifest(self, model_database=None): 'quoting': {}, 'vars': config_vars, 'tags': [], + 'severity': 'ERROR', }, 'sources': [], 'depends_on': {'macros': [], 'nodes': ['model.test.model']}, @@ -984,7 +987,7 @@ def expected_seeded_manifest(self, model_database=None): 'original_file_path': schema_yml_path, 'package_name': 'test', 'path': _normalize('schema_test/unique_model_id.sql'), - 'raw_sql': "{{ test_unique(model=ref('model'), column_name='id') }}", + 'raw_sql': "{{ config(severity='ERROR') }}{{ test_unique(model=ref('model'), column_name='id') }}", 'refs': [['model']], 'resource_type': 'test', 'root_path': os.getcwd(), @@ -1945,6 +1948,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'quoting': {}, 'vars': config_vars, 'tags': [], + 'severity': 'ERROR', }, 'sources': [], 'depends_on': {'macros': [], 'nodes': ['model.test.model']}, @@ -1958,7 +1962,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'original_file_path': schema_yml_path, 'package_name': 'test', 'path': _normalize('schema_test/not_null_model_id.sql'), - 'raw_sql': "{{ test_not_null(model=ref('model'), column_name='id') }}", + 'raw_sql': "{{ config(severity='ERROR') }}{{ test_not_null(model=ref('model'), column_name='id') }}", 'refs': [['model']], 'resource_type': 'test', 'root_path': os.getcwd(), @@ -1992,6 +1996,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'quoting': {}, 'vars': config_vars, 'tags': [], + 'severity': 'ERROR', }, 'sources': [], 'depends_on': {'macros': [], 'nodes': ['model.test.model']}, @@ -2005,7 +2010,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'original_file_path': schema_yml_path, 'package_name': 'test', 'path': _normalize('schema_test/nothing_model_.sql'), - 'raw_sql': "{{ test.test_nothing(model=ref('model'), ) }}", + 'raw_sql': "{{ config(severity='ERROR') }}{{ test.test_nothing(model=ref('model'), ) }}", 'refs': [['model']], 'resource_type': 'test', 'root_path': os.getcwd(), @@ -2040,6 +2045,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'quoting': {}, 'vars': config_vars, 'tags': [], + 'severity': 'ERROR', }, 'sources': [], 'depends_on': {'macros': [], 'nodes': ['model.test.model']}, @@ -2053,7 +2059,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'original_file_path': schema_yml_path, 'package_name': 'test', 'path': _normalize('schema_test/unique_model_id.sql'), - 'raw_sql': "{{ test_unique(model=ref('model'), column_name='id') }}", + 'raw_sql': "{{ config(severity='ERROR') }}{{ test_unique(model=ref('model'), column_name='id') }}", 'refs': [['model']], 'resource_type': 'test', 'root_path': os.getcwd(), diff --git a/test/integration/045_test_severity_tests/data/null_seed.csv b/test/integration/045_test_severity_tests/data/null_seed.csv new file mode 100644 index 00000000000..b26a87430ac --- /dev/null +++ b/test/integration/045_test_severity_tests/data/null_seed.csv @@ -0,0 +1,21 @@ +id,first_name,last_name,email,gender,ip_address,updated_at +1,Judith,Kennedy,jkennedy0@phpbb.com,Female,54.60.24.128,2015-12-24 12:19:28 +2,Arthur,Kelly,akelly1@eepurl.com,Male,62.56.24.215,2015-10-28 16:22:15 +3,Rachel,Moreno,rmoreno2@msu.edu,Female,31.222.249.23,2016-04-05 02:05:30 +4,Ralph,Turner,rturner3@hp.com,Male,157.83.76.114,2016-08-08 00:06:51 +5,Laura,Gonzales,lgonzales4@howstuffworks.com,Female,30.54.105.168,2016-09-01 08:25:38 +6,Katherine,Lopez,null,Female,169.138.46.89,2016-08-30 18:52:11 +7,Jeremy,Hamilton,jhamilton6@mozilla.org,Male,231.189.13.133,2016-07-17 02:09:46 +8,Heather,Rose,hrose7@goodreads.com,Female,87.165.201.65,2015-12-29 22:03:56 +9,Gregory,Kelly,gkelly8@trellian.com,Male,154.209.99.7,2016-03-24 21:18:16 +10,Rachel,Lopez,rlopez9@themeforest.net,Female,237.165.82.71,2016-08-20 15:44:49 +11,Donna,Welch,dwelcha@shutterfly.com,Female,103.33.110.138,2016-02-27 01:41:48 +12,Russell,Lawrence,rlawrenceb@qq.com,Male,189.115.73.4,2016-06-11 03:07:09 +13,Michelle,Montgomery,mmontgomeryc@scientificamerican.com,Female,243.220.95.82,2016-06-18 16:27:19 +14,Walter,Castillo,null,Male,71.159.238.196,2016-10-06 01:55:44 +15,Robin,Mills,rmillse@vkontakte.ru,Female,172.190.5.50,2016-10-31 11:41:21 +16,Raymond,Holmes,rholmesf@usgs.gov,Male,148.153.166.95,2016-10-03 08:16:38 +17,Gary,Bishop,gbishopg@plala.or.jp,Male,161.108.182.13,2016-08-29 19:35:20 +18,Anna,Riley,arileyh@nasa.gov,Female,253.31.108.22,2015-12-11 04:34:27 +19,Sarah,Knight,sknighti@foxnews.com,Female,222.220.3.177,2016-09-26 00:49:06 +20,Phyllis,Fox,pfoxj@creativecommons.org,Female,163.191.232.95,2016-08-21 10:35:19 diff --git a/test/integration/045_test_severity_tests/models/model.sql b/test/integration/045_test_severity_tests/models/model.sql new file mode 100644 index 00000000000..3e29210ab0a --- /dev/null +++ b/test/integration/045_test_severity_tests/models/model.sql @@ -0,0 +1 @@ +select * from {{ source('source', 'nulls') }} diff --git a/test/integration/045_test_severity_tests/models/schema.yml b/test/integration/045_test_severity_tests/models/schema.yml new file mode 100644 index 00000000000..133e8057c5a --- /dev/null +++ b/test/integration/045_test_severity_tests/models/schema.yml @@ -0,0 +1,19 @@ +version: 2 +models: + - name: model + columns: + - name: email + tests: + - not_null: + severity: WARN +sources: + - name: source + schema: "{{ var('test_run_schema') }}" + tables: + - name: nulls + identifier: null_seed + columns: + - name: email + tests: + - not_null: + severity: WARN diff --git a/test/integration/045_test_severity_tests/test_severity.py b/test/integration/045_test_severity_tests/test_severity.py new file mode 100644 index 00000000000..7d747bc4f4b --- /dev/null +++ b/test/integration/045_test_severity_tests/test_severity.py @@ -0,0 +1,43 @@ +from test.integration.base import DBTIntegrationTest, use_profile + +class TestSeverity(DBTIntegrationTest): + @property + def schema(self): + return "severity_045" + + @property + def models(self): + return "test/integration/045_test_severity_tests/models" + + @property + def project_config(self): + return { + 'data-paths': ['test/integration/045_test_severity_tests/data'], + } + + def run_dbt_with_vars(self, cmd, *args, **kwargs): + cmd.extend(['--vars', + '{{test_run_schema: {}}}'.format(self.unique_schema())]) + return self.run_dbt(cmd, *args, **kwargs) + + @use_profile('postgres') + def test_postgres_severity_warnings(self): + self.run_dbt_with_vars(['seed'], strict=False) + self.run_dbt_with_vars(['run'], strict=False) + results = self.run_dbt_with_vars(['test'], strict=False) + self.assertEqual(len(results), 2) + self.assertFalse(results[0].fail) + self.assertEqual(results[0].status, 2) + self.assertFalse(results[1].fail) + self.assertEqual(results[1].status, 2) + + @use_profile('postgres') + def test_postgres_severity_warnings_errors(self): + self.run_dbt_with_vars(['seed'], strict=False) + self.run_dbt_with_vars(['run'], strict=False) + results = self.run_dbt_with_vars(['test'], expect_pass=False) + self.assertEqual(len(results), 2) + self.assertTrue(results[0].fail) + self.assertEqual(results[0].status, 2) + self.assertTrue(results[1].fail) + self.assertEqual(results[1].status, 2) diff --git a/test/unit/test_parser.py b/test/unit/test_parser.py index 854d607dd4e..263355cef2c 100644 --- a/test/unit/test_parser.py +++ b/test/unit/test_parser.py @@ -8,17 +8,16 @@ import dbt.parser from dbt.parser import ModelParser, MacroParser, DataTestParser, SchemaParser, ParserUtils from dbt.parser.source_config import SourceConfig -from dbt.utils import timestring -from dbt.config import RuntimeConfig +from dbt.utils import timestring, deep_merge from dbt.node_types import NodeType from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.parsed import ParsedNode, ParsedMacro, \ ParsedNodePatch, ParsedSourceDefinition -from dbt.contracts.graph.unparsed import UnparsedNode from .utils import config_from_parts_or_dicts + def get_os_path(unix_path): return os.path.normpath(unix_path) @@ -165,6 +164,9 @@ def setUp(self): 'tags': [], } + self.test_config = deep_merge(self.model_config, {'severity': 'ERROR'}) + self.warn_test_config = deep_merge(self.model_config, {'severity': 'WARN'}) + self.disabled_config = { 'enabled': False, 'materialized': 'view', @@ -232,11 +234,11 @@ def setUp(self): refs=[], sources=[['my_source', 'my_table']], depends_on={'nodes': [], 'macros': []}, - config=self.model_config, + config=self.test_config, path=get_os_path( 'schema_test/source_accepted_values_my_source_my_table_id__a__b.sql'), tags=['schema'], - raw_sql="{{ test_accepted_values(model=source('my_source', 'my_table'), column_name='id', values=['a', 'b']) }}", + raw_sql="{{ config(severity='ERROR') }}{{ test_accepted_values(model=source('my_source', 'my_table'), column_name='id', values=['a', 'b']) }}", description='', columns={}, column_name='id' @@ -255,11 +257,11 @@ def setUp(self): refs=[], sources=[['my_source', 'my_table']], depends_on={'nodes': [], 'macros': []}, - config=self.model_config, + config=self.test_config, original_file_path='test_one.yml', path=get_os_path('schema_test/source_not_null_my_source_my_table_id.sql'), tags=['schema'], - raw_sql="{{ test_not_null(model=source('my_source', 'my_table'), column_name='id') }}", + raw_sql="{{ config(severity='ERROR') }}{{ test_not_null(model=source('my_source', 'my_table'), column_name='id') }}", description='', columns={}, column_name='id' @@ -280,10 +282,10 @@ def setUp(self): refs=[['model_two']], sources=[['my_source', 'my_table']], depends_on={'nodes': [], 'macros': []}, - config=self.model_config, + config=self.test_config, path=get_os_path('schema_test/source_relationships_my_source_my_table_id__id__ref_model_two_.sql'), # noqa tags=['schema'], - raw_sql="{{ test_relationships(model=source('my_source', 'my_table'), column_name='id', from='id', to=ref('model_two')) }}", + raw_sql="{{ config(severity='ERROR') }}{{ test_relationships(model=source('my_source', 'my_table'), column_name='id', from='id', to=ref('model_two')) }}", description='', columns={}, column_name='id' @@ -294,19 +296,19 @@ def setUp(self): database='test', schema='analytics', resource_type='test', - unique_id='test.root.source_some_test_my_source_my_table_value', - fqn=['root', 'schema_test', 'source_some_test_my_source_my_table_value'], + unique_id='test.snowplow.source_some_test_my_source_my_table_value', + fqn=['snowplow', 'schema_test', 'source_some_test_my_source_my_table_value'], empty=False, - package_name='root', + package_name='snowplow', original_file_path='test_one.yml', root_path=get_os_path('/usr/src/app'), refs=[], sources=[['my_source', 'my_table']], depends_on={'nodes': [], 'macros': []}, - config=self.model_config, + config=self.warn_test_config, path=get_os_path('schema_test/source_some_test_my_source_my_table_value.sql'), tags=['schema'], - raw_sql="{{ test_some_test(model=source('my_source', 'my_table'), key='value') }}", + raw_sql="{{ config(severity='WARN') }}{{ snowplow.test_some_test(model=source('my_source', 'my_table'), key='value') }}", description='', columns={} ), @@ -324,11 +326,11 @@ def setUp(self): refs=[], sources=[['my_source', 'my_table']], depends_on={'nodes': [], 'macros': []}, - config=self.model_config, + config=self.warn_test_config, original_file_path='test_one.yml', path=get_os_path('schema_test/source_unique_my_source_my_table_id.sql'), tags=['schema'], - raw_sql="{{ test_unique(model=source('my_source', 'my_table'), column_name='id') }}", + raw_sql="{{ config(severity='WARN') }}{{ test_unique(model=source('my_source', 'my_table'), column_name='id') }}", description='', columns={}, column_name='id' @@ -352,11 +354,11 @@ def setUp(self): refs=[['model_one']], sources=[], depends_on={'nodes': [], 'macros': []}, - config=self.model_config, + config=self.test_config, path=get_os_path( 'schema_test/accepted_values_model_one_id__a__b.sql'), tags=['schema'], - raw_sql="{{ test_accepted_values(model=ref('model_one'), column_name='id', values=['a', 'b']) }}", + raw_sql="{{ config(severity='ERROR') }}{{ test_accepted_values(model=ref('model_one'), column_name='id', values=['a', 'b']) }}", description='', columns={}, column_name='id' @@ -375,11 +377,11 @@ def setUp(self): refs=[['model_one']], sources=[], depends_on={'nodes': [], 'macros': []}, - config=self.model_config, + config=self.test_config, original_file_path='test_one.yml', path=get_os_path('schema_test/not_null_model_one_id.sql'), tags=['schema'], - raw_sql="{{ test_not_null(model=ref('model_one'), column_name='id') }}", + raw_sql="{{ config(severity='ERROR') }}{{ test_not_null(model=ref('model_one'), column_name='id') }}", description='', columns={}, column_name='id' @@ -400,10 +402,10 @@ def setUp(self): refs=[['model_one'], ['model_two']], sources=[], depends_on={'nodes': [], 'macros': []}, - config=self.model_config, + config=self.test_config, path=get_os_path('schema_test/relationships_model_one_id__id__ref_model_two_.sql'), # noqa tags=['schema'], - raw_sql="{{ test_relationships(model=ref('model_one'), column_name='id', from='id', to=ref('model_two')) }}", + raw_sql="{{ config(severity='ERROR') }}{{ test_relationships(model=ref('model_one'), column_name='id', from='id', to=ref('model_two')) }}", description='', columns={}, column_name='id' @@ -414,19 +416,19 @@ def setUp(self): database='test', schema='analytics', resource_type='test', - unique_id='test.root.some_test_model_one_value', - fqn=['root', 'schema_test', 'some_test_model_one_value'], + unique_id='test.snowplow.some_test_model_one_value', + fqn=['snowplow', 'schema_test', 'some_test_model_one_value'], empty=False, - package_name='root', + package_name='snowplow', original_file_path='test_one.yml', root_path=get_os_path('/usr/src/app'), refs=[['model_one']], sources=[], depends_on={'nodes': [], 'macros': []}, - config=self.model_config, + config=self.warn_test_config, path=get_os_path('schema_test/some_test_model_one_value.sql'), tags=['schema'], - raw_sql="{{ test_some_test(model=ref('model_one'), key='value') }}", + raw_sql="{{ config(severity='WARN') }}{{ snowplow.test_some_test(model=ref('model_one'), key='value') }}", description='', columns={} ), @@ -444,11 +446,11 @@ def setUp(self): refs=[['model_one']], sources=[], depends_on={'nodes': [], 'macros': []}, - config=self.model_config, + config=self.warn_test_config, original_file_path='test_one.yml', path=get_os_path('schema_test/unique_model_one_id.sql'), tags=['schema'], - raw_sql="{{ test_unique(model=ref('model_one'), column_name='id') }}", + raw_sql="{{ config(severity='WARN') }}{{ test_unique(model=ref('model_one'), column_name='id') }}", description='', columns={}, column_name='id' @@ -461,9 +463,10 @@ def setUp(self): original_file_path='test_one.yml', columns={ 'id': { - 'name': 'id', - 'description': 'user ID', - }}, + 'name': 'id', + 'description': 'user ID', + }, + }, docrefs=[], ) @@ -500,7 +503,8 @@ def test__source_schema(self): - name: id description: user ID tests: - - unique + - unique: + severity: WARN - not_null - accepted_values: values: @@ -510,8 +514,9 @@ def test__source_schema(self): from: id to: ref('model_two') tests: - - some_test: + - snowplow.some_test: key: value + severity: WARN ''') parser = SchemaParser( self.root_project_config, @@ -552,7 +557,8 @@ def test__model_schema(self): - name: id description: user ID tests: - - unique + - unique: + severity: WARN - not_null - accepted_values: values: @@ -562,7 +568,8 @@ def test__model_schema(self): from: id to: ref('model_two') tests: - - some_test: + - snowplow.some_test: + severity: WARN key: value ''') parser = SchemaParser( @@ -606,7 +613,8 @@ def test__mixed_schema(self): - name: id description: user ID tests: - - unique + - unique: + severity: WARN - not_null - accepted_values: values: @@ -616,7 +624,8 @@ def test__mixed_schema(self): from: id to: ref('model_two') tests: - - some_test: + - snowplow.some_test: + severity: WARN key: value sources: - name: my_source @@ -648,7 +657,8 @@ def test__mixed_schema(self): - name: id description: user ID tests: - - unique + - unique: + severity: WARN - not_null - accepted_values: values: @@ -658,7 +668,8 @@ def test__mixed_schema(self): from: id to: ref('model_two') tests: - - some_test: + - snowplow.some_test: + severity: WARN key: value ''') parser = SchemaParser( @@ -725,7 +736,8 @@ def test__source_schema_invalid_test_strict(self): - name: id description: user ID tests: - - unique + - unique: + severity: WARN - not_null - accepted_values: # this test is invalid - values: @@ -735,7 +747,8 @@ def test__source_schema_invalid_test_strict(self): from: id to: ref('model_two') tests: - - some_test: + - snowplow.some_test: + severity: WARN key: value ''') parser = SchemaParser( @@ -787,7 +800,8 @@ def test__source_schema_invalid_test_not_strict(self): - name: id description: user ID tests: - - unique + - unique: + severity: WARN - not_null - accepted_values: # this test is invalid - values: @@ -797,7 +811,8 @@ def test__source_schema_invalid_test_not_strict(self): from: id to: ref('model_two') tests: - - some_test: + - snowplow.some_test: + severity: WARN key: value ''') parser = SchemaParser( @@ -928,6 +943,7 @@ def setUp(self): 'column_types': {}, 'tags': [], } + self.test_config = deep_merge(self.model_config, {'severity': 'ERROR'}) self.disabled_config = { 'enabled': False, @@ -940,7 +956,6 @@ def setUp(self): 'tags': [], } - def test__single_model(self): models = [{ 'name': 'model_one', @@ -2263,7 +2278,7 @@ def test__simple_data_test(self): 'nodes': [], 'macros': [] }, - config=self.model_config, + config=self.test_config, path='no_events.sql', original_file_path='no_events.sql', root_path=get_os_path('/usr/src/app'),