From 81118d904a876276d8a890eb207e1e2a813c344a Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Thu, 24 Mar 2022 09:19:54 -0500 Subject: [PATCH] Convert source tests (#4935) * convert 059 to new test framework * remove replaced tests * WIP, has pre-commit errors * WIP, has pre-commit errors * one failing test, most issued resolved * fixed final test and cleaned up fixtures * remove converted tests * updated test to work on windows * remove config version --- .../042_sources_tests/error_models/model.sql | 1 - .../042_sources_tests/error_models/schema.yml | 12 - .../filtered_models/schema.yml | 18 - .../042_sources_tests/macros/macro.sql | 14 - .../malformed_models/descendant_model.sql | 1 - .../malformed_models/schema.yml | 14 - .../malformed_schema_tests/model.sql | 1 - .../malformed_schema_tests/schema.yml | 14 - .../models/descendant_model.sql | 1 - .../models/ephemeral_model.sql | 3 - .../models/multi_source_model.sql | 2 - .../models/nonsource_descendant.sql | 1 - .../042_sources_tests/models/schema.yml | 77 --- .../042_sources_tests/models/view_model.sql | 3 - .../override_freshness_models/schema.yml | 42 -- .../seeds/expected_multi_source.csv | 4 - .../seeds/other_source_table.csv | 4 - .../042_sources_tests/seeds/other_table.csv | 4 - .../042_sources_tests/test_sources.py | 557 ------------------ .../dupe-models/schema1.yml | 26 - .../dupe-models/schema2.yml | 26 - .../local_dependency/dbt_project.yml | 11 - .../local_dependency/models/my_model.sql | 8 - .../local_dependency/models/schema.yml | 43 -- .../seeds/keep/never_fresh.csv | 51 -- .../seeds/keep/snapshot_freshness_base.csv | 101 ---- .../local_dependency/seeds/my_other_seed.csv | 4 - .../local_dependency/seeds/my_seed.csv | 4 - .../models/schema.yml | 26 - .../seeds/expected_result.csv | 5 - .../seeds/my_real_other_seed.csv | 5 - .../seeds/my_real_seed.csv | 6 - .../test_source_overrides.py | 185 ------ tests/functional/source_overrides/fixtures.py | 377 ++++++++++++ .../test_simple_source_override.py | 149 +++++ .../test_source_overrides_duplicate_model.py | 67 +++ .../functional/sources/common_source_setup.py | 66 +++ .../functional/sources/data}/seed.sql | 0 .../functional/sources/fixtures.py | 255 +++++++- .../functional/sources/test_simple_source.py | 211 +++++++ .../sources/test_source_freshness.py | 375 ++++++++++++ 41 files changed, 1499 insertions(+), 1275 deletions(-) delete mode 100644 test/integration/042_sources_tests/error_models/model.sql delete mode 100644 test/integration/042_sources_tests/error_models/schema.yml delete mode 100644 test/integration/042_sources_tests/filtered_models/schema.yml delete mode 100644 test/integration/042_sources_tests/macros/macro.sql delete mode 100644 test/integration/042_sources_tests/malformed_models/descendant_model.sql delete mode 100644 test/integration/042_sources_tests/malformed_models/schema.yml delete mode 100644 test/integration/042_sources_tests/malformed_schema_tests/model.sql delete mode 100644 test/integration/042_sources_tests/malformed_schema_tests/schema.yml delete mode 100644 test/integration/042_sources_tests/models/descendant_model.sql delete mode 100644 test/integration/042_sources_tests/models/ephemeral_model.sql delete mode 100644 test/integration/042_sources_tests/models/multi_source_model.sql delete mode 100644 test/integration/042_sources_tests/models/nonsource_descendant.sql delete mode 100644 test/integration/042_sources_tests/models/schema.yml delete mode 100644 test/integration/042_sources_tests/models/view_model.sql delete mode 100644 test/integration/042_sources_tests/override_freshness_models/schema.yml delete mode 100644 test/integration/042_sources_tests/seeds/expected_multi_source.csv delete mode 100644 test/integration/042_sources_tests/seeds/other_source_table.csv delete mode 100644 test/integration/042_sources_tests/seeds/other_table.csv delete mode 100644 test/integration/042_sources_tests/test_sources.py delete mode 100644 test/integration/059_source_overrides_tests/dupe-models/schema1.yml delete mode 100644 test/integration/059_source_overrides_tests/dupe-models/schema2.yml delete mode 100644 test/integration/059_source_overrides_tests/local_dependency/dbt_project.yml delete mode 100644 test/integration/059_source_overrides_tests/local_dependency/models/my_model.sql delete mode 100644 test/integration/059_source_overrides_tests/local_dependency/models/schema.yml delete mode 100644 test/integration/059_source_overrides_tests/local_dependency/seeds/keep/never_fresh.csv delete mode 100644 test/integration/059_source_overrides_tests/local_dependency/seeds/keep/snapshot_freshness_base.csv delete mode 100644 test/integration/059_source_overrides_tests/local_dependency/seeds/my_other_seed.csv delete mode 100644 test/integration/059_source_overrides_tests/local_dependency/seeds/my_seed.csv delete mode 100644 test/integration/059_source_overrides_tests/models/schema.yml delete mode 100644 test/integration/059_source_overrides_tests/seeds/expected_result.csv delete mode 100644 test/integration/059_source_overrides_tests/seeds/my_real_other_seed.csv delete mode 100644 test/integration/059_source_overrides_tests/seeds/my_real_seed.csv delete mode 100644 test/integration/059_source_overrides_tests/test_source_overrides.py create mode 100644 tests/functional/source_overrides/fixtures.py create mode 100644 tests/functional/source_overrides/test_simple_source_override.py create mode 100644 tests/functional/source_overrides/test_source_overrides_duplicate_model.py create mode 100644 tests/functional/sources/common_source_setup.py rename {test/integration/042_sources_tests => tests/functional/sources/data}/seed.sql (100%) rename test/integration/042_sources_tests/seeds/source.csv => tests/functional/sources/fixtures.py (50%) create mode 100644 tests/functional/sources/test_simple_source.py create mode 100644 tests/functional/sources/test_source_freshness.py diff --git a/test/integration/042_sources_tests/error_models/model.sql b/test/integration/042_sources_tests/error_models/model.sql deleted file mode 100644 index 55bbcba67b4..00000000000 --- a/test/integration/042_sources_tests/error_models/model.sql +++ /dev/null @@ -1 +0,0 @@ -select * from {{ source('test_source', 'test_table') }} diff --git a/test/integration/042_sources_tests/error_models/schema.yml b/test/integration/042_sources_tests/error_models/schema.yml deleted file mode 100644 index 69cf1f304a6..00000000000 --- a/test/integration/042_sources_tests/error_models/schema.yml +++ /dev/null @@ -1,12 +0,0 @@ -version: 2 -sources: - - name: test_source - loader: custom - freshness: - warn_after: {count: 10, period: hour} - error_after: {count: 1, period: day} - schema: invalid - tables: - - name: test_table - identifier: source - loaded_at_field: updated_at diff --git a/test/integration/042_sources_tests/filtered_models/schema.yml b/test/integration/042_sources_tests/filtered_models/schema.yml deleted file mode 100644 index edad7f6ecfb..00000000000 --- a/test/integration/042_sources_tests/filtered_models/schema.yml +++ /dev/null @@ -1,18 +0,0 @@ -version: 2 -sources: - - name: test_source - loader: custom - freshness: - warn_after: {count: 10, period: hour} - error_after: {count: 1, period: day} - filter: id > 1 - schema: "{{ var(env_var('DBT_TEST_SCHEMA_NAME_VARIABLE')) }}" - quoting: - identifier: True - tables: - - name: test_table - identifier: source - loaded_at_field: updated_at - freshness: - error_after: {count: 18, period: hour} - filter: id > 101 diff --git a/test/integration/042_sources_tests/macros/macro.sql b/test/integration/042_sources_tests/macros/macro.sql deleted file mode 100644 index a607a6e4ce7..00000000000 --- a/test/integration/042_sources_tests/macros/macro.sql +++ /dev/null @@ -1,14 +0,0 @@ -{% macro override_me() -%} - {{ exceptions.raise_compiler_error('this is a bad macro') }} -{%- endmacro %} - -{% macro happy_little_macro() -%} - {{ override_me() }} -{%- endmacro %} - - -{% macro vacuum_source(source_name, table_name) -%} - {% call statement('stmt', auto_begin=false, fetch_result=false) %} - vacuum {{ source(source_name, table_name) }} - {% endcall %} -{%- endmacro %} diff --git a/test/integration/042_sources_tests/malformed_models/descendant_model.sql b/test/integration/042_sources_tests/malformed_models/descendant_model.sql deleted file mode 100644 index 55bbcba67b4..00000000000 --- a/test/integration/042_sources_tests/malformed_models/descendant_model.sql +++ /dev/null @@ -1 +0,0 @@ -select * from {{ source('test_source', 'test_table') }} diff --git a/test/integration/042_sources_tests/malformed_models/schema.yml b/test/integration/042_sources_tests/malformed_models/schema.yml deleted file mode 100644 index 544d18d6560..00000000000 --- a/test/integration/042_sources_tests/malformed_models/schema.yml +++ /dev/null @@ -1,14 +0,0 @@ -version: 2 -sources: - - name: test_source - loader: custom - schema: "{{ var('test_run_schema') }}" - tables: - - name: test_table - identifier: source - tests: - - relationships: - # this is invalid (list of 3 1-key dicts instead of a single 3-key dict) - - column_name: favorite_color - - to: ref('descendant_model') - - field: favorite_color diff --git a/test/integration/042_sources_tests/malformed_schema_tests/model.sql b/test/integration/042_sources_tests/malformed_schema_tests/model.sql deleted file mode 100644 index 55bbcba67b4..00000000000 --- a/test/integration/042_sources_tests/malformed_schema_tests/model.sql +++ /dev/null @@ -1 +0,0 @@ -select * from {{ source('test_source', 'test_table') }} diff --git a/test/integration/042_sources_tests/malformed_schema_tests/schema.yml b/test/integration/042_sources_tests/malformed_schema_tests/schema.yml deleted file mode 100644 index d72ab2eeec4..00000000000 --- a/test/integration/042_sources_tests/malformed_schema_tests/schema.yml +++ /dev/null @@ -1,14 +0,0 @@ -version: 2 -sources: - - name: test_source - schema: "{{ var('test_run_schema') }}" - tables: - - name: test_table - identifier: source - columns: - - name: favorite_color - tests: - - relationships: - to: ref('model') - # this will get rendered as its literal - field: "{{ 'favorite' ~ 'color' }}" diff --git a/test/integration/042_sources_tests/models/descendant_model.sql b/test/integration/042_sources_tests/models/descendant_model.sql deleted file mode 100644 index 55bbcba67b4..00000000000 --- a/test/integration/042_sources_tests/models/descendant_model.sql +++ /dev/null @@ -1 +0,0 @@ -select * from {{ source('test_source', 'test_table') }} diff --git a/test/integration/042_sources_tests/models/ephemeral_model.sql b/test/integration/042_sources_tests/models/ephemeral_model.sql deleted file mode 100644 index 8de35cd3e21..00000000000 --- a/test/integration/042_sources_tests/models/ephemeral_model.sql +++ /dev/null @@ -1,3 +0,0 @@ -{{ config(materialized='ephemeral') }} - -select 1 as id diff --git a/test/integration/042_sources_tests/models/multi_source_model.sql b/test/integration/042_sources_tests/models/multi_source_model.sql deleted file mode 100644 index e310206b0b4..00000000000 --- a/test/integration/042_sources_tests/models/multi_source_model.sql +++ /dev/null @@ -1,2 +0,0 @@ -select * from {{ source('test_source', 'other_test_table')}} - join {{ source('other_source', 'test_table')}} using (id) diff --git a/test/integration/042_sources_tests/models/nonsource_descendant.sql b/test/integration/042_sources_tests/models/nonsource_descendant.sql deleted file mode 100644 index 97f2151c754..00000000000 --- a/test/integration/042_sources_tests/models/nonsource_descendant.sql +++ /dev/null @@ -1 +0,0 @@ -select * from {{ schema }}.source diff --git a/test/integration/042_sources_tests/models/schema.yml b/test/integration/042_sources_tests/models/schema.yml deleted file mode 100644 index f02eb13453d..00000000000 --- a/test/integration/042_sources_tests/models/schema.yml +++ /dev/null @@ -1,77 +0,0 @@ -version: 2 -models: - - name: descendant_model - columns: - - name: favorite_color - tests: - - relationships: - to: source('test_source', 'test_table') - field: favorite_color - -sources: - - name: test_source - loader: custom - freshness: - warn_after: {count: 10, period: hour} - error_after: {count: 1, period: day} - schema: "{{ var(env_var('DBT_TEST_SCHEMA_NAME_VARIABLE')) }}" - quoting: - identifier: True - tags: - - my_test_source_tag - tables: - - name: test_table - identifier: source - loaded_at_field: "{{ var('test_loaded_at') | as_text }}" - freshness: - error_after: {count: 18, period: hour} - tags: - - my_test_source_table_tag - columns: - - name: favorite_color - description: The favorite color - - name: id - description: The user ID - tests: - - unique - - not_null - tags: - - id_column - - name: first_name - description: The first name of the user - tests: [] - - name: email - description: The email address of the user - - name: ip_address - description: The last IP address the user logged in from - - name: updated_at - description: The last update time for this user - tests: - - relationships: - # do this as a table-level test, just to test out that aspect - column_name: favorite_color - to: ref('descendant_model') - field: favorite_color - - name: other_test_table - identifier: other_table - columns: - - name: id - tests: - - not_null - - unique - tags: - - id_column - - name: disabled_test_table - freshness: null - loaded_at_field: "{{ var('test_loaded_at') | as_text }}" - - name: other_source - schema: "{{ var('test_run_schema') }}" - quoting: - identifier: True - tables: - - name: test_table - identifier: other_source_table - - name: external_source - schema: "{{ var('test_run_alt_schema', var('test_run_schema')) }}" - tables: - - name: table diff --git a/test/integration/042_sources_tests/models/view_model.sql b/test/integration/042_sources_tests/models/view_model.sql deleted file mode 100644 index 3a84fcccc61..00000000000 --- a/test/integration/042_sources_tests/models/view_model.sql +++ /dev/null @@ -1,3 +0,0 @@ -{# See here: https://github.com/dbt-labs/dbt-core/pull/1729 #} - -select * from {{ ref('ephemeral_model') }} diff --git a/test/integration/042_sources_tests/override_freshness_models/schema.yml b/test/integration/042_sources_tests/override_freshness_models/schema.yml deleted file mode 100644 index dabcf0779a1..00000000000 --- a/test/integration/042_sources_tests/override_freshness_models/schema.yml +++ /dev/null @@ -1,42 +0,0 @@ -version: 2 -sources: - - name: test_source - loader: custom - freshness: # default freshness - warn_after: {count: 12, period: hour} - error_after: {count: 24, period: hour} - schema: "{{ var(env_var('DBT_TEST_SCHEMA_NAME_VARIABLE')) }}" - loaded_at_field: loaded_at - quoting: - identifier: True - tags: - - my_test_source_tag - tables: - - name: source_a - identifier: source - loaded_at_field: "{{ var('test_loaded_at') | as_text }}" - freshness: - warn_after: {count: 6, period: hour} - # use the default error_after defined above - - name: source_b - identifier: source - loaded_at_field: "{{ var('test_loaded_at') | as_text }}" - freshness: - warn_after: {count: 6, period: hour} - error_after: {} # use the default error_after defined above - - name: source_c - identifier: source - loaded_at_field: "{{ var('test_loaded_at') | as_text }}" - freshness: - warn_after: {count: 6, period: hour} - error_after: null # override: disable error_after for this table - - name: source_d - identifier: source - loaded_at_field: "{{ var('test_loaded_at') | as_text }}" - freshness: - warn_after: {count: 6, period: hour} - error_after: {count: 72, period: hour} # override: use this new behavior instead of error_after defined above - - name: source_e - identifier: source - loaded_at_field: "{{ var('test_loaded_at') | as_text }}" - freshness: null # override: disable freshness for this table diff --git a/test/integration/042_sources_tests/seeds/expected_multi_source.csv b/test/integration/042_sources_tests/seeds/expected_multi_source.csv deleted file mode 100644 index de9c1c01da2..00000000000 --- a/test/integration/042_sources_tests/seeds/expected_multi_source.csv +++ /dev/null @@ -1,4 +0,0 @@ -id,first_name,color -1,Larry,blue -2,Curly,red -3,Moe,green diff --git a/test/integration/042_sources_tests/seeds/other_source_table.csv b/test/integration/042_sources_tests/seeds/other_source_table.csv deleted file mode 100644 index a92b2cb8ee6..00000000000 --- a/test/integration/042_sources_tests/seeds/other_source_table.csv +++ /dev/null @@ -1,4 +0,0 @@ -id,color -1,blue -2,red -3,green diff --git a/test/integration/042_sources_tests/seeds/other_table.csv b/test/integration/042_sources_tests/seeds/other_table.csv deleted file mode 100644 index 56bdda92b65..00000000000 --- a/test/integration/042_sources_tests/seeds/other_table.csv +++ /dev/null @@ -1,4 +0,0 @@ -id,first_name -1,Larry -2,Curly -3,Moe diff --git a/test/integration/042_sources_tests/test_sources.py b/test/integration/042_sources_tests/test_sources.py deleted file mode 100644 index 883e20bf8f7..00000000000 --- a/test/integration/042_sources_tests/test_sources.py +++ /dev/null @@ -1,557 +0,0 @@ -import json -import os -from datetime import datetime, timedelta - -import yaml - -from dbt.exceptions import ParsingException -import dbt.tracking -import dbt.version -from test.integration.base import DBTIntegrationTest, use_profile, AnyFloat, \ - AnyStringWith - - -class BaseSourcesTest(DBTIntegrationTest): - @property - def schema(self): - return "sources_042" - - @property - def models(self): - return "models" - - @property - def project_config(self): - return { - 'config-version': 2, - 'seed-paths': ['seeds'], - 'quoting': {'database': True, 'schema': True, 'identifier': True}, - 'seeds': { - 'quote_columns': True, - }, - } - - def setUp(self): - super().setUp() - os.environ['DBT_TEST_SCHEMA_NAME_VARIABLE'] = 'test_run_schema' - - def tearDown(self): - del os.environ['DBT_TEST_SCHEMA_NAME_VARIABLE'] - super().tearDown() - - def run_dbt_with_vars(self, cmd, *args, **kwargs): - vars_dict = { - 'test_run_schema': self.unique_schema(), - 'test_loaded_at': self.adapter.quote('updated_at'), - } - cmd.extend(['--vars', yaml.safe_dump(vars_dict)]) - return self.run_dbt(cmd, *args, **kwargs) - - -class SuccessfulSourcesTest(BaseSourcesTest): - def setUp(self): - super().setUp() - self.run_dbt_with_vars(['seed']) - self.maxDiff = None - self._id = 101 - # this is the db initial value - self.last_inserted_time = "2016-09-19T14:45:51+00:00" - os.environ['DBT_ENV_CUSTOM_ENV_key'] = 'value' - - def tearDown(self): - super().tearDown() - del os.environ['DBT_ENV_CUSTOM_ENV_key'] - - def _set_updated_at_to(self, delta): - insert_time = datetime.utcnow() + delta - timestr = insert_time.strftime("%Y-%m-%d %H:%M:%S") - # favorite_color,id,first_name,email,ip_address,updated_at - insert_id = self._id - self._id += 1 - raw_sql = """INSERT INTO {schema}.{source} - ({quoted_columns}) - VALUES ( - 'blue',{id},'Jake','abc@example.com','192.168.1.1','{time}' - )""" - quoted_columns = ','.join( - self.adapter.quote(c) for c in - ('favorite_color', 'id', 'first_name', - 'email', 'ip_address', 'updated_at') - ) - self.run_sql( - raw_sql, - kwargs={ - 'schema': self.unique_schema(), - 'time': timestr, - 'id': insert_id, - 'source': self.adapter.quote('source'), - 'quoted_columns': quoted_columns, - } - ) - self.last_inserted_time = insert_time.strftime( - "%Y-%m-%dT%H:%M:%S+00:00") - - -class TestSources(SuccessfulSourcesTest): - @property - def project_config(self): - cfg = super().project_config - cfg.update({ - 'macro-paths': ['macros'], - }) - return cfg - - def _create_schemas(self): - super()._create_schemas() - self._create_schema_named(self.default_database, - self.alternative_schema()) - - def alternative_schema(self): - return self.unique_schema() + '_other' - - def setUp(self): - super().setUp() - self.run_sql( - 'create table {}.dummy_table (id int)'.format(self.unique_schema()) - ) - self.run_sql( - 'create view {}.external_view as (select * from {}.dummy_table)' - .format(self.alternative_schema(), self.unique_schema()) - ) - - def run_dbt_with_vars(self, cmd, *args, **kwargs): - vars_dict = { - 'test_run_schema': self.unique_schema(), - 'test_run_alt_schema': self.alternative_schema(), - 'test_loaded_at': self.adapter.quote('updated_at'), - } - cmd.extend(['--vars', yaml.safe_dump(vars_dict)]) - return self.run_dbt(cmd, *args, **kwargs) - - @use_profile('postgres') - def test_postgres_basic_source_def(self): - results = self.run_dbt_with_vars(['run']) - self.assertEqual(len(results), 4) - self.assertManyTablesEqual( - ['source', 'descendant_model', 'nonsource_descendant'], - ['expected_multi_source', 'multi_source_model']) - results = self.run_dbt_with_vars(['test']) - self.assertEqual(len(results), 6) - print(results) - - @use_profile('postgres') - def test_postgres_source_selector(self): - # only one of our models explicitly depends upon a source - results = self.run_dbt_with_vars([ - 'run', - '--models', - 'source:test_source.test_table+' - ]) - self.assertEqual(len(results), 1) - self.assertTablesEqual('source', 'descendant_model') - self.assertTableDoesNotExist('nonsource_descendant') - self.assertTableDoesNotExist('multi_source_model') - - # do the same thing, but with tags - results = self.run_dbt_with_vars([ - 'run', - '--models', - 'tag:my_test_source_table_tag+' - ]) - self.assertEqual(len(results), 1) - - results = self.run_dbt_with_vars([ - 'test', - '--models', - 'source:test_source.test_table+' - ]) - self.assertEqual(len(results), 4) - - results = self.run_dbt_with_vars([ - 'test', '--models', 'tag:my_test_source_table_tag+' - ]) - self.assertEqual(len(results), 4) - - results = self.run_dbt_with_vars([ - 'test', '--models', 'tag:my_test_source_tag+' - ]) - # test_table + other_test_table - self.assertEqual(len(results), 6) - - results = self.run_dbt_with_vars([ - 'test', '--models', 'tag:id_column' - ]) - # all 4 id column tests - self.assertEqual(len(results), 4) - - @use_profile('postgres') - def test_postgres_empty_source_def(self): - # sources themselves can never be selected, so nothing should be run - results = self.run_dbt_with_vars([ - 'run', - '--models', - 'source:test_source.test_table' - ]) - self.assertTableDoesNotExist('nonsource_descendant') - self.assertTableDoesNotExist('multi_source_model') - self.assertTableDoesNotExist('descendant_model') - self.assertEqual(len(results), 0) - - @use_profile('postgres') - def test_postgres_source_only_def(self): - results = self.run_dbt_with_vars([ - 'run', '--models', 'source:other_source+' - ]) - self.assertEqual(len(results), 1) - self.assertTablesEqual('expected_multi_source', 'multi_source_model') - self.assertTableDoesNotExist('nonsource_descendant') - self.assertTableDoesNotExist('descendant_model') - - results = self.run_dbt_with_vars([ - 'run', '--models', 'source:test_source+' - ]) - self.assertEqual(len(results), 2) - self.assertManyTablesEqual( - ['source', 'descendant_model'], - ['expected_multi_source', 'multi_source_model']) - self.assertTableDoesNotExist('nonsource_descendant') - - @use_profile('postgres') - def test_postgres_source_childrens_parents(self): - results = self.run_dbt_with_vars([ - 'run', '--models', '@source:test_source' - ]) - self.assertEqual(len(results), 2) - self.assertManyTablesEqual( - ['source', 'descendant_model'], - ['expected_multi_source', 'multi_source_model'], - ) - self.assertTableDoesNotExist('nonsource_descendant') - - @use_profile('postgres') - def test_postgres_run_operation_source(self): - kwargs = '{"source_name": "test_source", "table_name": "test_table"}' - self.run_dbt_with_vars([ - 'run-operation', 'vacuum_source', '--args', kwargs - ]) - - -class TestSourceFreshness(SuccessfulSourcesTest): - - def _assert_freshness_results(self, path, state): - self.assertTrue(os.path.exists(path)) - with open(path) as fp: - data = json.load(fp) - - assert set(data) == {'metadata', 'results', 'elapsed_time'} - assert 'generated_at' in data['metadata'] - assert isinstance(data['elapsed_time'], float) - self.assertBetween(data['metadata']['generated_at'], - self.freshness_start_time) - assert data['metadata']['dbt_schema_version'] == 'https://schemas.getdbt.com/dbt/sources/v3.json' - assert data['metadata']['dbt_version'] == dbt.version.__version__ - assert data['metadata']['invocation_id'] == dbt.tracking.active_user.invocation_id - key = 'key' - if os.name == 'nt': - key = key.upper() - assert data['metadata']['env'] == {key: 'value'} - - last_inserted_time = self.last_inserted_time - - self.assertEqual(len(data['results']), 1) - - self.assertEqual(data['results'], [ - { - 'unique_id': 'source.test.test_source.test_table', - 'max_loaded_at': last_inserted_time, - 'snapshotted_at': AnyStringWith(), - 'max_loaded_at_time_ago_in_s': AnyFloat(), - 'status': state, - 'criteria': { - 'filter': None, - 'warn_after': {'count': 10, 'period': 'hour'}, - 'error_after': {'count': 18, 'period': 'hour'}, - }, - 'adapter_response': {}, - 'thread_id': AnyStringWith('Thread-'), - 'execution_time': AnyFloat(), - 'timing': [ - { - 'name': 'compile', - 'started_at': AnyStringWith(), - 'completed_at': AnyStringWith(), - }, - { - 'name': 'execute', - 'started_at': AnyStringWith(), - 'completed_at': AnyStringWith(), - } - ] - } - ]) - - def _run_source_freshness(self): - # test_source.test_table should have a loaded_at field of `updated_at` - # and a freshness of warn_after: 10 hours, error_after: 18 hours - # by default, our data set is way out of date! - self.freshness_start_time = datetime.utcnow() - results = self.run_dbt_with_vars( - ['source', 'freshness', '-o', 'target/error_source.json'], - expect_pass=False - ) - self.assertEqual(len(results), 1) - self.assertEqual(results[0].status, 'error') - self._assert_freshness_results('target/error_source.json', 'error') - - self._set_updated_at_to(timedelta(hours=-12)) - self.freshness_start_time = datetime.utcnow() - results = self.run_dbt_with_vars( - ['source', 'freshness', '-o', 'target/warn_source.json'], - ) - self.assertEqual(len(results), 1) - self.assertEqual(results[0].status, 'warn') - self._assert_freshness_results('target/warn_source.json', 'warn') - - self._set_updated_at_to(timedelta(hours=-2)) - self.freshness_start_time = datetime.utcnow() - results = self.run_dbt_with_vars( - ['source', 'freshness', '-o', 'target/pass_source.json'], - ) - self.assertEqual(len(results), 1) - self.assertEqual(results[0].status, 'pass') - self._assert_freshness_results('target/pass_source.json', 'pass') - - @use_profile('postgres') - def test_postgres_source_freshness(self): - self._run_source_freshness() - - @use_profile('postgres') - def test_postgres_source_snapshot_freshness(self): - """Ensures that the deprecated command `source snapshot-freshness` - aliases to `source freshness` command. - """ - self.freshness_start_time = datetime.utcnow() - results = self.run_dbt_with_vars( - ['source', 'snapshot-freshness', '-o', 'target/error_source.json'], - expect_pass=False - ) - self.assertEqual(len(results), 1) - self.assertEqual(results[0].status, 'error') - self._assert_freshness_results('target/error_source.json', 'error') - - self._set_updated_at_to(timedelta(hours=-12)) - self.freshness_start_time = datetime.utcnow() - results = self.run_dbt_with_vars( - ['source', 'snapshot-freshness', '-o', 'target/warn_source.json'], - ) - self.assertEqual(len(results), 1) - self.assertEqual(results[0].status, 'warn') - self._assert_freshness_results('target/warn_source.json', 'warn') - - self._set_updated_at_to(timedelta(hours=-2)) - self.freshness_start_time = datetime.utcnow() - results = self.run_dbt_with_vars( - ['source', 'snapshot-freshness', '-o', 'target/pass_source.json'], - ) - self.assertEqual(len(results), 1) - self.assertEqual(results[0].status, 'pass') - self._assert_freshness_results('target/pass_source.json', 'pass') - - @use_profile('postgres') - def test_postgres_source_freshness_selection_select(self): - """Tests node selection using the --select argument.""" - self._set_updated_at_to(timedelta(hours=-2)) - self.freshness_start_time = datetime.utcnow() - # select source directly - results = self.run_dbt_with_vars( - ['source', 'freshness', '--select', - 'source:test_source.test_table', '-o', 'target/pass_source.json'], - ) - self.assertEqual(len(results), 1) - self.assertEqual(results[0].status, 'pass') - self._assert_freshness_results('target/pass_source.json', 'pass') - - @use_profile('postgres') - def test_postgres_source_freshness_selection_exclude(self): - """Tests node selection using the --select argument. It 'excludes' the - only source in the project so it should return no results.""" - self._set_updated_at_to(timedelta(hours=-2)) - self.freshness_start_time = datetime.utcnow() - # exclude source directly - results = self.run_dbt_with_vars( - ['source', 'freshness', '--exclude', - 'source:test_source.test_table', '-o', 'target/exclude_source.json'], - ) - self.assertEqual(len(results), 0) - - @use_profile('postgres') - def test_postgres_source_freshness_selection_graph_operation(self): - """Tests node selection using the --select argument with graph - operations. `+descendant_model` == select all nodes `descendant_model` - depends on. - """ - self._set_updated_at_to(timedelta(hours=-2)) - self.freshness_start_time = datetime.utcnow() - # select model ancestors - results = self.run_dbt_with_vars( - ['source', 'freshness', '--select', - '+descendant_model', '-o', 'target/ancestor_source.json'] - ) - self.assertEqual(len(results), 1) - self.assertEqual(results[0].status, 'pass') - self._assert_freshness_results('target/ancestor_source.json', 'pass') - -class TestOverrideSourceFreshness(SuccessfulSourcesTest): - - @property - def models(self): - return "override_freshness_models" - - @staticmethod - def get_result_from_unique_id(data, unique_id): - try: - return list(filter(lambda x : x['unique_id'] == unique_id, data['results']))[0] - except IndexError: - raise f"No result for the given unique_id. unique_id={unique_id}" - - def _run_override_source_freshness(self): - self._set_updated_at_to(timedelta(hours=-30)) - self.freshness_start_time = datetime.utcnow() - - path = 'target/pass_source.json' - results = self.run_dbt_with_vars( - ['source', 'freshness', '-o', path], - expect_pass=False - ) - self.assertEqual(len(results), 4) # freshness disabled for source_e - - self.assertTrue(os.path.exists(path)) - with open(path) as fp: - data = json.load(fp) - - result_source_a = self.get_result_from_unique_id(data, 'source.test.test_source.source_a') - self.assertEqual(result_source_a['status'], 'error') - self.assertEqual( - result_source_a['criteria'], - { - 'warn_after': {'count': 6, 'period': 'hour'}, - 'error_after': {'count': 24, 'period': 'hour'}, - 'filter': None - } - ) - - result_source_b = self.get_result_from_unique_id(data, 'source.test.test_source.source_b') - self.assertEqual(result_source_b['status'], 'error') - self.assertEqual( - result_source_b['criteria'], - { - 'warn_after': {'count': 6, 'period': 'hour'}, - 'error_after': {'count': 24, 'period': 'hour'}, - 'filter': None - } - ) - - result_source_c = self.get_result_from_unique_id(data, 'source.test.test_source.source_c') - self.assertEqual(result_source_c['status'], 'warn') - self.assertEqual( - result_source_c['criteria'], - { - 'warn_after': {'count': 6, 'period': 'hour'}, - 'error_after': None, - 'filter': None - } - ) - - result_source_d = self.get_result_from_unique_id(data, 'source.test.test_source.source_d') - self.assertEqual(result_source_d['status'], 'warn') - self.assertEqual( - result_source_d['criteria'], - { - 'warn_after': {'count': 6, 'period': 'hour'}, - 'error_after': {'count': 72, 'period': 'hour'}, - 'filter': None - } - ) - - @use_profile('postgres') - def test_postgres_override_source_freshness(self): - self._run_override_source_freshness() - -class TestSourceFreshnessErrors(SuccessfulSourcesTest): - @property - def models(self): - return "error_models" - - @use_profile('postgres') - def test_postgres_error(self): - results = self.run_dbt_with_vars( - ['source', 'freshness'], - expect_pass=False - ) - self.assertEqual(len(results), 1) - self.assertEqual(results[0].status, 'runtime error') - - -class TestSourceFreshnessFilter(SuccessfulSourcesTest): - @property - def models(self): - return 'filtered_models' - - @use_profile('postgres') - def test_postgres_all_records(self): - # all records are filtered out - self.run_dbt_with_vars( - ['source', 'freshness'], expect_pass=False) - # we should insert a record with #101 that's fresh, but will still fail - # because the filter excludes it - self._set_updated_at_to(timedelta(hours=-2)) - self.run_dbt_with_vars( - ['source', 'freshness'], expect_pass=False) - - # we should now insert a record with #102 that's fresh, and the filter - # includes it - self._set_updated_at_to(timedelta(hours=-2)) - results = self.run_dbt_with_vars( - ['source', 'freshness'], expect_pass=True) - - -class TestMalformedSources(BaseSourcesTest): - # even seeds should fail, because parsing is what's raising - @property - def models(self): - return "malformed_models" - - @use_profile('postgres') - def test_postgres_malformed_schema_will_break_run(self): - with self.assertRaises(ParsingException): - self.run_dbt_with_vars(['seed']) - - -class TestRenderingInSourceTests(BaseSourcesTest): - @property - def models(self): - return "malformed_schema_tests" - - @use_profile('postgres') - def test_postgres_render_in_source_tests(self): - self.run_dbt_with_vars(['seed']) - self.run_dbt_with_vars(['run']) - # syntax error at or near "{", because the test isn't rendered - self.run_dbt_with_vars(['test'], expect_pass=False) - - -class TestUnquotedSources(SuccessfulSourcesTest): - @property - def project_config(self): - cfg = super().project_config - cfg['quoting'] = { - 'identifier': False, - 'schema': False, - 'database': False, - } - return cfg - - @use_profile('postgres') - def test_postgres_catalog(self): - self.run_dbt_with_vars(['run']) - self.run_dbt_with_vars(['docs', 'generate']) diff --git a/test/integration/059_source_overrides_tests/dupe-models/schema1.yml b/test/integration/059_source_overrides_tests/dupe-models/schema1.yml deleted file mode 100644 index 778618d51f8..00000000000 --- a/test/integration/059_source_overrides_tests/dupe-models/schema1.yml +++ /dev/null @@ -1,26 +0,0 @@ -version: 2 -sources: - - name: my_source - overrides: localdep - schema: "{{ target.schema }}" - database: "{{ target.database }}" - freshness: - error_after: {count: 3, period: day} - tables: - - name: my_table - identifier: my_real_seed - # on the override, the "color" column is only unique, it can be null! - columns: - - name: id - tests: - - not_null - - unique - - name: color - tests: - - unique - - name: my_other_table - identifier: my_real_other_seed - - name: snapshot_freshness - identifier: snapshot_freshness_base - freshness: - error_after: {count: 1, period: day} diff --git a/test/integration/059_source_overrides_tests/dupe-models/schema2.yml b/test/integration/059_source_overrides_tests/dupe-models/schema2.yml deleted file mode 100644 index 778618d51f8..00000000000 --- a/test/integration/059_source_overrides_tests/dupe-models/schema2.yml +++ /dev/null @@ -1,26 +0,0 @@ -version: 2 -sources: - - name: my_source - overrides: localdep - schema: "{{ target.schema }}" - database: "{{ target.database }}" - freshness: - error_after: {count: 3, period: day} - tables: - - name: my_table - identifier: my_real_seed - # on the override, the "color" column is only unique, it can be null! - columns: - - name: id - tests: - - not_null - - unique - - name: color - tests: - - unique - - name: my_other_table - identifier: my_real_other_seed - - name: snapshot_freshness - identifier: snapshot_freshness_base - freshness: - error_after: {count: 1, period: day} diff --git a/test/integration/059_source_overrides_tests/local_dependency/dbt_project.yml b/test/integration/059_source_overrides_tests/local_dependency/dbt_project.yml deleted file mode 100644 index 2822f88d541..00000000000 --- a/test/integration/059_source_overrides_tests/local_dependency/dbt_project.yml +++ /dev/null @@ -1,11 +0,0 @@ -config-version: 2 -name: localdep - -version: '1.0' - -profile: 'default' - -seeds: - quote_columns: False - -seed-paths: ['seeds'] diff --git a/test/integration/059_source_overrides_tests/local_dependency/models/my_model.sql b/test/integration/059_source_overrides_tests/local_dependency/models/my_model.sql deleted file mode 100644 index 5be6422876f..00000000000 --- a/test/integration/059_source_overrides_tests/local_dependency/models/my_model.sql +++ /dev/null @@ -1,8 +0,0 @@ -{{ config(materialized='table') }} -with colors as ( - select id, color from {{ source('my_source', 'my_table') }} -), -letters as ( - select id, letter from {{ source('my_source', 'my_other_table') }} -) -select letter, color from colors join letters using (id) diff --git a/test/integration/059_source_overrides_tests/local_dependency/models/schema.yml b/test/integration/059_source_overrides_tests/local_dependency/models/schema.yml deleted file mode 100644 index d4f8bef6b73..00000000000 --- a/test/integration/059_source_overrides_tests/local_dependency/models/schema.yml +++ /dev/null @@ -1,43 +0,0 @@ -version: 2 -sources: - - name: my_source - schema: invalid_schema - database: invalid_database - freshness: - error_after: {count: 3, period: hour} - tables: - - name: my_table - identifier: my_seed - columns: - - name: id - tests: - - unique - - not_null - - name: color - tests: - - unique - - not_null - - name: my_other_table - identifier: my_other_seed - columns: - - name: id - tests: - - unique - - not_null - - name: letter - tests: - - unique - - not_null - - name: snapshot_freshness - identifier: snapshot_freshness_base - loaded_at_field: updated_at - freshness: - error_after: {count: 1, period: hour} - - name: my_other_source - schema: "{{ target.schema }}" - database: "{{ target.database }}" - freshness: - error_after: {count: 1, period: day} - tables: - - name: never_fresh - loaded_at_field: updated_at diff --git a/test/integration/059_source_overrides_tests/local_dependency/seeds/keep/never_fresh.csv b/test/integration/059_source_overrides_tests/local_dependency/seeds/keep/never_fresh.csv deleted file mode 100644 index d7fd6c7b91d..00000000000 --- a/test/integration/059_source_overrides_tests/local_dependency/seeds/keep/never_fresh.csv +++ /dev/null @@ -1,51 +0,0 @@ -favorite_color,id,first_name,email,ip_address,updated_at -blue,1,Larry,lking0@miitbeian.gov.cn,'69.135.206.194',2008-09-12 19:08:31 -blue,2,Larry,lperkins1@toplist.cz,'64.210.133.162',1978-05-09 04:15:14 -blue,3,Anna,amontgomery2@miitbeian.gov.cn,'168.104.64.114',2011-10-16 04:07:57 -blue,4,Sandra,sgeorge3@livejournal.com,'229.235.252.98',1973-07-19 10:52:43 -blue,5,Fred,fwoods4@google.cn,'78.229.170.124',2012-09-30 16:38:29 -blue,6,Stephen,shanson5@livejournal.com,'182.227.157.105',1995-11-07 21:40:50 -blue,7,William,wmartinez6@upenn.edu,'135.139.249.50',1982-09-05 03:11:59 -blue,8,Jessica,jlong7@hao123.com,'203.62.178.210',1991-10-16 11:03:15 -blue,9,Douglas,dwhite8@tamu.edu,'178.187.247.1',1979-10-01 09:49:48 -blue,10,Lisa,lcoleman9@nydailynews.com,'168.234.128.249',2011-05-26 07:45:49 -blue,11,Ralph,rfieldsa@home.pl,'55.152.163.149',1972-11-18 19:06:11 -blue,12,Louise,lnicholsb@samsung.com,'141.116.153.154',2014-11-25 20:56:14 -blue,13,Clarence,cduncanc@sfgate.com,'81.171.31.133',2011-11-17 07:02:36 -blue,14,Daniel,dfranklind@omniture.com,'8.204.211.37',1980-09-13 00:09:04 -blue,15,Katherine,klanee@auda.org.au,'176.96.134.59',1997-08-22 19:36:56 -blue,16,Billy,bwardf@wikia.com,'214.108.78.85',2003-10-19 02:14:47 -blue,17,Annie,agarzag@ocn.ne.jp,'190.108.42.70',1988-10-28 15:12:35 -blue,18,Shirley,scolemanh@fastcompany.com,'109.251.164.84',1988-08-24 10:50:57 -blue,19,Roger,rfrazieri@scribd.com,'38.145.218.108',1985-12-31 15:17:15 -blue,20,Lillian,lstanleyj@goodreads.com,'47.57.236.17',1970-06-08 02:09:05 -blue,21,Aaron,arodriguezk@nps.gov,'205.245.118.221',1985-10-11 23:07:49 -blue,22,Patrick,pparkerl@techcrunch.com,'19.8.100.182',2006-03-29 12:53:56 -blue,23,Phillip,pmorenom@intel.com,'41.38.254.103',2011-11-07 15:35:43 -blue,24,Henry,hgarcian@newsvine.com,'1.191.216.252',2008-08-28 08:30:44 -blue,25,Irene,iturnero@opera.com,'50.17.60.190',1994-04-01 07:15:02 -blue,26,Andrew,adunnp@pen.io,'123.52.253.176',2000-11-01 06:03:25 -blue,27,David,dgutierrezq@wp.com,'238.23.203.42',1988-01-25 07:29:18 -blue,28,Henry,hsanchezr@cyberchimps.com,'248.102.2.185',1983-01-01 13:36:37 -blue,29,Evelyn,epetersons@gizmodo.com,'32.80.46.119',1979-07-16 17:24:12 -blue,30,Tammy,tmitchellt@purevolume.com,'249.246.167.88',2001-04-03 10:00:23 -blue,31,Jacqueline,jlittleu@domainmarket.com,'127.181.97.47',1986-02-11 21:35:50 -blue,32,Earl,eortizv@opera.com,'166.47.248.240',1996-07-06 08:16:27 -blue,33,Juan,jgordonw@sciencedirect.com,'71.77.2.200',1987-01-31 03:46:44 -blue,34,Diane,dhowellx@nyu.edu,'140.94.133.12',1994-06-11 02:30:05 -blue,35,Randy,rkennedyy@microsoft.com,'73.255.34.196',2005-05-26 20:28:39 -blue,36,Janice,jriveraz@time.com,'22.214.227.32',1990-02-09 04:16:52 -blue,37,Laura,lperry10@diigo.com,'159.148.145.73',2015-03-17 05:59:25 -blue,38,Gary,gray11@statcounter.com,'40.193.124.56',1970-01-27 10:04:51 -blue,39,Jesse,jmcdonald12@typepad.com,'31.7.86.103',2009-03-14 08:14:29 -blue,40,Sandra,sgonzalez13@goodreads.com,'223.80.168.239',1993-05-21 14:08:54 -blue,41,Scott,smoore14@archive.org,'38.238.46.83',1980-08-30 11:16:56 -blue,42,Phillip,pevans15@cisco.com,'158.234.59.34',2011-12-15 23:26:31 -blue,43,Steven,sriley16@google.ca,'90.247.57.68',2011-10-29 19:03:28 -blue,44,Deborah,dbrown17@hexun.com,'179.125.143.240',1995-04-10 14:36:07 -blue,45,Lori,lross18@ow.ly,'64.80.162.180',1980-12-27 16:49:15 -blue,46,Sean,sjackson19@tumblr.com,'240.116.183.69',1988-06-12 21:24:45 -blue,47,Terry,tbarnes1a@163.com,'118.38.213.137',1997-09-22 16:43:19 -blue,48,Dorothy,dross1b@ebay.com,'116.81.76.49',2005-02-28 13:33:24 -blue,49,Samuel,swashington1c@house.gov,'38.191.253.40',1989-01-19 21:15:48 -blue,50,Ralph,rcarter1d@tinyurl.com,'104.84.60.174',2007-08-11 10:21:49 diff --git a/test/integration/059_source_overrides_tests/local_dependency/seeds/keep/snapshot_freshness_base.csv b/test/integration/059_source_overrides_tests/local_dependency/seeds/keep/snapshot_freshness_base.csv deleted file mode 100644 index a8f87412ef5..00000000000 --- a/test/integration/059_source_overrides_tests/local_dependency/seeds/keep/snapshot_freshness_base.csv +++ /dev/null @@ -1,101 +0,0 @@ -favorite_color,id,first_name,email,ip_address,updated_at -blue,1,Larry,lking0@miitbeian.gov.cn,'69.135.206.194',2008-09-12 19:08:31 -blue,2,Larry,lperkins1@toplist.cz,'64.210.133.162',1978-05-09 04:15:14 -blue,3,Anna,amontgomery2@miitbeian.gov.cn,'168.104.64.114',2011-10-16 04:07:57 -blue,4,Sandra,sgeorge3@livejournal.com,'229.235.252.98',1973-07-19 10:52:43 -blue,5,Fred,fwoods4@google.cn,'78.229.170.124',2012-09-30 16:38:29 -blue,6,Stephen,shanson5@livejournal.com,'182.227.157.105',1995-11-07 21:40:50 -blue,7,William,wmartinez6@upenn.edu,'135.139.249.50',1982-09-05 03:11:59 -blue,8,Jessica,jlong7@hao123.com,'203.62.178.210',1991-10-16 11:03:15 -blue,9,Douglas,dwhite8@tamu.edu,'178.187.247.1',1979-10-01 09:49:48 -blue,10,Lisa,lcoleman9@nydailynews.com,'168.234.128.249',2011-05-26 07:45:49 -blue,11,Ralph,rfieldsa@home.pl,'55.152.163.149',1972-11-18 19:06:11 -blue,12,Louise,lnicholsb@samsung.com,'141.116.153.154',2014-11-25 20:56:14 -blue,13,Clarence,cduncanc@sfgate.com,'81.171.31.133',2011-11-17 07:02:36 -blue,14,Daniel,dfranklind@omniture.com,'8.204.211.37',1980-09-13 00:09:04 -blue,15,Katherine,klanee@auda.org.au,'176.96.134.59',1997-08-22 19:36:56 -blue,16,Billy,bwardf@wikia.com,'214.108.78.85',2003-10-19 02:14:47 -blue,17,Annie,agarzag@ocn.ne.jp,'190.108.42.70',1988-10-28 15:12:35 -blue,18,Shirley,scolemanh@fastcompany.com,'109.251.164.84',1988-08-24 10:50:57 -blue,19,Roger,rfrazieri@scribd.com,'38.145.218.108',1985-12-31 15:17:15 -blue,20,Lillian,lstanleyj@goodreads.com,'47.57.236.17',1970-06-08 02:09:05 -blue,21,Aaron,arodriguezk@nps.gov,'205.245.118.221',1985-10-11 23:07:49 -blue,22,Patrick,pparkerl@techcrunch.com,'19.8.100.182',2006-03-29 12:53:56 -blue,23,Phillip,pmorenom@intel.com,'41.38.254.103',2011-11-07 15:35:43 -blue,24,Henry,hgarcian@newsvine.com,'1.191.216.252',2008-08-28 08:30:44 -blue,25,Irene,iturnero@opera.com,'50.17.60.190',1994-04-01 07:15:02 -blue,26,Andrew,adunnp@pen.io,'123.52.253.176',2000-11-01 06:03:25 -blue,27,David,dgutierrezq@wp.com,'238.23.203.42',1988-01-25 07:29:18 -blue,28,Henry,hsanchezr@cyberchimps.com,'248.102.2.185',1983-01-01 13:36:37 -blue,29,Evelyn,epetersons@gizmodo.com,'32.80.46.119',1979-07-16 17:24:12 -blue,30,Tammy,tmitchellt@purevolume.com,'249.246.167.88',2001-04-03 10:00:23 -blue,31,Jacqueline,jlittleu@domainmarket.com,'127.181.97.47',1986-02-11 21:35:50 -blue,32,Earl,eortizv@opera.com,'166.47.248.240',1996-07-06 08:16:27 -blue,33,Juan,jgordonw@sciencedirect.com,'71.77.2.200',1987-01-31 03:46:44 -blue,34,Diane,dhowellx@nyu.edu,'140.94.133.12',1994-06-11 02:30:05 -blue,35,Randy,rkennedyy@microsoft.com,'73.255.34.196',2005-05-26 20:28:39 -blue,36,Janice,jriveraz@time.com,'22.214.227.32',1990-02-09 04:16:52 -blue,37,Laura,lperry10@diigo.com,'159.148.145.73',2015-03-17 05:59:25 -blue,38,Gary,gray11@statcounter.com,'40.193.124.56',1970-01-27 10:04:51 -blue,39,Jesse,jmcdonald12@typepad.com,'31.7.86.103',2009-03-14 08:14:29 -blue,40,Sandra,sgonzalez13@goodreads.com,'223.80.168.239',1993-05-21 14:08:54 -blue,41,Scott,smoore14@archive.org,'38.238.46.83',1980-08-30 11:16:56 -blue,42,Phillip,pevans15@cisco.com,'158.234.59.34',2011-12-15 23:26:31 -blue,43,Steven,sriley16@google.ca,'90.247.57.68',2011-10-29 19:03:28 -blue,44,Deborah,dbrown17@hexun.com,'179.125.143.240',1995-04-10 14:36:07 -blue,45,Lori,lross18@ow.ly,'64.80.162.180',1980-12-27 16:49:15 -blue,46,Sean,sjackson19@tumblr.com,'240.116.183.69',1988-06-12 21:24:45 -blue,47,Terry,tbarnes1a@163.com,'118.38.213.137',1997-09-22 16:43:19 -blue,48,Dorothy,dross1b@ebay.com,'116.81.76.49',2005-02-28 13:33:24 -blue,49,Samuel,swashington1c@house.gov,'38.191.253.40',1989-01-19 21:15:48 -blue,50,Ralph,rcarter1d@tinyurl.com,'104.84.60.174',2007-08-11 10:21:49 -green,51,Wayne,whudson1e@princeton.edu,'90.61.24.102',1983-07-03 16:58:12 -green,52,Rose,rjames1f@plala.or.jp,'240.83.81.10',1995-06-08 11:46:23 -green,53,Louise,lcox1g@theglobeandmail.com,'105.11.82.145',2016-09-19 14:45:51 -green,54,Kenneth,kjohnson1h@independent.co.uk,'139.5.45.94',1976-08-17 11:26:19 -green,55,Donna,dbrown1i@amazon.co.uk,'19.45.169.45',2006-05-27 16:51:40 -green,56,Johnny,jvasquez1j@trellian.com,'118.202.238.23',1975-11-17 08:42:32 -green,57,Patrick,pramirez1k@tamu.edu,'231.25.153.198',1997-08-06 11:51:09 -green,58,Helen,hlarson1l@prweb.com,'8.40.21.39',1993-08-04 19:53:40 -green,59,Patricia,pspencer1m@gmpg.org,'212.198.40.15',1977-08-03 16:37:27 -green,60,Joseph,jspencer1n@marriott.com,'13.15.63.238',2005-07-23 20:22:06 -green,61,Phillip,pschmidt1o@blogtalkradio.com,'177.98.201.190',1976-05-19 21:47:44 -green,62,Joan,jwebb1p@google.ru,'105.229.170.71',1972-09-07 17:53:47 -green,63,Phyllis,pkennedy1q@imgur.com,'35.145.8.244',2000-01-01 22:33:37 -green,64,Katherine,khunter1r@smh.com.au,'248.168.205.32',1991-01-09 06:40:24 -green,65,Laura,lvasquez1s@wiley.com,'128.129.115.152',1997-10-23 12:04:56 -green,66,Juan,jdunn1t@state.gov,'44.228.124.51',2004-11-10 05:07:35 -green,67,Judith,jholmes1u@wiley.com,'40.227.179.115',1977-08-02 17:01:45 -green,68,Beverly,bbaker1v@wufoo.com,'208.34.84.59',2016-03-06 20:07:23 -green,69,Lawrence,lcarr1w@flickr.com,'59.158.212.223',1988-09-13 06:07:21 -green,70,Gloria,gwilliams1x@mtv.com,'245.231.88.33',1995-03-18 22:32:46 -green,71,Steven,ssims1y@cbslocal.com,'104.50.58.255',2001-08-05 21:26:20 -green,72,Betty,bmills1z@arstechnica.com,'103.177.214.220',1981-12-14 21:26:54 -green,73,Mildred,mfuller20@prnewswire.com,'151.158.8.130',2000-04-19 10:13:55 -green,74,Donald,dday21@icq.com,'9.178.102.255',1972-12-03 00:58:24 -green,75,Eric,ethomas22@addtoany.com,'85.2.241.227',1992-11-01 05:59:30 -green,76,Joyce,jarmstrong23@sitemeter.com,'169.224.20.36',1985-10-24 06:50:01 -green,77,Maria,mmartinez24@amazonaws.com,'143.189.167.135',2005-10-05 05:17:42 -green,78,Harry,hburton25@youtube.com,'156.47.176.237',1978-03-26 05:53:33 -green,79,Kevin,klawrence26@hao123.com,'79.136.183.83',1994-10-12 04:38:52 -green,80,David,dhall27@prweb.com,'133.149.172.153',1976-12-15 16:24:24 -green,81,Kathy,kperry28@twitter.com,'229.242.72.228',1979-03-04 02:58:56 -green,82,Adam,aprice29@elegantthemes.com,'13.145.21.10',1982-11-07 11:46:59 -green,83,Brandon,bgriffin2a@va.gov,'73.249.128.212',2013-10-30 05:30:36 -green,84,Henry,hnguyen2b@discovery.com,'211.36.214.242',1985-01-09 06:37:27 -green,85,Eric,esanchez2c@edublogs.org,'191.166.188.251',2004-05-01 23:21:42 -green,86,Jason,jlee2d@jimdo.com,'193.92.16.182',1973-01-08 09:05:39 -green,87,Diana,drichards2e@istockphoto.com,'19.130.175.245',1994-10-05 22:50:49 -green,88,Andrea,awelch2f@abc.net.au,'94.155.233.96',2002-04-26 08:41:44 -green,89,Louis,lwagner2g@miitbeian.gov.cn,'26.217.34.111',2003-08-25 07:56:39 -green,90,Jane,jsims2h@seesaa.net,'43.4.220.135',1987-03-20 20:39:04 -green,91,Larry,lgrant2i@si.edu,'97.126.79.34',2000-09-07 20:26:19 -green,92,Louis,ldean2j@prnewswire.com,'37.148.40.127',2011-09-16 20:12:14 -green,93,Jennifer,jcampbell2k@xing.com,'38.106.254.142',1988-07-15 05:06:49 -green,94,Wayne,wcunningham2l@google.com.hk,'223.28.26.187',2009-12-15 06:16:54 -green,95,Lori,lstevens2m@icq.com,'181.250.181.58',1984-10-28 03:29:19 -green,96,Judy,jsimpson2n@marriott.com,'180.121.239.219',1986-02-07 15:18:10 -green,97,Phillip,phoward2o@usa.gov,'255.247.0.175',2002-12-26 08:44:45 -green,98,Gloria,gwalker2p@usa.gov,'156.140.7.128',1997-10-04 07:58:58 -green,99,Paul,pjohnson2q@umn.edu,'183.59.198.197',1991-11-14 12:33:55 -green,100,Frank,fgreene2r@blogspot.com,'150.143.68.121',2010-06-12 23:55:39 diff --git a/test/integration/059_source_overrides_tests/local_dependency/seeds/my_other_seed.csv b/test/integration/059_source_overrides_tests/local_dependency/seeds/my_other_seed.csv deleted file mode 100644 index ec44ccd4238..00000000000 --- a/test/integration/059_source_overrides_tests/local_dependency/seeds/my_other_seed.csv +++ /dev/null @@ -1,4 +0,0 @@ -id,letter -1,r -2,g -3,b diff --git a/test/integration/059_source_overrides_tests/local_dependency/seeds/my_seed.csv b/test/integration/059_source_overrides_tests/local_dependency/seeds/my_seed.csv deleted file mode 100644 index 37493c909b8..00000000000 --- a/test/integration/059_source_overrides_tests/local_dependency/seeds/my_seed.csv +++ /dev/null @@ -1,4 +0,0 @@ -id,color -1,red -2,green -3,blue diff --git a/test/integration/059_source_overrides_tests/models/schema.yml b/test/integration/059_source_overrides_tests/models/schema.yml deleted file mode 100644 index 778618d51f8..00000000000 --- a/test/integration/059_source_overrides_tests/models/schema.yml +++ /dev/null @@ -1,26 +0,0 @@ -version: 2 -sources: - - name: my_source - overrides: localdep - schema: "{{ target.schema }}" - database: "{{ target.database }}" - freshness: - error_after: {count: 3, period: day} - tables: - - name: my_table - identifier: my_real_seed - # on the override, the "color" column is only unique, it can be null! - columns: - - name: id - tests: - - not_null - - unique - - name: color - tests: - - unique - - name: my_other_table - identifier: my_real_other_seed - - name: snapshot_freshness - identifier: snapshot_freshness_base - freshness: - error_after: {count: 1, period: day} diff --git a/test/integration/059_source_overrides_tests/seeds/expected_result.csv b/test/integration/059_source_overrides_tests/seeds/expected_result.csv deleted file mode 100644 index 2d75f7658bb..00000000000 --- a/test/integration/059_source_overrides_tests/seeds/expected_result.csv +++ /dev/null @@ -1,5 +0,0 @@ -letter,color -c,cyan -m,magenta -y,yellow -k,key diff --git a/test/integration/059_source_overrides_tests/seeds/my_real_other_seed.csv b/test/integration/059_source_overrides_tests/seeds/my_real_other_seed.csv deleted file mode 100644 index defeee5ce23..00000000000 --- a/test/integration/059_source_overrides_tests/seeds/my_real_other_seed.csv +++ /dev/null @@ -1,5 +0,0 @@ -id,letter -1,c -2,m -3,y -4,k diff --git a/test/integration/059_source_overrides_tests/seeds/my_real_seed.csv b/test/integration/059_source_overrides_tests/seeds/my_real_seed.csv deleted file mode 100644 index ff44257bd6b..00000000000 --- a/test/integration/059_source_overrides_tests/seeds/my_real_seed.csv +++ /dev/null @@ -1,6 +0,0 @@ -id,color -1,cyan -2,magenta -3,yellow -4,key -5,NULL diff --git a/test/integration/059_source_overrides_tests/test_source_overrides.py b/test/integration/059_source_overrides_tests/test_source_overrides.py deleted file mode 100644 index 0e4628acc31..00000000000 --- a/test/integration/059_source_overrides_tests/test_source_overrides.py +++ /dev/null @@ -1,185 +0,0 @@ -import os -from datetime import datetime, timedelta -from test.integration.base import DBTIntegrationTest, use_profile -from dbt.exceptions import CompilationException - - -class TestSourceOverrides(DBTIntegrationTest): - def setUp(self): - super().setUp() - self._id = 101 - - @property - def schema(self): - return "source_overrides_059" - - @property - def models(self): - return 'models' - - @property - def packages_config(self): - return { - 'packages': [ - {'local': 'local_dependency'}, - ], - } - - @property - def project_config(self): - return { - 'config-version': 2, - 'seeds': { - 'localdep': { - 'enabled': False, - 'keep': { - 'enabled': True, - } - }, - 'quote_columns': False, - }, - 'sources': { - 'localdep': { - 'my_other_source': { - 'enabled': False, - } - } - } - } - - def _set_updated_at_to(self, delta): - insert_time = datetime.utcnow() + delta - timestr = insert_time.strftime("%Y-%m-%d %H:%M:%S") - # favorite_color,id,first_name,email,ip_address,updated_at - insert_id = self._id - self._id += 1 - raw_sql = """INSERT INTO {schema}.{source} - ({quoted_columns}) - VALUES ( - 'blue',{id},'Jake','abc@example.com','192.168.1.1','{time}' - )""" - quoted_columns = ','.join( - self.adapter.quote(c) for c in - ('favorite_color', 'id', 'first_name', 'email', 'ip_address', 'updated_at') - ) - self.run_sql( - raw_sql, - kwargs={ - 'schema': self.unique_schema(), - 'time': timestr, - 'id': insert_id, - 'source': self.adapter.quote('snapshot_freshness_base'), - 'quoted_columns': quoted_columns, - } - ) - - @use_profile('postgres') - def test_postgres_source_overrides(self): - self.run_dbt(['deps']) - seed_results = self.run_dbt(['seed']) - assert len(seed_results) == 5 - - # There should be 7, as we disabled 1 test of the original 8 - test_results = self.run_dbt(['test']) - assert len(test_results) == 7 - - results = self.run_dbt(['run']) - assert len(results) == 1 - - self.assertTablesEqual('expected_result', 'my_model') - - # set the updated_at field of this seed to last week - self._set_updated_at_to(timedelta(days=-7)) - # if snapshot-freshness fails, freshness just didn't happen! - results = self.run_dbt( - ['source', 'snapshot-freshness'], expect_pass=False - ) - # we disabled my_other_source, so we only run the one freshness check - # in - self.assertEqual(len(results), 1) - # If snapshot-freshness passes, that means error_after was - # applied from the source override but not the source table override - self._set_updated_at_to(timedelta(days=-2)) - results = self.run_dbt( - ['source', 'snapshot-freshness'], expect_pass=False, - ) - self.assertEqual(len(results), 1) - - self._set_updated_at_to(timedelta(hours=-12)) - results = self.run_dbt( - ['source', 'snapshot-freshness'], expect_pass=True - ) - self.assertEqual(len(results), 1) - - self.use_default_project({ - 'sources': { - 'localdep': { - 'my_other_source': { - 'enabled': True, - } - } - } - }) - # enable my_other_source, snapshot freshness should fail due to the new - # not-fresh source - results = self.run_dbt( - ['source', 'snapshot-freshness'], expect_pass=False - ) - self.assertEqual(len(results), 2) - - -class TestSourceDuplicateOverrides(DBTIntegrationTest): - def setUp(self): - super().setUp() - self._id = 101 - - @property - def schema(self): - return "source_overrides_059" - - @property - def models(self): - return 'dupe-models' - - @property - def packages_config(self): - return { - 'packages': [ - {'local': 'local_dependency'}, - ], - } - - @property - def project_config(self): - return { - 'config-version': 2, - 'seeds': { - 'localdep': { - 'enabled': False, - 'keep': { - 'enabled': True, - } - }, - 'quote_columns': False, - }, - 'sources': { - 'localdep': { - 'my_other_source': { - 'enabled': False, - } - } - } - } - - @use_profile('postgres') - def test_postgres_source_duplicate_overrides(self): - self.run_dbt(['deps']) - with self.assertRaises(CompilationException) as exc: - self.run_dbt(['compile']) - - self.assertIn('dbt found two schema.yml entries for the same source named', str(exc.exception)) - self.assertIn('one of these files', str(exc.exception)) - schema1_path = os.path.join('dupe-models', 'schema1.yml') - schema2_path = os.path.join('dupe-models', 'schema2.yml') - self.assertIn(schema1_path, str(exc.exception)) - self.assertIn(schema2_path, str(exc.exception)) diff --git a/tests/functional/source_overrides/fixtures.py b/tests/functional/source_overrides/fixtures.py new file mode 100644 index 00000000000..6d4b17960d7 --- /dev/null +++ b/tests/functional/source_overrides/fixtures.py @@ -0,0 +1,377 @@ +import pytest + + +dupe_models__schema2_yml = """ +version: 2 +sources: + - name: my_source + overrides: localdep + schema: "{{ target.schema }}" + database: "{{ target.database }}" + freshness: + error_after: {count: 3, period: day} + tables: + - name: my_table + identifier: my_real_seed + # on the override, the "color" column is only unique, it can be null! + columns: + - name: id + tests: + - not_null + - unique + - name: color + tests: + - unique + - name: my_other_table + identifier: my_real_other_seed + - name: snapshot_freshness + identifier: snapshot_freshness_base + freshness: + error_after: {count: 1, period: day} + +""" + +dupe_models__schema1_yml = """ +version: 2 +sources: + - name: my_source + overrides: localdep + schema: "{{ target.schema }}" + database: "{{ target.database }}" + freshness: + error_after: {count: 3, period: day} + tables: + - name: my_table + identifier: my_real_seed + # on the override, the "color" column is only unique, it can be null! + columns: + - name: id + tests: + - not_null + - unique + - name: color + tests: + - unique + - name: my_other_table + identifier: my_real_other_seed + - name: snapshot_freshness + identifier: snapshot_freshness_base + freshness: + error_after: {count: 1, period: day} + +""" + +local_dependency__dbt_project_yml = """ +config-version: 2 +name: localdep + +version: '1.0' + +profile: 'default' + +seeds: + quote_columns: False + +seed-paths: ['seeds'] + +""" + +local_dependency__models__schema_yml = """ +version: 2 +sources: + - name: my_source + schema: invalid_schema + database: invalid_database + freshness: + error_after: {count: 3, period: hour} + tables: + - name: my_table + identifier: my_seed + columns: + - name: id + tests: + - unique + - not_null + - name: color + tests: + - unique + - not_null + - name: my_other_table + identifier: my_other_seed + columns: + - name: id + tests: + - unique + - not_null + - name: letter + tests: + - unique + - not_null + - name: snapshot_freshness + identifier: snapshot_freshness_base + loaded_at_field: updated_at + freshness: + error_after: {count: 1, period: hour} + - name: my_other_source + schema: "{{ target.schema }}" + database: "{{ target.database }}" + freshness: + error_after: {count: 1, period: day} + tables: + - name: never_fresh + loaded_at_field: updated_at + +""" + +local_dependency__models__my_model_sql = """ + +{{ config(materialized="table") }} + +with colors as ( + select id, color from {{ source('my_source', 'my_table') }} +), +letters as ( + select id, letter from {{ source('my_source', 'my_other_table') }} +) +select letter, color from colors join letters using (id) + +""" + +local_dependency__seeds__my_other_seed_csv = """id,letter +1,r +2,g +3,b +""" + +local_dependency__seeds__my_seed_csv = """id,color +1,red +2,green +3,blue +""" + +local_dependency__seeds__keep__never_fresh_csv = """favorite_color,id,first_name,email,ip_address,updated_at +blue,1,Larry,lking0@miitbeian.gov.cn,'69.135.206.194',2008-09-12 19:08:31 +blue,2,Larry,lperkins1@toplist.cz,'64.210.133.162',1978-05-09 04:15:14 +blue,3,Anna,amontgomery2@miitbeian.gov.cn,'168.104.64.114',2011-10-16 04:07:57 +blue,4,Sandra,sgeorge3@livejournal.com,'229.235.252.98',1973-07-19 10:52:43 +blue,5,Fred,fwoods4@google.cn,'78.229.170.124',2012-09-30 16:38:29 +blue,6,Stephen,shanson5@livejournal.com,'182.227.157.105',1995-11-07 21:40:50 +blue,7,William,wmartinez6@upenn.edu,'135.139.249.50',1982-09-05 03:11:59 +blue,8,Jessica,jlong7@hao123.com,'203.62.178.210',1991-10-16 11:03:15 +blue,9,Douglas,dwhite8@tamu.edu,'178.187.247.1',1979-10-01 09:49:48 +blue,10,Lisa,lcoleman9@nydailynews.com,'168.234.128.249',2011-05-26 07:45:49 +blue,11,Ralph,rfieldsa@home.pl,'55.152.163.149',1972-11-18 19:06:11 +blue,12,Louise,lnicholsb@samsung.com,'141.116.153.154',2014-11-25 20:56:14 +blue,13,Clarence,cduncanc@sfgate.com,'81.171.31.133',2011-11-17 07:02:36 +blue,14,Daniel,dfranklind@omniture.com,'8.204.211.37',1980-09-13 00:09:04 +blue,15,Katherine,klanee@auda.org.au,'176.96.134.59',1997-08-22 19:36:56 +blue,16,Billy,bwardf@wikia.com,'214.108.78.85',2003-10-19 02:14:47 +blue,17,Annie,agarzag@ocn.ne.jp,'190.108.42.70',1988-10-28 15:12:35 +blue,18,Shirley,scolemanh@fastcompany.com,'109.251.164.84',1988-08-24 10:50:57 +blue,19,Roger,rfrazieri@scribd.com,'38.145.218.108',1985-12-31 15:17:15 +blue,20,Lillian,lstanleyj@goodreads.com,'47.57.236.17',1970-06-08 02:09:05 +blue,21,Aaron,arodriguezk@nps.gov,'205.245.118.221',1985-10-11 23:07:49 +blue,22,Patrick,pparkerl@techcrunch.com,'19.8.100.182',2006-03-29 12:53:56 +blue,23,Phillip,pmorenom@intel.com,'41.38.254.103',2011-11-07 15:35:43 +blue,24,Henry,hgarcian@newsvine.com,'1.191.216.252',2008-08-28 08:30:44 +blue,25,Irene,iturnero@opera.com,'50.17.60.190',1994-04-01 07:15:02 +blue,26,Andrew,adunnp@pen.io,'123.52.253.176',2000-11-01 06:03:25 +blue,27,David,dgutierrezq@wp.com,'238.23.203.42',1988-01-25 07:29:18 +blue,28,Henry,hsanchezr@cyberchimps.com,'248.102.2.185',1983-01-01 13:36:37 +blue,29,Evelyn,epetersons@gizmodo.com,'32.80.46.119',1979-07-16 17:24:12 +blue,30,Tammy,tmitchellt@purevolume.com,'249.246.167.88',2001-04-03 10:00:23 +blue,31,Jacqueline,jlittleu@domainmarket.com,'127.181.97.47',1986-02-11 21:35:50 +blue,32,Earl,eortizv@opera.com,'166.47.248.240',1996-07-06 08:16:27 +blue,33,Juan,jgordonw@sciencedirect.com,'71.77.2.200',1987-01-31 03:46:44 +blue,34,Diane,dhowellx@nyu.edu,'140.94.133.12',1994-06-11 02:30:05 +blue,35,Randy,rkennedyy@microsoft.com,'73.255.34.196',2005-05-26 20:28:39 +blue,36,Janice,jriveraz@time.com,'22.214.227.32',1990-02-09 04:16:52 +blue,37,Laura,lperry10@diigo.com,'159.148.145.73',2015-03-17 05:59:25 +blue,38,Gary,gray11@statcounter.com,'40.193.124.56',1970-01-27 10:04:51 +blue,39,Jesse,jmcdonald12@typepad.com,'31.7.86.103',2009-03-14 08:14:29 +blue,40,Sandra,sgonzalez13@goodreads.com,'223.80.168.239',1993-05-21 14:08:54 +blue,41,Scott,smoore14@archive.org,'38.238.46.83',1980-08-30 11:16:56 +blue,42,Phillip,pevans15@cisco.com,'158.234.59.34',2011-12-15 23:26:31 +blue,43,Steven,sriley16@google.ca,'90.247.57.68',2011-10-29 19:03:28 +blue,44,Deborah,dbrown17@hexun.com,'179.125.143.240',1995-04-10 14:36:07 +blue,45,Lori,lross18@ow.ly,'64.80.162.180',1980-12-27 16:49:15 +blue,46,Sean,sjackson19@tumblr.com,'240.116.183.69',1988-06-12 21:24:45 +blue,47,Terry,tbarnes1a@163.com,'118.38.213.137',1997-09-22 16:43:19 +blue,48,Dorothy,dross1b@ebay.com,'116.81.76.49',2005-02-28 13:33:24 +blue,49,Samuel,swashington1c@house.gov,'38.191.253.40',1989-01-19 21:15:48 +blue,50,Ralph,rcarter1d@tinyurl.com,'104.84.60.174',2007-08-11 10:21:49 +""" + +local_dependency__seeds__keep__snapshot_freshness_base_csv = """favorite_color,id,first_name,email,ip_address,updated_at +blue,1,Larry,lking0@miitbeian.gov.cn,'69.135.206.194',2008-09-12 19:08:31 +blue,2,Larry,lperkins1@toplist.cz,'64.210.133.162',1978-05-09 04:15:14 +blue,3,Anna,amontgomery2@miitbeian.gov.cn,'168.104.64.114',2011-10-16 04:07:57 +blue,4,Sandra,sgeorge3@livejournal.com,'229.235.252.98',1973-07-19 10:52:43 +blue,5,Fred,fwoods4@google.cn,'78.229.170.124',2012-09-30 16:38:29 +blue,6,Stephen,shanson5@livejournal.com,'182.227.157.105',1995-11-07 21:40:50 +blue,7,William,wmartinez6@upenn.edu,'135.139.249.50',1982-09-05 03:11:59 +blue,8,Jessica,jlong7@hao123.com,'203.62.178.210',1991-10-16 11:03:15 +blue,9,Douglas,dwhite8@tamu.edu,'178.187.247.1',1979-10-01 09:49:48 +blue,10,Lisa,lcoleman9@nydailynews.com,'168.234.128.249',2011-05-26 07:45:49 +blue,11,Ralph,rfieldsa@home.pl,'55.152.163.149',1972-11-18 19:06:11 +blue,12,Louise,lnicholsb@samsung.com,'141.116.153.154',2014-11-25 20:56:14 +blue,13,Clarence,cduncanc@sfgate.com,'81.171.31.133',2011-11-17 07:02:36 +blue,14,Daniel,dfranklind@omniture.com,'8.204.211.37',1980-09-13 00:09:04 +blue,15,Katherine,klanee@auda.org.au,'176.96.134.59',1997-08-22 19:36:56 +blue,16,Billy,bwardf@wikia.com,'214.108.78.85',2003-10-19 02:14:47 +blue,17,Annie,agarzag@ocn.ne.jp,'190.108.42.70',1988-10-28 15:12:35 +blue,18,Shirley,scolemanh@fastcompany.com,'109.251.164.84',1988-08-24 10:50:57 +blue,19,Roger,rfrazieri@scribd.com,'38.145.218.108',1985-12-31 15:17:15 +blue,20,Lillian,lstanleyj@goodreads.com,'47.57.236.17',1970-06-08 02:09:05 +blue,21,Aaron,arodriguezk@nps.gov,'205.245.118.221',1985-10-11 23:07:49 +blue,22,Patrick,pparkerl@techcrunch.com,'19.8.100.182',2006-03-29 12:53:56 +blue,23,Phillip,pmorenom@intel.com,'41.38.254.103',2011-11-07 15:35:43 +blue,24,Henry,hgarcian@newsvine.com,'1.191.216.252',2008-08-28 08:30:44 +blue,25,Irene,iturnero@opera.com,'50.17.60.190',1994-04-01 07:15:02 +blue,26,Andrew,adunnp@pen.io,'123.52.253.176',2000-11-01 06:03:25 +blue,27,David,dgutierrezq@wp.com,'238.23.203.42',1988-01-25 07:29:18 +blue,28,Henry,hsanchezr@cyberchimps.com,'248.102.2.185',1983-01-01 13:36:37 +blue,29,Evelyn,epetersons@gizmodo.com,'32.80.46.119',1979-07-16 17:24:12 +blue,30,Tammy,tmitchellt@purevolume.com,'249.246.167.88',2001-04-03 10:00:23 +blue,31,Jacqueline,jlittleu@domainmarket.com,'127.181.97.47',1986-02-11 21:35:50 +blue,32,Earl,eortizv@opera.com,'166.47.248.240',1996-07-06 08:16:27 +blue,33,Juan,jgordonw@sciencedirect.com,'71.77.2.200',1987-01-31 03:46:44 +blue,34,Diane,dhowellx@nyu.edu,'140.94.133.12',1994-06-11 02:30:05 +blue,35,Randy,rkennedyy@microsoft.com,'73.255.34.196',2005-05-26 20:28:39 +blue,36,Janice,jriveraz@time.com,'22.214.227.32',1990-02-09 04:16:52 +blue,37,Laura,lperry10@diigo.com,'159.148.145.73',2015-03-17 05:59:25 +blue,38,Gary,gray11@statcounter.com,'40.193.124.56',1970-01-27 10:04:51 +blue,39,Jesse,jmcdonald12@typepad.com,'31.7.86.103',2009-03-14 08:14:29 +blue,40,Sandra,sgonzalez13@goodreads.com,'223.80.168.239',1993-05-21 14:08:54 +blue,41,Scott,smoore14@archive.org,'38.238.46.83',1980-08-30 11:16:56 +blue,42,Phillip,pevans15@cisco.com,'158.234.59.34',2011-12-15 23:26:31 +blue,43,Steven,sriley16@google.ca,'90.247.57.68',2011-10-29 19:03:28 +blue,44,Deborah,dbrown17@hexun.com,'179.125.143.240',1995-04-10 14:36:07 +blue,45,Lori,lross18@ow.ly,'64.80.162.180',1980-12-27 16:49:15 +blue,46,Sean,sjackson19@tumblr.com,'240.116.183.69',1988-06-12 21:24:45 +blue,47,Terry,tbarnes1a@163.com,'118.38.213.137',1997-09-22 16:43:19 +blue,48,Dorothy,dross1b@ebay.com,'116.81.76.49',2005-02-28 13:33:24 +blue,49,Samuel,swashington1c@house.gov,'38.191.253.40',1989-01-19 21:15:48 +blue,50,Ralph,rcarter1d@tinyurl.com,'104.84.60.174',2007-08-11 10:21:49 +green,51,Wayne,whudson1e@princeton.edu,'90.61.24.102',1983-07-03 16:58:12 +green,52,Rose,rjames1f@plala.or.jp,'240.83.81.10',1995-06-08 11:46:23 +green,53,Louise,lcox1g@theglobeandmail.com,'105.11.82.145',2016-09-19 14:45:51 +green,54,Kenneth,kjohnson1h@independent.co.uk,'139.5.45.94',1976-08-17 11:26:19 +green,55,Donna,dbrown1i@amazon.co.uk,'19.45.169.45',2006-05-27 16:51:40 +green,56,Johnny,jvasquez1j@trellian.com,'118.202.238.23',1975-11-17 08:42:32 +green,57,Patrick,pramirez1k@tamu.edu,'231.25.153.198',1997-08-06 11:51:09 +green,58,Helen,hlarson1l@prweb.com,'8.40.21.39',1993-08-04 19:53:40 +green,59,Patricia,pspencer1m@gmpg.org,'212.198.40.15',1977-08-03 16:37:27 +green,60,Joseph,jspencer1n@marriott.com,'13.15.63.238',2005-07-23 20:22:06 +green,61,Phillip,pschmidt1o@blogtalkradio.com,'177.98.201.190',1976-05-19 21:47:44 +green,62,Joan,jwebb1p@google.ru,'105.229.170.71',1972-09-07 17:53:47 +green,63,Phyllis,pkennedy1q@imgur.com,'35.145.8.244',2000-01-01 22:33:37 +green,64,Katherine,khunter1r@smh.com.au,'248.168.205.32',1991-01-09 06:40:24 +green,65,Laura,lvasquez1s@wiley.com,'128.129.115.152',1997-10-23 12:04:56 +green,66,Juan,jdunn1t@state.gov,'44.228.124.51',2004-11-10 05:07:35 +green,67,Judith,jholmes1u@wiley.com,'40.227.179.115',1977-08-02 17:01:45 +green,68,Beverly,bbaker1v@wufoo.com,'208.34.84.59',2016-03-06 20:07:23 +green,69,Lawrence,lcarr1w@flickr.com,'59.158.212.223',1988-09-13 06:07:21 +green,70,Gloria,gwilliams1x@mtv.com,'245.231.88.33',1995-03-18 22:32:46 +green,71,Steven,ssims1y@cbslocal.com,'104.50.58.255',2001-08-05 21:26:20 +green,72,Betty,bmills1z@arstechnica.com,'103.177.214.220',1981-12-14 21:26:54 +green,73,Mildred,mfuller20@prnewswire.com,'151.158.8.130',2000-04-19 10:13:55 +green,74,Donald,dday21@icq.com,'9.178.102.255',1972-12-03 00:58:24 +green,75,Eric,ethomas22@addtoany.com,'85.2.241.227',1992-11-01 05:59:30 +green,76,Joyce,jarmstrong23@sitemeter.com,'169.224.20.36',1985-10-24 06:50:01 +green,77,Maria,mmartinez24@amazonaws.com,'143.189.167.135',2005-10-05 05:17:42 +green,78,Harry,hburton25@youtube.com,'156.47.176.237',1978-03-26 05:53:33 +green,79,Kevin,klawrence26@hao123.com,'79.136.183.83',1994-10-12 04:38:52 +green,80,David,dhall27@prweb.com,'133.149.172.153',1976-12-15 16:24:24 +green,81,Kathy,kperry28@twitter.com,'229.242.72.228',1979-03-04 02:58:56 +green,82,Adam,aprice29@elegantthemes.com,'13.145.21.10',1982-11-07 11:46:59 +green,83,Brandon,bgriffin2a@va.gov,'73.249.128.212',2013-10-30 05:30:36 +green,84,Henry,hnguyen2b@discovery.com,'211.36.214.242',1985-01-09 06:37:27 +green,85,Eric,esanchez2c@edublogs.org,'191.166.188.251',2004-05-01 23:21:42 +green,86,Jason,jlee2d@jimdo.com,'193.92.16.182',1973-01-08 09:05:39 +green,87,Diana,drichards2e@istockphoto.com,'19.130.175.245',1994-10-05 22:50:49 +green,88,Andrea,awelch2f@abc.net.au,'94.155.233.96',2002-04-26 08:41:44 +green,89,Louis,lwagner2g@miitbeian.gov.cn,'26.217.34.111',2003-08-25 07:56:39 +green,90,Jane,jsims2h@seesaa.net,'43.4.220.135',1987-03-20 20:39:04 +green,91,Larry,lgrant2i@si.edu,'97.126.79.34',2000-09-07 20:26:19 +green,92,Louis,ldean2j@prnewswire.com,'37.148.40.127',2011-09-16 20:12:14 +green,93,Jennifer,jcampbell2k@xing.com,'38.106.254.142',1988-07-15 05:06:49 +green,94,Wayne,wcunningham2l@google.com.hk,'223.28.26.187',2009-12-15 06:16:54 +green,95,Lori,lstevens2m@icq.com,'181.250.181.58',1984-10-28 03:29:19 +green,96,Judy,jsimpson2n@marriott.com,'180.121.239.219',1986-02-07 15:18:10 +green,97,Phillip,phoward2o@usa.gov,'255.247.0.175',2002-12-26 08:44:45 +green,98,Gloria,gwalker2p@usa.gov,'156.140.7.128',1997-10-04 07:58:58 +green,99,Paul,pjohnson2q@umn.edu,'183.59.198.197',1991-11-14 12:33:55 +green,100,Frank,fgreene2r@blogspot.com,'150.143.68.121',2010-06-12 23:55:39 +""" + +models__schema_yml = """ +version: 2 +sources: + - name: my_source + overrides: localdep + schema: "{{ target.schema }}" + database: "{{ target.database }}" + freshness: + error_after: {count: 3, period: day} + tables: + - name: my_table + identifier: my_real_seed + # on the override, the "color" column is only unique, it can be null! + columns: + - name: id + tests: + - not_null + - unique + - name: color + tests: + - unique + - name: my_other_table + identifier: my_real_other_seed + - name: snapshot_freshness + identifier: snapshot_freshness_base + freshness: + error_after: {count: 1, period: day} + +""" + +seeds__expected_result_csv = """letter,color +c,cyan +m,magenta +y,yellow +k,key +""" + +seeds__my_real_other_seed_csv = """id,letter +1,c +2,m +3,y +4,k +""" + +seeds__my_real_seed_csv = """id,color +1,cyan +2,magenta +3,yellow +4,key +5,NULL +""" + + +@pytest.fixture(scope="class") +def local_dependency(): + return { + "dbt_project.yml": local_dependency__dbt_project_yml, + "models": { + "schema.yml": local_dependency__models__schema_yml, + "my_model.sql": local_dependency__models__my_model_sql, + }, + "seeds": { + "my_other_seed.csv": local_dependency__seeds__my_other_seed_csv, + "my_seed.csv": local_dependency__seeds__my_seed_csv, + "keep": { + "never_fresh.csv": local_dependency__seeds__keep__never_fresh_csv, + "snapshot_freshness_base.csv": local_dependency__seeds__keep__snapshot_freshness_base_csv, + }, + }, + } diff --git a/tests/functional/source_overrides/test_simple_source_override.py b/tests/functional/source_overrides/test_simple_source_override.py new file mode 100644 index 00000000000..749221dbfea --- /dev/null +++ b/tests/functional/source_overrides/test_simple_source_override.py @@ -0,0 +1,149 @@ +from datetime import datetime, timedelta +import pytest + +from dbt.tests.util import run_dbt, update_config_file +from dbt.tests.tables import TableComparison +from dbt.tests.fixtures.project import write_project_files +from tests.functional.source_overrides.fixtures import ( # noqa: F401 + local_dependency, + models__schema_yml, + seeds__expected_result_csv, + seeds__my_real_other_seed_csv, + seeds__my_real_seed_csv, +) + + +class TestSourceOverride: + @pytest.fixture(scope="class", autouse=True) + def setUp(self, project_root, local_dependency): # noqa: F811 + write_project_files(project_root, "local_dependency", local_dependency) + + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": models__schema_yml} + + @pytest.fixture(scope="class") + def seeds(self): + return { + "expected_result.csv": seeds__expected_result_csv, + "my_real_other_seed.csv": seeds__my_real_other_seed_csv, + "my_real_seed.csv": seeds__my_real_seed_csv, + } + + @pytest.fixture(scope="class") + def packages(self): + return { + "packages": [ + { + "local": "local_dependency", + }, + ] + } + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seeds": { + "localdep": { + "enabled": False, + "keep": { + "enabled": True, + }, + }, + "quote_columns": False, + }, + "sources": { + "localdep": { + "my_other_source": { + "enabled": False, + } + } + }, + } + + def _set_updated_at_to(self, insert_id, delta, project): + insert_time = datetime.utcnow() + delta + timestr = insert_time.strftime("%Y-%m-%d %H:%M:%S") + # favorite_color,id,first_name,email,ip_address,updated_at + + quoted_columns = ",".join( + project.adapter.quote(c) + for c in ("favorite_color", "id", "first_name", "email", "ip_address", "updated_at") + ) + + kwargs = { + "schema": project.test_schema, + "time": timestr, + "id": insert_id, + "source": project.adapter.quote("snapshot_freshness_base"), + "quoted_columns": quoted_columns, + } + + raw_sql = """INSERT INTO {schema}.{source} + ({quoted_columns}) + VALUES ( + 'blue',{id},'Jake','abc@example.com','192.168.1.1','{time}' + )""".format( + **kwargs + ) + + project.run_sql(raw_sql) + + return insert_id + 1 + + def test_source_overrides(self, project): + insert_id = 101 + + run_dbt(["deps"]) + + seed_results = run_dbt(["seed"]) + assert len(seed_results) == 5 + + # There should be 7, as we disabled 1 test of the original 8 + test_results = run_dbt(["test"]) + assert len(test_results) == 7 + + results = run_dbt(["run"]) + assert len(results) == 1 + + table_comp = TableComparison( + adapter=project.adapter, unique_schema=project.test_schema, database=project.database + ) + table_comp.assert_tables_equal("expected_result", "my_model") + + # set the updated_at field of this seed to last week + insert_id = self._set_updated_at_to(insert_id, timedelta(days=-7), project) + # if snapshot-freshness fails, freshness just didn't happen! + results = run_dbt(["source", "snapshot-freshness"], expect_pass=False) + # we disabled my_other_source, so we only run the one freshness check + # in + assert len(results) == 1 + # If snapshot-freshness passes, that means error_after was + # applied from the source override but not the source table override + insert_id = self._set_updated_at_to(insert_id, timedelta(days=-2), project) + results = run_dbt( + ["source", "snapshot-freshness"], + expect_pass=False, + ) + assert len(results) == 1 + + insert_id = self._set_updated_at_to(insert_id, timedelta(hours=-12), project) + results = run_dbt(["source", "snapshot-freshness"], expect_pass=True) + assert len(results) == 1 + + # update source to be enabled + new_source_config = { + "sources": { + "localdep": { + "my_other_source": { + "enabled": True, + } + } + } + } + update_config_file(new_source_config, project.project_root, "dbt_project.yml") + + # enable my_other_source, snapshot freshness should fail due to the new + # not-fresh source + results = run_dbt(["source", "snapshot-freshness"], expect_pass=False) + assert len(results) == 2 diff --git a/tests/functional/source_overrides/test_source_overrides_duplicate_model.py b/tests/functional/source_overrides/test_source_overrides_duplicate_model.py new file mode 100644 index 00000000000..cd35fd6f7c2 --- /dev/null +++ b/tests/functional/source_overrides/test_source_overrides_duplicate_model.py @@ -0,0 +1,67 @@ +import os +from dbt.exceptions import CompilationException +import pytest + +from dbt.tests.util import run_dbt +from dbt.tests.fixtures.project import write_project_files +from tests.functional.source_overrides.fixtures import ( # noqa: F401 + dupe_models__schema2_yml, + dupe_models__schema1_yml, + local_dependency, +) + + +class TestSourceOverrideDuplicates: + @pytest.fixture(scope="class", autouse=True) + def setUp(self, project_root, local_dependency): # noqa: F811 + write_project_files(project_root, "local_dependency", local_dependency) + + @pytest.fixture(scope="class") + def models(self): + return { + "schema2.yml": dupe_models__schema2_yml, + "schema1.yml": dupe_models__schema1_yml, + } + + @pytest.fixture(scope="class") + def packages(self): + return { + "packages": [ + { + "local": "local_dependency", + }, + ] + } + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seeds": { + "localdep": { + "enabled": False, + "keep": { + "enabled": True, + }, + }, + "quote_columns": False, + }, + "sources": { + "localdep": { + "my_other_source": { + "enabled": False, + } + } + }, + } + + def test_source_duplicate_overrides(self, project): + run_dbt(["deps"]) + with pytest.raises(CompilationException) as exc: + run_dbt(["compile"]) + + assert "dbt found two schema.yml entries for the same source named" in str(exc.value) + assert "one of these files" in str(exc.value) + schema1_path = os.path.join("models", "schema1.yml") + schema2_path = os.path.join("models", "schema2.yml") + assert schema1_path in str(exc.value) + assert schema2_path in str(exc.value) diff --git a/tests/functional/sources/common_source_setup.py b/tests/functional/sources/common_source_setup.py new file mode 100644 index 00000000000..62b68b85367 --- /dev/null +++ b/tests/functional/sources/common_source_setup.py @@ -0,0 +1,66 @@ +import os +import pytest +import yaml + +from dbt.tests.util import run_dbt +from tests.functional.sources.fixtures import ( + models__schema_yml, + models__view_model_sql, + models__ephemeral_model_sql, + models__descendant_model_sql, + models__multi_source_model_sql, + models__nonsource_descendant_sql, + seeds__source_csv, + seeds__other_table_csv, + seeds__expected_multi_source_csv, + seeds__other_source_table_csv, +) + + +class BaseSourcesTest: + @pytest.fixture(scope="class", autouse=True) + def setEnvVars(self): + os.environ["DBT_TEST_SCHEMA_NAME_VARIABLE"] = "test_run_schema" + + yield + + del os.environ["DBT_TEST_SCHEMA_NAME_VARIABLE"] + + @pytest.fixture(scope="class") + def models(self): + return { + "schema.yml": models__schema_yml, + "view_model.sql": models__view_model_sql, + "ephemeral_model.sql": models__ephemeral_model_sql, + "descendant_model.sql": models__descendant_model_sql, + "multi_source_model.sql": models__multi_source_model_sql, + "nonsource_descendant.sql": models__nonsource_descendant_sql, + } + + @pytest.fixture(scope="class") + def seeds(self): + return { + "source.csv": seeds__source_csv, + "other_table.csv": seeds__other_table_csv, + "expected_multi_source.csv": seeds__expected_multi_source_csv, + "other_source_table.csv": seeds__other_source_table_csv, + } + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "config-version": 2, + "seed-paths": ["seeds"], + "quoting": {"database": True, "schema": True, "identifier": True}, + "seeds": { + "quote_columns": True, + }, + } + + def run_dbt_with_vars(self, project, cmd, *args, **kwargs): + vars_dict = { + "test_run_schema": project.test_schema, + "test_loaded_at": project.adapter.quote("updated_at"), + } + cmd.extend(["--vars", yaml.safe_dump(vars_dict)]) + return run_dbt(cmd, *args, **kwargs) diff --git a/test/integration/042_sources_tests/seed.sql b/tests/functional/sources/data/seed.sql similarity index 100% rename from test/integration/042_sources_tests/seed.sql rename to tests/functional/sources/data/seed.sql diff --git a/test/integration/042_sources_tests/seeds/source.csv b/tests/functional/sources/fixtures.py similarity index 50% rename from test/integration/042_sources_tests/seeds/source.csv rename to tests/functional/sources/fixtures.py index a8f87412ef5..39734f7303c 100644 --- a/test/integration/042_sources_tests/seeds/source.csv +++ b/tests/functional/sources/fixtures.py @@ -1,4 +1,219 @@ -favorite_color,id,first_name,email,ip_address,updated_at +error_models__schema_yml = """version: 2 +sources: + - name: test_source + loader: custom + freshness: + warn_after: {count: 10, period: hour} + error_after: {count: 1, period: day} + schema: invalid + tables: + - name: test_table + identifier: source + loaded_at_field: updated_at +""" + +error_models__model_sql = """select * from {{ source('test_source', 'test_table') }} +""" + +override_freshness_models__schema_yml = """version: 2 +sources: + - name: test_source + loader: custom + freshness: # default freshness + warn_after: {count: 12, period: hour} + error_after: {count: 24, period: hour} + schema: "{{ var(env_var('DBT_TEST_SCHEMA_NAME_VARIABLE')) }}" + loaded_at_field: loaded_at + quoting: + identifier: True + tags: + - my_test_source_tag + tables: + - name: source_a + identifier: source + loaded_at_field: "{{ var('test_loaded_at') | as_text }}" + freshness: + warn_after: {count: 6, period: hour} + # use the default error_after defined above + - name: source_b + identifier: source + loaded_at_field: "{{ var('test_loaded_at') | as_text }}" + freshness: + warn_after: {count: 6, period: hour} + error_after: {} # use the default error_after defined above + - name: source_c + identifier: source + loaded_at_field: "{{ var('test_loaded_at') | as_text }}" + freshness: + warn_after: {count: 6, period: hour} + error_after: null # override: disable error_after for this table + - name: source_d + identifier: source + loaded_at_field: "{{ var('test_loaded_at') | as_text }}" + freshness: + warn_after: {count: 6, period: hour} + error_after: {count: 72, period: hour} # override: use this new behavior instead of error_after defined above + - name: source_e + identifier: source + loaded_at_field: "{{ var('test_loaded_at') | as_text }}" + freshness: null # override: disable freshness for this table +""" + +models__schema_yml = """version: 2 +models: + - name: descendant_model + columns: + - name: favorite_color + tests: + - relationships: + to: source('test_source', 'test_table') + field: favorite_color + +sources: + - name: test_source + loader: custom + freshness: + warn_after: {count: 10, period: hour} + error_after: {count: 1, period: day} + schema: "{{ var(env_var('DBT_TEST_SCHEMA_NAME_VARIABLE')) }}" + quoting: + identifier: True + tags: + - my_test_source_tag + tables: + - name: test_table + identifier: source + loaded_at_field: "{{ var('test_loaded_at') | as_text }}" + freshness: + error_after: {count: 18, period: hour} + tags: + - my_test_source_table_tag + columns: + - name: favorite_color + description: The favorite color + - name: id + description: The user ID + tests: + - unique + - not_null + tags: + - id_column + - name: first_name + description: The first name of the user + tests: [] + - name: email + description: The email address of the user + - name: ip_address + description: The last IP address the user logged in from + - name: updated_at + description: The last update time for this user + tests: + - relationships: + # do this as a table-level test, just to test out that aspect + column_name: favorite_color + to: ref('descendant_model') + field: favorite_color + - name: other_test_table + identifier: other_table + columns: + - name: id + tests: + - not_null + - unique + tags: + - id_column + - name: disabled_test_table + freshness: null + loaded_at_field: "{{ var('test_loaded_at') | as_text }}" + - name: other_source + schema: "{{ var('test_run_schema') }}" + quoting: + identifier: True + tables: + - name: test_table + identifier: other_source_table + - name: external_source + schema: "{{ var('test_run_alt_schema', var('test_run_schema')) }}" + tables: + - name: table +""" + +models__view_model_sql = """{# See here: https://github.com/dbt-labs/dbt-core/pull/1729 #} + +select * from {{ ref('ephemeral_model') }} +""" + +models__ephemeral_model_sql = """{{ config(materialized='ephemeral') }} + +select 1 as id +""" + +models__descendant_model_sql = """select * from {{ source('test_source', 'test_table') }} +""" + +models__multi_source_model_sql = """select * from {{ source('test_source', 'other_test_table')}} + join {{ source('other_source', 'test_table')}} using (id) +""" + +models__nonsource_descendant_sql = """select * from {{ schema }}.source +""" + +malformed_models__schema_yml = """version: 2 +sources: + - name: test_source + loader: custom + schema: "{{ var('test_run_schema') }}" + tables: + - name: test_table + identifier: source + tests: + - relationships: + # this is invalid (list of 3 1-key dicts instead of a single 3-key dict) + - column_name: favorite_color + - to: ref('descendant_model') + - field: favorite_color +""" + +malformed_models__descendant_model_sql = """select * from {{ source('test_source', 'test_table') }} +""" + +filtered_models__schema_yml = """version: 2 +sources: + - name: test_source + loader: custom + freshness: + warn_after: {count: 10, period: hour} + error_after: {count: 1, period: day} + filter: id > 1 + schema: "{{ var(env_var('DBT_TEST_SCHEMA_NAME_VARIABLE')) }}" + quoting: + identifier: True + tables: + - name: test_table + identifier: source + loaded_at_field: updated_at + freshness: + error_after: {count: 18, period: hour} + filter: id > 101 +""" + +macros__macro_sql = """{% macro override_me() -%} + {{ exceptions.raise_compiler_error('this is a bad macro') }} +{%- endmacro %} + +{% macro happy_little_macro() -%} + {{ override_me() }} +{%- endmacro %} + + +{% macro vacuum_source(source_name, table_name) -%} + {% call statement('stmt', auto_begin=false, fetch_result=false) %} + vacuum {{ source(source_name, table_name) }} + {% endcall %} +{%- endmacro %} +""" + +seeds__source_csv = """favorite_color,id,first_name,email,ip_address,updated_at blue,1,Larry,lking0@miitbeian.gov.cn,'69.135.206.194',2008-09-12 19:08:31 blue,2,Larry,lperkins1@toplist.cz,'64.210.133.162',1978-05-09 04:15:14 blue,3,Anna,amontgomery2@miitbeian.gov.cn,'168.104.64.114',2011-10-16 04:07:57 @@ -99,3 +314,41 @@ green,98,Gloria,gwalker2p@usa.gov,'156.140.7.128',1997-10-04 07:58:58 green,99,Paul,pjohnson2q@umn.edu,'183.59.198.197',1991-11-14 12:33:55 green,100,Frank,fgreene2r@blogspot.com,'150.143.68.121',2010-06-12 23:55:39 +""" + +seeds__other_table_csv = """id,first_name +1,Larry +2,Curly +3,Moe +""" + +seeds__expected_multi_source_csv = """id,first_name,color +1,Larry,blue +2,Curly,red +3,Moe,green +""" + +seeds__other_source_table_csv = """id,color +1,blue +2,red +3,green +""" + +malformed_schema_tests__schema_yml = """version: 2 +sources: + - name: test_source + schema: "{{ var('test_run_schema') }}" + tables: + - name: test_table + identifier: source + columns: + - name: favorite_color + tests: + - relationships: + to: ref('model') + # this will get rendered as its literal + field: "{{ 'favorite' ~ 'color' }}" +""" + +malformed_schema_tests__model_sql = """select * from {{ source('test_source', 'test_table') }} +""" diff --git a/tests/functional/sources/test_simple_source.py b/tests/functional/sources/test_simple_source.py new file mode 100644 index 00000000000..c59ce2a185b --- /dev/null +++ b/tests/functional/sources/test_simple_source.py @@ -0,0 +1,211 @@ +import os +import pytest +import yaml +from dbt.exceptions import ParsingException + +from dbt.tests.util import run_dbt, update_config_file +from dbt.tests.tables import TableComparison +from tests.functional.sources.common_source_setup import ( + BaseSourcesTest, +) +from tests.functional.sources.fixtures import ( + macros__macro_sql, + malformed_models__schema_yml, + malformed_models__descendant_model_sql, + malformed_schema_tests__schema_yml, + malformed_schema_tests__model_sql, +) + + +class SuccessfulSourcesTest(BaseSourcesTest): + @pytest.fixture(scope="class", autouse=True) + def setUp(self, project): + self.run_dbt_with_vars(project, ["seed"]) + os.environ["DBT_ENV_CUSTOM_ENV_key"] = "value" + + yield + + del os.environ["DBT_ENV_CUSTOM_ENV_key"] + + @pytest.fixture(scope="class") + def macros(self): + return {"macro.sql": macros__macro_sql} + + def _create_schemas(self, project): + schema = self.alternative_schema(project.test_schema) + project.run_sql(f"drop schema if exists {schema} cascade") + project.run_sql(f"create schema {schema}") + + def alternative_schema(self, test_schema): + return test_schema + "_other" + + @pytest.fixture(scope="class", autouse=True) + def createDummyTables(self, project): + self._create_schemas(project) + project.run_sql("create table {}.dummy_table (id int)".format(project.test_schema)) + project.run_sql( + "create view {}.external_view as (select * from {}.dummy_table)".format( + self.alternative_schema(project.test_schema), project.test_schema + ) + ) + + def run_dbt_with_vars(self, project, cmd, *args, **kwargs): + vars_dict = { + "test_run_schema": project.test_schema, + "test_run_alt_schema": self.alternative_schema(project.test_schema), + "test_loaded_at": project.adapter.quote("updated_at"), + } + cmd.extend(["--vars", yaml.safe_dump(vars_dict)]) + return run_dbt(cmd, *args, **kwargs) + + +class TestBasicSource(SuccessfulSourcesTest): + def test_basic_source_def(self, project): + results = self.run_dbt_with_vars(project, ["run"]) + assert len(results) == 4 + table_comp = TableComparison( + adapter=project.adapter, unique_schema=project.test_schema, database=project.database + ) + table_comp.assert_many_tables_equal( + ["source", "descendant_model", "nonsource_descendant"], + ["expected_multi_source", "multi_source_model"], + ) + results = self.run_dbt_with_vars(project, ["test"]) + assert len(results) == 6 + print(results) + + +class TestSourceSelector(SuccessfulSourcesTest): + def test_source_selector(self, project): + # only one of our models explicitly depends upon a source + results = self.run_dbt_with_vars( + project, ["run", "--models", "source:test_source.test_table+"] + ) + assert len(results) == 1 + table_comp = TableComparison( + adapter=project.adapter, unique_schema=project.test_schema, database=project.database + ) + table_comp.assert_tables_equal("source", "descendant_model") + table_comp.assert_table_does_not_exist("nonsource_descendant") + table_comp.assert_table_does_not_exist("multi_source_model") + + # do the same thing, but with tags + results = self.run_dbt_with_vars( + project, ["run", "--models", "tag:my_test_source_table_tag+"] + ) + assert len(results) == 1 + + results = self.run_dbt_with_vars( + project, ["test", "--models", "source:test_source.test_table+"] + ) + assert len(results) == 4 + + results = self.run_dbt_with_vars( + project, ["test", "--models", "tag:my_test_source_table_tag+"] + ) + assert len(results) == 4 + + results = self.run_dbt_with_vars(project, ["test", "--models", "tag:my_test_source_tag+"]) + # test_table + other_test_table + assert len(results) == 6 + + results = self.run_dbt_with_vars(project, ["test", "--models", "tag:id_column"]) + # all 4 id column tests + assert len(results) == 4 + + +class TestEmptySource(SuccessfulSourcesTest): + def test_empty_source_def(self, project): + # sources themselves can never be selected, so nothing should be run + results = self.run_dbt_with_vars( + project, ["run", "--models", "source:test_source.test_table"] + ) + table_comp = TableComparison( + adapter=project.adapter, unique_schema=project.test_schema, database=project.database + ) + table_comp.assert_table_does_not_exist("nonsource_descendant") + table_comp.assert_table_does_not_exist("multi_source_model") + table_comp.assert_table_does_not_exist("descendant_model") + assert len(results) == 0 + + +class TestSourceDef(SuccessfulSourcesTest): + def test_source_only_def(self, project): + results = self.run_dbt_with_vars(project, ["run", "--models", "source:other_source+"]) + assert len(results) == 1 + table_comp = TableComparison( + adapter=project.adapter, unique_schema=project.test_schema, database=project.database + ) + table_comp.assert_tables_equal("expected_multi_source", "multi_source_model") + table_comp.assert_table_does_not_exist("nonsource_descendant") + table_comp.assert_table_does_not_exist("descendant_model") + + results = self.run_dbt_with_vars(project, ["run", "--models", "source:test_source+"]) + assert len(results) == 2 + table_comp.assert_many_tables_equal( + ["source", "descendant_model"], ["expected_multi_source", "multi_source_model"] + ) + table_comp.assert_table_does_not_exist("nonsource_descendant") + + +class TestSourceChildrenParents(SuccessfulSourcesTest): + def test_source_childrens_parents(self, project): + results = self.run_dbt_with_vars(project, ["run", "--models", "@source:test_source"]) + assert len(results) == 2 + table_comp = TableComparison( + adapter=project.adapter, unique_schema=project.test_schema, database=project.database + ) + table_comp.assert_many_tables_equal( + ["source", "descendant_model"], + ["expected_multi_source", "multi_source_model"], + ) + table_comp.assert_table_does_not_exist("nonsource_descendant") + + +class TestSourceRunOperation(SuccessfulSourcesTest): + def test_run_operation_source(self, project): + kwargs = '{"source_name": "test_source", "table_name": "test_table"}' + self.run_dbt_with_vars(project, ["run-operation", "vacuum_source", "--args", kwargs]) + + +class TestMalformedSources(BaseSourcesTest): + # even seeds should fail, because parsing is what's raising + @pytest.fixture(scope="class") + def models(self): + return { + "schema.yml": malformed_models__schema_yml, + "descendant_model.sql": malformed_models__descendant_model_sql, + } + + def test_malformed_schema_will_break_run(self, project): + with pytest.raises(ParsingException): + self.run_dbt_with_vars(project, ["seed"]) + + +class TestRenderingInSourceTests(BaseSourcesTest): + @pytest.fixture(scope="class") + def models(self): + return { + "schema.yml": malformed_schema_tests__schema_yml, + "model.sql": malformed_schema_tests__model_sql, + } + + def test_render_in_source_tests(self, project): + self.run_dbt_with_vars(project, ["seed"]) + self.run_dbt_with_vars(project, ["run"]) + # syntax error at or near "{", because the test isn't rendered + self.run_dbt_with_vars(project, ["test"], expect_pass=False) + + +class TestUnquotedSources(SuccessfulSourcesTest): + def test_catalog(self, project): + new_quoting_config = { + "quoting": { + "identifier": False, + "schema": False, + "database": False, + } + } + update_config_file(new_quoting_config, project.project_root, "dbt_project.yml") + self.run_dbt_with_vars(project, ["run"]) + self.run_dbt_with_vars(project, ["docs", "generate"]) diff --git a/tests/functional/sources/test_source_freshness.py b/tests/functional/sources/test_source_freshness.py new file mode 100644 index 00000000000..c2a23de5adc --- /dev/null +++ b/tests/functional/sources/test_source_freshness.py @@ -0,0 +1,375 @@ +import os +import json +import pytest +from datetime import datetime, timedelta + +import dbt.version +from tests.functional.sources.common_source_setup import BaseSourcesTest +from tests.functional.sources.fixtures import ( + error_models__schema_yml, + error_models__model_sql, + filtered_models__schema_yml, + override_freshness_models__schema_yml, +) + + +# put these here for now to get tests working +class AnyStringWith: + def __init__(self, contains=None): + self.contains = contains + + def __eq__(self, other): + if not isinstance(other, str): + return False + + if self.contains is None: + return True + + return self.contains in other + + def __repr__(self): + return "AnyStringWith<{!r}>".format(self.contains) + + +class AnyFloat: + """Any float. Use this in assertEqual() calls to assert that it is a float.""" + + def __eq__(self, other): + return isinstance(other, float) + + +class SuccessfulSourceFreshnessTest(BaseSourcesTest): + @pytest.fixture(scope="class", autouse=True) + def setUp(self, project): + self.run_dbt_with_vars(project, ["seed"]) + pytest._id = 101 + pytest.freshness_start_time = datetime.utcnow() + # this is the db initial value + pytest.last_inserted_time = "2016-09-19T14:45:51+00:00" + + os.environ["DBT_ENV_CUSTOM_ENV_key"] = "value" + + yield + + del os.environ["DBT_ENV_CUSTOM_ENV_key"] + + def _set_updated_at_to(self, project, delta): + insert_time = datetime.utcnow() + delta + timestr = insert_time.strftime("%Y-%m-%d %H:%M:%S") + # favorite_color,id,first_name,email,ip_address,updated_at + insert_id = pytest._id + pytest._id += 1 + quoted_columns = ",".join( + project.adapter.quote(c) + for c in ("favorite_color", "id", "first_name", "email", "ip_address", "updated_at") + ) + kwargs = { + "schema": project.test_schema, + "time": timestr, + "id": insert_id, + "source": project.adapter.quote("source"), + "quoted_columns": quoted_columns, + } + raw_sql = """INSERT INTO {schema}.{source} + ({quoted_columns}) + VALUES ( + 'blue',{id},'Jake','abc@example.com','192.168.1.1','{time}' + )""".format( + **kwargs + ) + project.run_sql(raw_sql) + pytest.last_inserted_time = insert_time.strftime("%Y-%m-%dT%H:%M:%S+00:00") + + def assertBetween(self, timestr, start, end=None): + datefmt = "%Y-%m-%dT%H:%M:%S.%fZ" + if end is None: + end = datetime.utcnow() + + parsed = datetime.strptime(timestr, datefmt) + + assert start <= parsed + assert end >= parsed + + def _assert_freshness_results(self, path, state): + assert os.path.exists(path) + with open(path) as fp: + data = json.load(fp) + + assert set(data) == {"metadata", "results", "elapsed_time"} + assert "generated_at" in data["metadata"] + assert isinstance(data["elapsed_time"], float) + self.assertBetween(data["metadata"]["generated_at"], pytest.freshness_start_time) + assert ( + data["metadata"]["dbt_schema_version"] + == "https://schemas.getdbt.com/dbt/sources/v3.json" + ) + assert data["metadata"]["dbt_version"] == dbt.version.__version__ + assert data["metadata"]["invocation_id"] == dbt.tracking.active_user.invocation_id + key = "key" + if os.name == "nt": + key = key.upper() + assert data["metadata"]["env"] == {key: "value"} + + last_inserted_time = pytest.last_inserted_time + + assert len(data["results"]) == 1 + + # TODO: replace below calls - could they be more sane? + assert data["results"] == [ + { + "unique_id": "source.test.test_source.test_table", + "max_loaded_at": last_inserted_time, + "snapshotted_at": AnyStringWith(), + "max_loaded_at_time_ago_in_s": AnyFloat(), + "status": state, + "criteria": { + "filter": None, + "warn_after": {"count": 10, "period": "hour"}, + "error_after": {"count": 18, "period": "hour"}, + }, + "adapter_response": {}, + "thread_id": AnyStringWith("Thread-"), + "execution_time": AnyFloat(), + "timing": [ + { + "name": "compile", + "started_at": AnyStringWith(), + "completed_at": AnyStringWith(), + }, + { + "name": "execute", + "started_at": AnyStringWith(), + "completed_at": AnyStringWith(), + }, + ], + } + ] + + +class TestSourceFreshness(SuccessfulSourceFreshnessTest): + def test_source_freshness(self, project): + # test_source.test_table should have a loaded_at field of `updated_at` + # and a freshness of warn_after: 10 hours, error_after: 18 hours + # by default, our data set is way out of date! + + results = self.run_dbt_with_vars( + project, ["source", "freshness", "-o", "target/error_source.json"], expect_pass=False + ) + assert len(results) == 1 + assert results[0].status == "error" + self._assert_freshness_results("target/error_source.json", "error") + + self._set_updated_at_to(project, timedelta(hours=-12)) + results = self.run_dbt_with_vars( + project, + ["source", "freshness", "-o", "target/warn_source.json"], + ) + assert len(results) == 1 + assert results[0].status == "warn" + self._assert_freshness_results("target/warn_source.json", "warn") + + self._set_updated_at_to(project, timedelta(hours=-2)) + results = self.run_dbt_with_vars( + project, + ["source", "freshness", "-o", "target/pass_source.json"], + ) + assert len(results) == 1 + assert results[0].status == "pass" + self._assert_freshness_results("target/pass_source.json", "pass") + + +class TestSourceSnapshotFreshness(SuccessfulSourceFreshnessTest): + def test_source_snapshot_freshness(self, project): + """Ensures that the deprecated command `source snapshot-freshness` + aliases to `source freshness` command. + """ + results = self.run_dbt_with_vars( + project, + ["source", "snapshot-freshness", "-o", "target/error_source.json"], + expect_pass=False, + ) + assert len(results) == 1 + assert results[0].status == "error" + self._assert_freshness_results("target/error_source.json", "error") + + self._set_updated_at_to(project, timedelta(hours=-12)) + results = self.run_dbt_with_vars( + project, + ["source", "snapshot-freshness", "-o", "target/warn_source.json"], + ) + assert len(results) == 1 + assert results[0].status == "warn" + self._assert_freshness_results("target/warn_source.json", "warn") + + self._set_updated_at_to(project, timedelta(hours=-2)) + results = self.run_dbt_with_vars( + project, + ["source", "snapshot-freshness", "-o", "target/pass_source.json"], + ) + assert len(results) == 1 + assert results[0].status == "pass" + self._assert_freshness_results("target/pass_source.json", "pass") + + +class TestSourceFreshnessSelection(SuccessfulSourceFreshnessTest): + def test_source_freshness_selection_select(self, project): + """Tests node selection using the --select argument.""" + self._set_updated_at_to(project, timedelta(hours=-2)) + # select source directly + results = self.run_dbt_with_vars( + project, + [ + "source", + "freshness", + "--select", + "source:test_source.test_table", + "-o", + "target/pass_source.json", + ], + ) + assert len(results) == 1 + assert results[0].status == "pass" + self._assert_freshness_results("target/pass_source.json", "pass") + + +class TestSourceFreshnessExclude(SuccessfulSourceFreshnessTest): + def test_source_freshness_selection_exclude(self, project): + """Tests node selection using the --select argument. It 'excludes' the + only source in the project so it should return no results.""" + self._set_updated_at_to(project, timedelta(hours=-2)) + # exclude source directly + results = self.run_dbt_with_vars( + project, + [ + "source", + "freshness", + "--exclude", + "source:test_source.test_table", + "-o", + "target/exclude_source.json", + ], + ) + assert len(results) == 0 + + +class TestSourceFreshnessGraph(SuccessfulSourceFreshnessTest): + def test_source_freshness_selection_graph_operation(self, project): + """Tests node selection using the --select argument with graph + operations. `+descendant_model` == select all nodes `descendant_model` + depends on. + """ + self._set_updated_at_to(project, timedelta(hours=-2)) + # select model ancestors + results = self.run_dbt_with_vars( + project, + [ + "source", + "freshness", + "--select", + "+descendant_model", + "-o", + "target/ancestor_source.json", + ], + ) + assert len(results) == 1 + assert results[0].status == "pass" + self._assert_freshness_results("target/ancestor_source.json", "pass") + + +class TestSourceFreshnessErrors(SuccessfulSourceFreshnessTest): + @pytest.fixture(scope="class") + def models(self): + return { + "schema.yml": error_models__schema_yml, + "model.sql": error_models__model_sql, + } + + def test_source_freshness_error(self, project): + results = self.run_dbt_with_vars(project, ["source", "freshness"], expect_pass=False) + assert len(results) == 1 + assert results[0].status == "runtime error" + + +class TestSourceFreshnessFilter(SuccessfulSourceFreshnessTest): + @pytest.fixture(scope="class") + def filtered_models(self): + return {"schema.yml": filtered_models__schema_yml} + + def test_source_freshness_all_records(self, project): + # all records are filtered out + self.run_dbt_with_vars(project, ["source", "freshness"], expect_pass=False) + # we should insert a record with _id=101 that's fresh, but will still fail + # because the filter excludes it + self._set_updated_at_to(project, timedelta(hours=-2)) + self.run_dbt_with_vars(project, ["source", "freshness"], expect_pass=False) + + # we should now insert a record with _id=102 that's fresh, and the filter + # includes it + self._set_updated_at_to(project, timedelta(hours=-2)) + self.run_dbt_with_vars(project, ["source", "freshness"], expect_pass=True) + + +class TestOverrideSourceFreshness(SuccessfulSourceFreshnessTest): + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": override_freshness_models__schema_yml} + + @staticmethod + def get_result_from_unique_id(data, unique_id): + try: + return list(filter(lambda x: x["unique_id"] == unique_id, data["results"]))[0] + except IndexError: + raise f"No result for the given unique_id. unique_id={unique_id}" + + def test_override_source_freshness(self, project): + self._set_updated_at_to(project, timedelta(hours=-30)) + + path = "target/pass_source.json" + results = self.run_dbt_with_vars( + project, ["source", "freshness", "-o", path], expect_pass=False + ) + assert len(results) == 4 # freshness disabled for source_e + + assert os.path.exists(path) + with open(path) as fp: + data = json.load(fp) + + result_source_a = self.get_result_from_unique_id(data, "source.test.test_source.source_a") + assert result_source_a["status"] == "error" + + expected = { + "warn_after": {"count": 6, "period": "hour"}, + "error_after": {"count": 24, "period": "hour"}, + "filter": None, + } + assert result_source_a["criteria"] == expected + + result_source_b = self.get_result_from_unique_id(data, "source.test.test_source.source_b") + assert result_source_b["status"] == "error" + + expected = { + "warn_after": {"count": 6, "period": "hour"}, + "error_after": {"count": 24, "period": "hour"}, + "filter": None, + } + assert result_source_b["criteria"] == expected + + result_source_c = self.get_result_from_unique_id(data, "source.test.test_source.source_c") + assert result_source_c["status"] == "warn" + + expected = { + "warn_after": {"count": 6, "period": "hour"}, + "error_after": None, + "filter": None, + } + assert result_source_c["criteria"] == expected + + result_source_d = self.get_result_from_unique_id(data, "source.test.test_source.source_d") + assert result_source_d["status"] == "warn" + + expected = { + "warn_after": {"count": 6, "period": "hour"}, + "error_after": {"count": 72, "period": "hour"}, + "filter": None, + } + assert result_source_d["criteria"] == expected