Er/1607 source paths

CHANGELOG.md

@@ -32,6 +32,7 @@
 - Change the default dbt packages installation directory to `dbt_packages` from `dbt_modules`.  Also rename `module-path` to `packages-install-path` to allow default overrides of package install directory.  Deprecation warning added for projects using the old `dbt_modules` name without specifying a `packages-install-path`.  ([#3523](https://github.com/dbt-labs/dbt/issues/3523))
 - Update the default project paths to be `analysis-paths = ['analyses']` and `test-paths = ['tests]`. Also have starter project set `analysis-paths: ['analyses']` from now on.  ([#2659](https://github.com/dbt-labs/dbt/issues/2659))
 - Define the data type of `sources` as an array of arrays of string in the manifest artifacts. ([#3966](https://github.com/dbt-labs/dbt/issues/3966), [#3967](https://github.com/dbt-labs/dbt/pull/3967))
+- Marked `source-paths` and `data-paths` as deprecated keys in `dbt_project.yml` in favor of `model-paths` and `seed-paths` respectively.([#1607](https://github.com/dbt-labs/dbt/issues/1607))
 
 Contributors:
 

core/dbt/config/project.py

@@ -9,6 +9,7 @@
 import hashlib
 import os
 
+from dbt import deprecations
 from dbt.clients.system import resolve_path_from_base
 from dbt.clients.system import path_exists
 from dbt.clients.system import load_file_contents
@@ -123,13 +124,13 @@ def _parse_versions(versions: Union[List[str], str]) -> List[VersionSpecifier]:
 
 
 def _all_source_paths(
-    source_paths: List[str],
-    data_paths: List[str],
+    model_paths: List[str],
+    seed_paths: List[str],
     snapshot_paths: List[str],
     analysis_paths: List[str],
     macro_paths: List[str],
 ) -> List[str]:
-    return list(chain(source_paths, data_paths, snapshot_paths, analysis_paths,
+    return list(chain(model_paths, seed_paths, snapshot_paths, analysis_paths,
                       macro_paths))
 
 
@@ -292,6 +293,21 @@ def render(self, renderer: DbtProjectYamlRenderer) -> 'Project':
                 exc.path = os.path.join(self.project_root, 'dbt_project.yml')
             raise
 
+    def check_config_path(self, project_dict, deprecated_path, exp_path):
+        if deprecated_path in project_dict:
+            if exp_path in project_dict:
+                msg = (
+                    '{deprecated_path} and {exp_path} cannot both be defined. The '
+                    '`{deprecated_path}` config has been deprecated in favor of `{exp_path}`. '
+                    'Please update your `dbt_project.yml` configuration to reflect this '
+                    'change.'
+                )
+                raise DbtProjectError(msg.format(deprecated_path=deprecated_path,
+                                                 exp_path=exp_path))
+            deprecations.warn('project_config_path',
+                              deprecated_path=deprecated_path,
+                              exp_path=exp_path)
+
     def create_project(self, rendered: RenderComponents) -> 'Project':
         unrendered = RenderComponents(
             project_dict=self.project_dict,
@@ -303,6 +319,9 @@ def create_project(self, rendered: RenderComponents) -> 'Project':
             verify_version=self.verify_version,
         )
 
+        self.check_config_path(rendered.project_dict, 'source-paths', 'model-paths')
+        self.check_config_path(rendered.project_dict, 'data-paths', 'seed-paths')
+
         try:
             ProjectContract.validate(rendered.project_dict)
             cfg = ProjectContract.from_dict(
@@ -324,15 +343,24 @@ def create_project(self, rendered: RenderComponents) -> 'Project':
         # to have been a cli argument.
         profile_name = cfg.profile
         # these are all the defaults
-        source_paths: List[str] = value_or(cfg.source_paths, ['models'])
+
+        # `source_paths` is deprecated but still allowed. Copy it into
+        # `model_paths` to simlify logic throughout the rest of the system.
+        model_paths: List[str] = value_or(cfg.model_paths
+                                          if 'model-paths' in rendered.project_dict
+                                          else cfg.source_paths, ['models'])
         macro_paths: List[str] = value_or(cfg.macro_paths, ['macros'])
-        data_paths: List[str] = value_or(cfg.data_paths, ['data'])
+        # `data_paths` is deprecated but still allowed. Copy it into
+        # `seed_paths` to simlify logic throughout the rest of the system.
+        seed_paths: List[str] = value_or(cfg.seed_paths
+                                         if 'seed-paths' in rendered.project_dict
+                                         else cfg.data_paths, ['seeds'])
         test_paths: List[str] = value_or(cfg.test_paths, ['tests'])
         analysis_paths: List[str] = value_or(cfg.analysis_paths, ['analyses'])
         snapshot_paths: List[str] = value_or(cfg.snapshot_paths, ['snapshots'])
 
         all_source_paths: List[str] = _all_source_paths(
-            source_paths, data_paths, snapshot_paths, analysis_paths,
+            model_paths, seed_paths, snapshot_paths, analysis_paths,
             macro_paths
         )
 
@@ -382,15 +410,14 @@ def create_project(self, rendered: RenderComponents) -> 'Project':
             # of dicts.
             manifest_selectors = SelectorDict.parse_from_selectors_list(
                 rendered.selectors_dict['selectors'])
-
         project = Project(
             project_name=name,
             version=version,
             project_root=project_root,
             profile_name=profile_name,
-            source_paths=source_paths,
+            model_paths=model_paths,
             macro_paths=macro_paths,
-            data_paths=data_paths,
+            seed_paths=seed_paths,
             test_paths=test_paths,
             analysis_paths=analysis_paths,
             docs_paths=docs_paths,
@@ -500,9 +527,9 @@ class Project:
     version: Union[SemverString, float]
     project_root: str
     profile_name: Optional[str]
-    source_paths: List[str]
+    model_paths: List[str]
     macro_paths: List[str]
-    data_paths: List[str]
+    seed_paths: List[str]
     test_paths: List[str]
     analysis_paths: List[str]
     docs_paths: List[str]
@@ -533,7 +560,7 @@ class Project:
     @property
     def all_source_paths(self) -> List[str]:
         return _all_source_paths(
-            self.source_paths, self.data_paths, self.snapshot_paths,
+            self.model_paths, self.seed_paths, self.snapshot_paths,
             self.analysis_paths, self.macro_paths
         )
 
@@ -561,9 +588,9 @@ def to_project_config(self, with_packages=False):
             'version': self.version,
             'project-root': self.project_root,
             'profile': self.profile_name,
-            'source-paths': self.source_paths,
+            'model-paths': self.model_paths,
             'macro-paths': self.macro_paths,
-            'data-paths': self.data_paths,
+            'seed-paths': self.seed_paths,
             'test-paths': self.test_paths,
             'analysis-paths': self.analysis_paths,
             'docs-paths': self.docs_paths,

core/dbt/config/runtime.py

@@ -87,9 +87,9 @@ def from_parts(
             project_name=project.project_name,
             version=project.version,
             project_root=project.project_root,
-            source_paths=project.source_paths,
+            model_paths=project.model_paths,
             macro_paths=project.macro_paths,
-            data_paths=project.data_paths,
+            seed_paths=project.seed_paths,
             test_paths=project.test_paths,
             analysis_paths=project.analysis_paths,
             docs_paths=project.docs_paths,
@@ -483,9 +483,9 @@ def from_parts(
             project_name=project.project_name,
             version=project.version,
             project_root=project.project_root,
-            source_paths=project.source_paths,
+            model_paths=project.model_paths,
             macro_paths=project.macro_paths,
-            data_paths=project.data_paths,
+            seed_paths=project.seed_paths,
             test_paths=project.test_paths,
             analysis_paths=project.analysis_paths,
             docs_paths=project.docs_paths,

core/dbt/contracts/project.py

@@ -174,8 +174,10 @@ class Project(HyphenatedDbtClassMixin, Replaceable):
     config_version: int
     project_root: Optional[str] = None
     source_paths: Optional[List[str]] = None
+    model_paths: Optional[List[str]] = None
     macro_paths: Optional[List[str]] = None
     data_paths: Optional[List[str]] = None
+    seed_paths: Optional[List[str]] = None
     test_paths: Optional[List[str]] = None
     analysis_paths: Optional[List[str]] = None
     docs_paths: Optional[List[str]] = None

core/dbt/deprecations.py

@@ -60,6 +60,14 @@ class PackageInstallPathDeprecation(DBTDeprecation):
     '''
 
 
+class ConfigPathDeprecation(DBTDeprecation):
+    _name = 'project_config_path'
+    _description = '''\
+    The `{deprecated_path}` config has been deprecated in favor of `{exp_path}`.
+    Please update your `dbt_project.yml` configuration to reflect this change.
+    '''
+
+
 _adapter_renamed_description = """\
 The adapter function `adapter.{old_name}` is deprecated and will be removed in
 a future release of dbt. Please use `adapter.{new_name}` instead.
@@ -98,6 +106,7 @@ def warn(name, *args, **kwargs):
 active_deprecations: Set[str] = set()
 
 deprecations_list: List[DBTDeprecation] = [
+    ConfigPathDeprecation(),
     PackageInstallPathDeprecation(),
     PackageRedirectDeprecation()
 ]

core/dbt/include/starter_project/dbt_project.yml

@@ -10,12 +10,12 @@ config-version: 2
 profile: 'default'
 
 # These configurations specify where dbt should look for different types of files.
-# The `source-paths` config, for example, states that models in this project can be
+# The `model-paths` config, for example, states that models in this project can be
 # found in the "models/" directory. You probably won't need to change these!
-source-paths: ["models"]
+model-paths: ["models"]
 analysis-paths: ["analyses"]
 test-paths: ["tests"]
-data-paths: ["data"]
+seed-paths: ["seeds"]
 macro-paths: ["macros"]
 snapshot-paths: ["snapshots"]
 

core/dbt/parser/read_files.py

@@ -125,7 +125,7 @@ def read_files(project, files, parser_files, saved_files):
     )
 
     project_files['ModelParser'] = read_files_for_parser(
-        project, files, project.source_paths, '.sql', ParseFileType.Model, saved_files
+        project, files, project.model_paths, '.sql', ParseFileType.Model, saved_files
     )
 
     project_files['SnapshotParser'] = read_files_for_parser(
@@ -141,7 +141,7 @@ def read_files(project, files, parser_files, saved_files):
     )
 
     project_files['SeedParser'] = read_files_for_parser(
-        project, files, project.data_paths, '.csv', ParseFileType.Seed, saved_files
+        project, files, project.seed_paths, '.csv', ParseFileType.Seed, saved_files
     )
 
     project_files['DocumentationParser'] = read_files_for_parser(

core/dbt/task/clean.py

@@ -22,7 +22,7 @@ def __is_protected_path(self, path):
         This function identifies protected paths, so as not to clean them.
         """
         abs_path = os.path.abspath(path)
-        protected_paths = self.config.source_paths + \
+        protected_paths = self.config.model_paths + \
             self.config.test_paths + ['.']
         protected_abs_paths = [os.path.abspath(p) for p in protected_paths]
         return abs_path in set(protected_abs_paths) or \

core/scripts/upgrade_dbt_schema_tests_v1_to_v2.py

@@ -137,13 +137,13 @@ def handle(parsed):
     validate_and_mutate_args(parsed)
     with open(os.path.join(parsed.search_directory, 'dbt_project.yml')) as fp:
         project = yaml.safe_load(fp)
-    source_dirs = project.get('source-paths', ['models'])
+    model_dirs = project.get('model-paths', ['models'])
     if parsed.apply:
         print('converting the following files to the v2 spec:')
     else:
         print('would convert the following files to the v2 spec:')
-    for source_dir in source_dirs:
-        search_path = os.path.join(parsed.search_directory, source_dir)
+    for model_dir in model_dirs:
+        search_path = os.path.join(parsed.search_directory, model_dir)
         convert_project(search_path, parsed.backup, parsed.apply,
                         parsed.extra_complex_tests)
     if not parsed.apply:

performance/projects/01_2000_simple_models/dbt_project.yml

@@ -11,20 +11,20 @@ config-version: 2
 profile: 'default'
 
 # These configurations specify where dbt should look for different types of files.
-# The `source-paths` config, for example, states that source models can be found
+# The `model-paths` config, for example, states that source models can be found
 # in the "models/" directory. You probably won't need to change these!
-source-paths: ["models"]
+model-paths: ["models"]
 analysis-paths: ["analyses"] 
 test-paths: ["tests"]
-data-paths: ["data"]
+seed-paths: ["seeds"]
 macro-paths: ["macros"]
 
 target-path: "target"  # directory which will store compiled SQL files
 clean-targets:         # directories to be removed by `dbt clean`
     - "target"
     - "dbt_packages"
 
-# You can define configurations for models in the `source-paths` directory here.
+# You can define configurations for models in the `model-paths` directory here.
 # Using these configurations, you can enable or disable models, change how they
 # are materialized, and more!
 

test/integration/001_simple_copy_test/test_simple_copy.py

@@ -39,7 +39,7 @@ class TestSimpleCopy(BaseTestSimpleCopy):
 
     @property
     def project_config(self):
-        return self.seed_quote_cfg_with({"data-paths": [self.dir("seed-initial")]})
+        return self.seed_quote_cfg_with({"seed-paths": [self.dir("seed-initial")]})
 
     @use_profile("postgres")
     def test__postgres__simple_copy(self):
@@ -50,7 +50,7 @@ def test__postgres__simple_copy(self):
 
         self.assertManyTablesEqual(["seed", "view_model", "incremental", "materialized", "get_and_ref"])
 
-        self.use_default_project({"data-paths": [self.dir("seed-update")]})
+        self.use_default_project({"seed-paths": [self.dir("seed-update")]})
         results = self.run_dbt(["seed"])
         self.assertEqual(len(results), 1)
         results = self.run_dbt()
@@ -93,7 +93,7 @@ def test__postgres__dbt_doesnt_run_empty_models(self):
 
     @use_profile("presto")
     def test__presto__simple_copy(self):
-        self.use_default_project({"data-paths": [self.dir("seed-initial")]})
+        self.use_default_project({"seed-paths": [self.dir("seed-initial")]})
 
         results = self.run_dbt(["seed"])
         self.assertEqual(len(results),  1)
@@ -113,7 +113,7 @@ def models(self):
 
     @property
     def project_config(self):
-        return self.seed_quote_cfg_with({"data-paths": [self.dir("seed-initial")]})
+        return self.seed_quote_cfg_with({"seed-paths": [self.dir("seed-initial")]})
 
     @use_profile("postgres")
     def test__postgres__simple_copy_loud(self):
@@ -124,7 +124,7 @@ def test__postgres__simple_copy_loud(self):
 
         self.assertManyTablesEqual(["seed", "VIEW_MODEL", "INCREMENTAL", "MATERIALIZED", "GET_AND_REF"])
 
-        self.use_default_project({"data-paths": [self.dir("seed-update")]})
+        self.use_default_project({"seed-paths": [self.dir("seed-update")]})
         results = self.run_dbt(["seed"])
         self.assertEqual(len(results),  1)
         results = self.run_dbt()
@@ -180,7 +180,7 @@ def project_config(self):
             'quoting': {
                 'database': True,
             },
-            "data-paths": [self.dir("seed-initial")],
+            "seed-paths": [self.dir("seed-initial")],
         })
 
     def seed_get_json(self, expect_pass=True):