Preload all project files at start of parsing [#3244]

CHANGELOG.md

@@ -26,6 +26,7 @@
 - General development environment clean up and improve experience running tests locally ([#3194](https://github.com/fishtown-analytics/dbt/issues/3194), [#3204](https://github.com/fishtown-analytics/dbt/pull/3204), [#3228](https://github.com/fishtown-analytics/dbt/pull/3228))
 - Add a new materialization for tests, update data tests to use test materialization when executing. ([#3154](https://github.com/fishtown-analytics/dbt/issues/3154), [#3181](https://github.com/fishtown-analytics/dbt/pull/3181))
 - Switch from externally storing parsing state in ParseResult object to using Manifest ([#3163](http://github.com/fishtown-analytics/dbt/issues/3163), [#3219](https://github.com/fishtown-analytics/dbt/pull/3219))
+- Switch from loading project files in separate parsers to loading in one place([#3244](http://github.com/fishtown-analytics/dbt/issues/3244), [#3248](https://github.com/fishtown-analytics/dbt/pull/3248))
 
 Contributors:
 - [@yu-iskw](https://github.com/yu-iskw) ([#2928](https://github.com/fishtown-analytics/dbt/pull/2928))

core/dbt/contracts/files.py

@@ -3,7 +3,7 @@
 from dataclasses import dataclass, field
 from typing import List, Optional, Union
 
-from dbt.dataclass_schema import dbtClassMixin
+from dbt.dataclass_schema import dbtClassMixin, StrEnum
 
 from dbt.exceptions import InternalException
 
@@ -14,6 +14,18 @@
 MAXIMUM_SEED_SIZE_NAME = '1MB'
 
 
+class ParseFileType(StrEnum):
+    Macro = 'macro'
+    Model = 'model'
+    Snapshot = 'snapshot'
+    Analysis = 'analysis'
+    Test = 'test'
+    Seed = 'seed'
+    Documentation = 'docs'
+    Schema = 'schema'
+    Hook = 'hook'
+
+
 @dataclass
 class FilePath(dbtClassMixin):
     searched_path: str
@@ -114,6 +126,10 @@ class SourceFile(dbtClassMixin):
     """Define a source file in dbt"""
     path: Union[FilePath, RemoteFile]  # the path information
     checksum: FileHash
+    # Seems like knowing which project the file came from would be useful
+    project_name: Optional[str] = None
+    # Parse file type: i.e. which parser will process this file
+    parse_file_type: Optional[ParseFileType] = None
     # we don't want to serialize this
     _contents: Optional[str] = None
     # the unique IDs contained in this file

core/dbt/contracts/graph/manifest.py

@@ -23,6 +23,7 @@
 from dbt.contracts.util import (
     BaseArtifactMetadata, MacroKey, SourceKey, ArtifactMixin, schema_version
 )
+from dbt.dataclass_schema import dbtClassMixin
 from dbt.exceptions import (
     InternalException, CompilationException,
     raise_duplicate_resource_name, raise_compiler_error, warn_or_error,
@@ -512,7 +513,7 @@ def _find_macros_by_name(
 
 
 @dataclass
-class ManifestStateCheck():
+class ManifestStateCheck(dbtClassMixin):
     vars_hash: FileHash
     profile_hash: FileHash
     project_hashes: MutableMapping[str, FileHash]
@@ -779,10 +780,11 @@ def deepcopy(self):
             macros={k: _deepcopy(v) for k, v in self.macros.items()},
             docs={k: _deepcopy(v) for k, v in self.docs.items()},
             exposures={k: _deepcopy(v) for k, v in self.exposures.items()},
-            selectors=self.root_project.manifest_selectors,
+            selectors={k: _deepcopy(v) for k, v in self.selectors.items()},
             metadata=self.metadata,
             disabled=[_deepcopy(n) for n in self.disabled],
             files={k: _deepcopy(v) for k, v in self.files.items()},
+            state_check=_deepcopy(self.state_check),
         )
 
     def writable_manifest(self):
@@ -1199,9 +1201,8 @@ def __reduce_ex__(self, protocol):
 
 
 class MacroManifest(MacroMethods):
-    def __init__(self, macros, files):
+    def __init__(self, macros):
         self.macros = macros
-        self.files = files
         self.metadata = ManifestMetadata()
         # This is returned by the 'graph' context property
         # in the ProviderContext class.

core/dbt/parser/analysis.py

@@ -3,15 +3,10 @@
 from dbt.contracts.graph.parsed import ParsedAnalysisNode
 from dbt.node_types import NodeType
 from dbt.parser.base import SimpleSQLParser
-from dbt.parser.search import FilesystemSearcher, FileBlock
+from dbt.parser.search import FileBlock
 
 
 class AnalysisParser(SimpleSQLParser[ParsedAnalysisNode]):
-    def get_paths(self):
-        return FilesystemSearcher(
-            self.project, self.project.analysis_paths, '.sql'
-        )
-
     def parse_from_dict(self, dct, validate=True) -> ParsedAnalysisNode:
         if validate:
             ParsedAnalysisNode.validate(dct)

core/dbt/parser/base.py

@@ -2,14 +2,13 @@
 import itertools
 import os
 from typing import (
-    List, Dict, Any, Iterable, Generic, TypeVar
+    List, Dict, Any, Generic, TypeVar
 )
 
 from dbt.dataclass_schema import ValidationError
 
 from dbt import utils
 from dbt.clients.jinja import MacroGenerator
-from dbt.clients.system import load_file_contents
 from dbt.context.providers import (
     generate_parser_model,
     generate_generate_component_name_macro,
@@ -20,9 +19,6 @@
 from dbt.context.context_config import (
     ContextConfig
 )
-from dbt.contracts.files import (
-    SourceFile, FilePath, FileHash
-)
 from dbt.contracts.graph.manifest import Manifest
 from dbt.contracts.graph.parsed import HasUniqueID, ManifestNodes
 from dbt.contracts.graph.unparsed import UnparsedNode
@@ -50,17 +46,6 @@ class BaseParser(Generic[FinalValue]):
     def __init__(self, project: Project, manifest: Manifest) -> None:
         self.project = project
         self.manifest = manifest
-        # this should be a superset of [x.path for x in self.manifest.files]
-        # because we fill it via search()
-        self.searched: List[FilePath] = []
-
-    @abc.abstractmethod
-    def get_paths(self) -> Iterable[FilePath]:
-        pass
-
-    def search(self) -> List[FilePath]:
-        self.searched = list(self.get_paths())
-        return self.searched
 
     @abc.abstractmethod
     def parse_file(self, block: FileBlock) -> None:
@@ -76,21 +61,6 @@ def generate_unique_id(self, resource_name: str) -> str:
                                  self.project.project_name,
                                  resource_name)
 
-    def load_file(
-        self,
-        path: FilePath,
-        *,
-        set_contents: bool = True,
-    ) -> SourceFile:
-        file_contents = load_file_contents(path.absolute_path, strip=False)
-        checksum = FileHash.from_contents(file_contents)
-        source_file = SourceFile(path=path, checksum=checksum)
-        if set_contents:
-            source_file.contents = file_contents.strip()
-        else:
-            source_file.contents = ''
-        return source_file
-
 
 class Parser(BaseParser[FinalValue], Generic[FinalValue]):
     def __init__(

core/dbt/parser/data_test.py

@@ -1,16 +1,11 @@
 from dbt.contracts.graph.parsed import ParsedDataTestNode
 from dbt.node_types import NodeType
 from dbt.parser.base import SimpleSQLParser
-from dbt.parser.search import FilesystemSearcher, FileBlock
+from dbt.parser.search import FileBlock
 from dbt.utils import get_pseudo_test_path
 
 
 class DataTestParser(SimpleSQLParser[ParsedDataTestNode]):
-    def get_paths(self):
-        return FilesystemSearcher(
-            self.project, self.project.test_paths, '.sql'
-        )
-
     def parse_from_dict(self, dct, validate=True) -> ParsedDataTestNode:
         if validate:
             ParsedDataTestNode.validate(dct)

core/dbt/parser/docs.py

@@ -7,21 +7,14 @@
 from dbt.node_types import NodeType
 from dbt.parser.base import Parser
 from dbt.parser.search import (
-    BlockContents, FileBlock, FilesystemSearcher, BlockSearcher
+    BlockContents, FileBlock, BlockSearcher
 )
 
 
 SHOULD_PARSE_RE = re.compile(r'{[{%]')
 
 
 class DocumentationParser(Parser[ParsedDocumentation]):
-    def get_paths(self):
-        return FilesystemSearcher(
-            project=self.project,
-            relative_dirs=self.project.docs_paths,
-            extension='.md',
-        )
-
     @property
     def resource_type(self) -> NodeType:
         return NodeType.Documentation
@@ -61,5 +54,3 @@ def parse_file(self, file_block: FileBlock):
         for block in searcher:
             for parsed in self.parse_block(block):
                 self.manifest.add_doc(file_block.file, parsed)
-        # mark the file as seen, even if there are no macros in it
-        self.manifest.get_file(file_block.file)

core/dbt/parser/hooks.py

@@ -70,13 +70,14 @@ class HookParser(SimpleParser[HookBlock, ParsedHookNode]):
     def transform(self, node):
         return node
 
-    def get_paths(self) -> List[FilePath]:
+    # Hooks are only in the dbt_project.yml file for the project
+    def get_path(self) -> FilePath:
         path = FilePath(
             project_root=self.project.project_root,
             searched_path='.',
             relative_path='dbt_project.yml',
         )
-        return [path]
+        return path
 
     def parse_from_dict(self, dct, validate=True) -> ParsedHookNode:
         if validate:

core/dbt/parser/macros.py

@@ -5,6 +5,7 @@
 from dbt.clients import jinja
 from dbt.contracts.graph.unparsed import UnparsedMacro
 from dbt.contracts.graph.parsed import ParsedMacro
+from dbt.contracts.files import FilePath
 from dbt.exceptions import CompilationException
 from dbt.logger import GLOBAL_LOGGER as logger
 from dbt.node_types import NodeType
@@ -14,12 +15,14 @@
 
 
 class MacroParser(BaseParser[ParsedMacro]):
-    def get_paths(self):
-        return FilesystemSearcher(
+    # This is only used when creating a MacroManifest separate
+    # from the normal parsing flow.
+    def get_paths(self) -> List[FilePath]:
+        return list(FilesystemSearcher(
             project=self.project,
             relative_dirs=self.project.macro_paths,
             extension='.sql',
-        )
+        ))
 
     @property
     def resource_type(self) -> NodeType: