From 74d1736475f1e32eb392900ed493120d0c033d79 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Thu, 11 Nov 2021 06:13:29 -0800 Subject: [PATCH] [#3689] Fix filesystem searcher and tests that mock it --- core/dbt/parser/macros.py | 6 ++--- core/dbt/parser/read_files.py | 6 ++--- core/dbt/parser/search.py | 41 +++++++++++++++-------------------- test/unit/test_graph.py | 23 +++++++------------- 4 files changed, 31 insertions(+), 45 deletions(-) diff --git a/core/dbt/parser/macros.py b/core/dbt/parser/macros.py index cd928fab0a3..5b5d8f21a91 100644 --- a/core/dbt/parser/macros.py +++ b/core/dbt/parser/macros.py @@ -11,7 +11,7 @@ from dbt.events.types import MacroFileParse from dbt.node_types import NodeType from dbt.parser.base import BaseParser -from dbt.parser.search import FileBlock, FilesystemSearcher +from dbt.parser.search import FileBlock, filesystem_search from dbt.utils import MACRO_PREFIX @@ -19,11 +19,11 @@ class MacroParser(BaseParser[ParsedMacro]): # This is only used when creating a MacroManifest separate # from the normal parsing flow. def get_paths(self) -> List[FilePath]: - return list(FilesystemSearcher( + return filesystem_search( project=self.project, relative_dirs=self.project.macro_paths, extension='.sql', - )) + ) @property def resource_type(self) -> NodeType: diff --git a/core/dbt/parser/read_files.py b/core/dbt/parser/read_files.py index 7b0c7de8afb..1b721c9599b 100644 --- a/core/dbt/parser/read_files.py +++ b/core/dbt/parser/read_files.py @@ -6,7 +6,7 @@ from dbt.parser.schemas import yaml_from_file, schema_file_keys, check_format_version from dbt.exceptions import CompilationException -from dbt.parser.search import FilesystemSearcher +from dbt.parser.search import filesystem_search from typing import Optional @@ -86,9 +86,9 @@ def load_seed_source_file(match: FilePath, project_name) -> SourceFile: # them into a bunch of FileSource objects def get_source_files(project, paths, extension, parse_file_type, saved_files): # file path list - fp_list = list(FilesystemSearcher( + fp_list = filesystem_search( project, paths, extension - )) + ) # file block list fb_list = [] for fp in fp_list: diff --git a/core/dbt/parser/search.py b/core/dbt/parser/search.py index 15a72f2517c..0d55f581742 100644 --- a/core/dbt/parser/search.py +++ b/core/dbt/parser/search.py @@ -63,31 +63,24 @@ def contents(self): return self.block.full_block -class FilesystemSearcher(Iterable[FilePath]): - def __init__( - self, project: Project, relative_dirs: List[str], extension: str - ) -> None: - self.project = project - self.relative_dirs = relative_dirs - self.extension = extension - - def __iter__(self) -> Iterator[FilePath]: - ext = "[!.#~]*" + self.extension - - root = self.project.project_root - - for result in find_matching(root, self.relative_dirs, ext): - if 'searched_path' not in result or 'relative_path' not in result: - raise InternalException( - 'Invalid result from find_matching: {}'.format(result) - ) - file_match = FilePath( - searched_path=result['searched_path'], - relative_path=result['relative_path'], - modification_time=result['modification_time'], - project_root=root, +def filesystem_search(project: Project, relative_dirs: List[str], extension: str): + ext = "[!.#~]*" + extension + root = project.project_root + file_path_list = [] + for result in find_matching(root, relative_dirs, ext): + if 'searched_path' not in result or 'relative_path' not in result: + raise InternalException( + 'Invalid result from find_matching: {}'.format(result) ) - yield file_match + file_match = FilePath( + searched_path=result['searched_path'], + relative_path=result['relative_path'], + modification_time=result['modification_time'], + project_root=root, + ) + file_path_list.append(file_match) + + return file_path_list Block = Union[BlockContents, FullBlock] diff --git a/test/unit/test_graph.py b/test/unit/test_graph.py index c68a7e68a25..a9d594075ce 100644 --- a/test/unit/test_graph.py +++ b/test/unit/test_graph.py @@ -29,8 +29,7 @@ class GraphTest(unittest.TestCase): def tearDown(self): self.write_gpickle_patcher.stop() - self.file_system_patcher.stop() - self.mock_filesystem_constructor.stop() + self.mock_filesystem_search.stop() self.mock_hook_constructor.stop() self.load_state_check.stop() self.load_source_file_patcher.stop() @@ -66,22 +65,16 @@ def mock_write_gpickle(graph, outfile): self.mock_write_gpickle = self.write_gpickle_patcher.start() self.mock_write_gpickle.side_effect = mock_write_gpickle - # Create file system patcher and filesystem searcher - self.file_system_patcher = patch.object( - dbt.parser.search.FilesystemSearcher, '__new__' - ) - self.mock_filesystem_constructor = self.file_system_patcher.start() - def filesystem_iter(iter_self): - if 'sql' not in iter_self.extension: + # Create file filesystem searcher + self.filesystem_search = patch('dbt.parser.read_files.filesystem_search') + def mock_filesystem_search(project, relative_dirs, extension): + if 'sql' not in extension: return [] - if 'models' not in iter_self.relative_dirs: + if 'models' not in relative_dirs: return [] return [model.path for model in self.mock_models] - def create_filesystem_searcher(cls, project, relative_dirs, extension): - result = MagicMock(project=project, relative_dirs=relative_dirs, extension=extension) - result.__iter__.side_effect = lambda: iter(filesystem_iter(result)) - return result - self.mock_filesystem_constructor.side_effect = create_filesystem_searcher + self.mock_filesystem_search = self.filesystem_search.start() + self.mock_filesystem_search.side_effect = mock_filesystem_search # Create HookParser patcher self.hook_patcher = patch.object(