[#3689] Fix filesystem searcher and tests that mock it (#4271)

.github/workflows/main.yml

@@ -77,7 +77,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.7, 3.8] # TODO: support unit testing for python 3.9 (https://github.com/dbt-labs/dbt/issues/3689)
+        python-version: [3.7, 3.8, 3.9]
 
     env:
       TOXENV: "unit"

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ### Under the hood
 Add --indirect-selection parameter to profiles.yml and builtin DBT_ env vars; stringified parameter to enable multi-modal use ([#3997](https://github.com/dbt-labs/dbt-core/issues/3997), [PR #4270](https://github.com/dbt-labs/dbt-core/pull/4270))
+- Fix filesystem searcher test failure on Python 3.9 ([#3689](https://github.com/dbt-labs/dbt-core/issues/3689), [#4271](https://github.com/dbt-labs/dbt-core/pull/4271))
 
 
 ## dbt-core 1.0.0rc1 (November 10, 2021)

core/dbt/parser/macros.py

@@ -11,19 +11,19 @@
 from dbt.events.types import MacroFileParse
 from dbt.node_types import NodeType
 from dbt.parser.base import BaseParser
-from dbt.parser.search import FileBlock, FilesystemSearcher
+from dbt.parser.search import FileBlock, filesystem_search
 from dbt.utils import MACRO_PREFIX
 
 
 class MacroParser(BaseParser[ParsedMacro]):
     # This is only used when creating a MacroManifest separate
     # from the normal parsing flow.
     def get_paths(self) -> List[FilePath]:
-        return list(FilesystemSearcher(
+        return filesystem_search(
             project=self.project,
             relative_dirs=self.project.macro_paths,
             extension='.sql',
-        ))
+        )
 
     @property
     def resource_type(self) -> NodeType:

core/dbt/parser/read_files.py

@@ -6,7 +6,7 @@
 
 from dbt.parser.schemas import yaml_from_file, schema_file_keys, check_format_version
 from dbt.exceptions import CompilationException
-from dbt.parser.search import FilesystemSearcher
+from dbt.parser.search import filesystem_search
 from typing import Optional
 
 
@@ -86,9 +86,9 @@ def load_seed_source_file(match: FilePath, project_name) -> SourceFile:
 # them into a bunch of FileSource objects
 def get_source_files(project, paths, extension, parse_file_type, saved_files):
     # file path list
-    fp_list = list(FilesystemSearcher(
+    fp_list = filesystem_search(
         project, paths, extension
-    ))
+    )
     # file block list
     fb_list = []
     for fp in fp_list:

core/dbt/parser/search.py

@@ -63,31 +63,24 @@ def contents(self):
         return self.block.full_block
 
 
-class FilesystemSearcher(Iterable[FilePath]):
-    def __init__(
-        self, project: Project, relative_dirs: List[str], extension: str
-    ) -> None:
-        self.project = project
-        self.relative_dirs = relative_dirs
-        self.extension = extension
-
-    def __iter__(self) -> Iterator[FilePath]:
-        ext = "[!.#~]*" + self.extension
-
-        root = self.project.project_root
-
-        for result in find_matching(root, self.relative_dirs, ext):
-            if 'searched_path' not in result or 'relative_path' not in result:
-                raise InternalException(
-                    'Invalid result from find_matching: {}'.format(result)
-                )
-            file_match = FilePath(
-                searched_path=result['searched_path'],
-                relative_path=result['relative_path'],
-                modification_time=result['modification_time'],
-                project_root=root,
+def filesystem_search(project: Project, relative_dirs: List[str], extension: str):
+    ext = "[!.#~]*" + extension
+    root = project.project_root
+    file_path_list = []
+    for result in find_matching(root, relative_dirs, ext):
+        if 'searched_path' not in result or 'relative_path' not in result:
+            raise InternalException(
+                'Invalid result from find_matching: {}'.format(result)
             )
-            yield file_match
+        file_match = FilePath(
+            searched_path=result['searched_path'],
+            relative_path=result['relative_path'],
+            modification_time=result['modification_time'],
+            project_root=root,
+        )
+        file_path_list.append(file_match)
+
+    return file_path_list
 
 
 Block = Union[BlockContents, FullBlock]

test/unit/test_graph.py

@@ -29,8 +29,7 @@ class GraphTest(unittest.TestCase):
 
     def tearDown(self):
         self.write_gpickle_patcher.stop()
-        self.file_system_patcher.stop()
-        self.mock_filesystem_constructor.stop()
+        self.mock_filesystem_search.stop()
         self.mock_hook_constructor.stop()
         self.load_state_check.stop()
         self.load_source_file_patcher.stop()
@@ -66,22 +65,16 @@ def mock_write_gpickle(graph, outfile):
         self.mock_write_gpickle = self.write_gpickle_patcher.start()
         self.mock_write_gpickle.side_effect = mock_write_gpickle
 
-        # Create file system patcher and filesystem searcher
-        self.file_system_patcher = patch.object(
-            dbt.parser.search.FilesystemSearcher, '__new__'
-        )
-        self.mock_filesystem_constructor = self.file_system_patcher.start()
-        def filesystem_iter(iter_self):
-            if 'sql' not in iter_self.extension:
+        # Create file filesystem searcher
+        self.filesystem_search = patch('dbt.parser.read_files.filesystem_search')
+        def mock_filesystem_search(project, relative_dirs, extension):
+            if 'sql' not in extension:
                 return []
-            if 'models' not in iter_self.relative_dirs:
+            if 'models' not in relative_dirs:
                 return []
             return [model.path for model in self.mock_models]
-        def create_filesystem_searcher(cls, project, relative_dirs, extension):
-            result = MagicMock(project=project, relative_dirs=relative_dirs, extension=extension)
-            result.__iter__.side_effect = lambda: iter(filesystem_iter(result))
-            return result
-        self.mock_filesystem_constructor.side_effect = create_filesystem_searcher
+        self.mock_filesystem_search = self.filesystem_search.start()
+        self.mock_filesystem_search.side_effect = mock_filesystem_search
 
         # Create HookParser patcher
         self.hook_patcher = patch.object(