make_target upgrades sources to EagerFilesetWithSpec (#5974)

This better simulates how the engine parses BUILD files, giving a more
faithful experience in tests.

I'm about to make it a warning/error to pass a list of strings as the
sources arg, so this will make tests which use make_target continue to
work after that.

Also, make cloc use base class scheduler instead of configuring its own.

tests/python/pants_test/backend/graph_info/tasks/test_cloc.py

@@ -8,9 +8,6 @@
 from pants.backend.graph_info.tasks.cloc import CountLinesOfCode
 from pants.backend.jvm.targets.java_library import JavaLibrary
 from pants.backend.python.targets.python_library import PythonLibrary
-from pants.base.file_system_project_tree import FileSystemProjectTree
-from pants.engine.fs import create_fs_rules
-from pants.engine.isolated_process import create_process_rules
 from pants_test.engine.scheduler_test_base import SchedulerTestBase
 from pants_test.task_test_base import ConsoleTaskTestBase
 
@@ -21,15 +18,19 @@ def task_type(cls):
     return CountLinesOfCode
 
   def test_counts(self):
-    dep_py_tgt = self.make_target('src/py/dep', PythonLibrary, sources=['dep.py'])
-    py_tgt = self.make_target('src/py/foo', PythonLibrary, dependencies=[dep_py_tgt],
-                              sources=['foo.py', 'bar.py'])
-    java_tgt = self.make_target('src/java/foo', JavaLibrary, sources=['Foo.java'])
     self.create_file('src/py/foo/foo.py', '# A comment.\n\nprint("some code")\n# Another comment.')
     self.create_file('src/py/foo/bar.py', '# A comment.\n\nprint("some more code")')
     self.create_file('src/py/dep/dep.py', 'print("a dependency")')
     self.create_file('src/java/foo/Foo.java', '// A comment. \n class Foo(){}\n')
     self.create_file('src/java/foo/Bar.java', '// We do not expect this file to appear in counts.')
+    dep_py_tgt = self.make_target('src/py/dep', PythonLibrary, sources=['dep.py'])
+    py_tgt = self.make_target(
+      'src/py/foo',
+      PythonLibrary,
+      dependencies=[dep_py_tgt],
+      sources=['foo.py', 'bar.py'],
+    )
+    java_tgt = self.make_target('src/java/foo', JavaLibrary, sources=['Foo.java'])
 
     def assert_counts(res, lang, files, blank, comment, code):
       for line in res:
@@ -43,41 +44,32 @@ def assert_counts(res, lang, files, blank, comment, code):
             return
       self.fail('Found no output line for {}'.format(lang))
 
-    scheduler = self.mk_configured_scheduler()
-
     res = self.execute_console_task(
       targets=[py_tgt, java_tgt],
       options={'transitive': True},
-      scheduler=scheduler,
+      scheduler=self.scheduler,
     )
     assert_counts(res, 'Python', files=3, blank=2, comment=3, code=3)
     assert_counts(res, 'Java', files=1, blank=0, comment=1, code=1)
 
     res = self.execute_console_task(
       targets=[py_tgt, java_tgt],
       options={'transitive': False},
-      scheduler=scheduler,
+      scheduler=self.scheduler,
     )
     assert_counts(res, 'Python', files=2, blank=2, comment=3, code=2)
     assert_counts(res, 'Java', files=1, blank=0, comment=1, code=1)
 
   def test_ignored(self):
-    py_tgt = self.make_target('src/py/foo', PythonLibrary, sources=['foo.py', 'empty.py'])
     self.create_file('src/py/foo/foo.py', 'print("some code")')
     self.create_file('src/py/foo/empty.py', '')
+    py_tgt = self.make_target('src/py/foo', PythonLibrary, sources=['foo.py', 'empty.py'])
 
     res = self.execute_console_task(
       targets=[py_tgt],
       options={'ignored': True},
-      scheduler=self.mk_configured_scheduler(),
+      scheduler=self.scheduler,
     )
     self.assertEquals(['Ignored the following files:',
                        'src/py/foo/empty.py: zero sized file'],
                       filter(None, res)[-2:])
-
-  def mk_configured_scheduler(self):
-    return self.mk_scheduler(
-      rules=create_fs_rules() + create_process_rules(),
-      project_tree=FileSystemProjectTree(self.build_root),
-      work_dir=self.pants_workdir
-    )

tests/python/pants_test/task/test_task.py

@@ -150,7 +150,12 @@ def _toggle_cache(self, enable_artifact_cache):
     )
 
   def _fixture(self, incremental, options=None):
-    target = self.make_target(':t', target_type=Files, sources=[self._filename])
+    target = self.make_target(
+      ':t',
+      target_type=Files,
+      sources=[self._filename],
+      make_missing_sources=False,
+    )
     context = self.context(options=options, target_roots=[target])
     task = self.create_task(context)
     task._incremental = incremental

tests/python/pants_test/test_base.py

@@ -23,6 +23,10 @@
 from pants.build_graph.build_file_aliases import BuildFileAliases
 from pants.build_graph.build_file_parser import BuildFileParser
 from pants.build_graph.target import Target
+from pants.engine.fs import PathGlobs
+from pants.engine.legacy.graph import HydratedField
+from pants.engine.legacy.structs import SourcesField
+from pants.engine.rules import RootRule
 from pants.init.engine_initializer import EngineInitializer
 from pants.init.util import clean_global_runtime_state
 from pants.option.options_bootstrapper import OptionsBootstrapper
@@ -193,6 +197,7 @@ def make_target(self,
                   dependencies=None,
                   derived_from=None,
                   synthetic=False,
+                  make_missing_sources=True,
                   **kwargs):
     """Creates a target and injects it into the test's build graph.
 
@@ -205,6 +210,13 @@ def make_target(self,
     :type derived_from: :class:`pants.build_graph.target.Target`
     """
     address = Address.parse(spec)
+
+    if make_missing_sources and 'sources' in kwargs:
+      for source in kwargs['sources']:
+        if '*' not in source:
+          self.create_file(os.path.join(address.spec_path, source), mode='a')
+      kwargs['sources'] = self.sources_for(kwargs['sources'], address.spec_path)
+
     target = target_type(name=address.target_name,
                          address=address,
                          build_graph=self.build_graph,
@@ -235,6 +247,17 @@ def make_target(self,
 
     return target
 
+  def sources_for(self, package_relative_path_globs, package_dir=''):
+    sources_field = SourcesField(
+      Address.parse('{}:_bogus_target_for_test'.format(package_dir)),
+      'sources',
+      {'globs': package_relative_path_globs},
+      None,
+      PathGlobs(tuple(os.path.join(package_dir, path) for path in package_relative_path_globs)),
+    )
+    field = self.scheduler.product_request(HydratedField, [sources_field])[0]
+    return field.value
+
   @classmethod
   def alias_groups(cls):
     """
@@ -339,6 +362,8 @@ def _init_engine(cls):
       native=init_native(),
       build_configuration=cls.build_config(),
       build_ignore_patterns=None,
+      # Required for sources_for:
+      rules=[RootRule(SourcesField)],
     ).new_session()
     cls._scheduler = graph_session.scheduler_session
     cls._build_graph, cls._address_mapper = graph_session.create_build_graph(