Allow arbitrary directories to be Snapshotted

We don't cache anything intermediate here, we just dump the Snapshot
into the Store and return it.

This is intended to be a short-term transition path to allow BinaryUtils
to be consumed in the v2 engine.

Some alternatives which seem reasonable:
 1. Using a "root" as a Variant in a regular PathGlobs -> Snapshot
    request. This seems reasonable to me, but I couldn't work out
    how to make Variants work.
 2. Exposing an intrinsic function to do this synchronously on
    scheduler, and returning a Snapshot, rather than exposing a
    custom type in a subject-product relationship. This seems
    perfectly reasonable, and is arguably a better model. The down side
    here is that the arg parsing of the globs is more fiddly to do in an
    intrinsic function (where we don't otherwise do that beyond a simple
    string) than in a product request (where we already have all of the
    tooling in place to make this easy).

src/python/pants/engine/fs.py

@@ -50,6 +50,18 @@ def create(relative_to, include, exclude=tuple()):
                      tuple(join(relative_to, f) for f in exclude))
 
 
+class PathGlobsAndRoot(datatype(['include', 'exclude', 'root'])):
+  """This type allows arbitrary files from the filesystem to be read.
+
+  It should be used with extreme caution. It only exists as a hacky transition tool for BinaryUtil
+  until we fully integrate BinaryUtil into the v2 model.
+
+  This can be used to request a globs rooted at root to be snapshotted as a
+  SnapshotFromOutsideBuildRoot, which itself wraps equivalent to a Snapshot.
+  """
+  pass
+
+
 class Snapshot(datatype(['fingerprint', 'digest_length', 'path_stats'])):
   """A Snapshot is a collection of Files and Dirs fingerprinted by their names/content.
 
@@ -86,6 +98,18 @@ def __str__(self):
     return repr(self)
 
 
+class SnapshotFromOutsideBuildRoot(datatype([("snapshot", Snapshot)])):
+  """This type allows arbitrary files from the filesystem to be read.
+
+  It should be used with extreme caution. It only exists as a hacky transition tool for BinaryUtil
+  until we fully integrate BinaryUtil into the v2 model.
+
+  This can be obtained by asking the scheduler for this as a product, using a PathGlobsAndRoot as a
+  subject.
+  """
+  pass
+
+
 FilesContent = Collection.of(FileContent)
 
 
@@ -105,4 +129,6 @@ def create_fs_rules():
   """Creates rules that consume the intrinsic filesystem types."""
   return [
     RootRule(PathGlobs),
+    RootRule(PathGlobsAndRoot),
+    RootRule(Snapshot),
   ]

src/python/pants/engine/native.py

@@ -183,6 +183,9 @@
                             Function,
                             Function,
                             Function,
+                            Function,
+                            TypeConstraint,
+                            TypeConstraint,
                             TypeConstraint,
                             TypeConstraint,
                             TypeConstraint,
@@ -706,6 +709,7 @@ def new_scheduler(self,
                     work_dir,
                     ignore_patterns,
                     construct_snapshot,
+                    construct_snapshot_from_outside_buildroot,
                     construct_file_content,
                     construct_files_content,
                     construct_path_stat,
@@ -717,7 +721,9 @@ def new_scheduler(self,
                     constraint_address,
                     constraint_variants,
                     constraint_path_globs,
+                    constraint_path_globs_and_root,
                     constraint_snapshot,
+                    constraint_snapshot_from_outside_buildroot,
                     constraint_files_content,
                     constraint_dir,
                     constraint_file,
@@ -736,6 +742,7 @@ def tc(constraint):
         tasks,
         # Constructors/functions.
         func(construct_snapshot),
+        func(construct_snapshot_from_outside_buildroot),
         func(construct_file_content),
         func(construct_files_content),
         func(construct_path_stat),
@@ -748,7 +755,9 @@ def tc(constraint):
         tc(constraint_has_products),
         tc(constraint_variants),
         tc(constraint_path_globs),
+        tc(constraint_path_globs_and_root),
         tc(constraint_snapshot),
+        tc(constraint_snapshot_from_outside_buildroot),
         tc(constraint_files_content),
         tc(constraint_dir),
         tc(constraint_file),

src/python/pants/engine/scheduler.py

@@ -14,7 +14,8 @@
 from pants.base.exceptions import TaskError
 from pants.base.project_tree import Dir, File, Link
 from pants.build_graph.address import Address
-from pants.engine.fs import FileContent, FilesContent, Path, PathGlobs, Snapshot
+from pants.engine.fs import (FileContent, FilesContent, Path, PathGlobs, PathGlobsAndRoot, Snapshot,
+                             SnapshotFromOutsideBuildRoot)
 from pants.engine.isolated_process import ExecuteProcessRequest, ExecuteProcessResult
 from pants.engine.native import Function, TypeConstraint, TypeId
 from pants.engine.nodes import Return, State, Throw
@@ -89,6 +90,7 @@ def __init__(self, native, build_root, work_dir, ignore_patterns, rule_index):
       work_dir,
       ignore_patterns,
       Snapshot,
+      SnapshotFromOutsideBuildRoot,
       FileContent,
       FilesContent,
       Path,
@@ -100,7 +102,9 @@ def __init__(self, native, build_root, work_dir, ignore_patterns, rule_index):
       constraint_for(Address),
       constraint_for(Variants),
       constraint_for(PathGlobs),
+      constraint_for(PathGlobsAndRoot),
       constraint_for(Snapshot),
+      constraint_for(SnapshotFromOutsideBuildRoot),
       constraint_for(FilesContent),
       constraint_for(Dir),
       constraint_for(File),

src/rust/engine/fs/src/lib.rs

@@ -371,8 +371,30 @@ impl PosixFS {
     pool: Arc<ResettablePool>,
     ignore_patterns: Vec<String>,
   ) -> Result<PosixFS, String> {
-    let root: &Path = root.as_ref();
-    let canonical_root = root
+    let ignore = create_ignore(&ignore_patterns).map_err(|e| {
+      format!(
+        "Could not parse build ignore inputs {:?}: {:?}",
+        ignore_patterns, e
+      )
+    })?;
+    Ok(PosixFS {
+      root: Self::canonicalize(root)?,
+      pool: pool,
+      ignore: ignore,
+    })
+  }
+
+  pub fn clone_with_root<P: AsRef<Path>>(&self, root: P) -> Result<PosixFS, String> {
+    Ok(PosixFS {
+      root: Self::canonicalize(root)?,
+      pool: self.pool.clone(),
+      ignore: self.ignore.clone(),
+    })
+  }
+
+  fn canonicalize<P: AsRef<Path>>(path: P) -> Result<Dir, String> {
+    let path: &Path = path.as_ref();
+    path
       .canonicalize()
       .and_then(|canonical| {
         canonical.metadata().and_then(|metadata| {
@@ -386,19 +408,7 @@ impl PosixFS {
           }
         })
       })
-      .map_err(|e| format!("Could not canonicalize root {:?}: {:?}", root, e))?;
-
-    let ignore = create_ignore(&ignore_patterns).map_err(|e| {
-      format!(
-        "Could not parse build ignore inputs {:?}: {:?}",
-        ignore_patterns, e
-      )
-    })?;
-    Ok(PosixFS {
-      root: canonical_root,
-      pool: pool,
-      ignore: ignore,
-    })
+      .map_err(|e| format!("Could not canonicalize root {:?}: {:?}", path, e))
   }
 
   fn scandir_sync(root: PathBuf, dir_relative_to_root: Dir) -> Result<Vec<Stat>, io::Error> {

src/rust/engine/src/lib.rs

@@ -186,6 +186,7 @@ pub extern "C" fn externs_val_for(key: Key) -> Value {
 pub extern "C" fn scheduler_create(
   tasks_ptr: *mut Tasks,
   construct_snapshot: Function,
+  construct_snapshot_from_outside_buildroot: Function,
   construct_file_content: Function,
   construct_files_content: Function,
   construct_path_stat: Function,
@@ -197,7 +198,9 @@ pub extern "C" fn scheduler_create(
   type_has_products: TypeConstraint,
   type_has_variants: TypeConstraint,
   type_path_globs: TypeConstraint,
+  type_path_globs_and_root: TypeConstraint,
   type_snapshot: TypeConstraint,
+  type_snapshot_from_outside_buildroot: TypeConstraint,
   type_files_content: TypeConstraint,
   type_dir: TypeConstraint,
   type_file: TypeConstraint,
@@ -218,6 +221,7 @@ pub extern "C" fn scheduler_create(
     .unwrap_or_else(|e| panic!("Failed to decode ignore patterns as UTF8: {:?}", e));
   let types = Types {
     construct_snapshot: construct_snapshot,
+    construct_snapshot_from_outside_buildroot: construct_snapshot_from_outside_buildroot,
     construct_file_content: construct_file_content,
     construct_files_content: construct_files_content,
     construct_path_stat: construct_path_stat,
@@ -229,7 +233,9 @@ pub extern "C" fn scheduler_create(
     has_products: type_has_products,
     has_variants: type_has_variants,
     path_globs: type_path_globs,
+    path_globs_and_root: type_path_globs_and_root,
     snapshot: type_snapshot,
+    snapshot_from_outside_buildroot: type_snapshot_from_outside_buildroot,
     files_content: type_files_content,
     dir: type_dir,
     file: type_file,

src/rust/engine/src/nodes.rs

@@ -278,6 +278,32 @@ impl Select {
       .to_boxed()
   }
 
+  fn snapshot_from_outside_buildroot(
+    &self,
+    context: &Context,
+    entry: &rule_graph::Entry,
+  ) -> NodeFuture<fs::Snapshot> {
+    let ref edges = context
+      .core
+      .rule_graph
+      .edges_for_inner(entry)
+      .expect("edges for Snapshot exist.");
+    // Compute PathGlobs for the subject.
+    let context = context.clone();
+    Select::new(
+      context.core.types.path_globs_and_root.clone(),
+      self.subject.clone(),
+      self.variants.clone(),
+      edges,
+    ).run(context.clone())
+      .and_then(move |path_globs_val| {
+        context.get(SnapshotFromOutsideBuildRoot {
+          subject: externs::key_for(path_globs_val),
+        })
+      })
+      .to_boxed()
+  }
+
   fn execute_process(
     &self,
     context: &Context,
@@ -331,6 +357,16 @@ impl Select {
               .map(move |snapshot| Snapshot::store_snapshot(&context, &snapshot))
               .to_boxed()
           }
+          &rule_graph::Rule::Intrinsic(Intrinsic {
+            kind: IntrinsicKind::SnapshotFromOutsideOfBuildRoot,
+            ..
+          }) => {
+            let context = context.clone();
+            self
+              .snapshot_from_outside_buildroot(&context, &entry)
+              .map(move |snapshot| SnapshotFromOutsideBuildRoot::store(&context, &snapshot))
+              .to_boxed()
+          }
           &rule_graph::Rule::Intrinsic(Intrinsic {
             kind: IntrinsicKind::FilesContent,
             ..
@@ -819,6 +855,80 @@ impl From<Snapshot> for NodeKey {
   }
 }
 
+#[derive(Clone, Debug, Eq, Hash, PartialEq)]
+pub struct SnapshotFromOutsideBuildRoot {
+  subject: Key,
+}
+
+impl SnapshotFromOutsideBuildRoot {
+  fn lift_path_globs_and_root(item: &Value) -> Result<(PathGlobs, PathBuf), String> {
+    Snapshot::lift_path_globs(item).map(|path_globs| {
+      (
+        path_globs,
+        PathBuf::from(externs::project_str(item, "root")),
+      )
+    })
+  }
+
+  fn create(
+    store: fs::Store,
+    vfs: &fs::PosixFS,
+    path_globs: PathGlobs,
+    root: &Path,
+  ) -> NodeFuture<fs::Snapshot> {
+    // Note that we don't use a Context here, and don't cache any intermediate steps, we just place
+    // the resultant Snapshot into the store and return it. This is important, because we're reading
+    // things from arbitrary filepaths which we don't want to cache in the graph, as we don't watch
+    // them for changes.
+    // We assume that this Snapshot is of an immutable piece of the filesystem.
+    let store2 = store.clone();
+    future::done(vfs.clone_with_root(root))
+      .map(|posix_fs| Arc::new(posix_fs))
+      .map(move |posix_fs| {
+        (
+          posix_fs.clone(),
+          fs::OneOffStoreFileByDigest::new(store, posix_fs),
+        )
+      })
+      .and_then(|(posix_fs, digester)| {
+        posix_fs
+          .expand(path_globs)
+          .map_err(|e| format!("PathGlobs expansion failed: {:?}", e))
+          .and_then(move |path_stats| {
+            fs::Snapshot::from_path_stats(store2, digester, path_stats)
+              .map_err(move |e| format!("Snapshot failed: {}", e))
+          })
+      })
+      .map_err(|e| throw(&e))
+      .to_boxed()
+  }
+
+  fn store(context: &Context, item: &fs::Snapshot) -> Value {
+    let snapshot = Snapshot::store_snapshot(context, item);
+    externs::unsafe_call(
+      &context.core.types.construct_snapshot_from_outside_buildroot,
+      &[snapshot],
+    )
+  }
+}
+
+impl Node for SnapshotFromOutsideBuildRoot {
+  type Output = fs::Snapshot;
+
+  fn run(self, context: Context) -> NodeFuture<fs::Snapshot> {
+    match Self::lift_path_globs_and_root(&externs::val_for(&self.subject)) {
+      Ok((pgs, root)) => Self::create(context.core.store.clone(), &context.core.vfs, pgs, &root),
+      Err(e) => err(throw(&e)),
+    }
+  }
+}
+
+impl From<SnapshotFromOutsideBuildRoot> for NodeKey {
+  fn from(n: SnapshotFromOutsideBuildRoot) -> Self {
+    NodeKey::SnapshotFromOutsideBuildRoot(n)
+  }
+}
+
 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
 pub struct Task {
   subject: Key,
@@ -948,6 +1058,7 @@ pub enum NodeKey {
   Scandir(Scandir),
   Select(Select),
   Snapshot(Snapshot),
+  SnapshotFromOutsideBuildRoot(SnapshotFromOutsideBuildRoot),
   Task(Task),
 }
 
@@ -976,6 +1087,9 @@ impl NodeKey {
         typstr(&s.product)
       ),
       &NodeKey::Snapshot(ref s) => format!("Snapshot({})", keystr(&s.0)),
+      &NodeKey::SnapshotFromOutsideBuildRoot(ref s) => {
+        format!("SnapshotFromOutsideBuildRoot({})", keystr(&s.subject))
+      }
     }
   }
 
@@ -988,6 +1102,7 @@ impl NodeKey {
       &NodeKey::Select(ref s) => typstr(&s.selector.product),
       &NodeKey::Task(ref s) => typstr(&s.product),
       &NodeKey::Snapshot(..) => "Snapshot".to_string(),
+      &NodeKey::SnapshotFromOutsideBuildRoot(..) => "SnapshotFromOutsideBuildRoot".to_string(),
       &NodeKey::DigestFile(..) => "DigestFile".to_string(),
       &NodeKey::ReadLink(..) => "LinkDest".to_string(),
       &NodeKey::Scandir(..) => "DirectoryListing".to_string(),
@@ -1010,6 +1125,7 @@ impl NodeKey {
       &NodeKey::ExecuteProcess { .. }
       | &NodeKey::Select { .. }
       | &NodeKey::Snapshot { .. }
+      | &NodeKey::SnapshotFromOutsideBuildRoot { .. }
       | &NodeKey::Task { .. } => None,
     }
   }
@@ -1026,6 +1142,7 @@ impl Node for NodeKey {
       NodeKey::Scandir(n) => n.run(context).map(|v| v.into()).to_boxed(),
       NodeKey::Select(n) => n.run(context).map(|v| v.into()).to_boxed(),
       NodeKey::Snapshot(n) => n.run(context).map(|v| v.into()).to_boxed(),
+      NodeKey::SnapshotFromOutsideBuildRoot(n) => n.run(context).map(|v| v.into()).to_boxed(),
       NodeKey::Task(n) => n.run(context).map(|v| v.into()).to_boxed(),
     }
   }