Auto merge of rust-lang#34366 - Diggsey:rust-src-pkg, r=brson

Produce source package in rust-installer format

See rust-lang-deprecated/rust-buildbot#102

There may be a better way to do this, wasn't sure how to clean-up the `rust-src-image` directory when it's used by multiple make rules.

mk/dist.mk

@@ -24,6 +24,7 @@ PKG_NAME := $(CFG_PACKAGE_NAME)
 STD_PKG_NAME := rust-std-$(CFG_PACKAGE_VERS)
 DOC_PKG_NAME := rust-docs-$(CFG_PACKAGE_VERS)
 MINGW_PKG_NAME := rust-mingw-$(CFG_PACKAGE_VERS)
+SRC_PKG_NAME := rust-src-$(CFG_PACKAGE_VERS)
 
 # License suitable for displaying in a popup
 LICENSE.txt: $(S)COPYRIGHT $(S)LICENSE-APACHE $(S)LICENSE-MIT
@@ -71,10 +72,10 @@ PKG_FILES := \
 
 UNROOTED_PKG_FILES := $(patsubst $(S)%,./%,$(PKG_FILES))
 
-$(PKG_TAR): $(PKG_FILES)
-	@$(call E, making dist dir)
-	$(Q)rm -Rf tmp/dist/$(PKG_NAME)
-	$(Q)mkdir -p tmp/dist/$(PKG_NAME)
+tmp/dist/$$(SRC_PKG_NAME)-image: $(PKG_FILES)
+	@$(call E, making src image)
+	$(Q)rm -Rf tmp/dist/$(SRC_PKG_NAME)-image
+	$(Q)mkdir -p tmp/dist/$(SRC_PKG_NAME)-image/lib/rustlib/src/rust
 	$(Q)tar \
          -C $(S) \
          -f - \
@@ -87,10 +88,11 @@ $(PKG_TAR): $(PKG_FILES)
          --exclude=*/llvm/test/*/*/*.ll \
          --exclude=*/llvm/test/*/*/*.td \
          --exclude=*/llvm/test/*/*/*.s \
-         -c $(UNROOTED_PKG_FILES) | tar -x -f - -C tmp/dist/$(PKG_NAME)
+         -c $(UNROOTED_PKG_FILES) | tar -x -f - -C tmp/dist/$(SRC_PKG_NAME)-image/lib/rustlib/src/rust
+
+$(PKG_TAR): tmp/dist/$$(SRC_PKG_NAME)-image
 	@$(call E, making $@)
-	$(Q)tar -czf $(PKG_TAR) -C tmp/dist $(PKG_NAME)
-	$(Q)rm -Rf tmp/dist/$(PKG_NAME)
+	$(Q)tar -czf $(PKG_TAR) -C tmp/dist/$(SRC_PKG_NAME)-image/lib/rustlib/src rust --transform 's,^rust,$(PKG_NAME),S'
 
 dist-tar-src: $(PKG_TAR)
 
@@ -259,6 +261,19 @@ endef
 $(foreach host,$(CFG_HOST),\
   $(eval $(call DEF_INSTALLER,$(host))))
 
+dist/$(SRC_PKG_NAME).tar.gz: tmp/dist/$(SRC_PKG_NAME)-image
+	@$(call E, build: $@)
+	$(Q)$(S)src/rust-installer/gen-installer.sh \
+		--product-name=Rust \
+		--rel-manifest-dir=rustlib \
+		--success-message=Awesome-Source. \
+		--image-dir=tmp/dist/$(SRC_PKG_NAME)-image \
+		--work-dir=tmp/dist \
+		--output-dir=dist \
+		--package-name=$(SRC_PKG_NAME) \
+		--component-name=rust-src \
+		--legacy-manifest-dirs=rustlib,cargo
+
 # When generating packages for the standard library, we've actually got a lot of
 # artifacts to choose from. Each of the CFG_HOST compilers will have a copy of
 # the standard library for each CFG_TARGET, but we only want to generate one
@@ -329,8 +344,8 @@ distcheck-docs: dist-docs
 # Primary targets (dist, distcheck)
 ######################################################################
 
-MAYBE_DIST_TAR_SRC=dist-tar-src
-MAYBE_DISTCHECK_TAR_SRC=distcheck-tar-src
+MAYBE_DIST_TAR_SRC=dist-tar-src dist/$(SRC_PKG_NAME).tar.gz
+MAYBE_DISTCHECK_TAR_SRC=distcheck-tar-src dist/$(SRC_PKG_NAME).tar.gz
 
 # FIXME #13224: On OS X don't produce tarballs simply because --exclude-vcs don't work.
 # This is a huge hack because I just don't have time to figure out another solution.

src/bootstrap/Cargo.toml

@@ -32,3 +32,4 @@ kernel32-sys = "0.2"
 gcc = { git = "https://github.com/alexcrichton/gcc-rs" }
 libc = "0.2"
 md5 = "0.1"
+regex = "0.1.73"

src/bootstrap/dist.rs

@@ -24,7 +24,8 @@ use std::path::{PathBuf, Path};
 use std::process::Command;
 
 use {Build, Compiler};
-use util::{cp_r, libdir, is_dylib};
+use util::{cp_r, libdir, is_dylib, cp_filtered, copy};
+use regex::{RegexSet, quote};
 
 fn package_vers(build: &Build) -> &str {
     match &build.config.channel[..] {
@@ -284,6 +285,119 @@ pub fn std(build: &Build, compiler: &Compiler, target: &str) {
     t!(fs::remove_dir_all(&image));
 }
 
+/// Creates the `rust-src` installer component and the plain source tarball
+pub fn rust_src(build: &Build) {
+    println!("Dist src");
+    let plain_name = format!("rustc-{}-src", package_vers(build));
+    let name = format!("rust-src-{}", package_vers(build));
+    let image = tmpdir(build).join(format!("{}-image", name));
+    let _ = fs::remove_dir_all(&image);
+
+    let dst = image.join("lib/rustlib/src");
+    let dst_src = dst.join("rust");
+    let plain_dst_src = dst.join(&plain_name);
+    t!(fs::create_dir_all(&dst_src));
+
+    // This is the set of root paths which will become part of the source package
+    let src_files = [
+        "COPYRIGHT",
+        "LICENSE-APACHE",
+        "LICENSE-MIT",
+        "CONTRIBUTING.md",
+        "README.md",
+        "RELEASES.md",
+        "configure",
+        "Makefile.in"
+    ];
+    let src_dirs = [
+        "man",
+        "src",
+        "mk"
+    ];
+
+    // Exclude paths matching these wildcard expressions
+    let excludes = [
+        // exclude-vcs
+        "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules", ".gitattributes", ".cvsignore",
+        ".svn", ".arch-ids", "{arch}", "=RELEASE-ID", "=meta-update", "=update", ".bzr",
+        ".bzrignore", ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs",
+        // extensions
+        "*~", "*.pyc",
+        // misc
+        "llvm/test/*/*.ll",
+        "llvm/test/*/*.td",
+        "llvm/test/*/*.s",
+        "llvm/test/*/*/*.ll",
+        "llvm/test/*/*/*.td",
+        "llvm/test/*/*/*.s"
+    ];
+
+    // Construct a set of regexes for efficiently testing whether paths match one of the above
+    // expressions.
+    let regex_set = t!(RegexSet::new(
+        // This converts a wildcard expression to a regex
+        excludes.iter().map(|&s| {
+            // Prefix ensures that matching starts on a path separator boundary
+            r"^(.*[\\/])?".to_owned() + (
+                // Escape the expression to produce a regex matching exactly that string
+                &quote(s)
+                // Replace slashes with a pattern matching either forward or backslash
+                .replace(r"/", r"[\\/]")
+                // Replace wildcards with a pattern matching a single path segment, ie. containing
+                // no slashes.
+                .replace(r"\*", r"[^\\/]*")
+            // Suffix anchors to the end of the path
+            ) + "$"
+        })
+    ));
+
+    // Create a filter which skips files which match the regex set or contain invalid unicode
+    let filter_fn = move |path: &Path| {
+        if let Some(path) = path.to_str() {
+            !regex_set.is_match(path)
+        } else {
+            false
+        }
+    };
+
+    // Copy the directories using our filter
+    for item in &src_dirs {
+        let dst = &dst_src.join(item);
+        t!(fs::create_dir(dst));
+        cp_filtered(&build.src.join(item), dst, &filter_fn);
+    }
+    // Copy the files normally
+    for item in &src_files {
+        copy(&build.src.join(item), &dst_src.join(item));
+    }
+
+    // Create source tarball in rust-installer format
+    let mut cmd = Command::new("sh");
+    cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/gen-installer.sh")))
+       .arg("--product-name=Rust")
+       .arg("--rel-manifest-dir=rustlib")
+       .arg("--success-message=Awesome-Source.")
+       .arg(format!("--image-dir={}", sanitize_sh(&image)))
+       .arg(format!("--work-dir={}", sanitize_sh(&tmpdir(build))))
+       .arg(format!("--output-dir={}", sanitize_sh(&distdir(build))))
+       .arg(format!("--package-name={}", name))
+       .arg("--component-name=rust-src")
+       .arg("--legacy-manifest-dirs=rustlib,cargo");
+    build.run(&mut cmd);
+
+    // Rename directory, so that root folder of tarball has the correct name
+    t!(fs::rename(&dst_src, &plain_dst_src));
+
+    // Create plain source tarball
+    let mut cmd = Command::new("tar");
+    cmd.arg("-czf").arg(sanitize_sh(&distdir(build).join(&format!("{}.tar.gz", plain_name))))
+       .arg(&plain_name)
+       .current_dir(&dst);
+    build.run(&mut cmd);
+
+    t!(fs::remove_dir_all(&image));
+}
+
 fn install(src: &Path, dstdir: &Path, perms: u32) {
     let dst = dstdir.join(src.file_name().unwrap());
     t!(fs::create_dir_all(dstdir));

src/bootstrap/lib.rs

@@ -26,6 +26,7 @@ extern crate md5;
 extern crate num_cpus;
 extern crate rustc_serialize;
 extern crate toml;
+extern crate regex;
 
 use std::cell::RefCell;
 use std::collections::HashMap;
@@ -451,6 +452,7 @@ impl Build {
                 DistMingw { _dummy } => dist::mingw(self, target.target),
                 DistRustc { stage } => dist::rustc(self, stage, target.target),
                 DistStd { compiler } => dist::std(self, &compiler, target.target),
+                DistSrc { _dummy } => dist::rust_src(self),
 
                 DebuggerScripts { stage } => {
                     let compiler = Compiler::new(stage, target.target);

src/bootstrap/step.rs

@@ -140,6 +140,7 @@ macro_rules! targets {
             (dist_mingw, DistMingw { _dummy: () }),
             (dist_rustc, DistRustc { stage: u32 }),
             (dist_std, DistStd { compiler: Compiler<'a> }),
+            (dist_src, DistSrc { _dummy: () }),
 
             // Misc targets
             (android_copy_libs, AndroidCopyLibs { compiler: Compiler<'a> }),
@@ -568,12 +569,14 @@ impl<'a> Step<'a> {
                     vec![self.libtest(compiler)]
                 }
             }
+            Source::DistSrc { _dummy: _ } => Vec::new(),
 
             Source::Dist { stage } => {
                 let mut base = Vec::new();
 
                 for host in build.config.host.iter() {
                     let host = self.target(host);
+                    base.push(host.dist_src(()));
                     base.push(host.dist_rustc(stage));
                     if host.target.contains("windows-gnu") {
                         base.push(host.dist_mingw(()));

src/bootstrap/util.rs

@@ -67,6 +67,35 @@ pub fn cp_r(src: &Path, dst: &Path) {
     }
 }
 
+/// Copies the `src` directory recursively to `dst`. Both are assumed to exist
+/// when this function is called. Unwanted files or directories can be skipped
+/// by returning `false` from the filter function.
+pub fn cp_filtered<F: Fn(&Path) -> bool>(src: &Path, dst: &Path, filter: &F) {
+    // Inner function does the actual work
+    fn recurse<F: Fn(&Path) -> bool>(src: &Path, dst: &Path, relative: &Path, filter: &F) {
+        for f in t!(fs::read_dir(src)) {
+            let f = t!(f);
+            let path = f.path();
+            let name = path.file_name().unwrap();
+            let dst = dst.join(name);
+            let relative = relative.join(name);
+            // Only copy file or directory if the filter function returns true
+            if filter(&relative) {
+                if t!(f.file_type()).is_dir() {
+                    let _ = fs::remove_dir_all(&dst);
+                    t!(fs::create_dir(&dst));
+                    recurse(&path, &dst, &relative, filter);
+                } else {
+                    let _ = fs::remove_file(&dst);
+                    copy(&path, &dst);
+                }
+            }
+        }
+    }
+    // Immediately recurse with an empty relative path
+    recurse(src, dst, Path::new(""), filter)
+}
+
 /// Given an executable called `name`, return the filename for the
 /// executable for a particular target.
 pub fn exe(name: &str, target: &str) -> String {