Auto merge of #38697 - alexcrichton:rollup, r=alexcrichton

Rollup of 25 pull requests

- Successful merges: #37149, #38491, #38517, #38559, #38587, #38609, #38611, #38622, #38628, #38630, #38631, #38632, #38635, #38647, #38649, #38655, #38659, #38660, #38662, #38665, #38671, #38674, #38676, #38693, #38695
- Failed merges: #38657, #38680

.travis.yml

@@ -19,7 +19,8 @@ matrix:
     - env: IMAGE=i686-gnu-nopt
     - env: IMAGE=x86_64-freebsd
     - env: IMAGE=x86_64-gnu
-    - env: IMAGE=x86_64-gnu-cargotest
+    - env: IMAGE=x86_64-gnu-full-bootstrap
+    - env: IMAGE=x86_64-gnu-aux
     - env: IMAGE=x86_64-gnu-debug
     - env: IMAGE=x86_64-gnu-nopt
     - env: IMAGE=x86_64-gnu-make
@@ -66,10 +67,10 @@ script:
       if [ "$ALLOW_PR" = "" ] && [ "$TRAVIS_BRANCH" != "auto" ]; then
           echo skipping, not a full build;
       elif [ "$TRAVIS_OS_NAME" = "osx" ]; then
-          git submodule update --init;
+          git submodule update --init &&
           src/ci/run.sh;
       else
-          git submodule update --init;
+          git submodule update --init &&
           src/ci/docker/run.sh $IMAGE;
       fi
 
@@ -78,7 +79,7 @@ before_cache:
   - docker history -q rust-ci |
     grep -v missing |
     xargs docker save |
-    gzip -9 > $HOME/docker/rust-ci.tar.gz
+    gzip > $HOME/docker/rust-ci.tar.gz
 before_install:
   - zcat $HOME/docker/rust-ci.tar.gz | docker load || true
 

README.md

@@ -82,8 +82,9 @@ build.
    # Install build tools needed for Rust. If you're building a 32-bit compiler,
    # then replace "x86_64" below with "i686". If you've already got git, python,
    # or CMake installed and in PATH you can remove them from this list. Note
-   # that it is important that the `python2` and `cmake` packages **not** used.
-   # The build has historically been known to fail with these packages.
+   # that it is important that you do **not** use the 'python2' and 'cmake'
+   # packages from the 'msys2' subsystem. The build has historically been known
+   # to fail with these packages.
    $ pacman -S git \
                make \
                diffutils \

appveyor.yml

@@ -21,7 +21,7 @@ environment:
   # MSVC cargotest
   - MSYS_BITS: 64
     NO_VENDOR: 1
-    RUST_CHECK_TARGET: check-cargotest
+    RUST_CHECK_TARGET: check-aux
     RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc
 
   # 32/64-bit MinGW builds.

configure

@@ -693,6 +693,7 @@ opt_nosave manage-submodules 1 "let the build manage the git submodules"
 opt_nosave clang 0 "prefer clang to gcc for building the runtime"
 opt_nosave jemalloc 1 "build liballoc with jemalloc"
 opt elf-tls 1 "elf thread local storage on platforms where supported"
+opt full-bootstrap 0 "build three compilers instead of two"
 
 valopt_nosave prefix "/usr/local" "set installation prefix"
 valopt_nosave local-rust-root "/usr/local" "set prefix for local rust binary"

src/bootstrap/check.rs

@@ -341,12 +341,22 @@ pub fn krate(build: &Build,
     println!("{} {} stage{} ({} -> {})", test_kind, name, compiler.stage,
              compiler.host, target);
 
+    // If we're not doing a full bootstrap but we're testing a stage2 version of
+    // libstd, then what we're actually testing is the libstd produced in
+    // stage1. Reflect that here by updating the compiler that we're working
+    // with automatically.
+    let compiler = if build.force_use_stage1(compiler, target) {
+        Compiler::new(1, compiler.host)
+    } else {
+        compiler.clone()
+    };
+
     // Build up the base `cargo test` command.
     //
     // Pass in some standard flags then iterate over the graph we've discovered
     // in `cargo metadata` with the maps above and figure out what `-p`
     // arguments need to get passed.
-    let mut cargo = build.cargo(compiler, mode, target, test_kind.subcommand());
+    let mut cargo = build.cargo(&compiler, mode, target, test_kind.subcommand());
     cargo.arg("--manifest-path")
          .arg(build.src.join(path).join("Cargo.toml"))
          .arg("--features").arg(features);
@@ -380,7 +390,7 @@ pub fn krate(build: &Build,
     // Note that to run the compiler we need to run with the *host* libraries,
     // but our wrapper scripts arrange for that to be the case anyway.
     let mut dylib_path = dylib_path();
-    dylib_path.insert(0, build.sysroot_libdir(compiler, target));
+    dylib_path.insert(0, build.sysroot_libdir(&compiler, target));
     cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap());
 
     if target.contains("android") {
@@ -399,10 +409,10 @@ pub fn krate(build: &Build,
 
     if target.contains("android") {
         build.run(&mut cargo);
-        krate_android(build, compiler, target, mode);
+        krate_android(build, &compiler, target, mode);
     } else if target.contains("emscripten") {
         build.run(&mut cargo);
-        krate_emscripten(build, compiler, target, mode);
+        krate_emscripten(build, &compiler, target, mode);
     } else {
         cargo.args(&build.flags.cmd.test_args());
         build.run(&mut cargo);

src/bootstrap/compile.rs

@@ -33,17 +33,12 @@ use {Build, Compiler, Mode};
 /// This will build the standard library for a particular stage of the build
 /// using the `compiler` targeting the `target` architecture. The artifacts
 /// created will also be linked into the sysroot directory.
-pub fn std<'a>(build: &'a Build, target: &str, compiler: &Compiler<'a>) {
-    println!("Building stage{} std artifacts ({} -> {})", compiler.stage,
-             compiler.host, target);
-
+pub fn std(build: &Build, target: &str, compiler: &Compiler) {
     let libdir = build.sysroot_libdir(compiler, target);
-    let _ = fs::remove_dir_all(&libdir);
     t!(fs::create_dir_all(&libdir));
 
-    // Some platforms have startup objects that may be required to produce the
-    // libstd dynamic library, for example.
-    build_startup_objects(build, target, &libdir);
+    println!("Building stage{} std artifacts ({} -> {})", compiler.stage,
+             compiler.host, target);
 
     let out_dir = build.cargo_out(compiler, Mode::Libstd, target);
     build.clear_if_dirty(&out_dir, &build.compiler_path(compiler));
@@ -65,29 +60,30 @@ pub fn std<'a>(build: &'a Build, target: &str, compiler: &Compiler<'a>) {
 
     build.run(&mut cargo);
     update_mtime(&libstd_stamp(build, &compiler, target));
-    std_link(build, target, compiler.stage, compiler.host);
 }
 
 /// Link all libstd rlibs/dylibs into the sysroot location.
 ///
-/// Links those artifacts generated in the given `stage` for `target` produced
-/// by `compiler` into `host`'s sysroot.
+/// Links those artifacts generated by `compiler` to a the `stage` compiler's
+/// sysroot for the specified `host` and `target`.
+///
+/// Note that this assumes that `compiler` has already generated the libstd
+/// libraries for `target`, and this method will find them in the relevant
+/// output directory.
 pub fn std_link(build: &Build,
-                target: &str,
-                stage: u32,
-                host: &str) {
-    let compiler = Compiler::new(stage, &build.config.build);
-    let target_compiler = Compiler::new(compiler.stage, host);
+                compiler: &Compiler,
+                target_compiler: &Compiler,
+                target: &str) {
+    println!("Copying stage{} std from stage{} ({} -> {} / {})",
+             target_compiler.stage,
+             compiler.stage,
+             compiler.host,
+             target_compiler.host,
+             target);
     let libdir = build.sysroot_libdir(&target_compiler, target);
     let out_dir = build.cargo_out(&compiler, Mode::Libstd, target);
 
-    // If we're linking one compiler host's output into another, then we weren't
-    // called from the `std` method above. In that case we clean out what's
-    // already there.
-    if host != compiler.host {
-        let _ = fs::remove_dir_all(&libdir);
-        t!(fs::create_dir_all(&libdir));
-    }
+    t!(fs::create_dir_all(&libdir));
     add_to_sysroot(&out_dir, &libdir);
 
     if target.contains("musl") && !target.contains("mips") {
@@ -110,20 +106,23 @@ fn copy_musl_third_party_objects(build: &Build, target: &str, into: &Path) {
 /// They don't require any library support as they're just plain old object
 /// files, so we just use the nightly snapshot compiler to always build them (as
 /// no other compilers are guaranteed to be available).
-fn build_startup_objects(build: &Build, target: &str, into: &Path) {
+pub fn build_startup_objects(build: &Build, for_compiler: &Compiler, target: &str) {
     if !target.contains("pc-windows-gnu") {
         return
     }
+
     let compiler = Compiler::new(0, &build.config.build);
     let compiler_path = build.compiler_path(&compiler);
+    let into = build.sysroot_libdir(for_compiler, target);
+    t!(fs::create_dir_all(&into));
 
     for file in t!(fs::read_dir(build.src.join("src/rtstartup"))) {
         let file = t!(file);
         let mut cmd = Command::new(&compiler_path);
         build.run(cmd.env("RUSTC_BOOTSTRAP", "1")
                      .arg("--target").arg(target)
                      .arg("--emit=obj")
-                     .arg("--out-dir").arg(into)
+                     .arg("--out-dir").arg(&into)
                      .arg(file.path()));
     }
 
@@ -137,7 +136,7 @@ fn build_startup_objects(build: &Build, target: &str, into: &Path) {
 /// This will build libtest and supporting libraries for a particular stage of
 /// the build using the `compiler` targeting the `target` architecture. The
 /// artifacts created will also be linked into the sysroot directory.
-pub fn test<'a>(build: &'a Build, target: &str, compiler: &Compiler<'a>) {
+pub fn test(build: &Build, target: &str, compiler: &Compiler) {
     println!("Building stage{} test artifacts ({} -> {})", compiler.stage,
              compiler.host, target);
     let out_dir = build.cargo_out(compiler, Mode::Libtest, target);
@@ -147,19 +146,19 @@ pub fn test<'a>(build: &'a Build, target: &str, compiler: &Compiler<'a>) {
          .arg(build.src.join("src/rustc/test_shim/Cargo.toml"));
     build.run(&mut cargo);
     update_mtime(&libtest_stamp(build, compiler, target));
-    test_link(build, target, compiler.stage, compiler.host);
 }
 
-/// Link all libtest rlibs/dylibs into the sysroot location.
-///
-/// Links those artifacts generated in the given `stage` for `target` produced
-/// by `compiler` into `host`'s sysroot.
+/// Same as `std_link`, only for libtest
 pub fn test_link(build: &Build,
-                 target: &str,
-                 stage: u32,
-                 host: &str) {
-    let compiler = Compiler::new(stage, &build.config.build);
-    let target_compiler = Compiler::new(compiler.stage, host);
+                 compiler: &Compiler,
+                 target_compiler: &Compiler,
+                 target: &str) {
+    println!("Copying stage{} test from stage{} ({} -> {} / {})",
+             target_compiler.stage,
+             compiler.stage,
+             compiler.host,
+             target_compiler.host,
+             target);
     let libdir = build.sysroot_libdir(&target_compiler, target);
     let out_dir = build.cargo_out(&compiler, Mode::Libtest, target);
     add_to_sysroot(&out_dir, &libdir);
@@ -170,7 +169,7 @@ pub fn test_link(build: &Build,
 /// This will build the compiler for a particular stage of the build using
 /// the `compiler` targeting the `target` architecture. The artifacts
 /// created will also be linked into the sysroot directory.
-pub fn rustc<'a>(build: &'a Build, target: &str, compiler: &Compiler<'a>) {
+pub fn rustc(build: &Build, target: &str, compiler: &Compiler) {
     println!("Building stage{} compiler artifacts ({} -> {})",
              compiler.stage, compiler.host, target);
 
@@ -222,20 +221,19 @@ pub fn rustc<'a>(build: &'a Build, target: &str, compiler: &Compiler<'a>) {
         cargo.env("CFG_DEFAULT_AR", s);
     }
     build.run(&mut cargo);
-
-    rustc_link(build, target, compiler.stage, compiler.host);
 }
 
-/// Link all librustc rlibs/dylibs into the sysroot location.
-///
-/// Links those artifacts generated in the given `stage` for `target` produced
-/// by `compiler` into `host`'s sysroot.
+/// Same as `std_link`, only for librustc
 pub fn rustc_link(build: &Build,
-                  target: &str,
-                  stage: u32,
-                  host: &str) {
-    let compiler = Compiler::new(stage, &build.config.build);
-    let target_compiler = Compiler::new(compiler.stage, host);
+                  compiler: &Compiler,
+                  target_compiler: &Compiler,
+                  target: &str) {
+    println!("Copying stage{} rustc from stage{} ({} -> {} / {})",
+             target_compiler.stage,
+             compiler.stage,
+             compiler.host,
+             target_compiler.host,
+             target);
     let libdir = build.sysroot_libdir(&target_compiler, target);
     let out_dir = build.cargo_out(&compiler, Mode::Librustc, target);
     add_to_sysroot(&out_dir, &libdir);
@@ -259,6 +257,17 @@ fn compiler_file(compiler: &Path, file: &str) -> PathBuf {
     PathBuf::from(out.trim())
 }
 
+pub fn create_sysroot(build: &Build, compiler: &Compiler) {
+    // nothing to do in stage0
+    if compiler.stage == 0 {
+        return
+    }
+
+    let sysroot = build.sysroot(compiler);
+    let _ = fs::remove_dir_all(&sysroot);
+    t!(fs::create_dir_all(&sysroot));
+}
+
 /// Prepare a new compiler from the artifacts in `stage`
 ///
 /// This will assemble a compiler in `build/$host/stage$stage`. The compiler
@@ -269,18 +278,17 @@ pub fn assemble_rustc(build: &Build, stage: u32, host: &str) {
     if stage == 0 {
         return
     }
+
+    println!("Copying stage{} compiler ({})", stage, host);
+
     // The compiler that we're assembling
     let target_compiler = Compiler::new(stage, host);
 
     // The compiler that compiled the compiler we're assembling
     let build_compiler = Compiler::new(stage - 1, &build.config.build);
 
-    // Clear out old files
-    let sysroot = build.sysroot(&target_compiler);
-    let _ = fs::remove_dir_all(&sysroot);
-    t!(fs::create_dir_all(&sysroot));
-
     // Link in all dylibs to the libdir
+    let sysroot = build.sysroot(&target_compiler);
     let sysroot_libdir = sysroot.join(libdir(host));
     t!(fs::create_dir_all(&sysroot_libdir));
     let src_libdir = build.sysroot_libdir(&build_compiler, host);

src/bootstrap/config.rs

@@ -46,6 +46,7 @@ pub struct Config {
     pub docs: bool,
     pub vendor: bool,
     pub target_config: HashMap<String, Target>,
+    pub full_bootstrap: bool,
 
     // llvm codegen options
     pub llvm_assertions: bool,
@@ -54,6 +55,7 @@ pub struct Config {
     pub llvm_version_check: bool,
     pub llvm_static_stdcpp: bool,
     pub llvm_link_shared: bool,
+    pub llvm_targets: Option<String>,
 
     // rust codegen options
     pub rust_optimize: bool,
@@ -134,6 +136,7 @@ struct Build {
     vendor: Option<bool>,
     nodejs: Option<String>,
     python: Option<String>,
+    full_bootstrap: Option<bool>,
 }
 
 /// TOML representation of various global install decisions.
@@ -152,6 +155,7 @@ struct Llvm {
     release_debuginfo: Option<bool>,
     version_check: Option<bool>,
     static_libstdcpp: Option<bool>,
+    targets: Option<String>,
 }
 
 #[derive(RustcDecodable)]
@@ -264,6 +268,7 @@ impl Config {
         set(&mut config.docs, build.docs);
         set(&mut config.submodules, build.submodules);
         set(&mut config.vendor, build.vendor);
+        set(&mut config.full_bootstrap, build.full_bootstrap);
 
         if let Some(ref install) = toml.install {
             config.prefix = install.prefix.clone();
@@ -285,6 +290,7 @@ impl Config {
             set(&mut config.llvm_release_debuginfo, llvm.release_debuginfo);
             set(&mut config.llvm_version_check, llvm.version_check);
             set(&mut config.llvm_static_stdcpp, llvm.static_libstdcpp);
+            config.llvm_targets = llvm.targets.clone();
         }
 
         if let Some(ref rust) = toml.rust {
@@ -393,6 +399,7 @@ impl Config {
                 ("NINJA", self.ninja),
                 ("CODEGEN_TESTS", self.codegen_tests),
                 ("VENDOR", self.vendor),
+                ("FULL_BOOTSTRAP", self.full_bootstrap),
             }
 
             match key {