Auto merge of #37269 - eddyb:rollup, r=eddyb

Rollup of 23 pull requests

- Successful merges: #36964, #37108, #37117, #37124, #37161, #37176, #37182, #37193, #37198, #37202, #37208, #37218, #37221, #37224, #37230, #37231, #37233, #37236, #37240, #37254, #37257, #37265, #37267
- Failed merges: #37213, #37220, #37261

appveyor.yml

@@ -0,0 +1,110 @@
+environment:
+  matrix:
+  # 32/64 bit MSVC
+  - MSYS_BITS: 64
+    TARGET: x86_64-pc-windows-msvc
+    CHECK: check
+    CONFIGURE_ARGS: --enable-llvm-assertions --enable-debug-assertions
+  - MSYS_BITS: 32
+    TARGET: i686-pc-windows-msvc
+    CHECK: check
+    CONFIGURE_ARGS: --enable-llvm-assertions --enable-debug-assertions
+
+  # MSVC rustbuild
+  - MSYS_BITS: 64
+    CONFIGURE_ARGS: --enable-rustbuild --enable-llvm-assertions --enable-debug-assertions
+    TARGET: x86_64-pc-windows-msvc
+    CHECK: check
+
+  # MSVC cargotest
+  - MSYS_BITS: 64
+    CONFIGURE_ARGS: --enable-rustbuild --enable-llvm-assertions --enable-debug-assertions
+    TARGET: x86_64-pc-windows-msvc
+    CHECK: check-cargotest
+
+  # 32/64-bit MinGW builds.
+  #
+  # The MinGW builds unfortunately have to both download a custom toolchain and
+  # avoid the one installed by AppVeyor by default. Interestingly, though, for
+  # different reasons!
+  #
+  # For 32-bit the installed gcc toolchain on AppVeyor uses the pthread
+  # threading model. This is unfortunately not what we want, and if we compile
+  # with it then there's lots of link errors in the standard library (undefined
+  # references to pthread symbols).
+  #
+  # For 64-bit the installed gcc toolchain is currently 5.3.0 which
+  # unfortunately segfaults on Windows with --enable-llvm-assertions (segfaults
+  # in LLVM). See rust-lang/rust#28445 for more information, but to work around
+  # this we go back in time to 4.9.2 specifically.
+  #
+  # Finally, note that the downloads below are all in the `rust-lang-ci` S3
+  # bucket, but they cleraly didn't originate there! The downloads originally
+  # came from the mingw-w64 SourceForge download site. Unfortunately
+  # SourceForge is notoriously flaky, so we mirror it on our own infrastructure.
+  #
+  # And as a final point of note, the 32-bit MinGW build using the makefiles do
+  # *not* use debug assertions and llvm assertions. This is because they take
+  # too long on appveyor and this is tested by rustbuild below.
+  - MSYS_BITS: 32
+    TARGET: i686-pc-windows-gnu
+    CHECK: check
+    MINGW_URL: https://s3.amazonaws.com/rust-lang-ci
+    MINGW_ARCHIVE: i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z
+    MINGW_DIR: mingw32
+
+  - MSYS_BITS: 32
+    CONFIGURE_ARGS: --enable-rustbuild --enable-llvm-assertions --enable-debug-assertions
+    TARGET: i686-pc-windows-gnu
+    CHECK: check
+    MINGW_URL: https://s3.amazonaws.com/rust-lang-ci
+    MINGW_ARCHIVE: i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z
+    MINGW_DIR: mingw32
+
+  - MSYS_BITS: 64
+    CONFIGURE_ARGS: --enable-llvm-assertions --enable-debug-assertions
+    TARGET: x86_64-pc-windows-gnu
+    CHECK: check
+    MINGW_URL: https://s3.amazonaws.com/rust-lang-ci
+    MINGW_ARCHIVE: x86_64-4.9.2-release-win32-seh-rt_v4-rev4.7z
+    MINGW_DIR: mingw64
+
+clone_depth: 1
+build: false
+
+install:
+  # If we need to download a custom MinGW, do so here and set the path
+  # appropriately.
+  #
+  # Note that this *also* means that we're not using what is typically
+  # /mingw32/bin/python2.7.exe, which is a "correct" python interpreter where
+  # /usr/bin/python2.7.exe is not. To ensure we use the right interpreter we
+  # move `C:\Python27` ahead in PATH and then also make sure the `python2.7.exe`
+  # file exists in there (which it doesn't by default).
+  - if defined MINGW_URL appveyor DownloadFile %MINGW_URL%/%MINGW_ARCHIVE%
+  - if defined MINGW_URL 7z x -y %MINGW_ARCHIVE% > nul
+  - if defined MINGW_URL set PATH=C:\Python27;%CD%\%MINGW_DIR%\bin;C:\msys64\usr\bin;%PATH%
+  - if defined MINGW_URL copy C:\Python27\python.exe C:\Python27\python2.7.exe
+
+  # Otherwise pull in the MinGW installed on appveyor
+  - if NOT defined MINGW_URL set PATH=C:\msys64\mingw%MSYS_BITS%\bin;C:\msys64\usr\bin;%PATH%
+
+test_script:
+  - sh ./configure
+          %CONFIGURE_ARGS%
+          --build=%TARGET%
+  - bash -c "make -j$(nproc)"
+  - bash -c "make %CHECK% -j$(nproc)"
+
+cache:
+  - build/%TARGET%/llvm -> src/rustllvm/llvm-auto-clean-trigger
+  - "%TARGET%/llvm -> src/rustllvm/llvm-auto-clean-trigger"
+
+branches:
+  only:
+    - auto
+
+# init:
+#   - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
+# on_finish:
+#   - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))

mk/llvm.mk

@@ -36,22 +36,27 @@ endif
 # If CFG_LLVM_ROOT is defined then we don't build LLVM ourselves
 ifeq ($(CFG_LLVM_ROOT),)
 
-LLVM_STAMP_$(1) = $$(CFG_LLVM_BUILD_DIR_$(1))/llvm-auto-clean-stamp
+LLVM_STAMP_$(1) = $(S)src/rustllvm/llvm-auto-clean-trigger
 LLVM_DONE_$(1) = $$(CFG_LLVM_BUILD_DIR_$(1))/llvm-finished-building
 
 $$(LLVM_CONFIG_$(1)): $$(LLVM_DONE_$(1))
 
-$$(LLVM_DONE_$(1)): $$(LLVM_DEPS_TARGET_$(1)) $$(LLVM_STAMP_$(1))
-	@$$(call E, cmake: llvm)
 ifneq ($$(CFG_NINJA),)
-	$$(Q)$$(CFG_NINJA) -C $$(CFG_LLVM_BUILD_DIR_$(1))
+BUILD_LLVM_$(1) := $$(CFG_NINJA) -C $$(CFG_LLVM_BUILD_DIR_$(1))
 else ifeq ($$(findstring msvc,$(1)),msvc)
-	$$(Q)$$(CFG_CMAKE) --build $$(CFG_LLVM_BUILD_DIR_$(1)) \
-		--config $$(LLVM_BUILD_CONFIG_MODE)
+BUILD_LLVM_$(1) := $$(CFG_CMAKE) --build $$(CFG_LLVM_BUILD_DIR_$(1)) \
+			--config $$(LLVM_BUILD_CONFIG_MODE)
 else
-	$$(Q)$$(MAKE) -C $$(CFG_LLVM_BUILD_DIR_$(1))
+BUILD_LLVM_$(1) := $$(MAKE) -C $$(CFG_LLVM_BUILD_DIR_$(1))
 endif
-	$$(Q)touch $$@
+
+$$(LLVM_DONE_$(1)): $$(LLVM_DEPS_TARGET_$(1)) $$(LLVM_STAMP_$(1))
+	@$$(call E, cmake: llvm)
+	$$(Q)if ! cmp $$(LLVM_STAMP_$(1)) $$(LLVM_DONE_$(1)); then \
+		$$(MAKE) clean-llvm$(1); \
+		$$(BUILD_LLVM_$(1)); \
+	fi
+	$$(Q)cp $$(LLVM_STAMP_$(1)) $$@
 
 ifneq ($$(CFG_NINJA),)
 clean-llvm$(1):
@@ -75,17 +80,6 @@ endif
 
 $$(LLVM_AR_$(1)): $$(LLVM_CONFIG_$(1))
 
-# This is used to independently force an LLVM clean rebuild
-# when we changed something not otherwise captured by builtin
-# dependencies. In these cases, commit a change that touches
-# the stamp in the source dir.
-$$(LLVM_STAMP_$(1)): $$(S)src/rustllvm/llvm-auto-clean-trigger
-	@$$(call E, make: cleaning llvm)
-	$$(Q)touch $$@.start_time
-	$$(Q)$$(MAKE) clean-llvm$(1)
-	@$$(call E, make: done cleaning llvm)
-	touch -r $$@.start_time $$@ && rm $$@.start_time
-
 ifeq ($$(CFG_ENABLE_LLVM_STATIC_STDCPP),1)
 LLVM_STDCPP_RUSTFLAGS_$(1) = -L "$$(dir $$(shell $$(CC_$(1)) $$(CFG_GCCISH_CFLAGS_$(1)) \
 					-print-file-name=lib$(CFG_STDCPP_NAME).a))"

mk/main.mk

@@ -53,17 +53,6 @@ endif
 # versions in the same place
 CFG_FILENAME_EXTRA=$(shell printf '%s' $(CFG_RELEASE)$(CFG_EXTRA_FILENAME) | $(CFG_HASH_COMMAND))
 
-# A magic value that allows the compiler to use unstable features during the
-# bootstrap even when doing so would normally be an error because of feature
-# staging or because the build turns on warnings-as-errors and unstable features
-# default to warnings. The build has to match this key in an env var.
-#
-# This value is keyed off the release to ensure that all compilers for one
-# particular release have the same bootstrap key. Note that this is
-# intentionally not "secure" by any definition, this is largely just a deterrent
-# from users enabling unstable features on the stable compiler.
-CFG_BOOTSTRAP_KEY=$(CFG_FILENAME_EXTRA)
-
 # If local-rust is the same as the current version, then force a local-rebuild
 ifdef CFG_ENABLE_LOCAL_RUST
 ifeq ($(CFG_RELEASE),\
@@ -73,14 +62,6 @@ ifeq ($(CFG_RELEASE),\
 endif
 endif
 
-# The stage0 compiler needs to use the previous key recorded in src/stage0.txt,
-# except for local-rebuild when it just uses the same current key.
-ifdef CFG_ENABLE_LOCAL_REBUILD
-CFG_BOOTSTRAP_KEY_STAGE0=$(CFG_BOOTSTRAP_KEY)
-else
-CFG_BOOTSTRAP_KEY_STAGE0=$(shell sed -ne 's/^rustc_key: //p' $(S)src/stage0.txt)
-endif
-
 # The name of the package to use for creating tarballs, installers etc.
 CFG_PACKAGE_NAME=rustc-$(CFG_PACKAGE_VERS)
 
@@ -387,13 +368,16 @@ CFG_INFO := $(info cfg: disabling unstable features (CFG_DISABLE_UNSTABLE_FEATUR
 # Turn on feature-staging
 export CFG_DISABLE_UNSTABLE_FEATURES
 # Subvert unstable feature lints to do the self-build
-export RUSTC_BOOTSTRAP_KEY:=$(CFG_BOOTSTRAP_KEY)
+export RUSTC_BOOTSTRAP
 endif
-export CFG_BOOTSTRAP_KEY
 ifdef CFG_MUSL_ROOT
 export CFG_MUSL_ROOT
 endif
 
+# FIXME: Transitionary measure to bootstrap using the old bootstrap logic.
+# Remove this once the bootstrap compiler uses the new login in Issue #36548.
+export RUSTC_BOOTSTRAP_KEY=62b3e239
+
 ######################################################################
 # Per-stage targets and runner
 ######################################################################

mk/target.mk

@@ -42,23 +42,6 @@ $(foreach host,$(CFG_HOST), \
    $(foreach crate,$(CRATES), \
     $(eval $(call RUST_CRATE_FULLDEPS,$(stage),$(target),$(host),$(crate)))))))
 
-# $(1) stage
-# $(2) target
-# $(3) host
-define DEFINE_BOOTSTRAP_KEY
-BOOTSTRAP_KEY$(1)_T_$(2)_H_$(3) := $$(CFG_BOOTSTRAP_KEY)
-ifeq ($(1),0)
-ifeq ($(3),$$(CFG_BUILD))
-BOOTSTRAP_KEY$(1)_T_$(2)_H_$(3) := $$(CFG_BOOTSTRAP_KEY_STAGE0)
-endif
-endif
-endef
-
-$(foreach host,$(CFG_TARGET), \
- $(foreach target,$(CFG_TARGET), \
-  $(foreach stage,$(STAGES), \
-   $(eval $(call DEFINE_BOOTSTRAP_KEY,$(stage),$(target),$(host))))))
-
 # RUST_TARGET_STAGE_N template: This defines how target artifacts are built
 # for all stage/target architecture combinations. This is one giant rule which
 # works as follows:
@@ -83,8 +66,6 @@ $(foreach host,$(CFG_TARGET), \
 define RUST_TARGET_STAGE_N
 
 $$(TLIB$(1)_T_$(2)_H_$(3))/stamp.$(4): CFG_COMPILER_HOST_TRIPLE = $(2)
-$$(TLIB$(1)_T_$(2)_H_$(3))/stamp.$(4): \
-	export RUSTC_BOOTSTRAP_KEY := $$(BOOTSTRAP_KEY$(1)_T_$(2)_H_$(3))
 $$(TLIB$(1)_T_$(2)_H_$(3))/stamp.$(4): \
 		$$(CRATEFILE_$(4)) \
 		$$(CRATE_FULLDEPS_$(1)_T_$(2)_H_$(3)_$(4)) \
@@ -132,8 +113,6 @@ endef
 # $(4) - name of the tool being built
 define TARGET_TOOL
 
-$$(TBIN$(1)_T_$(2)_H_$(3))/$(4)$$(X_$(2)): \
-	export RUSTC_BOOTSTRAP_KEY := $$(BOOTSTRAP_KEY$(1)_T_$(2)_H_$(3))
 $$(TBIN$(1)_T_$(2)_H_$(3))/$(4)$$(X_$(2)): \
 		$$(TOOL_SOURCE_$(4)) \
 		$$(TOOL_INPUTS_$(4)) \

src/bootstrap/check.rs

@@ -214,7 +214,7 @@ pub fn compiletest(build: &Build,
             }
         }
     }
-    build.add_bootstrap_key(compiler, &mut cmd);
+    build.add_bootstrap_key(&mut cmd);
 
     cmd.arg("--adb-path").arg("adb");
     cmd.arg("--adb-test-dir").arg(ADB_TEST_DIR);

src/bootstrap/compile.rs

@@ -119,7 +119,7 @@ fn build_startup_objects(build: &Build, target: &str, into: &Path) {
     for file in t!(fs::read_dir(build.src.join("src/rtstartup"))) {
         let file = t!(file);
         let mut cmd = Command::new(&compiler_path);
-        build.add_bootstrap_key(&compiler, &mut cmd);
+        build.add_bootstrap_key(&mut cmd);
         build.run(cmd.arg("--target").arg(target)
                      .arg("--emit=obj")
                      .arg("--out-dir").arg(into)
@@ -185,7 +185,6 @@ pub fn rustc<'a>(build: &'a Build, target: &str, compiler: &Compiler<'a>) {
     cargo.env("CFG_RELEASE", &build.release)
          .env("CFG_RELEASE_CHANNEL", &build.config.channel)
          .env("CFG_VERSION", &build.version)
-         .env("CFG_BOOTSTRAP_KEY", &build.bootstrap_key)
          .env("CFG_PREFIX", build.config.prefix.clone().unwrap_or(String::new()))
          .env("CFG_LIBDIR_RELATIVE", "lib");
 

src/bootstrap/config.rs

@@ -120,6 +120,7 @@ struct Build {
     rustc: Option<String>,
     compiler_docs: Option<bool>,
     docs: Option<bool>,
+    submodules: Option<bool>,
 }
 
 /// TOML representation of how the LLVM build is configured.
@@ -225,6 +226,7 @@ impl Config {
         config.cargo = build.cargo.map(PathBuf::from);
         set(&mut config.compiler_docs, build.compiler_docs);
         set(&mut config.docs, build.docs);
+        set(&mut config.submodules, build.submodules);
 
         if let Some(ref llvm) = toml.llvm {
             set(&mut config.ccache, llvm.ccache);

src/bootstrap/config.toml.example

@@ -76,6 +76,9 @@
 # library and facade crates.
 #compiler-docs = false
 
+# Indicate whether submodules are managed and updated automatically.
+#submodules = true
+
 # =============================================================================
 # Options for compiling Rust code itself
 # =============================================================================

src/bootstrap/lib.rs

@@ -662,7 +662,7 @@ impl Build {
              .env("RUSTDOC_REAL", self.rustdoc(compiler))
              .env("RUSTC_FLAGS", self.rustc_flags(target).join(" "));
 
-        self.add_bootstrap_key(compiler, &mut cargo);
+        self.add_bootstrap_key(&mut cargo);
 
         // Specify some various options for build scripts used throughout
         // the build.
@@ -871,16 +871,11 @@ impl Build {
     }
 
     /// Adds the compiler's bootstrap key to the environment of `cmd`.
-    fn add_bootstrap_key(&self, compiler: &Compiler, cmd: &mut Command) {
-        // In stage0 we're using a previously released stable compiler, so we
-        // use the stage0 bootstrap key. Otherwise we use our own build's
-        // bootstrap key.
-        let bootstrap_key = if compiler.is_snapshot(self) && !self.local_rebuild {
-            &self.bootstrap_key_stage0
-        } else {
-            &self.bootstrap_key
-        };
-        cmd.env("RUSTC_BOOTSTRAP_KEY", bootstrap_key);
+    fn add_bootstrap_key(&self, cmd: &mut Command) {
+        cmd.env("RUSTC_BOOTSTRAP", "");
+        // FIXME: Transitionary measure to bootstrap using the old bootstrap logic.
+        // Remove this once the bootstrap compiler uses the new login in Issue #36548.
+        cmd.env("RUSTC_BOOTSTRAP_KEY", "62b3e239");
     }
 
     /// Returns the compiler's libdir where it stores the dynamic libraries that