Skip to content

Commit

Permalink
Auto merge of #58897 - Mark-Simulacrum:tool-rework, r=alexcrichton
Browse files Browse the repository at this point in the history 
Rework how bootstrap tools are built

This makes bootstrap tools buildable and testable in stage 0 with the downloaded bootstrap compiler, futhermore, it makes it such that they cannot be built in any other stage.

Notably, this will also mean that compiletest may need to wait a cycle before it can use changes to `libtest`, as it no longer depends on the in-tree libtest.
  • Loading branch information
@bors
bors committed Mar 20, 2019
2 parents 82e2f3e + 03718ed commit 33b3b13
Show file tree
Hide file tree
Showing 5 changed files with 99 additions and 62 deletions.
15 changes: 7 additions & 8 deletions src/bootstrap/builder.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -670,28 +670,27 @@ impl<'a> Builder<'a> {
.map(|entry| entry.path())
}

pub fn rustdoc(&self, host: Interned<String>) -> PathBuf {
self.ensure(tool::Rustdoc { host })
pub fn rustdoc(&self, compiler: Compiler) -> PathBuf {
self.ensure(tool::Rustdoc { compiler })
}

pub fn rustdoc_cmd(&self, host: Interned<String>) -> Command {
pub fn rustdoc_cmd(&self, compiler: Compiler) -> Command {
let mut cmd = Command::new(&self.out.join("bootstrap/debug/rustdoc"));
let compiler = self.compiler(self.top_stage, host);
cmd.env("RUSTC_STAGE", compiler.stage.to_string())
.env("RUSTC_SYSROOT", self.sysroot(compiler))
// Note that this is *not* the sysroot_libdir because rustdoc must be linked
// equivalently to rustc.
.env("RUSTDOC_LIBDIR", self.rustc_libdir(compiler))
.env("CFG_RELEASE_CHANNEL", &self.config.channel)
.env("RUSTDOC_REAL", self.rustdoc(host))
.env("RUSTDOC_REAL", self.rustdoc(compiler))
.env("RUSTDOC_CRATE_VERSION", self.rust_version())
.env("RUSTC_BOOTSTRAP", "1");

// Remove make-related flags that can cause jobserver problems.
cmd.env_remove("MAKEFLAGS");
cmd.env_remove("MFLAGS");

if let Some(linker) = self.linker(host) {
if let Some(linker) = self.linker(compiler.host) {
cmd.env("RUSTC_TARGET_LINKER", linker);
}
cmd
Expand Down Expand Up @@ -753,7 +752,7 @@ impl<'a> Builder<'a> {
// This is the intended out directory for compiler documentation.
my_out = self.compiler_doc_out(target);
}
let rustdoc = self.rustdoc(compiler.host);
let rustdoc = self.rustdoc(compiler);
self.clear_if_dirty(&my_out, &rustdoc);
} else if cmd != "test" {
match mode {
Expand Down Expand Up @@ -910,7 +909,7 @@ impl<'a> Builder<'a> {
.env(
"RUSTDOC_REAL",
if cmd == "doc" || cmd == "rustdoc" || (cmd == "test" && want_rustdoc) {
self.rustdoc(compiler.host)
self.rustdoc(compiler)
} else {
PathBuf::from("/path/to/nowhere/rustdoc/not/required")
},
Expand Down
2 changes: 1 addition & 1 deletion src/bootstrap/dist.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -479,7 +479,7 @@ impl Step for Rustc {
t!(fs::create_dir_all(image.join("bin")));
builder.cp_r(&src.join("bin"), &image.join("bin"));

builder.install(&builder.rustdoc(compiler.host), &image.join("bin"), 0o755);
builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);

// Copy runtime DLLs needed by the compiler
if libdir != "bin" {
Expand Down
14 changes: 9 additions & 5 deletions src/bootstrap/doc.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -335,7 +335,7 @@ fn invoke_rustdoc(
let footer = builder.src.join("src/doc/footer.inc");
let version_info = out.join("version_info.html");

let mut cmd = builder.rustdoc_cmd(compiler.host);
let mut cmd = builder.rustdoc_cmd(compiler);

let out = out.join("book");

Expand Down Expand Up @@ -415,7 +415,7 @@ impl Step for Standalone {
}

let html = out.join(filename).with_extension("html");
let rustdoc = builder.rustdoc(compiler.host);
let rustdoc = builder.rustdoc(compiler);
if up_to_date(&path, &html) &&
up_to_date(&footer, &html) &&
up_to_date(&favicon, &html) &&
Expand All @@ -425,7 +425,7 @@ impl Step for Standalone {
continue
}

let mut cmd = builder.rustdoc_cmd(compiler.host);
let mut cmd = builder.rustdoc_cmd(compiler);
cmd.arg("--html-after-content").arg(&footer)
.arg("--html-before-content").arg(&version_info)
.arg("--html-in-header").arg(&favicon)
Expand Down Expand Up @@ -824,7 +824,7 @@ impl Step for Rustdoc {
builder.ensure(Rustc { stage, target });

// Build rustdoc.
builder.ensure(tool::Rustdoc { host: compiler.host });
builder.ensure(tool::Rustdoc { compiler: compiler });

// Symlink compiler docs to the output directory of rustdoc documentation.
let out_dir = builder.stage_out(compiler, Mode::ToolRustc)
Expand Down Expand Up @@ -883,7 +883,11 @@ impl Step for ErrorIndex {
builder.info(&format!("Documenting error index ({})", target));
let out = builder.doc_out(target);
t!(fs::create_dir_all(&out));
let mut index = builder.tool_cmd(Tool::ErrorIndex);
let compiler = builder.compiler(2, builder.config.build);
let mut index = tool::ErrorIndex::command(
builder,
compiler,
);
index.arg("html");
index.arg(out.join("error-index.html"));

Expand Down
18 changes: 9 additions & 9 deletions src/bootstrap/test.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ impl Step for Cargotest {
cmd.arg(&builder.initial_cargo)
.arg(&out_dir)
.env("RUSTC", builder.rustc(compiler))
.env("RUSTDOC", builder.rustdoc(compiler.host)),
.env("RUSTDOC", builder.rustdoc(compiler)),
);
}
}
Expand Down Expand Up @@ -414,7 +414,6 @@ impl Step for Miri {

#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
pub struct CompiletestTest {
stage: u32,
host: Interned<String>,
}

Expand All @@ -427,16 +426,14 @@ impl Step for CompiletestTest {

fn make_run(run: RunConfig<'_>) {
run.builder.ensure(CompiletestTest {
stage: run.builder.top_stage,
host: run.target,
});
}

/// Runs `cargo test` for compiletest.
fn run(self, builder: &Builder<'_>) {
let stage = self.stage;
let host = self.host;
let compiler = builder.compiler(stage, host);
let compiler = builder.compiler(0, host);

let mut cargo = tool::prepare_tool_cargo(builder,
compiler,
Expand Down Expand Up @@ -563,7 +560,7 @@ impl Step for RustdocTheme {
builder.sysroot_libdir(self.compiler, self.compiler.host),
)
.env("CFG_RELEASE_CHANNEL", &builder.config.channel)
.env("RUSTDOC_REAL", builder.rustdoc(self.compiler.host))
.env("RUSTDOC_REAL", builder.rustdoc(self.compiler))
.env("RUSTDOC_CRATE_VERSION", builder.rust_version())
.env("RUSTC_BOOTSTRAP", "1");
if let Some(linker) = builder.linker(self.compiler.host) {
Expand Down Expand Up @@ -1042,7 +1039,7 @@ impl Step for Compiletest {
|| mode == "js-doc-test"
{
cmd.arg("--rustdoc-path")
.arg(builder.rustdoc(compiler.host));
.arg(builder.rustdoc(compiler));
}

cmd.arg("--src-base")
Expand Down Expand Up @@ -1464,7 +1461,10 @@ impl Step for ErrorIndex {
t!(fs::create_dir_all(&dir));
let output = dir.join("error-index.md");

let mut tool = builder.tool_cmd(Tool::ErrorIndex);
let mut tool = tool::ErrorIndex::command(
builder,
builder.compiler(compiler.stage, builder.config.build),
);
tool.arg("markdown")
.arg(&output)
.env("CFG_BUILD", &builder.config.build)
Expand All @@ -1489,7 +1489,7 @@ fn markdown_test(builder: &Builder<'_>, compiler: Compiler, markdown: &Path) ->
}

builder.info(&format!("doc tests for: {}", markdown.display()));
let mut cmd = builder.rustdoc_cmd(compiler.host);
let mut cmd = builder.rustdoc_cmd(compiler);
builder.add_rust_test_threads(&mut cmd);
cmd.arg("--test");
cmd.arg(markdown);
Expand Down
112 changes: 73 additions & 39 deletions src/bootstrap/tool.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -251,9 +251,9 @@ pub fn prepare_tool_cargo(
cargo
}

macro_rules! tool {
macro_rules! bootstrap_tool {
($(
$name:ident, $path:expr, $tool_name:expr, $mode:expr
$name:ident, $path:expr, $tool_name:expr
$(,llvm_tools = $llvm:expr)*
$(,is_external_tool = $external:expr)*
;
Expand All @@ -267,10 +267,7 @@ macro_rules! tool {

impl Tool {
pub fn get_mode(&self) -> Mode {
let mode = match self {
$(Tool::$name => $mode,)+
};
mode
Mode::ToolBootstrap
}

/// Whether this tool requires LLVM to run
Expand All @@ -283,27 +280,15 @@ macro_rules! tool {

impl<'a> Builder<'a> {
pub fn tool_exe(&self, tool: Tool) -> PathBuf {
let stage = self.tool_default_stage(tool);
match tool {
$(Tool::$name =>
self.ensure($name {
compiler: self.compiler(stage, self.config.build),
compiler: self.compiler(0, self.config.build),
target: self.config.build,
}),
)+
}
}

pub fn tool_default_stage(&self, tool: Tool) -> u32 {
// Compile the error-index in the same stage as rustdoc to avoid
// recompiling rustdoc twice if we can. Otherwise compile
// everything else in stage0 as there's no need to rebootstrap
// everything.
match tool {
Tool::ErrorIndex if self.top_stage >= 2 => self.top_stage,
_ => 0,
}
}
}

$(
Expand All @@ -322,7 +307,8 @@ macro_rules! tool {

fn make_run(run: RunConfig<'_>) {
run.builder.ensure($name {
compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
// snapshot compiler
compiler: run.builder.compiler(0, run.builder.config.build),
target: run.target,
});
}
Expand All @@ -332,7 +318,7 @@ macro_rules! tool {
compiler: self.compiler,
target: self.target,
tool: $tool_name,
mode: $mode,
mode: Mode::ToolBootstrap,
path: $path,
is_optional_tool: false,
source_type: if false $(|| $external)* {
Expand All @@ -348,21 +334,67 @@ macro_rules! tool {
}
}

tool!(
Rustbook, "src/tools/rustbook", "rustbook", Mode::ToolBootstrap;
ErrorIndex, "src/tools/error_index_generator", "error_index_generator", Mode::ToolRustc;
UnstableBookGen, "src/tools/unstable-book-gen", "unstable-book-gen", Mode::ToolBootstrap;
Tidy, "src/tools/tidy", "tidy", Mode::ToolBootstrap;
Linkchecker, "src/tools/linkchecker", "linkchecker", Mode::ToolBootstrap;
CargoTest, "src/tools/cargotest", "cargotest", Mode::ToolBootstrap;
Compiletest, "src/tools/compiletest", "compiletest", Mode::ToolBootstrap, llvm_tools = true;
BuildManifest, "src/tools/build-manifest", "build-manifest", Mode::ToolBootstrap;
RemoteTestClient, "src/tools/remote-test-client", "remote-test-client", Mode::ToolBootstrap;
RustInstaller, "src/tools/rust-installer", "fabricate", Mode::ToolBootstrap,
is_external_tool = true;
RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes", Mode::ToolBootstrap;
bootstrap_tool!(
Rustbook, "src/tools/rustbook", "rustbook";
UnstableBookGen, "src/tools/unstable-book-gen", "unstable-book-gen";
Tidy, "src/tools/tidy", "tidy";
Linkchecker, "src/tools/linkchecker", "linkchecker";
CargoTest, "src/tools/cargotest", "cargotest";
Compiletest, "src/tools/compiletest", "compiletest", llvm_tools = true;
BuildManifest, "src/tools/build-manifest", "build-manifest";
RemoteTestClient, "src/tools/remote-test-client", "remote-test-client";
RustInstaller, "src/tools/rust-installer", "fabricate", is_external_tool = true;
RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes";
);

#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
pub struct ErrorIndex {
pub compiler: Compiler,
}

impl ErrorIndex {
pub fn command(builder: &Builder<'_>, compiler: Compiler) -> Command {
let mut cmd = Command::new(builder.ensure(ErrorIndex {
compiler
}));
add_lib_path(
vec![PathBuf::from(&builder.sysroot_libdir(compiler, compiler.host))],
&mut cmd,
);
cmd
}
}

impl Step for ErrorIndex {
type Output = PathBuf;

fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.path("src/tools/error_index_generator")
}

fn make_run(run: RunConfig<'_>) {
// Compile the error-index in the same stage as rustdoc to avoid
// recompiling rustdoc twice if we can.
let stage = if run.builder.top_stage >= 2 { run.builder.top_stage } else { 0 };
run.builder.ensure(ErrorIndex {
compiler: run.builder.compiler(stage, run.builder.config.build),
});
}

fn run(self, builder: &Builder<'_>) -> PathBuf {
builder.ensure(ToolBuild {
compiler: self.compiler,
target: self.compiler.host,
tool: "error_index_generator",
mode: Mode::ToolRustc,
path: "src/tools/error_index_generator",
is_optional_tool: false,
source_type: SourceType::InTree,
extra_features: Vec::new(),
}).expect("expected to build -- essential tool")
}
}

#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
pub struct RemoteTestServer {
pub compiler: Compiler,
Expand Down Expand Up @@ -399,7 +431,9 @@ impl Step for RemoteTestServer {

#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
pub struct Rustdoc {
pub host: Interned<String>,
/// This should only ever be 0 or 2.
/// We sometimes want to reference the "bootstrap" rustdoc, which is why this option is here.
pub compiler: Compiler,
}

impl Step for Rustdoc {
Expand All @@ -413,12 +447,12 @@ impl Step for Rustdoc {

fn make_run(run: RunConfig<'_>) {
run.builder.ensure(Rustdoc {
host: run.host,
compiler: run.builder.compiler(run.builder.top_stage, run.host),
});
}

fn run(self, builder: &Builder<'_>) -> PathBuf {
let target_compiler = builder.compiler(builder.top_stage, self.host);
let target_compiler = self.compiler;
if target_compiler.stage == 0 {
if !target_compiler.is_snapshot(builder) {
panic!("rustdoc in stage 0 must be snapshot rustdoc");
Expand Down Expand Up @@ -626,7 +660,7 @@ impl<'a> Builder<'a> {
/// `host`.
pub fn tool_cmd(&self, tool: Tool) -> Command {
let mut cmd = Command::new(self.tool_exe(tool));
let compiler = self.compiler(self.tool_default_stage(tool), self.config.build);
let compiler = self.compiler(0, self.config.build);
self.prepare_tool_cmd(compiler, tool, &mut cmd);
cmd
}
Expand All @@ -638,7 +672,7 @@ impl<'a> Builder<'a> {
fn prepare_tool_cmd(&self, compiler: Compiler, tool: Tool, cmd: &mut Command) {
let host = &compiler.host;
let mut lib_paths: Vec<PathBuf> = vec![
if compiler.stage == 0 && tool != Tool::ErrorIndex {
if compiler.stage == 0 {
self.build.rustc_snapshot_libdir()
} else {
PathBuf::from(&self.sysroot_libdir(compiler, compiler.host))
Expand Down

0 comments on commit 33b3b13

Please sign in to comment.