Skip to content

Commit

Permalink
Rollup merge of rust-lang#72999 - mati865:separate-self-contained-dir…
Browse files Browse the repository at this point in the history 
…, r=Mark-Simulacrum

Create self-contained directory and move there some of external binaries/libs

One of the steps to reach design described in rust-lang#68887 (comment)
This PR moves things around and allows link code to handle the new directory structure.
  • Loading branch information
@RalfJung
RalfJung committed Jun 19, 2020
2 parents 5e7eec2 + 43905cd commit ea3c309
Show file tree
Hide file tree
Showing 5 changed files with 162 additions and 61 deletions.
164 changes: 112 additions & 52 deletions src/bootstrap/compile.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ use crate::builder::Cargo;
use crate::dist;
use crate::native;
use crate::util::{exe, is_dylib, symlink_dir};
use crate::{Compiler, GitRepo, Mode};
use crate::{Compiler, DependencyType, GitRepo, Mode};

use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
use crate::cache::{Interned, INTERNER};
Expand Down Expand Up @@ -74,6 +74,7 @@ impl Step for Std {
// Even if we're not building std this stage, the new sysroot must
// still contain the third party objects needed by various targets.
copy_third_party_objects(builder, &compiler, target);
copy_self_contained_objects(builder, &compiler, target);

builder.ensure(StdLink {
compiler: compiler_to_use,
Expand All @@ -83,7 +84,8 @@ impl Step for Std {
return;
}

target_deps.extend(copy_third_party_objects(builder, &compiler, target).into_iter());
target_deps.extend(copy_third_party_objects(builder, &compiler, target));
target_deps.extend(copy_self_contained_objects(builder, &compiler, target));

let mut cargo = builder.cargo(compiler, Mode::Std, target, "build");
std_cargo(builder, target, compiler.stage, &mut cargo);
Expand All @@ -109,21 +111,76 @@ impl Step for Std {
}
}

fn copy_and_stamp(
builder: &Builder<'_>,
libdir: &Path,
sourcedir: &Path,
name: &str,
target_deps: &mut Vec<(PathBuf, DependencyType)>,
dependency_type: DependencyType,
) {
let target = libdir.join(name);
builder.copy(&sourcedir.join(name), &target);

target_deps.push((target, dependency_type));
}

/// Copies third party objects needed by various targets.
fn copy_third_party_objects(
builder: &Builder<'_>,
compiler: &Compiler,
target: Interned<String>,
) -> Vec<PathBuf> {
) -> Vec<(PathBuf, DependencyType)> {
let libdir = builder.sysroot_libdir(*compiler, target);

let mut target_deps = vec![];

let mut copy_and_stamp = |sourcedir: &Path, name: &str| {
let target = libdir.join(name);
builder.copy(&sourcedir.join(name), &target);
target_deps.push(target);
// Copies libunwind.a compiled to be linked with x86_64-fortanix-unknown-sgx.
//
// This target needs to be linked to Fortanix's port of llvm's libunwind.
// libunwind requires support for rwlock and printing to stderr,
// which is provided by std for this target.
if target == "x86_64-fortanix-unknown-sgx" {
let src_path_env = "X86_FORTANIX_SGX_LIBS";
let src =
env::var(src_path_env).unwrap_or_else(|_| panic!("{} not found in env", src_path_env));
copy_and_stamp(
builder,
&*libdir,
Path::new(&src),
"libunwind.a",
&mut target_deps,
DependencyType::Target,
);
}

if builder.config.sanitizers && compiler.stage != 0 {
// The sanitizers are only copied in stage1 or above,
// to avoid creating dependency on LLVM.
target_deps.extend(
copy_sanitizers(builder, &compiler, target)
.into_iter()
.map(|d| (d, DependencyType::Target)),
);
}

target_deps
}

/// Copies third party objects needed by various targets for self-contained linkage.
fn copy_self_contained_objects(
builder: &Builder<'_>,
compiler: &Compiler,
target: Interned<String>,
) -> Vec<(PathBuf, DependencyType)> {
// cfg(bootstrap)
// Remove when upgrading bootstrap compiler.
let libdir_self_contained = if compiler.stage == 0 {
builder.sysroot_libdir(*compiler, target).to_path_buf()
} else {
builder.sysroot_libdir(*compiler, target).join("self-contained")
};
t!(fs::create_dir_all(&libdir_self_contained));
let mut target_deps = vec![];

// Copies the CRT objects.
//
Expand All @@ -135,29 +192,32 @@ fn copy_third_party_objects(
if target.contains("musl") {
let srcdir = builder.musl_root(target).unwrap().join("lib");
for &obj in &["crt1.o", "Scrt1.o", "rcrt1.o", "crti.o", "crtn.o"] {
copy_and_stamp(&srcdir, obj);
copy_and_stamp(
builder,
&libdir_self_contained,
&srcdir,
obj,
&mut target_deps,
DependencyType::TargetSelfContained,
);
}
} else if target.ends_with("-wasi") {
let srcdir = builder.wasi_root(target).unwrap().join("lib/wasm32-wasi");
copy_and_stamp(&srcdir, "crt1.o");
}

// Copies libunwind.a compiled to be linked with x86_64-fortanix-unknown-sgx.
//
// This target needs to be linked to Fortanix's port of llvm's libunwind.
// libunwind requires support for rwlock and printing to stderr,
// which is provided by std for this target.
if target == "x86_64-fortanix-unknown-sgx" {
let src_path_env = "X86_FORTANIX_SGX_LIBS";
let src =
env::var(src_path_env).unwrap_or_else(|_| panic!("{} not found in env", src_path_env));
copy_and_stamp(Path::new(&src), "libunwind.a");
}

if builder.config.sanitizers && compiler.stage != 0 {
// The sanitizers are only copied in stage1 or above,
// to avoid creating dependency on LLVM.
target_deps.extend(copy_sanitizers(builder, &compiler, target));
copy_and_stamp(
builder,
&libdir_self_contained,
&srcdir,
"crt1.o",
&mut target_deps,
DependencyType::TargetSelfContained,
);
} else if target.contains("windows-gnu") {
for obj in ["crt2.o", "dllcrt2.o"].iter() {
let src = compiler_file(builder, builder.cc(target), target, obj);
let target = libdir_self_contained.join(obj);
builder.copy(&src, &target);
target_deps.push((target, DependencyType::TargetSelfContained));
}
}

target_deps
Expand Down Expand Up @@ -335,7 +395,7 @@ pub struct StartupObjects {
}

impl Step for StartupObjects {
type Output = Vec<PathBuf>;
type Output = Vec<(PathBuf, DependencyType)>;

fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.path("src/rtstartup")
Expand All @@ -354,7 +414,7 @@ impl Step for StartupObjects {
/// They don't require any library support as they're just plain old object
/// files, so we just use the nightly snapshot compiler to always build them (as
/// no other compilers are guaranteed to be available).
fn run(self, builder: &Builder<'_>) -> Vec<PathBuf> {
fn run(self, builder: &Builder<'_>) -> Vec<(PathBuf, DependencyType)> {
let for_compiler = self.compiler;
let target = self.target;
if !target.contains("windows-gnu") {
Expand Down Expand Up @@ -388,14 +448,7 @@ impl Step for StartupObjects {

let target = sysroot_dir.join((*file).to_string() + ".o");
builder.copy(dst_file, &target);
target_deps.push(target);
}

for obj in ["crt2.o", "dllcrt2.o"].iter() {
let src = compiler_file(builder, builder.cc(target), target, obj);
let target = sysroot_dir.join(obj);
builder.copy(&src, &target);
target_deps.push(target);
target_deps.push((target, DependencyType::Target));
}

target_deps
Expand Down Expand Up @@ -808,14 +861,17 @@ pub fn add_to_sysroot(
sysroot_host_dst: &Path,
stamp: &Path,
) {
let self_contained_dst = &sysroot_dst.join("self-contained");
t!(fs::create_dir_all(&sysroot_dst));
t!(fs::create_dir_all(&sysroot_host_dst));
for (path, host) in builder.read_stamp_file(stamp) {
if host {
builder.copy(&path, &sysroot_host_dst.join(path.file_name().unwrap()));
} else {
builder.copy(&path, &sysroot_dst.join(path.file_name().unwrap()));
}
t!(fs::create_dir_all(&self_contained_dst));
for (path, dependency_type) in builder.read_stamp_file(stamp) {
let dst = match dependency_type {
DependencyType::Host => sysroot_host_dst,
DependencyType::Target => sysroot_dst,
DependencyType::TargetSelfContained => self_contained_dst,
};
builder.copy(&path, &dst.join(path.file_name().unwrap()));
}
}

Expand All @@ -824,7 +880,7 @@ pub fn run_cargo(
cargo: Cargo,
tail_args: Vec<String>,
stamp: &Path,
additional_target_deps: Vec<PathBuf>,
additional_target_deps: Vec<(PathBuf, DependencyType)>,
is_check: bool,
) -> Vec<PathBuf> {
if builder.config.dry_run {
Expand Down Expand Up @@ -875,15 +931,15 @@ pub fn run_cargo(
if filename.starts_with(&host_root_dir) {
// Unless it's a proc macro used in the compiler
if crate_types.iter().any(|t| t == "proc-macro") {
deps.push((filename.to_path_buf(), true));
deps.push((filename.to_path_buf(), DependencyType::Host));
}
continue;
}

// If this was output in the `deps` dir then this is a precise file
// name (hash included) so we start tracking it.
if filename.starts_with(&target_deps_dir) {
deps.push((filename.to_path_buf(), false));
deps.push((filename.to_path_buf(), DependencyType::Target));
continue;
}

Expand Down Expand Up @@ -935,17 +991,21 @@ pub fn run_cargo(
let candidate = format!("{}.lib", path_to_add);
let candidate = PathBuf::from(candidate);
if candidate.exists() {
deps.push((candidate, false));
deps.push((candidate, DependencyType::Target));
}
}
deps.push((path_to_add.into(), false));
deps.push((path_to_add.into(), DependencyType::Target));
}

deps.extend(additional_target_deps.into_iter().map(|d| (d, false)));
deps.extend(additional_target_deps);
deps.sort();
let mut new_contents = Vec::new();
for (dep, proc_macro) in deps.iter() {
new_contents.extend(if *proc_macro { b"h" } else { b"t" });
for (dep, dependency_type) in deps.iter() {
new_contents.extend(match *dependency_type {
DependencyType::Host => b"h",
DependencyType::Target => b"t",
DependencyType::TargetSelfContained => b"s",
});
new_contents.extend(dep.to_str().unwrap().as_bytes());
new_contents.extend(b"\0");
}
Expand Down
24 changes: 19 additions & 5 deletions src/bootstrap/dist.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ use crate::channel;
use crate::compile;
use crate::tool::{self, Tool};
use crate::util::{exe, is_dylib, timeit};
use crate::{Compiler, Mode, LLVM_TOOLS};
use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
use time::{self, Timespec};

pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
Expand Down Expand Up @@ -306,7 +306,12 @@ fn make_win_dist(
}

//Copy platform tools to platform-specific bin directory
let target_bin_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("bin");
let target_bin_dir = plat_root
.join("lib")
.join("rustlib")
.join(target_triple)
.join("bin")
.join("self-contained");
fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
for src in target_tools {
builder.copy_to_folder(&src, &target_bin_dir);
Expand All @@ -321,7 +326,12 @@ fn make_win_dist(
);

//Copy platform libs to platform-specific lib directory
let target_lib_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("lib");
let target_lib_dir = plat_root
.join("lib")
.join("rustlib")
.join(target_triple)
.join("lib")
.join("self-contained");
fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
for src in target_libs {
builder.copy_to_folder(&src, &target_lib_dir);
Expand Down Expand Up @@ -652,9 +662,13 @@ fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
/// Copy stamped files into an image's `target/lib` directory.
fn copy_target_libs(builder: &Builder<'_>, target: &str, image: &Path, stamp: &Path) {
let dst = image.join("lib/rustlib").join(target).join("lib");
let self_contained_dst = dst.join("self-contained");
t!(fs::create_dir_all(&dst));
for (path, host) in builder.read_stamp_file(stamp) {
if !host || builder.config.build == target {
t!(fs::create_dir_all(&self_contained_dst));
for (path, dependency_type) in builder.read_stamp_file(stamp) {
if dependency_type == DependencyType::TargetSelfContained {
builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
} else if dependency_type == DependencyType::Target || builder.config.build == target {
builder.copy(&path, &dst.join(path.file_name().unwrap()));
}
}
Expand Down
22 changes: 19 additions & 3 deletions src/bootstrap/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,17 @@ impl Crate {
}
}

/// When building Rust various objects are handled differently.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum DependencyType {
/// Libraries originating from proc-macros.
Host,
/// Typical Rust libraries.
Target,
/// Non Rust libraries and objects shipped to ease usage of certain targets.
TargetSelfContained,
}

/// The various "modes" of invoking Cargo.
///
/// These entries currently correspond to the various output directories of the
Expand Down Expand Up @@ -1097,7 +1108,7 @@ impl Build {
ret
}

fn read_stamp_file(&self, stamp: &Path) -> Vec<(PathBuf, bool)> {
fn read_stamp_file(&self, stamp: &Path) -> Vec<(PathBuf, DependencyType)> {
if self.config.dry_run {
return Vec::new();
}
Expand All @@ -1110,9 +1121,14 @@ impl Build {
if part.is_empty() {
continue;
}
let host = part[0] as char == 'h';
let dependency_type = match part[0] as char {
'h' => DependencyType::Host,
's' => DependencyType::TargetSelfContained,
't' => DependencyType::Target,
_ => unreachable!(),
};
let path = PathBuf::from(t!(str::from_utf8(&part[1..])));
paths.push((path, host));
paths.push((path, dependency_type));
}
paths
}
Expand Down
7 changes: 7 additions & 0 deletions src/librustc_codegen_ssa/back/link.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1075,6 +1075,10 @@ fn get_object_file_path(sess: &Session, name: &str) -> PathBuf {
if file_path.exists() {
return file_path;
}
let file_path = fs.get_selfcontained_lib_path().join(name);
if file_path.exists() {
return file_path;
}
for search_path in fs.search_paths() {
let file_path = search_path.dir.join(name);
if file_path.exists() {
Expand Down Expand Up @@ -1470,6 +1474,9 @@ fn add_library_search_dirs(cmd: &mut dyn Linker, sess: &Session) {
// The location of crates will be determined as needed.
let lib_path = sess.target_filesearch(PathKind::All).get_lib_path();
cmd.include_path(&fix_windows_verbatim_for_gcc(&lib_path));

let lib_path = sess.target_filesearch(PathKind::All).get_selfcontained_lib_path();
cmd.include_path(&fix_windows_verbatim_for_gcc(&lib_path));
}

/// Add options making relocation sections in the produced ELF files read-only
Expand Down
Loading

0 comments on commit ea3c309

Please sign in to comment.