Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

rustc: Implement ThinLTO #44841

Merged
merged 1 commit into from
Oct 8, 2017
Merged

rustc: Implement ThinLTO #44841

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 6 additions & 14 deletions src/librustc/session/config.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -409,9 +409,7 @@ impl_stable_hash_for!(struct self::OutputFilenames {
outputs
});

/// Codegen unit names generated by the numbered naming scheme will contain this
/// marker right before the index of the codegen unit.
pub const NUMBERED_CODEGEN_UNIT_MARKER: &'static str = ".cgu-";
pub const RUST_CGU_EXT: &str = "rust-cgu";

impl OutputFilenames {
pub fn path(&self, flavor: OutputType) -> PathBuf {
Expand Down Expand Up @@ -442,22 +440,14 @@ impl OutputFilenames {
let mut extension = String::new();

if let Some(codegen_unit_name) = codegen_unit_name {
if codegen_unit_name.contains(NUMBERED_CODEGEN_UNIT_MARKER) {
// If we use the numbered naming scheme for modules, we don't want
// the files to look like <crate-name><extra>.<crate-name>.<index>.<ext>
// but simply <crate-name><extra>.<index>.<ext>
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@michaelwoerister mind helping me understand what was going on here? For ThinLTO we need to make sure that all the objects and such have unique names, which this comment seems to indicate we will have achieved. (although in practice I didn't see crate name hashes and such in those names)

With this name munging left in though I found that lots of objects were overwriting one another by accident, because I think the backend of shuffling files around was "getting weird". I couldn't find a downside to removing this logic, though, so I was curious if you knew what this was originally added for?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This logic looks like a measure for keeping filenames/paths short. We could probably just get rid of the "numbered" codegen unit naming scheme (incremental does not use that). That was only introduced to stay close to the then existing behavior. That would have the downside of codegen units having somewhat misleading names but they should be unique.

let marker_offset = codegen_unit_name.rfind(NUMBERED_CODEGEN_UNIT_MARKER)
.unwrap();
let index_offset = marker_offset + NUMBERED_CODEGEN_UNIT_MARKER.len();
extension.push_str(&codegen_unit_name[index_offset .. ]);
} else {
extension.push_str(codegen_unit_name);
};
extension.push_str(codegen_unit_name);
}

if !ext.is_empty() {
if !extension.is_empty() {
extension.push_str(".");
extension.push_str(RUST_CGU_EXT);
extension.push_str(".");
}

extension.push_str(ext);
Expand Down Expand Up @@ -1105,6 +1095,8 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options,
"run the non-lexical lifetimes MIR pass"),
trans_time_graph: bool = (false, parse_bool, [UNTRACKED],
"generate a graphical HTML report of time spent in trans and LLVM"),
thinlto: bool = (false, parse_bool, [TRACKED],
"enable ThinLTO when possible"),
}

pub fn default_lib_output() -> CrateType {
Expand Down
1 change: 1 addition & 0 deletions src/librustc_llvm/build.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,7 @@ fn main() {
"linker",
"asmparser",
"mcjit",
"lto",
"interpreter",
"instrumentation"];

Expand Down
56 changes: 56 additions & 0 deletions src/librustc_llvm/ffi.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -345,6 +345,20 @@ pub enum PassKind {
Module,
}

/// LLVMRustThinLTOData
pub enum ThinLTOData {}

/// LLVMRustThinLTOBuffer
pub enum ThinLTOBuffer {}

/// LLVMRustThinLTOModule
#[repr(C)]
pub struct ThinLTOModule {
pub identifier: *const c_char,
pub data: *const u8,
pub len: usize,
}

// Opaque pointer types
#[allow(missing_copy_implementations)]
pub enum Module_opaque {}
Expand Down Expand Up @@ -1271,6 +1285,9 @@ extern "C" {
PM: PassManagerRef,
Internalize: Bool,
RunInliner: Bool);
pub fn LLVMRustPassManagerBuilderPopulateThinLTOPassManager(
PMB: PassManagerBuilderRef,
PM: PassManagerRef) -> bool;

// Stuff that's in rustllvm/ because it's not upstream yet.

Expand Down Expand Up @@ -1685,4 +1702,43 @@ extern "C" {
pub fn LLVMRustModuleBufferLen(p: *const ModuleBuffer) -> usize;
pub fn LLVMRustModuleBufferFree(p: *mut ModuleBuffer);
pub fn LLVMRustModuleCost(M: ModuleRef) -> u64;

pub fn LLVMRustThinLTOAvailable() -> bool;
pub fn LLVMRustWriteThinBitcodeToFile(PMR: PassManagerRef,
M: ModuleRef,
BC: *const c_char) -> bool;
pub fn LLVMRustThinLTOBufferCreate(M: ModuleRef) -> *mut ThinLTOBuffer;
pub fn LLVMRustThinLTOBufferFree(M: *mut ThinLTOBuffer);
pub fn LLVMRustThinLTOBufferPtr(M: *const ThinLTOBuffer) -> *const c_char;
pub fn LLVMRustThinLTOBufferLen(M: *const ThinLTOBuffer) -> size_t;
pub fn LLVMRustCreateThinLTOData(
Modules: *const ThinLTOModule,
NumModules: c_uint,
PreservedSymbols: *const *const c_char,
PreservedSymbolsLen: c_uint,
) -> *mut ThinLTOData;
pub fn LLVMRustPrepareThinLTORename(
Data: *const ThinLTOData,
Module: ModuleRef,
) -> bool;
pub fn LLVMRustPrepareThinLTOResolveWeak(
Data: *const ThinLTOData,
Module: ModuleRef,
) -> bool;
pub fn LLVMRustPrepareThinLTOInternalize(
Data: *const ThinLTOData,
Module: ModuleRef,
) -> bool;
pub fn LLVMRustPrepareThinLTOImport(
Data: *const ThinLTOData,
Module: ModuleRef,
) -> bool;
pub fn LLVMRustFreeThinLTOData(Data: *mut ThinLTOData);
pub fn LLVMRustParseBitcodeForThinLTO(
Context: ContextRef,
Data: *const u8,
len: usize,
Identifier: *const c_char,
) -> ModuleRef;
pub fn LLVMGetModuleIdentifier(M: ModuleRef, size: *mut usize) -> *const c_char;
}
43 changes: 30 additions & 13 deletions src/librustc_trans/back/link.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ use super::rpath::RPathConfig;
use super::rpath;
use metadata::METADATA_FILENAME;
use rustc::session::config::{self, NoDebugInfo, OutputFilenames, OutputType, PrintRequest};
use rustc::session::config::RUST_CGU_EXT;
use rustc::session::filesearch;
use rustc::session::search_paths::PathKind;
use rustc::session::Session;
Expand Down Expand Up @@ -45,13 +46,9 @@ use syntax::attr;
/// The LLVM module name containing crate-metadata. This includes a `.` on
/// purpose, so it cannot clash with the name of a user-defined module.
pub const METADATA_MODULE_NAME: &'static str = "crate.metadata";
/// The name of the crate-metadata object file the compiler generates. Must
/// match up with `METADATA_MODULE_NAME`.
pub const METADATA_OBJ_NAME: &'static str = "crate.metadata.o";

// same as for metadata above, but for allocator shim
pub const ALLOCATOR_MODULE_NAME: &'static str = "crate.allocator";
pub const ALLOCATOR_OBJ_NAME: &'static str = "crate.allocator.o";

pub use rustc_trans_utils::link::{find_crate_name, filename_for_input, default_output_for_target,
invalid_output_for_target, build_link_meta, out_filename,
Expand Down Expand Up @@ -129,6 +126,14 @@ fn command_path(sess: &Session) -> OsString {
env::join_paths(new_path).unwrap()
}

fn metadata_obj(outputs: &OutputFilenames) -> PathBuf {
outputs.temp_path(OutputType::Object, Some(METADATA_MODULE_NAME))
}

fn allocator_obj(outputs: &OutputFilenames) -> PathBuf {
outputs.temp_path(OutputType::Object, Some(ALLOCATOR_MODULE_NAME))
}

pub fn remove(sess: &Session, path: &Path) {
match fs::remove_file(path) {
Ok(..) => {}
Expand Down Expand Up @@ -174,9 +179,9 @@ pub fn link_binary(sess: &Session,
remove(sess, &obj.object);
}
}
remove(sess, &outputs.with_extension(METADATA_OBJ_NAME));
remove(sess, &metadata_obj(outputs));
if trans.allocator_module.is_some() {
remove(sess, &outputs.with_extension(ALLOCATOR_OBJ_NAME));
remove(sess, &allocator_obj(outputs));
}
}

Expand Down Expand Up @@ -478,7 +483,7 @@ fn link_rlib<'a>(sess: &'a Session,

RlibFlavor::StaticlibBase => {
if trans.allocator_module.is_some() {
ab.add_file(&outputs.with_extension(ALLOCATOR_OBJ_NAME));
ab.add_file(&allocator_obj(outputs));
}
}
}
Expand Down Expand Up @@ -908,11 +913,11 @@ fn link_args(cmd: &mut Linker,
// object file, so we link that in here.
if crate_type == config::CrateTypeDylib ||
crate_type == config::CrateTypeProcMacro {
cmd.add_object(&outputs.with_extension(METADATA_OBJ_NAME));
cmd.add_object(&metadata_obj(outputs));
}

if trans.allocator_module.is_some() {
cmd.add_object(&outputs.with_extension(ALLOCATOR_OBJ_NAME));
cmd.add_object(&allocator_obj(outputs));
}

// Try to strip as much out of the generated object by removing unused
Expand Down Expand Up @@ -1265,11 +1270,23 @@ fn add_upstream_rust_crates(cmd: &mut Linker,
let canonical = f.replace("-", "_");
let canonical_name = name.replace("-", "_");

// Look for `.rust-cgu.o` at the end of the filename to conclude
// that this is a Rust-related object file.
fn looks_like_rust(s: &str) -> bool {
let path = Path::new(s);
let ext = path.extension().and_then(|s| s.to_str());
if ext != Some(OutputType::Object.extension()) {
return false
}
let ext2 = path.file_stem()
.and_then(|s| Path::new(s).extension())
.and_then(|s| s.to_str());
ext2 == Some(RUST_CGU_EXT)
}

let is_rust_object =
canonical.starts_with(&canonical_name) && {
let num = &f[name.len()..f.len() - 2];
num.len() > 0 && num[1..].parse::<u32>().is_ok()
};
canonical.starts_with(&canonical_name) &&
looks_like_rust(&f);

// If we've been requested to skip all native object files
// (those not generated by the rust compiler) then we can skip
Expand Down
Loading