diff --git a/Cargo.lock b/Cargo.lock index 149f02b4b8c94..f44644cffa62a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3559,6 +3559,7 @@ dependencies = [ "flate2", "libc", "log", + "measureme", "rustc", "rustc-demangle", "rustc_attr", diff --git a/src/librustc_codegen_llvm/Cargo.toml b/src/librustc_codegen_llvm/Cargo.toml index e7c0ee5ea763e..c5e862ffc179f 100644 --- a/src/librustc_codegen_llvm/Cargo.toml +++ b/src/librustc_codegen_llvm/Cargo.toml @@ -14,6 +14,7 @@ doctest = false bitflags = "1.0" flate2 = "1.0" libc = "0.2" +measureme = "0.7.1" log = "0.4" rustc = { path = "../librustc" } rustc-demangle = "0.1" diff --git a/src/librustc_codegen_llvm/back/lto.rs b/src/librustc_codegen_llvm/back/lto.rs index e3d69fc5c76df..d7297ed41769c 100644 --- a/src/librustc_codegen_llvm/back/lto.rs +++ b/src/librustc_codegen_llvm/back/lto.rs @@ -593,7 +593,7 @@ pub(crate) fn run_pass_manager( } else { opt_level }; - write::optimize_with_new_llvm_pass_manager(module, config, opt_level, opt_stage); + write::optimize_with_new_llvm_pass_manager(cgcx, module, config, opt_level, opt_stage); debug!("lto done"); return; } diff --git a/src/librustc_codegen_llvm/back/profiling.rs b/src/librustc_codegen_llvm/back/profiling.rs new file mode 100644 index 0000000000000..d56ddac699b09 --- /dev/null +++ b/src/librustc_codegen_llvm/back/profiling.rs @@ -0,0 +1,58 @@ +use measureme::{event_id::SEPARATOR_BYTE, EventId, StringComponent, StringId}; +use rustc_data_structures::profiling::{SelfProfiler, TimingGuard}; +use std::ffi::{c_void, CStr}; +use std::os::raw::c_char; +use std::sync::Arc; + +fn llvm_args_to_string_id(profiler: &SelfProfiler, pass_name: &str, ir_name: &str) -> EventId { + let pass_name = profiler.get_or_alloc_cached_string(pass_name); + let mut components = vec![StringComponent::Ref(pass_name)]; + // handle that LazyCallGraph::SCC is a comma separated list within parentheses + let parentheses: &[_] = &['(', ')']; + let trimed = ir_name.trim_matches(parentheses); + for part in trimed.split(", ") { + let demangled_ir_name = rustc_demangle::demangle(part).to_string(); + let ir_name = profiler.get_or_alloc_cached_string(demangled_ir_name); + components.push(StringComponent::Value(SEPARATOR_BYTE)); + components.push(StringComponent::Ref(ir_name)); + } + EventId::from_label(profiler.alloc_string(components.as_slice())) +} + +pub struct LlvmSelfProfiler<'a> { + profiler: Arc, + stack: Vec>, + llvm_pass_event_kind: StringId, +} + +impl<'a> LlvmSelfProfiler<'a> { + pub fn new(profiler: Arc) -> Self { + let llvm_pass_event_kind = profiler.alloc_string("LLVM Pass"); + Self { profiler, stack: Vec::default(), llvm_pass_event_kind } + } + + fn before_pass_callback(&'a mut self, pass_name: &str, ir_name: &str) { + let event_id = llvm_args_to_string_id(&self.profiler, pass_name, ir_name); + + self.stack.push(TimingGuard::start(&self.profiler, self.llvm_pass_event_kind, event_id)); + } + fn after_pass_callback(&mut self) { + self.stack.pop(); + } +} + +pub unsafe extern "C" fn selfprofile_before_pass_callback( + llvm_self_profiler: *mut c_void, + pass_name: *const c_char, + ir_name: *const c_char, +) { + let llvm_self_profiler = &mut *(llvm_self_profiler as *mut LlvmSelfProfiler<'_>); + let pass_name = CStr::from_ptr(pass_name).to_str().expect("valid UTF-8"); + let ir_name = CStr::from_ptr(ir_name).to_str().expect("valid UTF-8"); + llvm_self_profiler.before_pass_callback(pass_name, ir_name); +} + +pub unsafe extern "C" fn selfprofile_after_pass_callback(llvm_self_profiler: *mut c_void) { + let llvm_self_profiler = &mut *(llvm_self_profiler as *mut LlvmSelfProfiler<'_>); + llvm_self_profiler.after_pass_callback(); +} diff --git a/src/librustc_codegen_llvm/back/write.rs b/src/librustc_codegen_llvm/back/write.rs index 9008970847a59..a215ef81bc9eb 100644 --- a/src/librustc_codegen_llvm/back/write.rs +++ b/src/librustc_codegen_llvm/back/write.rs @@ -1,6 +1,9 @@ use crate::attributes; use crate::back::bytecode; use crate::back::lto::ThinBuffer; +use crate::back::profiling::{ + selfprofile_after_pass_callback, selfprofile_before_pass_callback, LlvmSelfProfiler, +}; use crate::base; use crate::common; use crate::consts; @@ -348,6 +351,7 @@ pub(crate) fn should_use_new_llvm_pass_manager(config: &ModuleConfig) -> bool { } pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( + cgcx: &CodegenContext, module: &ModuleCodegen, config: &ModuleConfig, opt_level: config::OptLevel, @@ -372,6 +376,13 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( None }; + let llvm_selfprofiler = if cgcx.prof.llvm_recording_enabled() { + let mut llvm_profiler = LlvmSelfProfiler::new(cgcx.prof.get_self_profiler().unwrap()); + &mut llvm_profiler as *mut _ as *mut c_void + } else { + std::ptr::null_mut() + }; + // FIXME: NewPM doesn't provide a facility to pass custom InlineParams. // We would have to add upstream support for this first, before we can support // config.inline_threshold and our more aggressive default thresholds. @@ -394,6 +405,9 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( sanitizer_options.as_ref(), pgo_gen_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), pgo_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), + llvm_selfprofiler, + selfprofile_before_pass_callback, + selfprofile_after_pass_callback, ); } @@ -428,10 +442,15 @@ pub(crate) unsafe fn optimize( _ if cgcx.opts.cg.linker_plugin_lto.enabled() => llvm::OptStage::PreLinkThinLTO, _ => llvm::OptStage::PreLinkNoLTO, }; - optimize_with_new_llvm_pass_manager(module, config, opt_level, opt_stage); + optimize_with_new_llvm_pass_manager(cgcx, module, config, opt_level, opt_stage); return Ok(()); } + if cgcx.prof.llvm_recording_enabled() { + diag_handler + .warn("`-Z self-profile-events = llvm` requires `-Z new-llvm-pass-manager`"); + } + // Create the two optimizing pass managers. These mirror what clang // does, and are by populated by LLVM's default PassManagerBuilder. // Each manager has a different set of passes, but they also share diff --git a/src/librustc_codegen_llvm/lib.rs b/src/librustc_codegen_llvm/lib.rs index 7aaa70d6ec4d3..b1085ba170330 100644 --- a/src/librustc_codegen_llvm/lib.rs +++ b/src/librustc_codegen_llvm/lib.rs @@ -44,6 +44,7 @@ mod back { pub mod archive; pub mod bytecode; pub mod lto; + mod profiling; pub mod write; } diff --git a/src/librustc_codegen_llvm/llvm/ffi.rs b/src/librustc_codegen_llvm/llvm/ffi.rs index f12bfe0e80ace..808094eca91d5 100644 --- a/src/librustc_codegen_llvm/llvm/ffi.rs +++ b/src/librustc_codegen_llvm/llvm/ffi.rs @@ -709,6 +709,10 @@ extern "C" { pub type ModuleBuffer; } +pub type SelfProfileBeforePassCallback = + unsafe extern "C" fn(*mut c_void, *const c_char, *const c_char); +pub type SelfProfileAfterPassCallback = unsafe extern "C" fn(*mut c_void); + extern "C" { pub fn LLVMRustInstallFatalErrorHandler(); @@ -1945,6 +1949,9 @@ extern "C" { SanitizerOptions: Option<&SanitizerOptions>, PGOGenPath: *const c_char, PGOUsePath: *const c_char, + llvm_selfprofiler: *mut c_void, + begin_callback: SelfProfileBeforePassCallback, + end_callback: SelfProfileAfterPassCallback, ); pub fn LLVMRustPrintModule( M: &'a Module, diff --git a/src/librustc_data_structures/profiling.rs b/src/librustc_data_structures/profiling.rs index debda9f0a0a24..1d0ac4f4907d1 100644 --- a/src/librustc_data_structures/profiling.rs +++ b/src/librustc_data_structures/profiling.rs @@ -127,6 +127,7 @@ bitflags::bitflags! { const QUERY_KEYS = 1 << 5; const FUNCTION_ARGS = 1 << 6; + const LLVM = 1 << 7; const DEFAULT = Self::GENERIC_ACTIVITIES.bits | Self::QUERY_PROVIDERS.bits | @@ -150,6 +151,7 @@ const EVENT_FILTERS_BY_NAME: &[(&str, EventFilter)] = &[ ("query-keys", EventFilter::QUERY_KEYS), ("function-args", EventFilter::FUNCTION_ARGS), ("args", EventFilter::ARGS), + ("llvm", EventFilter::LLVM), ]; /// Something that uniquely identifies a query invocation. @@ -364,6 +366,15 @@ impl SelfProfilerRef { pub fn enabled(&self) -> bool { self.profiler.is_some() } + + #[inline] + pub fn llvm_recording_enabled(&self) -> bool { + self.event_filter_mask.contains(EventFilter::LLVM) + } + #[inline] + pub fn get_self_profiler(&self) -> Option> { + self.profiler.clone() + } } pub struct SelfProfiler { diff --git a/src/librustc_session/options.rs b/src/librustc_session/options.rs index a794670d7b8fe..d3163fa356436 100644 --- a/src/librustc_session/options.rs +++ b/src/librustc_session/options.rs @@ -940,7 +940,7 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options, "specifies which kinds of events get recorded by the self profiler; for example: `-Z self-profile-events=default,query-keys` all options: none, all, default, generic-activity, query-provider, query-cache-hit - query-blocked, incr-cache-load, query-keys"), + query-blocked, incr-cache-load, query-keys, function-args, args, llvm"), emit_stack_sizes: bool = (false, parse_bool, [UNTRACKED], "emits a section containing stack size metadata"), plt: Option = (None, parse_opt_bool, [TRACKED], diff --git a/src/rustllvm/PassWrapper.cpp b/src/rustllvm/PassWrapper.cpp index 15e2251d76321..2c283149be8bc 100644 --- a/src/rustllvm/PassWrapper.cpp +++ b/src/rustllvm/PassWrapper.cpp @@ -640,6 +640,62 @@ LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR, return LLVMRustResult::Success; } +extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler + const char*, // pass name + const char*); // IR name +extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler + +#if LLVM_VERSION_GE(9, 0) + +std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) { + if (any_isa(WrappedIr)) + return any_cast(WrappedIr)->getName().str(); + if (any_isa(WrappedIr)) + return any_cast(WrappedIr)->getName().str(); + if (any_isa(WrappedIr)) + return any_cast(WrappedIr)->getName().str(); + if (any_isa(WrappedIr)) + return any_cast(WrappedIr)->getName(); + return ""; +} + + +void LLVMSelfProfileInitializeCallbacks( + PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler, + LLVMRustSelfProfileBeforePassCallback BeforePassCallback, + LLVMRustSelfProfileAfterPassCallback AfterPassCallback) { + PIC.registerBeforePassCallback([LlvmSelfProfiler, BeforePassCallback]( + StringRef Pass, llvm::Any Ir) { + std::string PassName = Pass.str(); + std::string IrName = LLVMRustwrappedIrGetName(Ir); + BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str()); + return true; + }); + + PIC.registerAfterPassCallback( + [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) { + AfterPassCallback(LlvmSelfProfiler); + }); + + PIC.registerAfterPassInvalidatedCallback( + [LlvmSelfProfiler, AfterPassCallback](StringRef Pass) { + AfterPassCallback(LlvmSelfProfiler); + }); + + PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback]( + StringRef Pass, llvm::Any Ir) { + std::string PassName = Pass.str(); + std::string IrName = LLVMRustwrappedIrGetName(Ir); + BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str()); + }); + + PIC.registerAfterAnalysisCallback( + [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) { + AfterPassCallback(LlvmSelfProfiler); + }); +} +#endif + enum class LLVMRustOptStage { PreLinkNoLTO, PreLinkThinLTO, @@ -666,7 +722,10 @@ LLVMRustOptimizeWithNewPassManager( bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize, bool DisableSimplifyLibCalls, LLVMRustSanitizerOptions *SanitizerOptions, - const char *PGOGenPath, const char *PGOUsePath) { + const char *PGOGenPath, const char *PGOUsePath, + void* LlvmSelfProfiler, + LLVMRustSelfProfileBeforePassCallback BeforePassCallback, + LLVMRustSelfProfileAfterPassCallback AfterPassCallback) { #if LLVM_VERSION_GE(9, 0) Module *TheModule = unwrap(ModuleRef); TargetMachine *TM = unwrap(TMRef); @@ -685,6 +744,10 @@ LLVMRustOptimizeWithNewPassManager( StandardInstrumentations SI; SI.registerCallbacks(PIC); + if (LlvmSelfProfiler){ + LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback); + } + Optional PGOOpt; if (PGOGenPath) { assert(!PGOUsePath);