[self-profiler] add selfprofiling to llvm

Cargo.lock

@@ -3559,6 +3559,7 @@ dependencies = [
  "flate2",
  "libc",
  "log",
+ "measureme",
  "rustc",
  "rustc-demangle",
  "rustc_attr",

src/librustc_codegen_llvm/Cargo.toml

@@ -14,6 +14,7 @@ doctest = false
 bitflags = "1.0"
 flate2 = "1.0"
 libc = "0.2"
+measureme = "0.7.1"
 log = "0.4"
 rustc = { path = "../librustc" }
 rustc-demangle = "0.1"

src/librustc_codegen_llvm/back/lto.rs

@@ -593,7 +593,7 @@ pub(crate) fn run_pass_manager(
             } else {
                 opt_level
             };
-            write::optimize_with_new_llvm_pass_manager(module, config, opt_level, opt_stage);
+            write::optimize_with_new_llvm_pass_manager(cgcx, module, config, opt_level, opt_stage);
             debug!("lto done");
             return;
         }

src/librustc_codegen_llvm/back/profiling.rs

@@ -0,0 +1,58 @@
+use measureme::{event_id::SEPARATOR_BYTE, EventId, StringComponent, StringId};
+use rustc_data_structures::profiling::{SelfProfiler, TimingGuard};
+use std::ffi::{c_void, CStr};
+use std::os::raw::c_char;
+use std::sync::Arc;
+
+fn llvm_args_to_string_id(profiler: &SelfProfiler, pass_name: &str, ir_name: &str) -> EventId {
+    let pass_name = profiler.get_or_alloc_cached_string(pass_name);
+    let mut components = vec![StringComponent::Ref(pass_name)];
+    // handle that LazyCallGraph::SCC is a comma separated list within parentheses
+    let parentheses: &[_] = &['(', ')'];
+    let trimed = ir_name.trim_matches(parentheses);
+    for part in trimed.split(", ") {
+        let demangled_ir_name = rustc_demangle::demangle(part).to_string();
+        let ir_name = profiler.get_or_alloc_cached_string(demangled_ir_name);
+        components.push(StringComponent::Value(SEPARATOR_BYTE));
+        components.push(StringComponent::Ref(ir_name));
+    }
+    EventId::from_label(profiler.alloc_string(components.as_slice()))
+}
+
+pub struct LlvmSelfProfiler<'a> {
+    profiler: Arc<SelfProfiler>,
+    stack: Vec<TimingGuard<'a>>,
+    llvm_pass_event_kind: StringId,
+}
+
+impl<'a> LlvmSelfProfiler<'a> {
+    pub fn new(profiler: Arc<SelfProfiler>) -> Self {
+        let llvm_pass_event_kind = profiler.alloc_string("LLVM Pass");
+        Self { profiler, stack: Vec::default(), llvm_pass_event_kind }
+    }
+
+    fn before_pass_callback(&'a mut self, pass_name: &str, ir_name: &str) {
+        let event_id = llvm_args_to_string_id(&self.profiler, pass_name, ir_name);
+
+        self.stack.push(TimingGuard::start(&self.profiler, self.llvm_pass_event_kind, event_id));
+    }
+    fn after_pass_callback(&mut self) {
+        self.stack.pop();
+    }
+}
+
+pub unsafe extern "C" fn selfprofile_before_pass_callback(
+    llvm_self_profiler: *mut c_void,
+    pass_name: *const c_char,
+    ir_name: *const c_char,
+) {
+    let llvm_self_profiler = &mut *(llvm_self_profiler as *mut LlvmSelfProfiler<'_>);
+    let pass_name = CStr::from_ptr(pass_name).to_str().expect("valid UTF-8");
+    let ir_name = CStr::from_ptr(ir_name).to_str().expect("valid UTF-8");
+    llvm_self_profiler.before_pass_callback(pass_name, ir_name);
+}
+
+pub unsafe extern "C" fn selfprofile_after_pass_callback(llvm_self_profiler: *mut c_void) {
+    let llvm_self_profiler = &mut *(llvm_self_profiler as *mut LlvmSelfProfiler<'_>);
+    llvm_self_profiler.after_pass_callback();
+}

src/librustc_codegen_llvm/back/write.rs

@@ -1,6 +1,9 @@
 use crate::attributes;
 use crate::back::bytecode;
 use crate::back::lto::ThinBuffer;
+use crate::back::profiling::{
+    selfprofile_after_pass_callback, selfprofile_before_pass_callback, LlvmSelfProfiler,
+};
 use crate::base;
 use crate::common;
 use crate::consts;
@@ -348,6 +351,7 @@ pub(crate) fn should_use_new_llvm_pass_manager(config: &ModuleConfig) -> bool {
 }
 
 pub(crate) unsafe fn optimize_with_new_llvm_pass_manager(
+    cgcx: &CodegenContext<LlvmCodegenBackend>,
     module: &ModuleCodegen<ModuleLlvm>,
     config: &ModuleConfig,
     opt_level: config::OptLevel,
@@ -372,6 +376,13 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager(
         None
     };
 
+    let llvm_selfprofiler = if cgcx.prof.llvm_recording_enabled() {
+        let mut llvm_profiler = LlvmSelfProfiler::new(cgcx.prof.get_self_profiler().unwrap());
+        &mut llvm_profiler as *mut _ as *mut c_void
+    } else {
+        std::ptr::null_mut()
+    };
+
     // FIXME: NewPM doesn't provide a facility to pass custom InlineParams.
     // We would have to add upstream support for this first, before we can support
     // config.inline_threshold and our more aggressive default thresholds.
@@ -394,6 +405,9 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager(
         sanitizer_options.as_ref(),
         pgo_gen_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
         pgo_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
+        llvm_selfprofiler,
+        selfprofile_before_pass_callback,
+        selfprofile_after_pass_callback,
     );
 }
 
@@ -428,10 +442,15 @@ pub(crate) unsafe fn optimize(
                 _ if cgcx.opts.cg.linker_plugin_lto.enabled() => llvm::OptStage::PreLinkThinLTO,
                 _ => llvm::OptStage::PreLinkNoLTO,
             };
-            optimize_with_new_llvm_pass_manager(module, config, opt_level, opt_stage);
+            optimize_with_new_llvm_pass_manager(cgcx, module, config, opt_level, opt_stage);
             return Ok(());
         }
 
+        if cgcx.prof.llvm_recording_enabled() {
+            diag_handler
+                .warn("`-Z self-profile-events = llvm` requires `-Z new-llvm-pass-manager`");
+        }
+
         // Create the two optimizing pass managers. These mirror what clang
         // does, and are by populated by LLVM's default PassManagerBuilder.
         // Each manager has a different set of passes, but they also share

src/librustc_codegen_llvm/lib.rs

@@ -44,6 +44,7 @@ mod back {
     pub mod archive;
     pub mod bytecode;
     pub mod lto;
+    mod profiling;
     pub mod write;
 }
 

src/librustc_codegen_llvm/llvm/ffi.rs

@@ -709,6 +709,10 @@ extern "C" {
     pub type ModuleBuffer;
 }
 
+pub type SelfProfileBeforePassCallback =
+    unsafe extern "C" fn(*mut c_void, *const c_char, *const c_char);
+pub type SelfProfileAfterPassCallback = unsafe extern "C" fn(*mut c_void);
+
 extern "C" {
     pub fn LLVMRustInstallFatalErrorHandler();
 
@@ -1945,6 +1949,9 @@ extern "C" {
         SanitizerOptions: Option<&SanitizerOptions>,
         PGOGenPath: *const c_char,
         PGOUsePath: *const c_char,
+        llvm_selfprofiler: *mut c_void,
+        begin_callback: SelfProfileBeforePassCallback,
+        end_callback: SelfProfileAfterPassCallback,
     );
     pub fn LLVMRustPrintModule(
         M: &'a Module,

src/librustc_data_structures/profiling.rs

@@ -127,6 +127,7 @@ bitflags::bitflags! {
 
         const QUERY_KEYS         = 1 << 5;
         const FUNCTION_ARGS      = 1 << 6;
+        const LLVM               = 1 << 7;
 
         const DEFAULT = Self::GENERIC_ACTIVITIES.bits |
                         Self::QUERY_PROVIDERS.bits |
@@ -150,6 +151,7 @@ const EVENT_FILTERS_BY_NAME: &[(&str, EventFilter)] = &[
     ("query-keys", EventFilter::QUERY_KEYS),
     ("function-args", EventFilter::FUNCTION_ARGS),
     ("args", EventFilter::ARGS),
+    ("llvm", EventFilter::LLVM),
 ];
 
 /// Something that uniquely identifies a query invocation.
@@ -364,6 +366,15 @@ impl SelfProfilerRef {
     pub fn enabled(&self) -> bool {
         self.profiler.is_some()
     }
+
+    #[inline]
+    pub fn llvm_recording_enabled(&self) -> bool {
+        self.event_filter_mask.contains(EventFilter::LLVM)
+    }
+    #[inline]
+    pub fn get_self_profiler(&self) -> Option<Arc<SelfProfiler>> {
+        self.profiler.clone()
+    }
 }
 
 pub struct SelfProfiler {

src/librustc_session/options.rs

@@ -940,7 +940,7 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options,
         "specifies which kinds of events get recorded by the self profiler;
         for example: `-Z self-profile-events=default,query-keys`
         all options: none, all, default, generic-activity, query-provider, query-cache-hit
-                     query-blocked, incr-cache-load, query-keys"),
+                     query-blocked, incr-cache-load, query-keys, function-args, args, llvm"),
     emit_stack_sizes: bool = (false, parse_bool, [UNTRACKED],
         "emits a section containing stack size metadata"),
     plt: Option<bool> = (None, parse_opt_bool, [TRACKED],

src/rustllvm/PassWrapper.cpp

@@ -640,6 +640,62 @@ LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR,
   return LLVMRustResult::Success;
 }
 
+extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler
+                                                      const char*,      // pass name
+                                                      const char*);     // IR name
+extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler
+
+#if LLVM_VERSION_GE(9, 0)
+
+std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) {
+  if (any_isa<const Module *>(WrappedIr))
+    return any_cast<const Module *>(WrappedIr)->getName().str();
+  if (any_isa<const Function *>(WrappedIr))
+    return any_cast<const Function *>(WrappedIr)->getName().str();
+  if (any_isa<const Loop *>(WrappedIr))
+    return any_cast<const Loop *>(WrappedIr)->getName().str();
+  if (any_isa<const LazyCallGraph::SCC *>(WrappedIr))
+    return any_cast<const LazyCallGraph::SCC *>(WrappedIr)->getName();
+  return "<UNKNOWN>";
+}
+
+
+void LLVMSelfProfileInitializeCallbacks(
+    PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler,
+    LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
+    LLVMRustSelfProfileAfterPassCallback AfterPassCallback) {
+  PIC.registerBeforePassCallback([LlvmSelfProfiler, BeforePassCallback](
+                                     StringRef Pass, llvm::Any Ir) {
+    std::string PassName = Pass.str();
+    std::string IrName = LLVMRustwrappedIrGetName(Ir);
+    BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
+    return true;
+  });
+
+  PIC.registerAfterPassCallback(
+      [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) {
+        AfterPassCallback(LlvmSelfProfiler);
+      });
+
+  PIC.registerAfterPassInvalidatedCallback(
+      [LlvmSelfProfiler, AfterPassCallback](StringRef Pass) {
+        AfterPassCallback(LlvmSelfProfiler);
+      });
+
+  PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback](
+                                         StringRef Pass, llvm::Any Ir) {
+    std::string PassName = Pass.str();
+    std::string IrName = LLVMRustwrappedIrGetName(Ir);
+    BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
+  });
+
+  PIC.registerAfterAnalysisCallback(
+      [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) {
+        AfterPassCallback(LlvmSelfProfiler);
+      });
+}
+#endif
+
 enum class LLVMRustOptStage {
   PreLinkNoLTO,
   PreLinkThinLTO,
@@ -666,7 +722,10 @@ LLVMRustOptimizeWithNewPassManager(
     bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize,
     bool DisableSimplifyLibCalls,
     LLVMRustSanitizerOptions *SanitizerOptions,
-    const char *PGOGenPath, const char *PGOUsePath) {
+    const char *PGOGenPath, const char *PGOUsePath,
+    void* LlvmSelfProfiler,
+    LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
+    LLVMRustSelfProfileAfterPassCallback AfterPassCallback) {
 #if LLVM_VERSION_GE(9, 0)
   Module *TheModule = unwrap(ModuleRef);
   TargetMachine *TM = unwrap(TMRef);
@@ -685,6 +744,10 @@ LLVMRustOptimizeWithNewPassManager(
   StandardInstrumentations SI;
   SI.registerCallbacks(PIC);
 
+  if (LlvmSelfProfiler){
+    LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback);
+  }
+
   Optional<PGOOptions> PGOOpt;
   if (PGOGenPath) {
     assert(!PGOUsePath);