From 7c5b6024e2bafed863a1304671da92b42c765f0d Mon Sep 17 00:00:00 2001 From: YangKeao Date: Mon, 25 Oct 2021 21:01:03 +1100 Subject: [PATCH 01/11] avoid deadlock by skipping sampling in libc, libgcc and pthread Signed-off-by: YangKeao --- Cargo.toml | 3 ++ README.md | 9 +++++ examples/backtrace_while_sampling.rs | 30 ++++++++++++++ src/profiler.rs | 60 ++++++++++++++++++++++++++-- 4 files changed, 98 insertions(+), 4 deletions(-) create mode 100644 examples/backtrace_while_sampling.rs diff --git a/Cargo.toml b/Cargo.toml index 3bd4bd0c..d23b1c1e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ default = ["cpp"] flamegraph = ["inferno"] protobuf = ["prost", "prost-derive", "prost-build"] cpp = ["symbolic-demangle/cpp"] +ignore-libc = ["findshlibs"] [dependencies] backtrace = "0.3" @@ -30,6 +31,8 @@ prost = { version = "0.9", optional = true } prost-derive = { version = "0.9", optional = true } criterion = {version = "0.3", optional = true} +findshlibs = {version = "0.10", optional = true} + [dependencies.symbolic-demangle] version = "8.0" default-features = false diff --git a/README.md b/README.md index 8575b356..88314279 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,15 @@ FRAME: backtrace::backtrace::trace::h3e91a3123a3049a5 -> FRAME: pprof::profiler: FRAME: backtrace::backtrace::trace::h3e91a3123a3049a5 -> FRAME: pprof::profiler::perf_signal_handler::h7b995c4ab2e66493 -> FRAME: Unknown -> FRAME: prime_number::main::h47f1058543990c8b -> FRAME: std::rt::lang_start::{{closure}}::h4262e250f8024b06 -> FRAME: std::rt::lang_start_internal::{{closure}}::h812f70926ebbddd0 -> std::panicking::try::do_call::h3210e2ce6a68897b -> FRAME: __rust_maybe_catch_panic -> FRAME: std::panicking::try::h28c2e2ec1c3871ce -> std::panic::catch_unwind::h05e542185e35aabf -> std::rt::lang_start_internal::hd7efcfd33686f472 -> FRAME: main -> FRAME: __libc_start_main -> FRAME: _start -> FRAME: Unknown -> THREAD: prime_number 1 ``` + +## Features + +- `cpp` enables the cpp demangle. +- `flamegraph` enables the flamegraph report format. +- `protobuf` enables the pprof protobuf report format. +- `ignore-libc` will ignore the `libc`, `libgcc` and `pthread` while sampling. + This feature will avoid deadlock in the implementation of `_Unwind_Backtrace`. + ## Flamegraph ```toml diff --git a/examples/backtrace_while_sampling.rs b/examples/backtrace_while_sampling.rs new file mode 100644 index 00000000..78ea2614 --- /dev/null +++ b/examples/backtrace_while_sampling.rs @@ -0,0 +1,30 @@ +// Copyright 2021 TiKV Project Authors. Licensed under Apache-2.0. + +use pprof; +use std::fs::File; + +fn deep_recursive(depth: i32) { + if depth > 0 { + deep_recursive(depth - 1); + } else { + backtrace::Backtrace::new(); + } +} + +fn main() { + let guard = pprof::ProfilerGuard::new(1000).unwrap(); + + for _ in 0..10000 { + deep_recursive(20); + } + + match guard.report().build() { + Ok(report) => { + let file = File::create("flamegraph.svg").unwrap(); + report.flamegraph(file).unwrap(); + + println!("report: {:?}", &report); + } + Err(_) => {} + }; +} diff --git a/src/profiler.rs b/src/profiler.rs index ca990a62..94aa8538 100644 --- a/src/profiler.rs +++ b/src/profiler.rs @@ -7,6 +7,9 @@ use backtrace::Frame; use nix::sys::signal; use parking_lot::RwLock; +#[cfg(feature = "ignore-libc")] +use findshlibs::{Segment, TargetSharedLibrary, SharedLibrary}; + use crate::collector::Collector; use crate::error::{Error, Result}; use crate::frames::UnresolvedFrames; @@ -23,6 +26,9 @@ pub struct Profiler { sample_counter: i32, running: bool, + + #[cfg(feature = "ignore-libc")] + blacklist_segments: Vec<(usize, usize)>, } /// RAII structure used to stop profiling when dropped. It is the only interface to access profiler. @@ -118,9 +124,18 @@ fn write_thread_name(current_thread: libc::pthread_t, name: &mut [libc::c_char]) #[no_mangle] #[allow(clippy::uninit_assumed_init)] -extern "C" fn perf_signal_handler(_signal: c_int) { +extern "C" fn perf_signal_handler(_signal: c_int, _siginfo: *mut libc::siginfo_t, ucontext: *mut libc::c_void) { if let Some(mut guard) = PROFILER.try_write() { if let Ok(profiler) = guard.as_mut() { + #[cfg(all(feature = "ignore-libc", target_arch = "x86_64" ))] + { + let ucontext: *mut libc::ucontext_t = ucontext as *mut libc::ucontext_t; + let addr = unsafe {(*ucontext).uc_mcontext.gregs[libc::REG_RIP as usize] as usize}; + if profiler.is_blacklisted(addr) { + return; + } + } + let mut bt: [Frame; MAX_DEPTH] = unsafe { std::mem::MaybeUninit::uninit().assume_init() }; let mut index = 0; @@ -151,12 +166,47 @@ extern "C" fn perf_signal_handler(_signal: c_int) { impl Profiler { fn new() -> Result { + #[cfg(feature = "ignore-libc")] + let blacklist_segments = { + let mut segments = Vec::new(); + TargetSharedLibrary::each(|shlib| { + if shlib.name().to_str().and_then(|name| { + if name.contains("libc") || name.contains("libgcc_s") || name.contains("libpthread") { + return Some(()) + } + + None + }).is_some() { + for seg in shlib.segments() { + let avam = seg.actual_virtual_memory_address(shlib); + let start = avam.0; + let end = start + seg.len(); + segments.push((start, end)); + } + } + }); + segments + }; + Ok(Profiler { data: Collector::new()?, sample_counter: 0, running: false, + + #[cfg(feature = "ignore-libc")] + blacklist_segments, }) } + + #[cfg(feature = "ignore-libc")] + fn is_blacklisted(&self, addr: usize) -> bool { + for libs in &self.blacklist_segments { + if addr > libs.0 && addr < libs.1 { + return true; + } + } + false + } } impl Profiler { @@ -193,8 +243,9 @@ impl Profiler { } fn register_signal_handler(&self) -> Result<()> { - let handler = signal::SigHandler::Handler(perf_signal_handler); - unsafe { signal::signal(signal::SIGPROF, handler) }?; + let handler = signal::SigHandler::SigAction(perf_signal_handler); + let sigaction = signal::SigAction::new(handler, signal::SaFlags::SA_SIGINFO, signal::SigSet::empty()); + unsafe { signal::sigaction(signal::SIGPROF, &sigaction) }?; Ok(()) } @@ -220,6 +271,7 @@ mod tests { use super::*; use std::cell::RefCell; use std::ffi::c_void; + use std::ptr::null_mut; #[cfg(not(target_os = "linux"))] #[allow(clippy::wrong_self_convention)] @@ -307,7 +359,7 @@ mod tests { for i in 2..50000 { if is_prime_number(i, &prime_numbers) { _v += 1; - perf_signal_handler(27); + perf_signal_handler(27, null_mut(), null_mut()); } } unsafe { From 46a3996759081ce1dd0625a154c5ebb0227a1723 Mon Sep 17 00:00:00 2001 From: YangKeao Date: Mon, 25 Oct 2021 21:08:17 +1100 Subject: [PATCH 02/11] make clippy happy Signed-off-by: YangKeao --- src/profiler.rs | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/src/profiler.rs b/src/profiler.rs index 94aa8538..e500a11f 100644 --- a/src/profiler.rs +++ b/src/profiler.rs @@ -8,7 +8,7 @@ use nix::sys::signal; use parking_lot::RwLock; #[cfg(feature = "ignore-libc")] -use findshlibs::{Segment, TargetSharedLibrary, SharedLibrary}; +use findshlibs::{Segment, SharedLibrary, TargetSharedLibrary}; use crate::collector::Collector; use crate::error::{Error, Result}; @@ -124,13 +124,18 @@ fn write_thread_name(current_thread: libc::pthread_t, name: &mut [libc::c_char]) #[no_mangle] #[allow(clippy::uninit_assumed_init)] -extern "C" fn perf_signal_handler(_signal: c_int, _siginfo: *mut libc::siginfo_t, ucontext: *mut libc::c_void) { +extern "C" fn perf_signal_handler( + _signal: c_int, + _siginfo: *mut libc::siginfo_t, + ucontext: *mut libc::c_void, +) { if let Some(mut guard) = PROFILER.try_write() { if let Ok(profiler) = guard.as_mut() { - #[cfg(all(feature = "ignore-libc", target_arch = "x86_64" ))] + #[cfg(all(feature = "ignore-libc", target_arch = "x86_64"))] { let ucontext: *mut libc::ucontext_t = ucontext as *mut libc::ucontext_t; - let addr = unsafe {(*ucontext).uc_mcontext.gregs[libc::REG_RIP as usize] as usize}; + let addr = + unsafe { (*ucontext).uc_mcontext.gregs[libc::REG_RIP as usize] as usize }; if profiler.is_blacklisted(addr) { return; } @@ -164,19 +169,30 @@ extern "C" fn perf_signal_handler(_signal: c_int, _siginfo: *mut libc::siginfo_t } } +#[cfg(feature = "ignore-libc")] +const SHLIB_BLACKLIST: [&str; 3] = ["libc", "libgcc_s", "libpthread"]; + impl Profiler { fn new() -> Result { #[cfg(feature = "ignore-libc")] let blacklist_segments = { let mut segments = Vec::new(); TargetSharedLibrary::each(|shlib| { - if shlib.name().to_str().and_then(|name| { - if name.contains("libc") || name.contains("libgcc_s") || name.contains("libpthread") { - return Some(()) + let in_blacklist = match shlib.name().to_str() { + Some(name) => { + let mut in_blacklist = false; + for blocked_name in SHLIB_BLACKLIST.iter() { + if name.contains(blocked_name) { + in_blacklist = true; + } + } + + in_blacklist } - None - }).is_some() { + None => false, + }; + if in_blacklist { for seg in shlib.segments() { let avam = seg.actual_virtual_memory_address(shlib); let start = avam.0; @@ -244,7 +260,11 @@ impl Profiler { fn register_signal_handler(&self) -> Result<()> { let handler = signal::SigHandler::SigAction(perf_signal_handler); - let sigaction = signal::SigAction::new(handler, signal::SaFlags::SA_SIGINFO, signal::SigSet::empty()); + let sigaction = signal::SigAction::new( + handler, + signal::SaFlags::SA_SIGINFO, + signal::SigSet::empty(), + ); unsafe { signal::sigaction(signal::SIGPROF, &sigaction) }?; Ok(()) From a640ff5f61287c51fbbb5df03c47a502cae7e626 Mon Sep 17 00:00:00 2001 From: YangKeao Date: Mon, 25 Oct 2021 21:14:20 +1100 Subject: [PATCH 03/11] skip libc only for linux Signed-off-by: YangKeao --- README.md | 2 +- src/profiler.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 88314279..594535e1 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ FRAME: backtrace::backtrace::trace::h3e91a3123a3049a5 -> FRAME: pprof::profiler: - `cpp` enables the cpp demangle. - `flamegraph` enables the flamegraph report format. - `protobuf` enables the pprof protobuf report format. -- `ignore-libc` will ignore the `libc`, `libgcc` and `pthread` while sampling. +- `ignore-libc` will ignore the `libc`, `libgcc` and `pthread` while sampling. THIS FEATURE IS ONLY AVAILABLE ON LINUX. This feature will avoid deadlock in the implementation of `_Unwind_Backtrace`. ## Flamegraph diff --git a/src/profiler.rs b/src/profiler.rs index e500a11f..f0394537 100644 --- a/src/profiler.rs +++ b/src/profiler.rs @@ -131,7 +131,7 @@ extern "C" fn perf_signal_handler( ) { if let Some(mut guard) = PROFILER.try_write() { if let Ok(profiler) = guard.as_mut() { - #[cfg(all(feature = "ignore-libc", target_arch = "x86_64"))] + #[cfg(all(feature = "ignore-libc", target_arch = "x86_64", target_os = "linux"))] { let ucontext: *mut libc::ucontext_t = ucontext as *mut libc::ucontext_t; let addr = From ff4e24e23448756ad2ee8c14b886f57d1fed34c2 Mon Sep 17 00:00:00 2001 From: YangKeao Date: Tue, 26 Oct 2021 16:22:17 +1100 Subject: [PATCH 04/11] add null check in the signal handler Signed-off-by: YangKeao --- src/profiler.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/profiler.rs b/src/profiler.rs index f0394537..80c3c6b7 100644 --- a/src/profiler.rs +++ b/src/profiler.rs @@ -132,7 +132,7 @@ extern "C" fn perf_signal_handler( if let Some(mut guard) = PROFILER.try_write() { if let Ok(profiler) = guard.as_mut() { #[cfg(all(feature = "ignore-libc", target_arch = "x86_64", target_os = "linux"))] - { + if !ucontext.is_null() { let ucontext: *mut libc::ucontext_t = ucontext as *mut libc::ucontext_t; let addr = unsafe { (*ucontext).uc_mcontext.gregs[libc::REG_RIP as usize] as usize }; From a12b5ea343da1ed16c5ac95b66e5a2a141b89582 Mon Sep 17 00:00:00 2001 From: YangKeao Date: Thu, 28 Oct 2021 15:46:48 +1100 Subject: [PATCH 05/11] add more cfg to remove blacklist on non-linux platform Signed-off-by: YangKeao --- src/profiler.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/profiler.rs b/src/profiler.rs index 80c3c6b7..6d6ae3bc 100644 --- a/src/profiler.rs +++ b/src/profiler.rs @@ -27,7 +27,7 @@ pub struct Profiler { running: bool, - #[cfg(feature = "ignore-libc")] + #[cfg(all(feature = "ignore-libc", target_arch = "x86_64", target_os = "linux"))] blacklist_segments: Vec<(usize, usize)>, } @@ -169,12 +169,12 @@ extern "C" fn perf_signal_handler( } } -#[cfg(feature = "ignore-libc")] +#[cfg(all(feature = "ignore-libc", target_arch = "x86_64", target_os = "linux"))] const SHLIB_BLACKLIST: [&str; 3] = ["libc", "libgcc_s", "libpthread"]; impl Profiler { fn new() -> Result { - #[cfg(feature = "ignore-libc")] + #[cfg(all(feature = "ignore-libc", target_arch = "x86_64", target_os = "linux"))] let blacklist_segments = { let mut segments = Vec::new(); TargetSharedLibrary::each(|shlib| { @@ -209,12 +209,12 @@ impl Profiler { sample_counter: 0, running: false, - #[cfg(feature = "ignore-libc")] + #[cfg(all(feature = "ignore-libc", target_arch = "x86_64", target_os = "linux"))] blacklist_segments, }) } - #[cfg(feature = "ignore-libc")] + #[cfg(all(feature = "ignore-libc", target_arch = "x86_64", target_os = "linux"))] fn is_blacklisted(&self, addr: usize) -> bool { for libs in &self.blacklist_segments { if addr > libs.0 && addr < libs.1 { From 872f5dceb781471b4256520a9d10664016a76c6c Mon Sep 17 00:00:00 2001 From: YangKeao Date: Thu, 28 Oct 2021 15:51:01 +1100 Subject: [PATCH 06/11] fix clippy warning for test Signed-off-by: YangKeao --- src/profiler.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/profiler.rs b/src/profiler.rs index 6d6ae3bc..985d85c1 100644 --- a/src/profiler.rs +++ b/src/profiler.rs @@ -293,10 +293,6 @@ mod tests { use std::ffi::c_void; use std::ptr::null_mut; - #[cfg(not(target_os = "linux"))] - #[allow(clippy::wrong_self_convention)] - static mut __malloc_hook: Option *mut c_void> = None; - extern "C" { #[cfg(target_os = "linux")] static mut __malloc_hook: Option *mut c_void>; @@ -308,6 +304,7 @@ mod tests { static FLAG: RefCell = RefCell::new(false); } + #[cfg(target_os = "linux")] extern "C" fn malloc_hook(size: usize) -> *mut c_void { unsafe { __malloc_hook = None; @@ -365,6 +362,7 @@ mod tests { prime_numbers } + #[cfg(target_os = "linux")] #[test] fn malloc_free() { trigger_lazy(); From 59bad74e3c5f1d1641b7d509423734f08cc52faf Mon Sep 17 00:00:00 2001 From: YangKeao Date: Fri, 29 Oct 2021 14:46:52 +1100 Subject: [PATCH 07/11] fix clippy Signed-off-by: YangKeao --- examples/malloc_hook.rs | 1 + src/collector.rs | 45 ++++++++++++++++++++++++----------------- src/profiler.rs | 11 +++++++--- 3 files changed, 36 insertions(+), 21 deletions(-) diff --git a/examples/malloc_hook.rs b/examples/malloc_hook.rs index 4d23fc33..2957c36d 100644 --- a/examples/malloc_hook.rs +++ b/examples/malloc_hook.rs @@ -7,6 +7,7 @@ use std::ffi::c_void; #[cfg(not(target_os = "linux"))] #[allow(clippy::wrong_self_convention)] +#[allow(non_upper_case_globals)] static mut __malloc_hook: Option *mut c_void> = None; extern "C" { diff --git a/src/collector.rs b/src/collector.rs index ec11c1b9..71b897e9 100644 --- a/src/collector.rs +++ b/src/collector.rs @@ -247,12 +247,25 @@ impl Collector { } } +#[cfg(test)] +mod test_utils { + use super::*; + use std::collections::BTreeMap; + + pub fn add_map(hashmap: &mut BTreeMap, entry: &Entry) { + match hashmap.get_mut(&entry.item) { + None => { + hashmap.insert(entry.item, entry.count); + } + Some(count) => *count += entry.count, + } + } +} + #[cfg(test)] mod tests { use super::*; - use std::cell::RefCell; use std::collections::BTreeMap; - use std::ffi::c_void; #[test] fn stack_hash_counter() { @@ -272,15 +285,6 @@ mod tests { }); } - fn add_map(hashmap: &mut BTreeMap, entry: &Entry) { - match hashmap.get_mut(&entry.item) { - None => { - hashmap.insert(entry.item, entry.count); - } - Some(count) => *count += entry.count, - } - } - #[test] fn evict_test() { let mut stack_hash_counter = StackHashCounter::::default(); @@ -291,14 +295,14 @@ mod tests { match stack_hash_counter.add(item, 1) { None => {} Some(evict) => { - add_map(&mut real_map, &evict); + test_utils::add_map(&mut real_map, &evict); } } } } stack_hash_counter.iter().for_each(|entry| { - add_map(&mut real_map, &entry); + test_utils::add_map(&mut real_map, &entry); }); for item in 0..(1 << 10) * 4 { @@ -326,7 +330,7 @@ mod tests { } collector.try_iter().unwrap().for_each(|entry| { - add_map(&mut real_map, &entry); + test_utils::add_map(&mut real_map, &entry); }); for item in 0..(1 << 12) * 4 { @@ -341,10 +345,15 @@ mod tests { } } } +} - #[cfg(not(target_os = "linux"))] - #[allow(clippy::wrong_self_convention)] - static mut __malloc_hook: Option *mut c_void> = None; +#[cfg(test)] +#[cfg(target_os = "linux")] +mod malloc_free_test { + use super::*; + use std::cell::RefCell; + use std::collections::BTreeMap; + use std::ffi::c_void; extern "C" { #[cfg(target_os = "linux")] @@ -397,7 +406,7 @@ mod tests { }); collector.try_iter().unwrap().for_each(|entry| { - add_map(&mut real_map, &entry); + test_utils::add_map(&mut real_map, &entry); }); for item in 0..(1 << 10) * 4 { diff --git a/src/profiler.rs b/src/profiler.rs index 985d85c1..335ed782 100644 --- a/src/profiler.rs +++ b/src/profiler.rs @@ -7,7 +7,7 @@ use backtrace::Frame; use nix::sys::signal; use parking_lot::RwLock; -#[cfg(feature = "ignore-libc")] +#[cfg(all(feature = "ignore-libc", target_arch = "x86_64", target_os = "linux"))] use findshlibs::{Segment, SharedLibrary, TargetSharedLibrary}; use crate::collector::Collector; @@ -124,6 +124,10 @@ fn write_thread_name(current_thread: libc::pthread_t, name: &mut [libc::c_char]) #[no_mangle] #[allow(clippy::uninit_assumed_init)] +#[cfg_attr( + not(all(feature = "ignore-libc", target_arch = "x86_64", target_os = "linux")), + allow(unused_variables) +)] extern "C" fn perf_signal_handler( _signal: c_int, _siginfo: *mut libc::siginfo_t, @@ -287,14 +291,16 @@ impl Profiler { } #[cfg(test)] +#[cfg(target_os = "linux")] mod tests { use super::*; + use std::cell::RefCell; use std::ffi::c_void; + use std::ptr::null_mut; extern "C" { - #[cfg(target_os = "linux")] static mut __malloc_hook: Option *mut c_void>; fn malloc(size: usize) -> *mut c_void; @@ -304,7 +310,6 @@ mod tests { static FLAG: RefCell = RefCell::new(false); } - #[cfg(target_os = "linux")] extern "C" fn malloc_hook(size: usize) -> *mut c_void { unsafe { __malloc_hook = None; From 1b7b26c5818e7a0fef187ee8f38fd52fae9b3aa4 Mon Sep 17 00:00:00 2001 From: YangKeao Date: Fri, 29 Oct 2021 20:03:54 +1100 Subject: [PATCH 08/11] use a configurable option to set blacklist Signed-off-by: YangKeao --- Cargo.toml | 4 +- src/lib.rs | 2 +- src/profiler.rs | 161 ++++++++++++++++++++++++++++++------------------ 3 files changed, 104 insertions(+), 63 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d23b1c1e..17a49ffe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,6 @@ default = ["cpp"] flamegraph = ["inferno"] protobuf = ["prost", "prost-derive", "prost-build"] cpp = ["symbolic-demangle/cpp"] -ignore-libc = ["findshlibs"] [dependencies] backtrace = "0.3" @@ -25,14 +24,13 @@ nix = "0.23" parking_lot = "0.11" tempfile = "3.1" thiserror = "1.0" +findshlibs = "0.10" inferno = { version = "0.10", default-features = false, features = ["nameattr"], optional = true } prost = { version = "0.9", optional = true } prost-derive = { version = "0.9", optional = true } criterion = {version = "0.3", optional = true} -findshlibs = {version = "0.10", optional = true} - [dependencies.symbolic-demangle] version = "8.0" default-features = false diff --git a/src/lib.rs b/src/lib.rs index 32da691a..956ccf9d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -40,7 +40,7 @@ mod timer; pub use self::collector::{Collector, StackHashCounter}; pub use self::error::{Error, Result}; pub use self::frames::{Frames, Symbol}; -pub use self::profiler::ProfilerGuard; +pub use self::profiler::{ProfilerGuard, ProfilerGuardBuilder}; pub use self::report::{Report, ReportBuilder}; #[cfg(feature = "flamegraph")] diff --git a/src/profiler.rs b/src/profiler.rs index 335ed782..454b5265 100644 --- a/src/profiler.rs +++ b/src/profiler.rs @@ -7,7 +7,7 @@ use backtrace::Frame; use nix::sys::signal; use parking_lot::RwLock; -#[cfg(all(feature = "ignore-libc", target_arch = "x86_64", target_os = "linux"))] +#[cfg(all(target_arch = "x86_64", target_os = "linux"))] use findshlibs::{Segment, SharedLibrary, TargetSharedLibrary}; use crate::collector::Collector; @@ -27,10 +27,93 @@ pub struct Profiler { running: bool, - #[cfg(all(feature = "ignore-libc", target_arch = "x86_64", target_os = "linux"))] + #[cfg(all(target_arch = "x86_64", target_os = "linux"))] blacklist_segments: Vec<(usize, usize)>, } +pub struct ProfilerGuardBuilder { + frequency: c_int, + #[cfg(all(target_arch = "x86_64", target_os = "linux"))] + blacklist_segments: Vec<(usize, usize)>, +} + +impl Default for ProfilerGuardBuilder { + fn default() -> ProfilerGuardBuilder { + ProfilerGuardBuilder { + frequency: 99, + + #[cfg(all(target_arch = "x86_64", target_os = "linux"))] + blacklist_segments: Vec::new(), + } + } +} + +impl ProfilerGuardBuilder { + pub fn frequency(self, frequency: c_int) -> Self { + Self { frequency, ..self } + } + #[cfg(all(target_arch = "x86_64", target_os = "linux"))] + pub fn blacklist>(self, blacklist: &[T]) -> Self { + let blacklist_segments = { + let mut segments = Vec::new(); + TargetSharedLibrary::each(|shlib| { + let in_blacklist = match shlib.name().to_str() { + Some(name) => { + let mut in_blacklist = false; + for blocked_name in blacklist.iter() { + if name.contains(blocked_name.as_ref()) { + in_blacklist = true; + } + } + + in_blacklist + } + + None => false, + }; + if in_blacklist { + for seg in shlib.segments() { + let avam = seg.actual_virtual_memory_address(shlib); + let start = avam.0; + let end = start + seg.len(); + segments.push((start, end)); + } + } + }); + segments + }; + + Self { + blacklist_segments, + ..self + } + } + pub fn build(self) -> Result> { + trigger_lazy(); + + match PROFILER.write().as_mut() { + Err(err) => { + log::error!("Error in creating profiler: {}", err); + Err(Error::CreatingError) + } + Ok(profiler) => { + #[cfg(all(target_arch = "x86_64", target_os = "linux"))] + { + profiler.blacklist_segments = self.blacklist_segments; + } + + match profiler.start() { + Ok(()) => Ok(ProfilerGuard::<'static> { + profiler: &PROFILER, + timer: Some(Timer::new(self.frequency)), + }), + Err(err) => Err(err), + } + } + } + } +} + /// RAII structure used to stop profiling when dropped. It is the only interface to access profiler. pub struct ProfilerGuard<'a> { profiler: &'a RwLock>, @@ -45,21 +128,7 @@ fn trigger_lazy() { impl ProfilerGuard<'_> { /// Start profiling with given sample frequency. pub fn new(frequency: c_int) -> Result> { - trigger_lazy(); - - match PROFILER.write().as_mut() { - Err(err) => { - log::error!("Error in creating profiler: {}", err); - Err(Error::CreatingError) - } - Ok(profiler) => match profiler.start() { - Ok(()) => Ok(ProfilerGuard::<'static> { - profiler: &PROFILER, - timer: Some(Timer::new(frequency)), - }), - Err(err) => Err(err), - }, - } + ProfilerGuardBuilder::default().frequency(frequency).build() } /// Generate a report @@ -74,10 +143,17 @@ impl<'a> Drop for ProfilerGuard<'a> { match self.profiler.write().as_mut() { Err(_) => {} - Ok(profiler) => match profiler.stop() { - Ok(()) => {} - Err(err) => log::error!("error while stopping profiler {}", err), - }, + Ok(profiler) => { + #[cfg(all(target_arch = "x86_64", target_os = "linux"))] + { + profiler.blacklist_segments = Vec::new(); + } + + match profiler.stop() { + Ok(()) => {} + Err(err) => log::error!("error while stopping profiler {}", err), + } + } } } } @@ -125,7 +201,7 @@ fn write_thread_name(current_thread: libc::pthread_t, name: &mut [libc::c_char]) #[no_mangle] #[allow(clippy::uninit_assumed_init)] #[cfg_attr( - not(all(feature = "ignore-libc", target_arch = "x86_64", target_os = "linux")), + not(all(target_arch = "x86_64", target_os = "linux")), allow(unused_variables) )] extern "C" fn perf_signal_handler( @@ -135,7 +211,7 @@ extern "C" fn perf_signal_handler( ) { if let Some(mut guard) = PROFILER.try_write() { if let Ok(profiler) = guard.as_mut() { - #[cfg(all(feature = "ignore-libc", target_arch = "x86_64", target_os = "linux"))] + #[cfg(all(target_arch = "x86_64", target_os = "linux"))] if !ucontext.is_null() { let ucontext: *mut libc::ucontext_t = ucontext as *mut libc::ucontext_t; let addr = @@ -173,52 +249,19 @@ extern "C" fn perf_signal_handler( } } -#[cfg(all(feature = "ignore-libc", target_arch = "x86_64", target_os = "linux"))] -const SHLIB_BLACKLIST: [&str; 3] = ["libc", "libgcc_s", "libpthread"]; - impl Profiler { fn new() -> Result { - #[cfg(all(feature = "ignore-libc", target_arch = "x86_64", target_os = "linux"))] - let blacklist_segments = { - let mut segments = Vec::new(); - TargetSharedLibrary::each(|shlib| { - let in_blacklist = match shlib.name().to_str() { - Some(name) => { - let mut in_blacklist = false; - for blocked_name in SHLIB_BLACKLIST.iter() { - if name.contains(blocked_name) { - in_blacklist = true; - } - } - - in_blacklist - } - - None => false, - }; - if in_blacklist { - for seg in shlib.segments() { - let avam = seg.actual_virtual_memory_address(shlib); - let start = avam.0; - let end = start + seg.len(); - segments.push((start, end)); - } - } - }); - segments - }; - Ok(Profiler { data: Collector::new()?, sample_counter: 0, running: false, - #[cfg(all(feature = "ignore-libc", target_arch = "x86_64", target_os = "linux"))] - blacklist_segments, + #[cfg(all(target_arch = "x86_64", target_os = "linux"))] + blacklist_segments: Vec::new(), }) } - #[cfg(all(feature = "ignore-libc", target_arch = "x86_64", target_os = "linux"))] + #[cfg(all(target_arch = "x86_64", target_os = "linux"))] fn is_blacklisted(&self, addr: usize) -> bool { for libs in &self.blacklist_segments { if addr > libs.0 && addr < libs.1 { From d2b1fb6b98d3ff33ccf3427b85f0be10c2dfbd57 Mon Sep 17 00:00:00 2001 From: YangKeao Date: Fri, 29 Oct 2021 20:46:44 +1100 Subject: [PATCH 09/11] add document and don't reset blacklist_segments Signed-off-by: YangKeao --- README.md | 8 ++++++-- examples/backtrace_while_sampling.rs | 6 +++++- src/lib.rs | 23 +++++++++++++++++++---- src/profiler.rs | 20 ++++++++------------ 4 files changed, 38 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 594535e1..598e0b97 100644 --- a/README.md +++ b/README.md @@ -37,8 +37,6 @@ FRAME: backtrace::backtrace::trace::h3e91a3123a3049a5 -> FRAME: pprof::profiler: - `cpp` enables the cpp demangle. - `flamegraph` enables the flamegraph report format. - `protobuf` enables the pprof protobuf report format. -- `ignore-libc` will ignore the `libc`, `libgcc` and `pthread` while sampling. THIS FEATURE IS ONLY AVAILABLE ON LINUX. - This feature will avoid deadlock in the implementation of `_Unwind_Backtrace`. ## Flamegraph @@ -215,6 +213,12 @@ Unfortunately, there is no 100% robust stack tracing method. [Some related resea > libgcc's unwind method is not safe to use from signal handlers. One particular cause of deadlock is when profiling tick happens when program is propagating thrown exception. +This can be resolved by adding a blacklist: + +```rust +let guard = pprof::ProfilerGuardBuilder::default().frequency(1000).blacklist(&["libc", "libgcc", "pthread"]).build().unwrap(); +``` + ### Signal Safety Signal safety is hard to guarantee. But it's not *that* hard. diff --git a/examples/backtrace_while_sampling.rs b/examples/backtrace_while_sampling.rs index 78ea2614..f3f0463d 100644 --- a/examples/backtrace_while_sampling.rs +++ b/examples/backtrace_while_sampling.rs @@ -12,7 +12,11 @@ fn deep_recursive(depth: i32) { } fn main() { - let guard = pprof::ProfilerGuard::new(1000).unwrap(); + let guard = pprof::ProfilerGuardBuilder::default() + .frequency(1000) + .blacklist(&["libc", "libgcc", "pthread"]) + .build() + .unwrap(); for _ in 0..10000 { deep_recursive(20); diff --git a/src/lib.rs b/src/lib.rs index 956ccf9d..e36e87fe 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,9 +2,9 @@ //! pprof-rs is an integrated profiler for rust program. //! -//! This crate provides a programable interface to start/stop/report a profiler dynamically. With the -//! help of this crate, you can easily integrate a profiler into your rust program in a modern, convenient -//! way. +//! This crate provides a programable interface to start/stop/report a profiler +//! dynamically. With the help of this crate, you can easily integrate a +//! profiler into your rust program in a modern, convenient way. //! //! A sample usage is: //! @@ -21,7 +21,22 @@ //!}; //! ``` //! -//! You can find more details in [README.md](https://github.com/tikv/pprof-rs/blob/master/README.md) +//! More configuration can be passed through `ProfilerGuardBuilder`: +//! +//! ```rust +//! let guard = pprof::ProfilerGuardBuilder::default().frequency(1000).blacklist(&["libc", "libgcc", "pthread"]).build().unwrap(); +//! ``` +//! +//! The frequency means the sampler frequency, and the `blacklist` means the +//! profiler will ignore the sample whose first frame is from library containing +//! these strings. +//! +//! Skipping `libc`, `libgcc` and `libpthread` could be a solution to the +//! possible deadlock inside the `_Unwind_Backtrace`, and keep the signal +//! safety. +//! +//! You can find more details in +//! [README.md](https://github.com/tikv/pprof-rs/blob/master/README.md) /// Define the MAX supported stack depth. TODO: make this variable mutable. pub const MAX_DEPTH: usize = 32; diff --git a/src/profiler.rs b/src/profiler.rs index 454b5265..e92e5b31 100644 --- a/src/profiler.rs +++ b/src/profiler.rs @@ -7,7 +7,10 @@ use backtrace::Frame; use nix::sys::signal; use parking_lot::RwLock; -#[cfg(all(target_arch = "x86_64", target_os = "linux"))] +#[cfg(all( + any(target_arch = "x86_64", target_arch = "aarch64", target_arch = "arm"), + target_os = "linux" +))] use findshlibs::{Segment, SharedLibrary, TargetSharedLibrary}; use crate::collector::Collector; @@ -143,17 +146,10 @@ impl<'a> Drop for ProfilerGuard<'a> { match self.profiler.write().as_mut() { Err(_) => {} - Ok(profiler) => { - #[cfg(all(target_arch = "x86_64", target_os = "linux"))] - { - profiler.blacklist_segments = Vec::new(); - } - - match profiler.stop() { - Ok(()) => {} - Err(err) => log::error!("error while stopping profiler {}", err), - } - } + Ok(profiler) => match profiler.stop() { + Ok(()) => {} + Err(err) => log::error!("error while stopping profiler {}", err), + }, } } } From 1fc974a60b1f5fb8345de0a71f95787141ea4a51 Mon Sep 17 00:00:00 2001 From: YangKeao Date: Fri, 29 Oct 2021 21:05:07 +1100 Subject: [PATCH 10/11] support aarch64 Signed-off-by: YangKeao --- Cargo.toml | 1 + src/profiler.rs | 49 ++++++++++++++++++++++++++++++++++++------------- 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 17a49ffe..4767cfaa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ parking_lot = "0.11" tempfile = "3.1" thiserror = "1.0" findshlibs = "0.10" +cfg-if = "1.0" inferno = { version = "0.10", default-features = false, features = ["nameattr"], optional = true } prost = { version = "0.9", optional = true } diff --git a/src/profiler.rs b/src/profiler.rs index e92e5b31..b0f5955e 100644 --- a/src/profiler.rs +++ b/src/profiler.rs @@ -7,10 +7,7 @@ use backtrace::Frame; use nix::sys::signal; use parking_lot::RwLock; -#[cfg(all( - any(target_arch = "x86_64", target_arch = "aarch64", target_arch = "arm"), - target_os = "linux" -))] +#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] use findshlibs::{Segment, SharedLibrary, TargetSharedLibrary}; use crate::collector::Collector; @@ -30,13 +27,13 @@ pub struct Profiler { running: bool, - #[cfg(all(target_arch = "x86_64", target_os = "linux"))] + #[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64")))] blacklist_segments: Vec<(usize, usize)>, } pub struct ProfilerGuardBuilder { frequency: c_int, - #[cfg(all(target_arch = "x86_64", target_os = "linux"))] + #[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64")))] blacklist_segments: Vec<(usize, usize)>, } @@ -45,7 +42,7 @@ impl Default for ProfilerGuardBuilder { ProfilerGuardBuilder { frequency: 99, - #[cfg(all(target_arch = "x86_64", target_os = "linux"))] + #[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64")))] blacklist_segments: Vec::new(), } } @@ -55,7 +52,7 @@ impl ProfilerGuardBuilder { pub fn frequency(self, frequency: c_int) -> Self { Self { frequency, ..self } } - #[cfg(all(target_arch = "x86_64", target_os = "linux"))] + #[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64")))] pub fn blacklist>(self, blacklist: &[T]) -> Self { let blacklist_segments = { let mut segments = Vec::new(); @@ -100,7 +97,7 @@ impl ProfilerGuardBuilder { Err(Error::CreatingError) } Ok(profiler) => { - #[cfg(all(target_arch = "x86_64", target_os = "linux"))] + #[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64")))] { profiler.blacklist_segments = self.blacklist_segments; } @@ -197,7 +194,7 @@ fn write_thread_name(current_thread: libc::pthread_t, name: &mut [libc::c_char]) #[no_mangle] #[allow(clippy::uninit_assumed_init)] #[cfg_attr( - not(all(target_arch = "x86_64", target_os = "linux")), + not(all(any(target_arch = "x86_64", target_arch = "aarch64"))), allow(unused_variables) )] extern "C" fn perf_signal_handler( @@ -207,11 +204,37 @@ extern "C" fn perf_signal_handler( ) { if let Some(mut guard) = PROFILER.try_write() { if let Ok(profiler) = guard.as_mut() { - #[cfg(all(target_arch = "x86_64", target_os = "linux"))] + #[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64")))] if !ucontext.is_null() { let ucontext: *mut libc::ucontext_t = ucontext as *mut libc::ucontext_t; + + #[cfg(all(target_arch = "x86_64", target_os = "linux"))] let addr = unsafe { (*ucontext).uc_mcontext.gregs[libc::REG_RIP as usize] as usize }; + + #[cfg(all(target_arch = "x86_64", target_os = "macos"))] + let addr = unsafe { + let mcontext = (*ucontext).uc_mcontext; + if mcontext.is_null() { + 0 + } else { + (*mcontext).__ss.__rip as usize + } + }; + + #[cfg(all(target_arch = "aarch64", target_os = "linux"))] + let addr = unsafe { (*ucontext).uc_mcontext.pc as usize }; + + #[cfg(all(target_arch = "aarch64", target_os = "macos"))] + let addr = unsafe { + let mcontext = (*ucontext).uc_mcontext; + if mcontext.is_null() { + 0 + } else { + (*mcontext).__ss.__pc as usize + } + }; + if profiler.is_blacklisted(addr) { return; } @@ -252,12 +275,12 @@ impl Profiler { sample_counter: 0, running: false, - #[cfg(all(target_arch = "x86_64", target_os = "linux"))] + #[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64")))] blacklist_segments: Vec::new(), }) } - #[cfg(all(target_arch = "x86_64", target_os = "linux"))] + #[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64")))] fn is_blacklisted(&self, addr: usize) -> bool { for libs in &self.blacklist_segments { if addr > libs.0 && addr < libs.1 { From 6e7bc098e50441906744200b16e69adc8e07df0f Mon Sep 17 00:00:00 2001 From: YangKeao Date: Mon, 1 Nov 2021 17:59:38 +1100 Subject: [PATCH 11/11] replace blacklist into blocklist Signed-off-by: YangKeao --- README.md | 4 ++-- examples/backtrace_while_sampling.rs | 2 +- src/lib.rs | 4 ++-- src/profiler.rs | 34 ++++++++++++++-------------- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 598e0b97..e3df7a37 100644 --- a/README.md +++ b/README.md @@ -213,10 +213,10 @@ Unfortunately, there is no 100% robust stack tracing method. [Some related resea > libgcc's unwind method is not safe to use from signal handlers. One particular cause of deadlock is when profiling tick happens when program is propagating thrown exception. -This can be resolved by adding a blacklist: +This can be resolved by adding a blocklist: ```rust -let guard = pprof::ProfilerGuardBuilder::default().frequency(1000).blacklist(&["libc", "libgcc", "pthread"]).build().unwrap(); +let guard = pprof::ProfilerGuardBuilder::default().frequency(1000).blocklist(&["libc", "libgcc", "pthread"]).build().unwrap(); ``` ### Signal Safety diff --git a/examples/backtrace_while_sampling.rs b/examples/backtrace_while_sampling.rs index f3f0463d..90b6d613 100644 --- a/examples/backtrace_while_sampling.rs +++ b/examples/backtrace_while_sampling.rs @@ -14,7 +14,7 @@ fn deep_recursive(depth: i32) { fn main() { let guard = pprof::ProfilerGuardBuilder::default() .frequency(1000) - .blacklist(&["libc", "libgcc", "pthread"]) + .blocklist(&["libc", "libgcc", "pthread"]) .build() .unwrap(); diff --git a/src/lib.rs b/src/lib.rs index e36e87fe..54ec07c4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -24,10 +24,10 @@ //! More configuration can be passed through `ProfilerGuardBuilder`: //! //! ```rust -//! let guard = pprof::ProfilerGuardBuilder::default().frequency(1000).blacklist(&["libc", "libgcc", "pthread"]).build().unwrap(); +//! let guard = pprof::ProfilerGuardBuilder::default().frequency(1000).blocklist(&["libc", "libgcc", "pthread"]).build().unwrap(); //! ``` //! -//! The frequency means the sampler frequency, and the `blacklist` means the +//! The frequency means the sampler frequency, and the `blocklist` means the //! profiler will ignore the sample whose first frame is from library containing //! these strings. //! diff --git a/src/profiler.rs b/src/profiler.rs index b0f5955e..529603c8 100644 --- a/src/profiler.rs +++ b/src/profiler.rs @@ -28,13 +28,13 @@ pub struct Profiler { running: bool, #[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64")))] - blacklist_segments: Vec<(usize, usize)>, + blocklist_segments: Vec<(usize, usize)>, } pub struct ProfilerGuardBuilder { frequency: c_int, #[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64")))] - blacklist_segments: Vec<(usize, usize)>, + blocklist_segments: Vec<(usize, usize)>, } impl Default for ProfilerGuardBuilder { @@ -43,7 +43,7 @@ impl Default for ProfilerGuardBuilder { frequency: 99, #[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64")))] - blacklist_segments: Vec::new(), + blocklist_segments: Vec::new(), } } } @@ -53,25 +53,25 @@ impl ProfilerGuardBuilder { Self { frequency, ..self } } #[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64")))] - pub fn blacklist>(self, blacklist: &[T]) -> Self { - let blacklist_segments = { + pub fn blocklist>(self, blocklist: &[T]) -> Self { + let blocklist_segments = { let mut segments = Vec::new(); TargetSharedLibrary::each(|shlib| { - let in_blacklist = match shlib.name().to_str() { + let in_blocklist = match shlib.name().to_str() { Some(name) => { - let mut in_blacklist = false; - for blocked_name in blacklist.iter() { + let mut in_blocklist = false; + for blocked_name in blocklist.iter() { if name.contains(blocked_name.as_ref()) { - in_blacklist = true; + in_blocklist = true; } } - in_blacklist + in_blocklist } None => false, }; - if in_blacklist { + if in_blocklist { for seg in shlib.segments() { let avam = seg.actual_virtual_memory_address(shlib); let start = avam.0; @@ -84,7 +84,7 @@ impl ProfilerGuardBuilder { }; Self { - blacklist_segments, + blocklist_segments, ..self } } @@ -99,7 +99,7 @@ impl ProfilerGuardBuilder { Ok(profiler) => { #[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64")))] { - profiler.blacklist_segments = self.blacklist_segments; + profiler.blocklist_segments = self.blocklist_segments; } match profiler.start() { @@ -235,7 +235,7 @@ extern "C" fn perf_signal_handler( } }; - if profiler.is_blacklisted(addr) { + if profiler.is_blocklisted(addr) { return; } } @@ -276,13 +276,13 @@ impl Profiler { running: false, #[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64")))] - blacklist_segments: Vec::new(), + blocklist_segments: Vec::new(), }) } #[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64")))] - fn is_blacklisted(&self, addr: usize) -> bool { - for libs in &self.blacklist_segments { + fn is_blocklisted(&self, addr: usize) -> bool { + for libs in &self.blocklist_segments { if addr > libs.0 && addr < libs.1 { return true; }