Skip to content

Commit

Permalink
fix(libsinsp_e2e): make tests more stable
Browse files Browse the repository at this point in the history 
Signed-off-by: Roberto Scolaro <[email protected]>
  • Loading branch information
@therealbobo
therealbobo committed Sep 25, 2024
1 parent f87b40f commit f4ebf18
Show file tree
Hide file tree
Showing 7 changed files with 118 additions and 144 deletions.
12 changes: 2 additions & 10 deletions test/libsinsp_e2e/container/container.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ TEST_F(sys_call_test, container_cgroups) {

if(ctid >= 0) {
if(ctid == 0) {
sleep(1);
sleep(5);
// _exit prevents asan from complaining for a false positive memory leak.
_exit(0);
} else {
Expand Down Expand Up @@ -251,7 +251,7 @@ static void run_container_docker_test(bool fork_after_container_start) {
ASSERT_TRUE(false);
}

sleep(2);
sleep(10);

ASSERT_TRUE(system("docker kill libsinsp_docker > /dev/null 2>&1 || true") == 0);
ASSERT_TRUE(system("docker rm -v libsinsp_docker > /dev/null 2>&1") == 0);
Expand Down Expand Up @@ -607,14 +607,6 @@ static void healthcheck_helper(
};

run_callback_t test = [&](concurrent_object_handle<sinsp> inspector_handle) {
// Setting dropping mode preserves the execs but
// reduces the chances that we'll drop events during
// the docker fetch.
{
std::scoped_lock inspector_handle_lock(inspector_handle);
inspector_handle->start_dropping_mode(1);
}

int rc = dhelper.run_container("cont_health_ut", "/bin/sh -c '/bin/sleep 10'");

ASSERT_TRUE(exited_early || (rc == 0));
Expand Down
131 changes: 47 additions & 84 deletions test/libsinsp_e2e/event_capture.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ limitations under the License.
#include <libsinsp/sinsp.h>
#include <libscap/scap_engines.h>
#include <libsinsp/sinsp_cycledumper.h>
#include <string>
#include <unistd.h>

std::string event_capture::m_engine_string = KMOD_ENGINE;
Expand All @@ -31,41 +32,43 @@ unsigned long event_capture::m_buffer_dim = DEFAULT_DRIVER_BUFFER_BYTES_DIM;
bool event_capture::inspector_ok = false;

concurrent_object_handle<sinsp> event_capture::get_inspector_handle() {
return {get_inspector(), m_inspector_mutex};
return {m_inspector.get(), m_inspector_mutex};
}

void event_capture::init_inspector() {
get_inspector()->m_thread_manager->set_max_thread_table_size(m_max_thread_table_size);
get_inspector()->m_thread_timeout_ns = m_thread_timeout_ns;
get_inspector()->set_auto_threads_purging_interval_s(m_inactive_thread_scan_time_ns);
get_inspector()->set_auto_threads_purging(false);

get_inspector()->set_get_procs_cpu_from_driver(true);
m_inspector = std::unique_ptr<sinsp>(new sinsp);
m_inspector->m_thread_manager->set_max_thread_table_size(m_max_thread_table_size);
m_inspector->m_thread_timeout_ns = m_thread_timeout_ns;
m_inspector->set_auto_threads_purging_interval_s(m_inactive_thread_scan_time_ns);
m_inspector->set_auto_threads_purging(false);

ASSERT_FALSE(get_inspector()->is_capture());
ASSERT_FALSE(get_inspector()->is_live());
ASSERT_FALSE(get_inspector()->is_nodriver());
m_inspector->set_get_procs_cpu_from_driver(true);

ASSERT_FALSE(m_inspector->is_capture());
ASSERT_FALSE(m_inspector->is_live());
ASSERT_FALSE(m_inspector->is_nodriver());

try {
if(m_mode == SINSP_MODE_NODRIVER) {
get_inspector()->open_nodriver();
m_inspector->open_nodriver();
} else {
open_engine(event_capture::get_engine(), {});
}
} catch(sinsp_exception& e) {
m_start_failed = true;
m_start_failure_message =
"couldn't open inspector (maybe driver hasn't been loaded yet?) err=" +
get_inspector()->getlasterr() + " exception=" + e.what();
m_inspector->getlasterr() + " exception=" + e.what();
{
m_capture_started = true;
m_condition_started.notify_one();
m_condition_started.notify_all();
}
return;
}

get_inspector()->set_debug_mode(true);
get_inspector()->set_hostname_and_port_resolution_mode(false);
m_inspector->set_debug_mode(true);
m_inspector->set_hostname_and_port_resolution_mode(false);
}

void event_capture::capture() {
Expand All @@ -75,25 +78,21 @@ void event_capture::capture() {
{
std::scoped_lock init_lock(m_inspector_mutex, m_object_state_mutex);

if(!inspector_ok) {
init_inspector();
if(!m_start_failed) {
inspector_ok = true;
} else {
std::cerr << m_start_failure_message << std::endl;
return;
}
init_inspector();
if(m_start_failed) {
std::cerr << m_start_failure_message << std::endl;
return;
}

m_param.m_inspector = get_inspector();
m_param.m_inspector = m_inspector.get();

m_before_open(get_inspector());
m_before_open(m_inspector.get());

get_inspector()->start_capture();
m_inspector->start_capture();
if(m_mode != SINSP_MODE_NODRIVER) {
m_dump_filename = std::string(LIBSINSP_TEST_CAPTURES_PATH) +
test_info->test_case_name() + "_" + test_info->name() + ".scap";
dumper = std::make_unique<sinsp_cycledumper>(get_inspector(),
dumper = std::make_unique<sinsp_cycledumper>(m_inspector.get(),
m_dump_filename.c_str(),
0,
0,
Expand All @@ -103,83 +102,47 @@ void event_capture::capture() {
}
} // End init synchronized section

bool signaled_start = false;
sinsp_evt* event;
bool result = true;
int32_t next_result = SCAP_SUCCESS;
while(result && !::testing::Test::HasFatalFailure()) {
{
std::lock_guard<std::mutex> lock(m_object_state_mutex);
if(m_capture_stopped) {
break;
}
}
uint64_t timeouts = 0;

while(!::testing::Test::HasFatalFailure()) {
{
std::scoped_lock inspector_next_lock(m_inspector_mutex);
next_result = get_inspector()->next(&event);
next_result = m_inspector->next(&event);
}
if(SCAP_SUCCESS == next_result) {
result = handle_event(event);
if(m_mode != SINSP_MODE_NODRIVER && m_dump) {
dumper->dump(event);
}
}
if(!signaled_start) {
signaled_start = true;
{
std::lock_guard<std::mutex> lock(m_object_state_mutex);
if(next_result == SCAP_SUCCESS) {
if(!m_capture_started) {
m_capture_started = true;
m_condition_started.notify_all();
}
m_condition_started.notify_one();
}
}

if(m_mode != SINSP_MODE_NODRIVER) {
uint32_t n_timeouts = 0;
while(result && !::testing::Test::HasFatalFailure()) {
{
std::scoped_lock inspector_next_lock(m_inspector_mutex);
next_result = get_inspector()->next(&event);
}
if(next_result == SCAP_TIMEOUT) {
n_timeouts++;

if(n_timeouts < m_max_timeouts) {
continue;
} else {
if((strcmp(event->get_name(), "open") == 0 || strcmp(event->get_name(), "openat") == 0)
&& event->get_param_by_name("name")->as<std::string>() == "/tmp/test.lock") {
if(event->get_direction() == SCAP_ED_OUT ) {
break;
} else {
continue;
}
} else {
handle_event(event);
}

if(next_result == SCAP_FILTERED_EVENT) {
continue;
}
if(next_result != SCAP_SUCCESS) {
break;
}
if(m_dump) {
if(m_mode != SINSP_MODE_NODRIVER && m_dump) {
dumper->dump(event);
}
result = handle_event(event);
}
{
std::scoped_lock inspector_next_lock(m_inspector_mutex);
while(SCAP_SUCCESS == get_inspector()->next(&event)) {
// just consume the remaining events
}
}
}

{ // Begin teardown synchronized section
std::scoped_lock teardown_lock(m_inspector_mutex, m_object_state_mutex);
m_before_close(get_inspector());
m_before_close(m_inspector.get());

get_inspector()->stop_capture();
m_inspector->stop_capture();
if(m_mode != SINSP_MODE_NODRIVER) {
dumper->close();
}

m_capture_stopped = true;
m_condition_stopped.notify_all();
} // End teardown synchronized section

m_condition_stopped.notify_one();
Expand All @@ -190,7 +153,7 @@ void event_capture::stop_capture() {
std::scoped_lock init_lock(m_inspector_mutex, m_object_state_mutex);
get_inspector()->stop_capture();
m_capture_stopped = true;
m_condition_stopped.notify_one();
m_condition_stopped.notify_all();
}
}

Expand Down Expand Up @@ -249,7 +212,7 @@ void event_capture::open_engine(const std::string& engine_string,
}
#ifdef HAS_ENGINE_KMOD
else if(!engine_string.compare(KMOD_ENGINE)) {
get_inspector()->open_kmod(m_buffer_dim);
m_inspector->open_kmod(m_buffer_dim);
}
#endif
#ifdef HAS_ENGINE_BPF
Expand All @@ -259,12 +222,12 @@ void event_capture::open_engine(const std::string& engine_string,
<< std::endl;
exit(EXIT_FAILURE);
}
get_inspector()->open_bpf(event_capture::get_engine_path().c_str(), m_buffer_dim);
m_inspector->open_bpf(event_capture::get_engine_path().c_str(), m_buffer_dim);
}
#endif
#ifdef HAS_ENGINE_MODERN_BPF
else if(!engine_string.compare(MODERN_BPF_ENGINE)) {
get_inspector()->open_modern_bpf(m_buffer_dim);
m_inspector->open_modern_bpf(m_buffer_dim);
}
#endif
else {
Expand Down
19 changes: 12 additions & 7 deletions test/libsinsp_e2e/event_capture.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,11 +119,6 @@ class event_capture {

static bool always_continue() { return true; }

sinsp* get_inspector() {
static sinsp inspector = sinsp();
return &inspector;
}

static void run(run_callback_t run_function,
captured_event_callback_t captured_event_callback,
event_filter_t filter,
Expand All @@ -134,7 +129,7 @@ class event_capture {
uint64_t thread_timeout_ns = (uint64_t)60 * 1000 * 1000 * 1000,
uint64_t inactive_thread_scan_time_ns = (uint64_t)60 * 1000 * 1000 * 1000,
sinsp_mode_t mode = SINSP_MODE_LIVE,
uint64_t max_timeouts = 3,
uint64_t max_timeouts = 10,
bool dump = true) {
event_capture capturing;
{ // Synchronized section
Expand All @@ -158,14 +153,22 @@ class event_capture {

if(!capturing.m_start_failed.load()) {
run_function(capturing.get_inspector_handle());
capturing.stop_capture();
// This lseek is just used to be sure that all the events
// created in the run_function were consumed.
capturing.m_fd = open("/tmp/test.lock", O_CREAT|O_RDONLY, S_IRWXU);
capturing.wait_for_capture_stop();
} else {
std::unique_lock<std::mutex> error_lookup_lock(capturing.m_object_state_mutex);
GTEST_MESSAGE_(capturing.m_start_failure_message.c_str(),
::testing::TestPartResult::kFatalFailure);
}

if(capturing.m_fd != -1)
{
close(capturing.m_fd);
unlink("/tmp/test.lock");
}

thread.join();
}

Expand Down Expand Up @@ -208,8 +211,10 @@ class event_capture {
std::string m_start_failure_message;
std::string m_dump_filename;
callback_param m_param;
std::unique_ptr<sinsp> m_inspector;
static bool inspector_ok;
sinsp_mode_t m_mode;
uint64_t m_max_timeouts;
bool m_dump;
int64_t m_fd = -1;
};
8 changes: 7 additions & 1 deletion test/libsinsp_e2e/forking.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,10 @@ TEST_F(sys_call_test, forking_while_scap_stopped) {
int ctid; // child tid
int xstatus = 33; // child exit value

if(event_capture::m_engine_string != KMOD_ENGINE) {
GTEST_SKIP() << "This is test only works with the kmod engine";
}

//
// FILTER
//
Expand Down Expand Up @@ -283,7 +287,9 @@ TEST_F(sys_call_test, forking_process_expired) {
event_capture::always_continue,
131072,
5 * ONE_SECOND_IN_NS,
ONE_SECOND_IN_NS);
ONE_SECOND_IN_NS,
SINSP_MODE_LIVE,
20);
});

EXPECT_TRUE(sleep_caught);
Expand Down
10 changes: 3 additions & 7 deletions test/libsinsp_e2e/process.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -739,9 +739,7 @@ TEST_F(sys_call_test, procinfo_processchild_cpuload) {
if(tinfo->m_tid == ctid) {
uint64_t tcpu;

const sinsp_evt_param* parinfo = e->get_param(0);
// tcpu = *(uint64_t*)parinfo->m_val;
memcpy(&tcpu, parinfo->m_val, sizeof(uint64_t));
memcpy(&tcpu, e->get_param(0)->m_val, sizeof(uint64_t));

uint64_t delta = tcpu - lastcpu;

Expand Down Expand Up @@ -812,8 +810,7 @@ TEST_F(sys_call_test, procinfo_two_processchilds_cpuload) {
if(tinfo->m_tid == ctid) {
uint64_t tcpu;

const sinsp_evt_param* parinfo = e->get_param(0);
tcpu = *(uint64_t*)parinfo->m_val;
memcpy(&tcpu, e->get_param(0)->m_val, sizeof(uint64_t));

uint64_t delta = tcpu - lastcpu;

Expand All @@ -828,8 +825,7 @@ TEST_F(sys_call_test, procinfo_two_processchilds_cpuload) {
} else if(tinfo->m_tid == ctid1) {
uint64_t tcpu;

const sinsp_evt_param* parinfo = e->get_param(0);
tcpu = *(uint64_t*)parinfo->m_val;
memcpy(&tcpu, e->get_param(0)->m_val, sizeof(uint64_t));

uint64_t delta = tcpu - lastcpu1;

Expand Down
Loading

0 comments on commit f4ebf18

Please sign in to comment.