From 1dbfe3e50affc4542dd4ec881c16ce00649f1e1b Mon Sep 17 00:00:00 2001 From: Mark Stemm Date: Mon, 5 Dec 2016 10:11:41 -0800 Subject: [PATCH 1/2] Add ability to write capture stats to a file. With -s, periodically fetch capture stats from the inspector and write them to the provided file. Separate class StatsFileWriter handles the details. It does rely on a timer + SIGALRM handler so you can only practically create a single object, but it does keep the code/state separate. The output format has a sample number, the set of current stats, a delta with the difference from the prior sample, and the percentage of events dropped during that sample. --- userspace/falco/CMakeLists.txt | 2 +- userspace/falco/falco.cpp | 28 ++++++++- userspace/falco/statsfilewriter.cpp | 90 +++++++++++++++++++++++++++++ userspace/falco/statsfilewriter.h | 30 ++++++++++ 4 files changed, 146 insertions(+), 4 deletions(-) create mode 100644 userspace/falco/statsfilewriter.cpp create mode 100644 userspace/falco/statsfilewriter.h diff --git a/userspace/falco/CMakeLists.txt b/userspace/falco/CMakeLists.txt index 41988076522..9314d051959 100644 --- a/userspace/falco/CMakeLists.txt +++ b/userspace/falco/CMakeLists.txt @@ -9,7 +9,7 @@ include_directories("${CURL_INCLUDE_DIR}") include_directories("${YAMLCPP_INCLUDE_DIR}") include_directories("${DRAIOS_DEPENDENCIES_DIR}/yaml-${DRAIOS_YAML_VERSION}/target/include") -add_executable(falco configuration.cpp logger.cpp falco_outputs.cpp falco.cpp) +add_executable(falco configuration.cpp logger.cpp falco_outputs.cpp statsfilewriter.cpp falco.cpp) target_link_libraries(falco falco_engine sinsp) target_link_libraries(falco diff --git a/userspace/falco/falco.cpp b/userspace/falco/falco.cpp index e15bf2b2da8..b55b2c6ae95 100644 --- a/userspace/falco/falco.cpp +++ b/userspace/falco/falco.cpp @@ -36,6 +36,7 @@ along with falco. If not, see . #include "configuration.h" #include "falco_engine.h" #include "config_falco.h" +#include "statsfilewriter.h" bool g_terminate = false; // @@ -97,6 +98,8 @@ static void usage() " -P, --pidfile When run as a daemon, write pid to specified file\n" " -r Rules file (defaults to value set in configuration file, or /etc/falco_rules.yaml).\n" " Can be specified multiple times to read from multiple files.\n" + " -s If specified, write statistics related to falco's reading/processing of events\n" + " to this file. (Only useful in live mode).\n" " -v Verbose output.\n" "\n" ); @@ -124,11 +127,23 @@ std::list cmdline_options; // uint64_t do_inspect(falco_engine *engine, falco_outputs *outputs, - sinsp* inspector) + sinsp* inspector, + string &stats_filename) { uint64_t num_evts = 0; int32_t res; sinsp_evt* ev; + StatsFileWriter writer; + + if (stats_filename != "") + { + string errstr; + + if (!writer.init(inspector, stats_filename, 5, errstr)) + { + throw falco_exception(errstr); + } + } // // Loop through the events @@ -138,6 +153,8 @@ uint64_t do_inspect(falco_engine *engine, res = inspector->next(&ev); + writer.handle(); + if (g_terminate) { break; @@ -202,6 +219,7 @@ int falco_init(int argc, char **argv) string pidfilename = "/var/run/falco.pid"; bool describe_all_rules = false; string describe_rule = ""; + string stats_filename = ""; bool verbose = false; bool all_events = false; string* k8s_api = 0; @@ -246,7 +264,7 @@ int falco_init(int argc, char **argv) // Parse the args // while((op = getopt_long(argc, argv, - "hc:AdD:e:k:K:Ll:m:o:P:p:r:vw:", + "hc:AdD:e:k:K:Ll:m:o:P:p:r:s:vw:", long_options, &long_index)) != -1) { switch(op) @@ -318,6 +336,9 @@ int falco_init(int argc, char **argv) case 'r': rules_filenames.push_back(optarg); break; + case 's': + stats_filename = optarg; + break; case 'v': verbose = true; break; @@ -588,7 +609,8 @@ int falco_init(int argc, char **argv) num_evts = do_inspect(engine, outputs, - inspector); + inspector, + stats_filename); duration = ((double)clock()) / CLOCKS_PER_SEC - duration; diff --git a/userspace/falco/statsfilewriter.cpp b/userspace/falco/statsfilewriter.cpp new file mode 100644 index 00000000000..e50032b598e --- /dev/null +++ b/userspace/falco/statsfilewriter.cpp @@ -0,0 +1,90 @@ +#include +#include + +#include "statsfilewriter.h" + +using namespace std; + +static bool g_save_stats = false; +static void timer_handler (int signum) +{ + g_save_stats = true; +} + +StatsFileWriter::StatsFileWriter() + : m_num_stats(0), m_inspector(NULL) +{ +} + +StatsFileWriter::~StatsFileWriter() +{ + m_output.close(); +} + +bool StatsFileWriter::init(sinsp *inspector, string &filename, uint32_t interval_sec, string &errstr) +{ + struct itimerval timer; + struct sigaction handler; + + m_inspector = inspector; + + m_output.exceptions ( ofstream::failbit | ofstream::badbit ); + m_output.open(filename, ios_base::app); + + memset (&handler, 0, sizeof (handler)); + handler.sa_handler = &timer_handler; + if (sigaction(SIGALRM, &handler, NULL) == -1) + { + errstr = string("Could not set up signal handler for periodic timer: ") + strerror(errno); + return false; + } + + timer.it_value.tv_sec = interval_sec; + timer.it_value.tv_usec = 0; + timer.it_interval = timer.it_value; + if (setitimer(ITIMER_REAL, &timer, NULL) == -1) + { + errstr = string("Could not set up periodic timer: ") + strerror(errno); + return false; + } + + return true; +} + +void StatsFileWriter::handle() +{ + if (g_save_stats) + { + scap_stats cstats; + scap_stats delta; + + g_save_stats = false; + m_num_stats++; + m_inspector->get_capture_stats(&cstats); + + if(m_num_stats == 1) + { + delta = cstats; + } + else + { + delta.n_evts = cstats.n_evts - m_last_stats.n_evts; + delta.n_drops = cstats.n_drops - m_last_stats.n_drops; + delta.n_preemptions = cstats.n_preemptions - m_last_stats.n_preemptions; + } + + m_output << "{\"sample\": " << m_num_stats << + ", \"cur\": {" << + "\"events\": " << cstats.n_evts << + ", \"drops\": " << cstats.n_drops << + ", \"preemptions\": " << cstats.n_preemptions << + "}, \"delta\": {" << + "\"events\": " << delta.n_evts << + ", \"drops\": " << delta.n_drops << + ", \"preemptions\": " << delta.n_preemptions << + "}, \"drop_pct\": " << (delta.n_evts == 0 ? 0 : (100.0*delta.n_drops/delta.n_evts)) << + "}," << endl; + + m_last_stats = cstats; + } +} diff --git a/userspace/falco/statsfilewriter.h b/userspace/falco/statsfilewriter.h new file mode 100644 index 00000000000..8c95a0aa01d --- /dev/null +++ b/userspace/falco/statsfilewriter.h @@ -0,0 +1,30 @@ +#pragma once + +#include + +#include + +// Periodically collects scap stats files and writes them to a file as +// json. + +class StatsFileWriter { +public: + StatsFileWriter(); + virtual ~StatsFileWriter(); + + // Returns success as bool. On false fills in errstr. + bool init(sinsp *inspector, std::string &filename, + uint32_t interval_sec, + string &errstr); + + // Should be called often (like for each event in a sinsp + // loop). + void handle(); + +protected: + uint32_t m_num_stats; + sinsp *m_inspector; + std::ofstream m_output; + + scap_stats m_last_stats; +}; From ca1d263f9b5a25e302db3579cfdf6b12a56a4b66 Mon Sep 17 00:00:00 2001 From: Mark Stemm Date: Mon, 5 Dec 2016 11:59:55 -0800 Subject: [PATCH 2/2] Add ability to write "extra" stuff to stats file. When run via scripts like run_performance_tests.sh, it's useful to include extra info like the test being run and the specific program variant to the stats file. So support that via the environment. Environment keys starting with FALCO_STATS_EXTRA_XXX will have the XXX and environment value added to the stats file. It's undocumented as I doubt other programs will need this functionality and it keeps the docs simpler. --- test/run_performance_tests.sh | 14 +++++++++--- userspace/falco/statsfilewriter.cpp | 34 +++++++++++++++++++++++++++-- userspace/falco/statsfilewriter.h | 4 +++- 3 files changed, 46 insertions(+), 6 deletions(-) diff --git a/test/run_performance_tests.sh b/test/run_performance_tests.sh index b9885c1fc36..4663709b025 100644 --- a/test/run_performance_tests.sh +++ b/test/run_performance_tests.sh @@ -143,13 +143,13 @@ function start_subject_prog() { if [ -z $RULES_FILE ]; then RULES_FILE=$SOURCE/../output/rules.yaml fi - sudo $ROOT/test_mm -s $SOURCE/search_order.yaml -r $RULES_FILE > ./prog-output.txt 2>&1 & + sudo FALCO_STATS_EXTRA_variant=$VARIANT FALCO_STATS_EXTRA_benchmark=$live_test $ROOT/test_mm -S $SOURCE/search_order.yaml -s $STATS_FILE -r $RULES_FILE > ./prog-output.txt 2>&1 & elif [[ $ROOT == *"falco"* ]]; then echo " starting falco..." if [ -z $RULES_FILE ]; then RULES_FILE=$SOURCE/rules/falco_rules.yaml fi - sudo $ROOT/userspace/falco/falco -c $SOURCE/falco.yaml -r $RULES_FILE --option=stdout_output.enabled=false > ./prog-output.txt -A 2>&1 & + sudo FALCO_STATS_EXTRA_variant=$VARIANT FALCO_STATS_EXTRA_benchmark=$live_test $ROOT/userspace/falco/falco -c $SOURCE/falco.yaml -s $STATS_FILE -r $RULES_FILE --option=stdout_output.enabled=false > ./prog-output.txt -A 2>&1 & elif [[ $ROOT == *"sysdig"* ]]; then echo " starting sysdig..." sudo $ROOT/userspace/sysdig/sysdig -N -z evt.type=none & @@ -323,6 +323,7 @@ usage() { echo " -r/--root: root directory containing falco/sysdig binaries (i.e. where you ran 'cmake')" echo " -s/--source: root directory containing falco/sysdig source code" echo " -R/--results: append test results to this file" + echo " -S/--stats: append capture statistics to this file (only works for falco/test_mm)" echo " -o/--output: append program output to this file" echo " -U/--rules: path to rules file (only applicable for falco/test_mm)" echo " -t/--test: test to run. Argument has the following format:" @@ -342,7 +343,7 @@ usage() { echo " -F/--falco-agent: When running an agent, whether or not to enable falco" } -OPTS=`getopt -o hv:r:s:R:o:U:t:T: --long help,variant:,root:,source:,results:,output:,rules:,test:,tracedir:,agent-autodrop:,falco-agent: -n $0 -- "$@"` +OPTS=`getopt -o hv:r:s:R:S:o:U:t:T: --long help,variant:,root:,source:,results:,stats:,output:,rules:,test:,tracedir:,agent-autodrop:,falco-agent: -n $0 -- "$@"` if [ $? != 0 ]; then echo "Exiting" >&2 @@ -356,6 +357,7 @@ ROOT=`dirname $0`/../build SOURCE=$ROOT SCRIPTDIR=`dirname $0` RESULTS_FILE=`dirname $0`/results.json +STATS_FILE=`dirname $0`/capture_stats.json OUTPUT_FILE=`dirname $0`/program-output.txt RULES_FILE= TEST=trace:all @@ -371,6 +373,7 @@ while true; do -r | --root ) ROOT="$2"; shift 2;; -s | --source ) SOURCE="$2"; shift 2;; -R | --results ) RESULTS_FILE="$2"; shift 2;; + -S | --stats ) STATS_FILE="$2"; shift 2;; -o | --output ) OUTPUT_FILE="$2"; shift 2;; -U | --rules ) RULES_FILE="$2"; shift 2;; -t | --test ) TEST="$2"; shift 2;; @@ -405,6 +408,11 @@ if [ -z $RESULTS_FILE ]; then exit 1 fi +if [ -z $STATS_FILE ]; then + echo "An output file for capture statistics must be provided. Not continuing." + exit 1 +fi + if [ -z $OUTPUT_FILE ]; then echo "An file for program output must be provided. Not continuing." exit 1 diff --git a/userspace/falco/statsfilewriter.cpp b/userspace/falco/statsfilewriter.cpp index e50032b598e..8f01b4efb2a 100644 --- a/userspace/falco/statsfilewriter.cpp +++ b/userspace/falco/statsfilewriter.cpp @@ -11,6 +11,8 @@ static void timer_handler (int signum) g_save_stats = true; } +extern char **environ; + StatsFileWriter::StatsFileWriter() : m_num_stats(0), m_inspector(NULL) { @@ -48,6 +50,30 @@ bool StatsFileWriter::init(sinsp *inspector, string &filename, uint32_t interval return false; } + // (Undocumented) feature. Take any environment keys prefixed + // with FALCO_STATS_EXTRA_XXX and add them to the output. Used by + // run_performance_tests.sh. + for(uint32_t i=0; environ[i]; i++) + { + char *p = strstr(environ[i], "="); + if(!p) + { + errstr = string("Could not find environment separator in ") + string(environ[i]); + return false; + } + string key(environ[i], p-environ[i]); + string val(p+1, strlen(environ[i])-(p-environ[i])-1); + if(key.compare(0, 18, "FALCO_STATS_EXTRA_") == 0) + { + string sub = key.substr(18); + if (m_extra != "") + { + m_extra += ", "; + } + m_extra += "\"" + sub + "\": " + "\"" + val + "\""; + } + } + return true; } @@ -73,8 +99,12 @@ void StatsFileWriter::handle() delta.n_preemptions = cstats.n_preemptions - m_last_stats.n_preemptions; } - m_output << "{\"sample\": " << m_num_stats << - ", \"cur\": {" << + m_output << "{\"sample\": " << m_num_stats; + if(m_extra != "") + { + m_output << ", " << m_extra; + } + m_output << ", \"cur\": {" << "\"events\": " << cstats.n_evts << ", \"drops\": " << cstats.n_drops << ", \"preemptions\": " << cstats.n_preemptions << diff --git a/userspace/falco/statsfilewriter.h b/userspace/falco/statsfilewriter.h index 8c95a0aa01d..57d91d74857 100644 --- a/userspace/falco/statsfilewriter.h +++ b/userspace/falco/statsfilewriter.h @@ -1,6 +1,8 @@ #pragma once #include +#include +#include #include @@ -25,6 +27,6 @@ class StatsFileWriter { uint32_t m_num_stats; sinsp *m_inspector; std::ofstream m_output; - + std::string m_extra; scap_stats m_last_stats; };