Start tracking k8s liveness/readiness probes

K8s has a similar but not identical method as docker for container
health checks. They are called liveness/readiness probes and are a part
of the pod specification, and not a part of the image.

Luckily, the pod configuration *is* a part of the container metadata as
stringified json, with a label
"annotation.kubectl.kubernetes.io/last-applied-configuration", so we can
use that label to identify liveness/readiness probes.

A new class sinsp_container_info::container_health_probe represents one
of these health probes. It has a probe
type (healthcheck/liveness/readiness), methods to parse the actual
executable/arguments/etc out of the container json and methods to add
the json back to a larger json object.

The container info now has a list of possible health probe objects and
iterates over them when dumping the container to json.

For threads, switch everything to use a threadinfo category instead of a
simple bool for has healthcheck. The possible values for the category
are CAT_NONE, CAT_HEALTHCHECK, and
CAT_LIVENESS_PROBE. identify_healthcheck becomes identify_category() but
otherwise behaves the same (passing categories down and checking the
args list otherwise).

The filterchecks aren't quite as generic as the threadinfo categories to
keep the filtering simple. A new field
proc.is_container_{liveness,readiness}_probe checks for k8s
liveness/readiness probes, and container.{liveness,readiness}_probe
prints the exe + args.

userspace/libsinsp/container.cpp

@@ -17,6 +17,8 @@ limitations under the License.
 
 */
 
+#include <algorithm>
+
 #include "container_engine/cri.h"
 #include "container_engine/docker.h"
 #include "container_engine/rkt.h"
@@ -128,8 +130,8 @@ bool sinsp_container_manager::resolve_container(sinsp_threadinfo* tinfo, bool qu
 
 #endif // CYGWING_AGENT
 
-	// Also identify if this thread is part of a container healthcheck
-	identify_healthcheck(tinfo);
+	// Also possibly set the category for the threadinfo
+	identify_category(tinfo);
 
 	return matches;
 }
@@ -165,9 +167,9 @@ string sinsp_container_manager::container_to_json(const sinsp_container_info& co
 
 	container["Mounts"] = mounts;
 
-	if(!container_info.m_healthcheck_obj.isNull())
+	for(auto &probe : container_info.m_health_probes)
 	{
-		container["Healthcheck"] = container_info.m_healthcheck_obj;
+		probe.add_to_json(container);
 	}
 
 	char addrbuff[100];
@@ -282,15 +284,14 @@ string sinsp_container_manager::get_container_name(sinsp_threadinfo* tinfo)
 	return res;
 }
 
-void sinsp_container_manager::identify_healthcheck(sinsp_threadinfo *tinfo)
+void sinsp_container_manager::identify_category(sinsp_threadinfo *tinfo)
 {
-	// This thread is a part of a container healthcheck if its
-	// parent thread is part of a health check.
+	// Categories are passed from parent to child threads
 	sinsp_threadinfo* ptinfo = tinfo->get_parent_thread();
 
-	if(ptinfo && ptinfo->m_is_container_healthcheck)
+	if(ptinfo && ptinfo->m_category != sinsp_threadinfo::CAT_NONE)
 	{
-		tinfo->m_is_container_healthcheck = true;
+		tinfo->m_category = ptinfo->m_category;
 		return;
 	}
 
@@ -301,17 +302,17 @@ void sinsp_container_manager::identify_healthcheck(sinsp_threadinfo *tinfo)
 		return;
 	}
 
-	// Otherwise, the thread is a part of a container healthcheck if:
+	// Otherwise, the thread is a part of a container health probe if:
 	//
-	// 1. the comm and args match the container's healthcheck
+	// 1. the comm and args match one of the container's health probes
 	// 2. we traverse the parent state and do *not* find vpid=1,
 	//    or find a process not in a container
 	//
-	// This indicates the initial process of the healthcheck.
+	// This indicates the initial process of the health probe.
+
+	sinsp_container_info::container_health_probe::probe_type ptype;
 
-	if(!cinfo->m_has_healthcheck ||
-	   cinfo->m_healthcheck_exe != tinfo->m_exe ||
-	   cinfo->m_healthcheck_args != tinfo->m_args)
+	if((ptype = cinfo->match_health_probe(tinfo)) == sinsp_container_info::container_health_probe::PT_NONE)
 	{
 		return;
 	}
@@ -339,7 +340,21 @@ void sinsp_container_manager::identify_healthcheck(sinsp_threadinfo *tinfo)
 
 	if(!found_container_init)
 	{
-		tinfo->m_is_container_healthcheck = true;
+		// Each health probe type maps to a command category
+		switch(ptype)
+		{
+		case sinsp_container_info::container_health_probe::PT_NONE:
+			break;
+		case sinsp_container_info::container_health_probe::PT_HEALTHCHECK:
+			tinfo->m_category = sinsp_threadinfo::CAT_HEALTHCHECK;
+			break;
+		case sinsp_container_info::container_health_probe::PT_LIVENESS_PROBE:
+			tinfo->m_category = sinsp_threadinfo::CAT_LIVENESS_PROBE;
+			break;
+		case sinsp_container_info::container_health_probe::PT_READINESS_PROBE:
+			tinfo->m_category = sinsp_threadinfo::CAT_READINESS_PROBE;
+			break;
+		}
 	}
 }
 
@@ -378,4 +393,4 @@ void sinsp_container_manager::set_cri_timeout(int64_t timeout_ms)
 #if defined(HAS_CAPTURE)
 	libsinsp::container_engine::cri::set_cri_timeout(timeout_ms);
 #endif
-}
+}

userspace/libsinsp/container.h

@@ -46,12 +46,11 @@ class sinsp_container_manager
 	void dump_containers(scap_dumper_t* dumper);
 	string get_container_name(sinsp_threadinfo* tinfo);
 
-	// Set tinfo's is_container_healthcheck attribute to true if
-	// it is identified as a container healthcheck. It will *not*
-	// set it to false by default, so a threadinfo that is
-	// initially identified as a health check will remain one
+	// Set tinfo's m_category based on the container context.  It
+	// will *not* change any category to NONE, so a threadinfo
+	// that initially has a category will retain its category
 	// across execs e.g. "sh -c /bin/true" execing /bin/true.
-	void identify_healthcheck(sinsp_threadinfo *tinfo);
+	void identify_category(sinsp_threadinfo *tinfo);
 
 	bool container_exists(const string& container_id) const {
 		return m_containers.find(container_id) != m_containers.end();

userspace/libsinsp/container_engine/docker_common.cpp

@@ -87,19 +87,17 @@ bool docker::parse_docker(sinsp_container_manager* manager, sinsp_container_info
 		container->m_imageid = imgstr.substr(cpos + 1);
 	}
 
-	container->parse_healthcheck(config_obj["Healthcheck"]);
-
-	// Saving full healthcheck for container event parsing/writing
-	container->m_healthcheck_obj = config_obj["Healthcheck"];
+	// Add any health checks described in the container config/labels.
+	sinsp_container_info::container_health_probe::add_health_probes(config_obj, container->m_health_probes);
 
 	// containers can be spawned using just the imageID as image name,
 	// with or without the hash prefix (e.g. sha256:)
 	bool no_name = !container->m_imageid.empty() &&
-		       strncmp(container->m_image.c_str(), container->m_imageid.c_str(),
-			       MIN(container->m_image.length(), container->m_imageid.length())) == 0;
+		strncmp(container->m_image.c_str(), container->m_imageid.c_str(),
+			MIN(container->m_image.length(), container->m_imageid.length())) == 0;
 	no_name |= !imgstr.empty() &&
-		   strncmp(container->m_image.c_str(), imgstr.c_str(),
-			   MIN(container->m_image.length(), imgstr.length())) == 0;
+		strncmp(container->m_image.c_str(), imgstr.c_str(),
+			MIN(container->m_image.length(), imgstr.length())) == 0;
 
 	if(!no_name || !m_query_image_info)
 	{
@@ -288,3 +286,4 @@ bool docker::parse_docker(sinsp_container_manager* manager, sinsp_container_info
 #endif
 	return true;
 }
+