Start tracking k8s liveness/readiness probes

K8s has a similar but not identical method as docker for container
health checks. They are called liveness/readiness probes and are a part
of the pod specification, and not a part of the image.

Luckily, the pod configuration *is* a part of the container metadata as
stringified json, with a label
"annotation.kubectl.kubernetes.io/last-applied-configuration", so we can
use that label to identify liveness/readiness probes.

New method docker::get_k8s_pod_spec() grabs the pod configuration out of
the container labels. From there, a
docker::parse_liveness_readiness_probe() parses the liveness/readiness
json (they have the same format, differing only by property name). The
values are saved/loaded in json just as the healthcheck value was saved.

Now that we have 3 different ways to run commands as a "health check",
make the actual command/args a generic "health probe" property of the
container.

For threads, switch everything to use a threadinfo category instead of a
simple bool for has healthcheck. The possible values for the category
are CAT_NONE, CAT_HEALTHCHECK, and
CAT_LIVENESS_PROBE. identify_healthcheck becomes identify_category() but
otherwise behaves the same (passing categories down and checking the
args list otherwise).

The filterchecks aren't quite as generic as the threadinfo categories to
keep the filtering simple. A new field proc.is_container_liveness_probe
checks for k8s liveness probes, and container.liveness_probe prints the
exe + args.

userspace/libsinsp/container.cpp

@@ -128,8 +128,8 @@ bool sinsp_container_manager::resolve_container(sinsp_threadinfo* tinfo, bool qu
 
 #endif // CYGWING_AGENT
 
-	// Also identify if this thread is part of a container healthcheck
-	identify_healthcheck(tinfo);
+	// Also possibly set the category for the threadinfo
+	identify_category(tinfo);
 
 	return matches;
 }
@@ -170,6 +170,16 @@ string sinsp_container_manager::container_to_json(const sinsp_container_info& co
 		container["Healthcheck"] = container_info.m_healthcheck_obj;
 	}
 
+	if(!container_info.m_liveness_probe_obj.isNull())
+	{
+		container["livenessProbe"] = container_info.m_liveness_probe_obj;
+	}
+
+	if(!container_info.m_readiness_probe_obj.isNull())
+	{
+		container["readinessProbe"] = container_info.m_readiness_probe_obj;
+	}
+
 	char addrbuff[100];
 	uint32_t iph = htonl(container_info.m_container_ip);
 	inet_ntop(AF_INET, &iph, addrbuff, sizeof(addrbuff));
@@ -282,15 +292,14 @@ string sinsp_container_manager::get_container_name(sinsp_threadinfo* tinfo)
 	return res;
 }
 
-void sinsp_container_manager::identify_healthcheck(sinsp_threadinfo *tinfo)
+void sinsp_container_manager::identify_category(sinsp_threadinfo *tinfo)
 {
-	// This thread is a part of a container healthcheck if its
-	// parent thread is part of a health check.
+	// Categories are passed from parent to child threads
 	sinsp_threadinfo* ptinfo = tinfo->get_parent_thread();
 
-	if(ptinfo && ptinfo->m_is_container_healthcheck)
+	if(ptinfo && ptinfo->m_category != sinsp_threadinfo::CAT_NONE)
 	{
-		tinfo->m_is_container_healthcheck = true;
+		tinfo->m_category = ptinfo->m_category;
 		return;
 	}
 
@@ -301,17 +310,17 @@ void sinsp_container_manager::identify_healthcheck(sinsp_threadinfo *tinfo)
 		return;
 	}
 
-	// Otherwise, the thread is a part of a container healthcheck if:
+	// Otherwise, the thread is a part of a container health probe if:
 	//
-	// 1. the comm and args match the container's healthcheck
+	// 1. the comm and args match the container's health probe
 	// 2. we traverse the parent state and do *not* find vpid=1,
 	//    or find a process not in a container
 	//
-	// This indicates the initial process of the healthcheck.
+	// This indicates the initial process of the health probe
 
-	if(!cinfo->m_has_healthcheck ||
-	   cinfo->m_healthcheck_exe != tinfo->m_exe ||
-	   cinfo->m_healthcheck_args != tinfo->m_args)
+	if(!cinfo->m_has_health_probe ||
+	   cinfo->m_health_probe_exe != tinfo->m_exe ||
+	   cinfo->m_health_probe_args != tinfo->m_args)
 	{
 		return;
 	}
@@ -339,7 +348,22 @@ void sinsp_container_manager::identify_healthcheck(sinsp_threadinfo *tinfo)
 
 	if(!found_container_init)
 	{
-		tinfo->m_is_container_healthcheck = true;
+		if (!cinfo->m_liveness_probe_obj.isNull())
+		{
+			tinfo->m_category = sinsp_threadinfo::CAT_LIVENESS_PROBE;
+		}
+		else if (!cinfo->m_readiness_probe_obj.isNull())
+		{
+			tinfo->m_category = sinsp_threadinfo::CAT_READINESS_PROBE;
+		}
+		else if(!cinfo->m_healthcheck_obj.isNull())
+		{
+			tinfo->m_category = sinsp_threadinfo::CAT_HEALTHCHECK;
+		}
+		else
+		{
+			g_logger.format(sinsp_logger::SEV_ERROR, "Thread had liveness probe but no associated config object?");
+		}
 	}
 }
 
@@ -378,4 +402,4 @@ void sinsp_container_manager::set_cri_timeout(int64_t timeout_ms)
 #if defined(HAS_CAPTURE)
 	libsinsp::container_engine::cri::set_cri_timeout(timeout_ms);
 #endif
-}
+}

userspace/libsinsp/container.h

@@ -46,12 +46,11 @@ class sinsp_container_manager
 	void dump_containers(scap_dumper_t* dumper);
 	string get_container_name(sinsp_threadinfo* tinfo);
 
-	// Set tinfo's is_container_healthcheck attribute to true if
-	// it is identified as a container healthcheck. It will *not*
-	// set it to false by default, so a threadinfo that is
-	// initially identified as a health check will remain one
+	// Set tinfo's m_category based on the container context.  It
+	// will *not* change any category to NONE, so a threadinfo
+	// that initially has a category will retain its category
 	// across execs e.g. "sh -c /bin/true" execing /bin/true.
-	void identify_healthcheck(sinsp_threadinfo *tinfo);
+	void identify_category(sinsp_threadinfo *tinfo);
 
 	bool container_exists(const string& container_id) const {
 		return m_containers.find(container_id) != m_containers.end();

userspace/libsinsp/container_engine/docker.h

@@ -58,6 +58,7 @@ class docker
 	docker_response get_docker(sinsp_container_manager* manager, const std::string& url, std::string &json);
 	std::string build_request(const std::string& url);
 	bool parse_docker(sinsp_container_manager* manager, sinsp_container_info *container, sinsp_threadinfo* tinfo);
+	Json::Value get_k8s_pod_spec(const Json::Value &config_obj);
 
 	static std::string m_api_version;
 	static bool m_query_image_info;

userspace/libsinsp/container_engine/docker_common.cpp

@@ -89,17 +89,29 @@ bool docker::parse_docker(sinsp_container_manager* manager, sinsp_container_info
 
 	container->parse_healthcheck(config_obj["Healthcheck"]);
 
-	// Saving full healthcheck for container event parsing/writing
+	// Saving full object for container event parsing/writing
 	container->m_healthcheck_obj = config_obj["Healthcheck"];
 
+	// For k8s liveness/readiness probes, we need to fetch a
+	// specific label that contains the pod spec, which is
+	// stringified json
+	Json::Value pod_spec_obj = get_k8s_pod_spec(config_obj);
+
+	container->parse_liveness_readiness_probe(pod_spec_obj["livenessProbe"]);
+	container->parse_liveness_readiness_probe(pod_spec_obj["readinessProbe"]);
+
+	// Saving full object for container event parsing/writing
+	container->m_liveness_probe_obj = pod_spec_obj["livenessProbe"];
+	container->m_readiness_probe_obj = pod_spec_obj["readinessProbe"];
+
 	// containers can be spawned using just the imageID as image name,
 	// with or without the hash prefix (e.g. sha256:)
 	bool no_name = !container->m_imageid.empty() &&
-		       strncmp(container->m_image.c_str(), container->m_imageid.c_str(),
-			       MIN(container->m_image.length(), container->m_imageid.length())) == 0;
+		strncmp(container->m_image.c_str(), container->m_imageid.c_str(),
+			MIN(container->m_image.length(), container->m_imageid.length())) == 0;
 	no_name |= !imgstr.empty() &&
-		   strncmp(container->m_image.c_str(), imgstr.c_str(),
-			   MIN(container->m_image.length(), imgstr.length())) == 0;
+		strncmp(container->m_image.c_str(), imgstr.c_str(),
+			MIN(container->m_image.length(), imgstr.length())) == 0;
 
 	if(!no_name || !m_query_image_info)
 	{
@@ -288,3 +300,55 @@ bool docker::parse_docker(sinsp_container_manager* manager, sinsp_container_info
 #endif
 	return true;
 }
+
+Json::Value docker::get_k8s_pod_spec(const Json::Value &config_obj)
+{
+	std::string cfg_str;
+	Json::Value spec;
+	Json::Reader reader;
+
+	if(config_obj.isNull())
+	{
+		return spec;
+	}
+
+	const Json::Value &labels = config_obj["Labels"];
+
+	if(labels.isNull())
+	{
+		return spec;
+	}
+
+	// The pod spec is stored as a stringified json label on the container
+	cfg_str = labels["annotation.kubectl.kubernetes.io/last-applied-configuration"].asString();
+
+	if(cfg_str == "")
+	{
+		return spec;
+	}
+
+	Json::Value cfg;
+	if(!reader.parse(cfg_str.c_str(), cfg))
+	{
+		g_logger.format(sinsp_logger::SEV_WARNING, "Could not parse pod config '%s'", cfg_str.c_str());
+		return spec;
+	}
+
+	Json::Value &sobj = cfg["spec"];
+
+	if(sobj.isNull())
+	{
+		return spec;
+	}
+
+	Json::Value &containers = sobj["containers"];
+
+	if(containers.isNull())
+	{
+		return spec;
+	}
+
+	spec = containers[0];
+
+	return spec;
+}

userspace/libsinsp/container_info.cpp

@@ -59,7 +59,7 @@ const sinsp_container_info::container_mount_info *sinsp_container_info::mount_by
 	return NULL;
 }
 
-std::string sinsp_container_info::normalize_healthcheck_arg(const std::string &arg)
+std::string sinsp_container_info::normalize_health_probe_arg(const std::string &arg)
 {
 	std::string ret = arg;
 
@@ -85,26 +85,53 @@ void sinsp_container_info::parse_healthcheck(const Json::Value &healthcheck_obj)
 {
 	if(!healthcheck_obj.isNull())
 	{
+		m_healthcheck_obj = healthcheck_obj;
+
 		const Json::Value &test_obj = healthcheck_obj["Test"];
 
 		if(!test_obj.isNull() && test_obj.isArray() && test_obj.size() >= 2)
 		{
 			if(test_obj[0].asString() == "CMD")
 			{
-				m_has_healthcheck = true;
-				m_healthcheck_exe = normalize_healthcheck_arg(test_obj[1].asString());
+				m_has_health_probe = true;
+				m_health_probe_exe = normalize_health_probe_arg(test_obj[1].asString());
 
 				for(uint32_t i = 2; i < test_obj.size(); i++)
 				{
-					m_healthcheck_args.push_back(normalize_healthcheck_arg(test_obj[i].asString()));
+					m_health_probe_args.push_back(normalize_health_probe_arg(test_obj[i].asString()));
 				}
 			}
 			else if(test_obj[0].asString() == "CMD-SHELL")
 			{
-				m_has_healthcheck = true;
-				m_healthcheck_exe = "/bin/sh";
-				m_healthcheck_args.push_back("-c");
-				m_healthcheck_args.push_back(test_obj[1].asString());
+				m_has_health_probe = true;
+				m_health_probe_exe = "/bin/sh";
+				m_health_probe_args.push_back("-c");
+				m_health_probe_args.push_back(test_obj[1].asString());
+			}
+		}
+	}
+}
+
+void sinsp_container_info::parse_liveness_readiness_probe(const Json::Value &probe_obj)
+{
+	if(!probe_obj.isNull())
+	{
+		const Json::Value &exec_obj = probe_obj["exec"];
+
+		if(exec_obj.isNull())
+		{
+			return;
+		}
+
+		const Json::Value command_obj = exec_obj["command"];
+
+		if(!command_obj.isNull() && command_obj.isArray())
+		{
+			m_has_health_probe = true;
+			m_health_probe_exe = normalize_health_probe_arg(command_obj[0].asString());
+			for(uint32_t i = 1; i < command_obj.size(); i++)
+			{
+				m_health_probe_args.push_back(normalize_health_probe_arg(command_obj[i].asString()));
 			}
 		}
 	}

userspace/libsinsp/container_info.h

@@ -130,17 +130,18 @@ class sinsp_container_info
 		m_cpu_shares(1024),
 		m_cpu_quota(0),
 		m_cpu_period(100000),
-		m_has_healthcheck(false),
-		m_healthcheck_exe(""),
+		m_has_health_probe(false),
+		m_health_probe_exe(""),
 		m_is_pod_sandbox(false)
 #ifdef HAS_ANALYZER
 		,m_metadata_deadline(0)
 #endif
 	{
 	}
 
-	std::string normalize_healthcheck_arg(const std::string &arg);
+	std::string normalize_health_probe_arg(const std::string &arg);
 	void parse_healthcheck(const Json::Value &config_obj);
+	void parse_liveness_readiness_probe(const Json::Value &config_obj);
 
 	const std::vector<std::string>& get_env() const { return m_env; }
 
@@ -173,9 +174,12 @@ class sinsp_container_info
 	int64_t m_cpu_quota;
 	int64_t m_cpu_period;
 	Json::Value m_healthcheck_obj;
-	bool m_has_healthcheck;
-	std::string m_healthcheck_exe;
-	std::vector<std::string> m_healthcheck_args;
+	Json::Value m_liveness_probe_obj;
+	Json::Value m_readiness_probe_obj;
+	bool m_has_health_probe;
+	std::string m_health_probe_exe;
+	std::vector<std::string> m_health_probe_args;
+
 	bool m_is_pod_sandbox;
 #ifdef HAS_ANALYZER
 	std::string m_sysdig_agent_conf;

userspace/libsinsp/filterchecks.cpp

@@ -1810,6 +1810,8 @@ const filtercheck_field_info sinsp_filter_check_thread_fields[] =
 	{PT_CHARBUF, EPF_TABLE_ONLY, PF_NA, "thread.nametid", "this field chains the process name and tid of a thread and can be used as a specific identifier of a thread for a specific execve."},
 	{PT_INT64, EPF_NONE, PF_ID, "proc.vpgid", "the process group id of the process generating the event, as seen from its current PID namespace."},
 	{PT_BOOL, EPF_NONE, PF_NA, "proc.is_container_healthcheck", "true if this process is running as a part of the container's health check."},
+	{PT_BOOL, EPF_NONE, PF_NA, "proc.is_container_liveness_probe", "true if this process is running as a part of the container's liveness probe."},
+	{PT_BOOL, EPF_NONE, PF_NA, "proc.is_container_readiness_probe", "true if this process is running as a part of the container's readiness probe."},
 };
 
 sinsp_filter_check_thread::sinsp_filter_check_thread()
@@ -2599,7 +2601,13 @@ uint8_t* sinsp_filter_check_thread::extract(sinsp_evt *evt, OUT uint32_t* len, b
 		m_tstr = tinfo->get_comm() + to_string(evt->get_tid());
 		RETURN_EXTRACT_STRING(m_tstr);
 	case TYPE_IS_CONTAINER_HEALTHCHECK:
-		m_tbool = tinfo->m_is_container_healthcheck;
+		m_tbool = (tinfo->m_category == sinsp_threadinfo::CAT_HEALTHCHECK);
+		RETURN_EXTRACT_VAR(m_tbool);
+	case TYPE_IS_CONTAINER_LIVENESS_PROBE:
+		m_tbool = (tinfo->m_category == sinsp_threadinfo::CAT_LIVENESS_PROBE);
+		RETURN_EXTRACT_VAR(m_tbool);
+	case TYPE_IS_CONTAINER_READINESS_PROBE:
+		m_tbool = (tinfo->m_category == sinsp_threadinfo::CAT_READINESS_PROBE);
 		RETURN_EXTRACT_VAR(m_tbool);
 	default:
 		ASSERT(false);
@@ -5900,7 +5908,9 @@ const filtercheck_field_info sinsp_filter_check_container_fields[] =
 	{PT_CHARBUF, EPF_NONE, PF_NA, "container.image.repository", "the container image repository (e.g. sysdig/sysdig)."},
 	{PT_CHARBUF, EPF_NONE, PF_NA, "container.image.tag", "the container image tag (e.g. stable, latest)."},
 	{PT_CHARBUF, EPF_NONE, PF_NA, "container.image.digest", "the container image registry digest (e.g. sha256:d977378f890d445c15e51795296e4e5062f109ce6da83e0a355fc4ad8699d27)."},
-	{PT_CHARBUF, EPF_NONE, PF_NA, "container.healthcheck", "The container's health check. Will be the null value (\"N/A\") if no healthcheck configured, \"NONE\" if configured but explicitly not created, and the healthcheck command line otherwise"}
+	{PT_CHARBUF, EPF_NONE, PF_NA, "container.healthcheck", "The container's health check. Will be the null value (\"N/A\") if no healthcheck configured, \"NONE\" if configured but explicitly not created, and the healthcheck command line otherwise"},
+	{PT_CHARBUF, EPF_NONE, PF_NA, "container.liveness_probe", "The container's liveness probe. Will be the null value (\"N/A\") if no liveness probe configured, the liveness probe command line otherwise"},
+	{PT_CHARBUF, EPF_NONE, PF_NA, "container.readiness_probe", "The container's readiness probe. Will be the null value (\"N/A\") if no readiness probe configured, the readiness probe command line otherwise"}
 };
 
 sinsp_filter_check_container::sinsp_filter_check_container()
@@ -6325,6 +6335,8 @@ uint8_t* sinsp_filter_check_container::extract(sinsp_evt *evt, OUT uint32_t* len
 		}
 		break;
 	case TYPE_CONTAINER_HEALTHCHECK:
+	case TYPE_CONTAINER_LIVENESS_PROBE:
+	case TYPE_CONTAINER_READINESS_PROBE:
 		if(tinfo->m_container_id.empty())
 		{
 			return NULL;
@@ -6338,21 +6350,23 @@ uint8_t* sinsp_filter_check_container::extract(sinsp_evt *evt, OUT uint32_t* len
 				return NULL;
 			}
 
-			if(container_info->m_healthcheck_obj.isNull())
+			if((m_field_id == TYPE_CONTAINER_HEALTHCHECK && container_info->m_healthcheck_obj.isNull()) ||
+			   (m_field_id == TYPE_CONTAINER_LIVENESS_PROBE && container_info->m_liveness_probe_obj.isNull()) ||
+			   (m_field_id == TYPE_CONTAINER_READINESS_PROBE && container_info->m_readiness_probe_obj.isNull()))
 			{
 				return NULL;
 			}
 
-			if(!container_info->m_has_healthcheck)
+			if(!container_info->m_has_health_probe)
 			{
 				m_tstr = "NONE";
 
 				RETURN_EXTRACT_STRING(m_tstr);
 			}
 
-			m_tstr = container_info->m_healthcheck_exe;
+			m_tstr = container_info->m_health_probe_exe;
 
-			for(auto &arg : container_info->m_healthcheck_args)
+			for(auto &arg : container_info->m_health_probe_args)
 			{
 				m_tstr += " ";
 				m_tstr += arg;