diff --git a/userspace/libsinsp/container.cpp b/userspace/libsinsp/container.cpp index 4c0b5d966e..803130b0a3 100644 --- a/userspace/libsinsp/container.cpp +++ b/userspace/libsinsp/container.cpp @@ -128,8 +128,8 @@ bool sinsp_container_manager::resolve_container(sinsp_threadinfo* tinfo, bool qu #endif // CYGWING_AGENT - // Also identify if this thread is part of a container healthcheck - identify_healthcheck(tinfo); + // Also possibly set the category for the threadinfo + identify_category(tinfo); return matches; } @@ -170,6 +170,16 @@ string sinsp_container_manager::container_to_json(const sinsp_container_info& co container["Healthcheck"] = container_info.m_healthcheck_obj; } + if(!container_info.m_liveness_probe_obj.isNull()) + { + container["livenessProbe"] = container_info.m_liveness_probe_obj; + } + + if(!container_info.m_readiness_probe_obj.isNull()) + { + container["readinessProbe"] = container_info.m_readiness_probe_obj; + } + char addrbuff[100]; uint32_t iph = htonl(container_info.m_container_ip); inet_ntop(AF_INET, &iph, addrbuff, sizeof(addrbuff)); @@ -282,15 +292,14 @@ string sinsp_container_manager::get_container_name(sinsp_threadinfo* tinfo) return res; } -void sinsp_container_manager::identify_healthcheck(sinsp_threadinfo *tinfo) +void sinsp_container_manager::identify_category(sinsp_threadinfo *tinfo) { - // This thread is a part of a container healthcheck if its - // parent thread is part of a health check. + // Categories are passed from parent to child threads sinsp_threadinfo* ptinfo = tinfo->get_parent_thread(); - if(ptinfo && ptinfo->m_is_container_healthcheck) + if(ptinfo && ptinfo->m_category != sinsp_threadinfo::CAT_NONE) { - tinfo->m_is_container_healthcheck = true; + tinfo->m_category = ptinfo->m_category; return; } @@ -301,17 +310,17 @@ void sinsp_container_manager::identify_healthcheck(sinsp_threadinfo *tinfo) return; } - // Otherwise, the thread is a part of a container healthcheck if: + // Otherwise, the thread is a part of a container health probe if: // - // 1. the comm and args match the container's healthcheck + // 1. the comm and args match the container's health probe // 2. we traverse the parent state and do *not* find vpid=1, // or find a process not in a container // - // This indicates the initial process of the healthcheck. + // This indicates the initial process of the health probe - if(!cinfo->m_has_healthcheck || - cinfo->m_healthcheck_exe != tinfo->m_exe || - cinfo->m_healthcheck_args != tinfo->m_args) + if(!cinfo->m_has_health_probe || + cinfo->m_health_probe_exe != tinfo->m_exe || + cinfo->m_health_probe_args != tinfo->m_args) { return; } @@ -339,7 +348,22 @@ void sinsp_container_manager::identify_healthcheck(sinsp_threadinfo *tinfo) if(!found_container_init) { - tinfo->m_is_container_healthcheck = true; + if (!cinfo->m_liveness_probe_obj.isNull()) + { + tinfo->m_category = sinsp_threadinfo::CAT_LIVENESS_PROBE; + } + else if (!cinfo->m_readiness_probe_obj.isNull()) + { + tinfo->m_category = sinsp_threadinfo::CAT_READINESS_PROBE; + } + else if(!cinfo->m_healthcheck_obj.isNull()) + { + tinfo->m_category = sinsp_threadinfo::CAT_HEALTHCHECK; + } + else + { + g_logger.format(sinsp_logger::SEV_ERROR, "Thread had liveness probe but no associated config object?"); + } } } @@ -378,4 +402,4 @@ void sinsp_container_manager::set_cri_timeout(int64_t timeout_ms) #if defined(HAS_CAPTURE) libsinsp::container_engine::cri::set_cri_timeout(timeout_ms); #endif -} \ No newline at end of file +} diff --git a/userspace/libsinsp/container.h b/userspace/libsinsp/container.h index f741b2d0a7..2f091919ae 100644 --- a/userspace/libsinsp/container.h +++ b/userspace/libsinsp/container.h @@ -46,12 +46,11 @@ class sinsp_container_manager void dump_containers(scap_dumper_t* dumper); string get_container_name(sinsp_threadinfo* tinfo); - // Set tinfo's is_container_healthcheck attribute to true if - // it is identified as a container healthcheck. It will *not* - // set it to false by default, so a threadinfo that is - // initially identified as a health check will remain one + // Set tinfo's m_category based on the container context. It + // will *not* change any category to NONE, so a threadinfo + // that initially has a category will retain its category // across execs e.g. "sh -c /bin/true" execing /bin/true. - void identify_healthcheck(sinsp_threadinfo *tinfo); + void identify_category(sinsp_threadinfo *tinfo); bool container_exists(const string& container_id) const { return m_containers.find(container_id) != m_containers.end(); diff --git a/userspace/libsinsp/container_engine/docker.h b/userspace/libsinsp/container_engine/docker.h index 02d74db64e..07236956f9 100644 --- a/userspace/libsinsp/container_engine/docker.h +++ b/userspace/libsinsp/container_engine/docker.h @@ -58,6 +58,7 @@ class docker docker_response get_docker(sinsp_container_manager* manager, const std::string& url, std::string &json); std::string build_request(const std::string& url); bool parse_docker(sinsp_container_manager* manager, sinsp_container_info *container, sinsp_threadinfo* tinfo); + Json::Value get_k8s_pod_spec(const Json::Value &config_obj); static std::string m_api_version; static bool m_query_image_info; diff --git a/userspace/libsinsp/container_engine/docker_common.cpp b/userspace/libsinsp/container_engine/docker_common.cpp index 75a554ddb9..0fd82227c2 100644 --- a/userspace/libsinsp/container_engine/docker_common.cpp +++ b/userspace/libsinsp/container_engine/docker_common.cpp @@ -89,17 +89,29 @@ bool docker::parse_docker(sinsp_container_manager* manager, sinsp_container_info container->parse_healthcheck(config_obj["Healthcheck"]); - // Saving full healthcheck for container event parsing/writing + // Saving full object for container event parsing/writing container->m_healthcheck_obj = config_obj["Healthcheck"]; + // For k8s liveness/readiness probes, we need to fetch a + // specific label that contains the pod spec, which is + // stringified json + Json::Value pod_spec_obj = get_k8s_pod_spec(config_obj); + + container->parse_liveness_readiness_probe(pod_spec_obj["livenessProbe"]); + container->parse_liveness_readiness_probe(pod_spec_obj["readinessProbe"]); + + // Saving full object for container event parsing/writing + container->m_liveness_probe_obj = pod_spec_obj["livenessProbe"]; + container->m_readiness_probe_obj = pod_spec_obj["readinessProbe"]; + // containers can be spawned using just the imageID as image name, // with or without the hash prefix (e.g. sha256:) bool no_name = !container->m_imageid.empty() && - strncmp(container->m_image.c_str(), container->m_imageid.c_str(), - MIN(container->m_image.length(), container->m_imageid.length())) == 0; + strncmp(container->m_image.c_str(), container->m_imageid.c_str(), + MIN(container->m_image.length(), container->m_imageid.length())) == 0; no_name |= !imgstr.empty() && - strncmp(container->m_image.c_str(), imgstr.c_str(), - MIN(container->m_image.length(), imgstr.length())) == 0; + strncmp(container->m_image.c_str(), imgstr.c_str(), + MIN(container->m_image.length(), imgstr.length())) == 0; if(!no_name || !m_query_image_info) { @@ -288,3 +300,55 @@ bool docker::parse_docker(sinsp_container_manager* manager, sinsp_container_info #endif return true; } + +Json::Value docker::get_k8s_pod_spec(const Json::Value &config_obj) +{ + std::string cfg_str; + Json::Value spec; + Json::Reader reader; + + if(config_obj.isNull()) + { + return spec; + } + + const Json::Value &labels = config_obj["Labels"]; + + if(labels.isNull()) + { + return spec; + } + + // The pod spec is stored as a stringified json label on the container + cfg_str = labels["annotation.kubectl.kubernetes.io/last-applied-configuration"].asString(); + + if(cfg_str == "") + { + return spec; + } + + Json::Value cfg; + if(!reader.parse(cfg_str.c_str(), cfg)) + { + g_logger.format(sinsp_logger::SEV_WARNING, "Could not parse pod config '%s'", cfg_str.c_str()); + return spec; + } + + Json::Value &sobj = cfg["spec"]; + + if(sobj.isNull()) + { + return spec; + } + + Json::Value &containers = sobj["containers"]; + + if(containers.isNull()) + { + return spec; + } + + spec = containers[0]; + + return spec; +} diff --git a/userspace/libsinsp/container_info.cpp b/userspace/libsinsp/container_info.cpp index 61ccc667df..d4d755765d 100644 --- a/userspace/libsinsp/container_info.cpp +++ b/userspace/libsinsp/container_info.cpp @@ -59,7 +59,7 @@ const sinsp_container_info::container_mount_info *sinsp_container_info::mount_by return NULL; } -std::string sinsp_container_info::normalize_healthcheck_arg(const std::string &arg) +std::string sinsp_container_info::normalize_health_probe_arg(const std::string &arg) { std::string ret = arg; @@ -85,26 +85,53 @@ void sinsp_container_info::parse_healthcheck(const Json::Value &healthcheck_obj) { if(!healthcheck_obj.isNull()) { + m_healthcheck_obj = healthcheck_obj; + const Json::Value &test_obj = healthcheck_obj["Test"]; if(!test_obj.isNull() && test_obj.isArray() && test_obj.size() >= 2) { if(test_obj[0].asString() == "CMD") { - m_has_healthcheck = true; - m_healthcheck_exe = normalize_healthcheck_arg(test_obj[1].asString()); + m_has_health_probe = true; + m_health_probe_exe = normalize_health_probe_arg(test_obj[1].asString()); for(uint32_t i = 2; i < test_obj.size(); i++) { - m_healthcheck_args.push_back(normalize_healthcheck_arg(test_obj[i].asString())); + m_health_probe_args.push_back(normalize_health_probe_arg(test_obj[i].asString())); } } else if(test_obj[0].asString() == "CMD-SHELL") { - m_has_healthcheck = true; - m_healthcheck_exe = "/bin/sh"; - m_healthcheck_args.push_back("-c"); - m_healthcheck_args.push_back(test_obj[1].asString()); + m_has_health_probe = true; + m_health_probe_exe = "/bin/sh"; + m_health_probe_args.push_back("-c"); + m_health_probe_args.push_back(test_obj[1].asString()); + } + } + } +} + +void sinsp_container_info::parse_liveness_readiness_probe(const Json::Value &probe_obj) +{ + if(!probe_obj.isNull()) + { + const Json::Value &exec_obj = probe_obj["exec"]; + + if(exec_obj.isNull()) + { + return; + } + + const Json::Value command_obj = exec_obj["command"]; + + if(!command_obj.isNull() && command_obj.isArray()) + { + m_has_health_probe = true; + m_health_probe_exe = normalize_health_probe_arg(command_obj[0].asString()); + for(uint32_t i = 1; i < command_obj.size(); i++) + { + m_health_probe_args.push_back(normalize_health_probe_arg(command_obj[i].asString())); } } } diff --git a/userspace/libsinsp/container_info.h b/userspace/libsinsp/container_info.h index 0171268e6c..6dee2276d3 100644 --- a/userspace/libsinsp/container_info.h +++ b/userspace/libsinsp/container_info.h @@ -130,8 +130,8 @@ class sinsp_container_info m_cpu_shares(1024), m_cpu_quota(0), m_cpu_period(100000), - m_has_healthcheck(false), - m_healthcheck_exe(""), + m_has_health_probe(false), + m_health_probe_exe(""), m_is_pod_sandbox(false) #ifdef HAS_ANALYZER ,m_metadata_deadline(0) @@ -139,8 +139,9 @@ class sinsp_container_info { } - std::string normalize_healthcheck_arg(const std::string &arg); + std::string normalize_health_probe_arg(const std::string &arg); void parse_healthcheck(const Json::Value &config_obj); + void parse_liveness_readiness_probe(const Json::Value &config_obj); const std::vector& get_env() const { return m_env; } @@ -173,9 +174,12 @@ class sinsp_container_info int64_t m_cpu_quota; int64_t m_cpu_period; Json::Value m_healthcheck_obj; - bool m_has_healthcheck; - std::string m_healthcheck_exe; - std::vector m_healthcheck_args; + Json::Value m_liveness_probe_obj; + Json::Value m_readiness_probe_obj; + bool m_has_health_probe; + std::string m_health_probe_exe; + std::vector m_health_probe_args; + bool m_is_pod_sandbox; #ifdef HAS_ANALYZER std::string m_sysdig_agent_conf; diff --git a/userspace/libsinsp/filterchecks.cpp b/userspace/libsinsp/filterchecks.cpp index f80cb4af33..0bff9f37a6 100644 --- a/userspace/libsinsp/filterchecks.cpp +++ b/userspace/libsinsp/filterchecks.cpp @@ -1810,6 +1810,8 @@ const filtercheck_field_info sinsp_filter_check_thread_fields[] = {PT_CHARBUF, EPF_TABLE_ONLY, PF_NA, "thread.nametid", "this field chains the process name and tid of a thread and can be used as a specific identifier of a thread for a specific execve."}, {PT_INT64, EPF_NONE, PF_ID, "proc.vpgid", "the process group id of the process generating the event, as seen from its current PID namespace."}, {PT_BOOL, EPF_NONE, PF_NA, "proc.is_container_healthcheck", "true if this process is running as a part of the container's health check."}, + {PT_BOOL, EPF_NONE, PF_NA, "proc.is_container_liveness_probe", "true if this process is running as a part of the container's liveness probe."}, + {PT_BOOL, EPF_NONE, PF_NA, "proc.is_container_readiness_probe", "true if this process is running as a part of the container's readiness probe."}, }; sinsp_filter_check_thread::sinsp_filter_check_thread() @@ -2599,7 +2601,13 @@ uint8_t* sinsp_filter_check_thread::extract(sinsp_evt *evt, OUT uint32_t* len, b m_tstr = tinfo->get_comm() + to_string(evt->get_tid()); RETURN_EXTRACT_STRING(m_tstr); case TYPE_IS_CONTAINER_HEALTHCHECK: - m_tbool = tinfo->m_is_container_healthcheck; + m_tbool = (tinfo->m_category == sinsp_threadinfo::CAT_HEALTHCHECK); + RETURN_EXTRACT_VAR(m_tbool); + case TYPE_IS_CONTAINER_LIVENESS_PROBE: + m_tbool = (tinfo->m_category == sinsp_threadinfo::CAT_LIVENESS_PROBE); + RETURN_EXTRACT_VAR(m_tbool); + case TYPE_IS_CONTAINER_READINESS_PROBE: + m_tbool = (tinfo->m_category == sinsp_threadinfo::CAT_READINESS_PROBE); RETURN_EXTRACT_VAR(m_tbool); default: ASSERT(false); @@ -5900,7 +5908,9 @@ const filtercheck_field_info sinsp_filter_check_container_fields[] = {PT_CHARBUF, EPF_NONE, PF_NA, "container.image.repository", "the container image repository (e.g. sysdig/sysdig)."}, {PT_CHARBUF, EPF_NONE, PF_NA, "container.image.tag", "the container image tag (e.g. stable, latest)."}, {PT_CHARBUF, EPF_NONE, PF_NA, "container.image.digest", "the container image registry digest (e.g. sha256:d977378f890d445c15e51795296e4e5062f109ce6da83e0a355fc4ad8699d27)."}, - {PT_CHARBUF, EPF_NONE, PF_NA, "container.healthcheck", "The container's health check. Will be the null value (\"N/A\") if no healthcheck configured, \"NONE\" if configured but explicitly not created, and the healthcheck command line otherwise"} + {PT_CHARBUF, EPF_NONE, PF_NA, "container.healthcheck", "The container's health check. Will be the null value (\"N/A\") if no healthcheck configured, \"NONE\" if configured but explicitly not created, and the healthcheck command line otherwise"}, + {PT_CHARBUF, EPF_NONE, PF_NA, "container.liveness_probe", "The container's liveness probe. Will be the null value (\"N/A\") if no liveness probe configured, the liveness probe command line otherwise"}, + {PT_CHARBUF, EPF_NONE, PF_NA, "container.readiness_probe", "The container's readiness probe. Will be the null value (\"N/A\") if no readiness probe configured, the readiness probe command line otherwise"} }; sinsp_filter_check_container::sinsp_filter_check_container() @@ -6325,6 +6335,8 @@ uint8_t* sinsp_filter_check_container::extract(sinsp_evt *evt, OUT uint32_t* len } break; case TYPE_CONTAINER_HEALTHCHECK: + case TYPE_CONTAINER_LIVENESS_PROBE: + case TYPE_CONTAINER_READINESS_PROBE: if(tinfo->m_container_id.empty()) { return NULL; @@ -6338,21 +6350,23 @@ uint8_t* sinsp_filter_check_container::extract(sinsp_evt *evt, OUT uint32_t* len return NULL; } - if(container_info->m_healthcheck_obj.isNull()) + if((m_field_id == TYPE_CONTAINER_HEALTHCHECK && container_info->m_healthcheck_obj.isNull()) || + (m_field_id == TYPE_CONTAINER_LIVENESS_PROBE && container_info->m_liveness_probe_obj.isNull()) || + (m_field_id == TYPE_CONTAINER_READINESS_PROBE && container_info->m_readiness_probe_obj.isNull())) { return NULL; } - if(!container_info->m_has_healthcheck) + if(!container_info->m_has_health_probe) { m_tstr = "NONE"; RETURN_EXTRACT_STRING(m_tstr); } - m_tstr = container_info->m_healthcheck_exe; + m_tstr = container_info->m_health_probe_exe; - for(auto &arg : container_info->m_healthcheck_args) + for(auto &arg : container_info->m_health_probe_args) { m_tstr += " "; m_tstr += arg; diff --git a/userspace/libsinsp/filterchecks.h b/userspace/libsinsp/filterchecks.h index 6e108e592e..f4e6031790 100644 --- a/userspace/libsinsp/filterchecks.h +++ b/userspace/libsinsp/filterchecks.h @@ -339,6 +339,8 @@ class sinsp_filter_check_thread : public sinsp_filter_check TYPE_NAMETID = 44, TYPE_VPGID = 45, TYPE_IS_CONTAINER_HEALTHCHECK = 46, + TYPE_IS_CONTAINER_LIVENESS_PROBE = 47, + TYPE_IS_CONTAINER_READINESS_PROBE = 48, }; sinsp_filter_check_thread(); @@ -724,6 +726,8 @@ class sinsp_filter_check_container : public sinsp_filter_check TYPE_CONTAINER_IMAGE_TAG, TYPE_CONTAINER_IMAGE_DIGEST, TYPE_CONTAINER_HEALTHCHECK, + TYPE_CONTAINER_LIVENESS_PROBE, + TYPE_CONTAINER_READINESS_PROBE, }; sinsp_filter_check_container(); diff --git a/userspace/libsinsp/parsers.cpp b/userspace/libsinsp/parsers.cpp index b9017c4c2c..cbecb2db9b 100644 --- a/userspace/libsinsp/parsers.cpp +++ b/userspace/libsinsp/parsers.cpp @@ -2261,7 +2261,7 @@ inline void sinsp_parser::add_socket(sinsp_evt *evt, int64_t fd, uint32_t domain * * Preconditions: evt->m_fdinfo == nullptr and * evt->m_tinfo != nullptr - * + * */ inline void sinsp_parser::infer_sendto_fdinfo(sinsp_evt* const evt) { @@ -4534,6 +4534,12 @@ void sinsp_parser::parse_container_json_evt(sinsp_evt *evt) libsinsp::container_engine::docker::parse_json_mounts(container["Mounts"], container_info.m_mounts); container_info.parse_healthcheck(container["Healthcheck"]); + container_info.parse_liveness_readiness_probe(container["livenessProbe"]); + container_info.parse_liveness_readiness_probe(container["readinessProbe"]); + + container_info.m_liveness_probe_obj = container["livenessProbe"]; + container_info.m_readiness_probe_obj = container["readinessProbe"]; + const Json::Value& contip = container["ip"]; if(!contip.isNull() && contip.isConvertibleTo(Json::stringValue)) { diff --git a/userspace/libsinsp/threadinfo.cpp b/userspace/libsinsp/threadinfo.cpp index ef68be9ba1..6738a636d8 100644 --- a/userspace/libsinsp/threadinfo.cpp +++ b/userspace/libsinsp/threadinfo.cpp @@ -90,7 +90,7 @@ void sinsp_threadinfo::init() m_lastevent_data = NULL; m_parent_loop_detected = false; m_tty = 0; - m_is_container_healthcheck = false; + m_category = CAT_NONE; m_blprogram = NULL; m_loginuid = 0; } @@ -424,7 +424,7 @@ void sinsp_threadinfo::init(scap_threadinfo* pi) m_clone_ts = pi->clone_ts; m_tty = pi->tty; m_loginuid = pi->loginuid; - m_is_container_healthcheck = false; + m_category = CAT_NONE; set_cgroups(pi->cgroups, pi->cgroups_len); m_root = pi->root; diff --git a/userspace/libsinsp/threadinfo.h b/userspace/libsinsp/threadinfo.h index ca90478513..ce7b8cd07a 100644 --- a/userspace/libsinsp/threadinfo.h +++ b/userspace/libsinsp/threadinfo.h @@ -265,8 +265,16 @@ class SINSP_PUBLIC sinsp_threadinfo int32_t m_tty; int32_t m_loginuid; ///< loginuid (auid) - // If true, this thread is part of a container health check - bool m_is_container_healthcheck; + // In some cases, a threadinfo has a category that identfies + // why it was run. + enum command_category { + CAT_NONE = 0, + CAT_HEALTHCHECK, + CAT_LIVENESS_PROBE, + CAT_READINESS_PROBE + }; + + command_category m_category; // // State for multi-event processing