From 4b2c59996aa42e290e942d5d7ac15458f7a2b40d Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 29 Nov 2023 15:02:46 +0100 Subject: [PATCH 1/2] Add memory/cgroup fields to metricbeat Kibana module --- metricbeat/docs/fields.asciidoc | 44 +++++ metricbeat/module/kibana/fields.go | 2 +- .../module/kibana/stats/_meta/fields.yml | 18 ++ .../kibana/stats/_meta/test/stats.8120.json | 172 ++++++++++++++++++ metricbeat/module/kibana/stats/data.go | 14 ++ 5 files changed, 249 insertions(+), 1 deletion(-) create mode 100644 metricbeat/module/kibana/stats/_meta/test/stats.8120.json diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 32d73808e31..f0b6a3eb7b4 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -42743,6 +42743,36 @@ type: long -- +*`kibana.stats.os.cpuacct.control_group`*:: ++ +-- +type: keyword + +-- + +*`kibana.stats.os.cpuacct.usage_nanos`*:: ++ +-- +type: long + +-- + + +*`kibana.stats.os.cgroup_memory.current_in_bytes`*:: ++ +-- +type: long + +-- + +*`kibana.stats.os.cgroup_memory.swap_current_in_bytes`*:: ++ +-- +type: long + +-- + + *`kibana.stats.os.load.1m`*:: + -- @@ -42788,6 +42818,20 @@ type: long -- +*`kibana.stats.process.memory.array_buffers.bytes`*:: ++ +-- +type: long + +-- + +*`kibana.stats.process.memory.external.bytes`*:: ++ +-- +type: long + +-- + *`kibana.stats.process.uptime.ms`*:: + -- diff --git a/metricbeat/module/kibana/fields.go b/metricbeat/module/kibana/fields.go index 790e70dbe9b..670132dd90e 100644 --- a/metricbeat/module/kibana/fields.go +++ b/metricbeat/module/kibana/fields.go @@ -32,5 +32,5 @@ func init() { // AssetKibana returns asset data. // This is the base64 encoded zlib format compressed contents of module/kibana. func AssetKibana() string { - return "eJzsWs2O47gRvvdTFPoylxkhfZjD+hAkmF0gQTCDYLKDHILAoMWyxQxFalmkp3ufPqB+bFkiJVqWdxuL8aHRkFRffawqVhV/3sFXfNnAV7Fjij0AWGElbuDxH/WDxwcAjpQbUVmh1Qb+/AAA0LyEUnMn8QGACm3sNtdqLw4b2DNJ/qlBiYxwAwcPTGitUAfawH8eieTjW3gsrK0e//sAsBcoOW1q7HegWIkdoy1ZZql+AWBfKo9mtKvaJ33BvrAVJZJlZXV600kzKRj1nlbMFht4/MtJ4nEE1jDJDFKlFeHWf5qV7DkRuxWvBzIGyUqKKfQSjpZoCQN08JqyEkttXrK9QdwKtd29WFykaA6qU1kZnSNR5io/bv9dKaQUi3ReYoXs133RUiuQVRmJX3ErRSnsLTrDiFl40LlWuTMGVT01FOZ+Ci0a8gxSZNwGSXAvQ2i3NdtbXD0HHjGCpkxqxrOncmF8DaWHwO9vAn4fB366DfkpAN3ZEI/eclLrastRspdb/DHEmpoQvW+dFVL8ynwgZSy34ogrkZgATqQjuLwHmQvYRCq9/+/AKITeETP4i0OylFltmVxWamqEAcAInwtqc8qitNBpCcGcdfWKHt21dA5UsSMadlgUTZfq2PEwUamdEzxRB6E5ihyzVuAS6Oo+ByUjK3JCZvIiy6UjiyYLkPmKL9+04eMa1YhsWaQ49Un4X6AZ7H5tU9giQosIJVojcsp6354awx1a1ns+HGXUQNMcY1B9uFFvNW2uvqg+ouEOV6OSa6dslInU6hAVHRaPeTJThPrI1fs/Bd932HupWYh0D+GHH65AGMajcTLQpiyPRo/3PRS/h2ICQielNMcV86KH++MmxT0T0hkcCkcC51S+njF3QwMnyPmSrJ1NkLpw5lpJpXblHzOjvHY/drtIK3ixbvOAKQ7GqXeeyqQ7D6nOHDSD03YfEG2pCUWWqRzhy5e//xhU4v+upWSE1SkRiuPzUi2fWImg9622NwRCWTSKyQBsp7DQNMzki0flsaIjs4YpqrSxW8a5QQpHYLpW39WjeUMnrXVceQVB9Uc0dLmkXKI0hHKaJ4pVVOiwNXdaS2RD9Rcq/l2gLdCALbBTt3NCchAE7ITePAsTCOWgpc70hkUmbRGC7TRKnTOZPilOzcHQSZ2Qj9cDmnEK6u1IwyvMP6+nmDi6XHsvITKVi5I4xLLxcHkOsSX6+VeKg2GNR60Ztd0THoez1++b2ufV3ifZj/UuTf/ZfYe/QlWYD5yw9BrRs2KdCQ0jjUQY9dIE4W+mTLBW5RrT/p1r2WxQJlU3Hda/JKNzQdboqzL6WPxzU5cWo1SS2b025c0AtxJpzrOi4rfsXMSOV8daglsrfah6E38lLEfIb4A691tsbNU1zPYUCoozdMHkfju/6fN+FZSnZTDJR8AwZeu5EuvKHRpfZHMpUFnoqYCScQSr66TXVI0MPmmLYAtmYWf0N0JDkDMFhIpD6aQVlUQg4f9lCrWjC0Sru0Oj3sc1MjAi9L2sBa1yfFsXJVvgSw1v8J0jBCTLdlJQgbwPm4Wb8ubgLDnlzVjqnw1c119f1XgmnnfPOvM0+0bXFpJFp456L0HIL4V4JL5njAXwk9cDXk+zrQtCQXNfA3Ot+FBpgF743LTPMJwgZqn9XCDULYQP+3rN5AMcz4QFgdIWBJdCHTzx8/vK6KPgvhFopkWdU2Mgxikl1CEbcZirrqPD9Cs8k2AAgB9d00OdLNAjXjCCHaJqWQAJ3/RK5ucsMnIGS1R2PKjTIoCPVtC/OXXP4WriUwG3Hv8PuqycRd5XB0cmHYLByiChsj7sThHausHqZlT1mNOGNkg/BbLxbLlpJnU50SNHEuN8sNdTKJIJZ/oR37sxu4GYcII7fq4ncD0AJqXOmXeN1V318LM/jN7vhH439n/zvD0D2L2kU47eOvvtiH9kzxloyYEqlmPN6NL+nzTH7H/UDeptgiOiRXFmOAl0v9TYfjb2jBwtZ4P7MWt1IACf296pPmO9qhO5WHoF7uqsZahzP9ldDmqaxW9osKca42up4V2ltZg1E12N+EU817u4s8x/n1uI5h7xQmcNrgwtscZfm+tLpyE1BWS6G+uVjssrUksIfGTPonTlFQSCN5O2zQJlmTN+6iPBhxrpDY224VNc0qyim6K8JZ1/xYmJNNz/j9AbxmbXeUXQBwt6Lu9GpOmjkmj84tAh3wan1QpMGvjxrI3ur119pPKvGqH1f+/N9acn9999D694X+05hScWptzu8WaNo5PJX7XJ3EbRsDtebbO5jptM+zwrZX3zc33vt+i1soj3wxVmcbfxsYEbzwdIugSWsGcFK1T1D42WnsL4xhAEri5HKcU2HxM4dV2a95Woy86YR9ou5nSvBvPbtUl8IdAd9Vu1aYtCQu92T6YX5P4fAAD//3zyQmA=" + return "eJzsWk+v27gRv79PMfAll0ToO+SwPhQtsgu0KBIU6QY9FIVAS2OLDUVqOaTzvJ++oP7YskRKtCzvPiziw8ODpPnNjzPDmeGfd/AVT1v4yndMsicAw43ALWz+UT/YPAHkSJnmleFKbuHPTwAAzUsoVW4FPgFQobRJMyX3/LCFPRPknmoUyAi3cHDAhMZweaAt/GdDJDZvYVMYU23++wSw5yhy2tbY70CyEjtGKRlmqH4BYE6VQ9PKVu2TvmBf2PASybCyOr/ppJngjHpPK2aKLWz+cpbYjMAaJolGqpQkTN2nScleIrFb8XogY5CkpJBCJ2FpiRY/QAevKCmxVPqU7DViymW6OxlcpGgOqlNZaZUhUWIrN273XcmF4It0XmP57Nd90VIrkFUJ8V8xFbzk5h6dfsTEP+hMycxqjbKeGhIzN4UWDXkGKTBujcRzJ0No0prtPa6eAw8YQVEiFMuT53JhfA2lh8Dv7wJ+HwZ+vg/52QPd2RCPznJCqSrNUbDTPf4YYk1NiN631nDBf2UukBKWGX7ElUhMAEfS4bl4BJkr2Egqvf8fwMiH3hHT+ItFMpQYZZhYVmpqhAHACD/n1OaURWmh0+KDuejqFT16aOkcqGJH1OywKJqu1bHjYaJSW8vzSB2E+sgzTFqBa6Cb+xwUjAzPCJnOiiQTlgzqxEPmK56+KZ2Pa1QjkrJAceqTcD9PM9j92qawRYQWEUo0mmeU9L49N4Y7NKz3fDjKoIGmOYag+nCj3mraXH1RdUSdW1yNSqasNEEmQslDUHRYPObJTBHqI1fv/+R932HvhWI+0j2EH364AWEYj9oKT5uyPBod3vdQ/B6KEQidlFQ5rpgXHdwfNynuGRdW41A4EDjn8vWCmR0aOELOlWRlTYTUlTPXSiq1K/+YGeW1+7HbRVrBi3WbB0zmoK1856hMuvMQ68xBMzht9wHRlhqXZJjMEL58+fuPXiXu71pKRlidEi5zfFmq5RMrEdS+1faGgEuDWjLhge0UFoqGmXzxqBxWcGRGM0mV0iZlea6R/BEYr9V19ajf0FlrHVdOgVf9ETVdLymXKPWhnOeJZBUVym/NnVIC2VD9lYp/F2gK1GAK7NTtLBc5cAJ2Rm+e+Qn4ctBSZzrDIhOm8MF2GoXKmIifFOfmYOikTsjF6wH1OAX1dqThFeaf11NMLF2vvZcQmcpFURxC2Xi4PIfQEv3yK/lBs8ajRo/a7gmPw8Xrj03t82ofk+zHepem/+Sxw1+hKswHjl96jehZsc74hhFHwo96bQL/N1MmWKtyjWn/zrVsNiijqpvy61+S0XNORqubMvpY/HNTlxajVIKZvdLl3QD3EmnOs4Li9+xchI5Xx1q8Wyt9qHoTfyUsS5jfAXXeLKosy7LwdtE9lsuUNFqJNIQBs+6FYR+SSiZHk+iW4dZc0geGS3fEu46X6Rur0vshL9018xn5/lE/+1LABbpgYp/Ob/G9XwXleRlM9IE/TNl6rqGy5Q61a6kywVEa6KmAkuUIRtUlrukREvikDIIpmIGdVt8INUHGJBDKHEorDK8EAnH3L5OoLF0hGtUdEfY+rpGBEaFbuRhQMsO3dQtiCjzV8BrfWUJAMmwnOBWY92ET/xKsOSaNLnAzlvpnA9etpm5aZkTebph15gCPac1O6c7u96jpTix8aRrppTDjezPRolN3Da5ByK3F88CUm/EfwE9ODzg9zbkCcAnNhSHMlMyHSj30/Af3fYb+nDVL7ecCoe5h3UysF+1uzuGFMCeQygDPBZcHR/zyvtLqyHPXiTYztS7qIRBtpeTykIw4zLV3o9scN3gmwgAAP9qmiT9boEe8YAQ7RNmyAOJu1SWYSyPIyGosUZrxoM6r0Hy0hfObU3ccbiY+FXDr8f+gysoazPvq4MiERdBYaSSUxoXdOUJbNxjVjKoec9zQBlmnQDaeLXfNpC5NO+RArp4P9noKBTLhTKvkFg/MbCEkHOGOn+sJXA+ACaEy5lxjVFfQ3Oz3o/db8d+N/d8cb8cAdqd4ysFrj78d8Y/sJQElcqCKZVgzurb/J5Vj8j/qBvU2whHBojgznAi6X2psNxt7Rg6Ws8EFrbWaIoDPbTtXH/Lf1Bxdrf09l8XWMtSlxe1upzX96zfU2FON4cX88LLcWsyaiS5H/AKe690cW+a/zy1Ec5F9obMGd9aWWOOvzf2585CaAjLdjfVKx/UdvSUEPrIXXtryBgLeq3Fps2Za5oyf+kjwoUZ6Q6NzoBiXNNs4TVFOSWVfcWIiDQ+gAvSGsdl1XgH0wY5SLh5GpOmjomj8YtFinnqn1QpMGvjxrA1u8N58pvevGqH1f+/N7cd3jz/+8S/CX+1BmSPmp9weMiSNo6PJ33TK0UbRsDte7bSjjptEuTwrRH31eH3vt+i1soD3/RVmcbfxsYEbzweIuoUYsY0GK1T1D42WnsLwXhV47s4HKYX2QyM4dV2a8xWvy86YR9zG6nSvBvM7yVF8wdMd9Vu1aYtCRO/2SKZX5P4fAAD//3pp2g4=" } diff --git a/metricbeat/module/kibana/stats/_meta/fields.yml b/metricbeat/module/kibana/stats/_meta/fields.yml index eda272fe1fb..f216cb93e86 100644 --- a/metricbeat/module/kibana/stats/_meta/fields.yml +++ b/metricbeat/module/kibana/stats/_meta/fields.yml @@ -72,6 +72,20 @@ type: long - name: used_in_bytes type: long + - name: cpuacct + type: group + fields: + - name: control_group + type: keyword + - name: usage_nanos + type: long + - name: cgroup_memory + type: group + fields: + - name: current_in_bytes + type: long + - name: swap_current_in_bytes + type: long - name: load type: group fields: @@ -92,6 +106,10 @@ fields: - name: memory.resident_set_size.bytes type: long + - name: memory.array_buffers.bytes + type: long + - name: memory.external.bytes + type: long - name: uptime.ms type: long - name: event_loop_delay.ms diff --git a/metricbeat/module/kibana/stats/_meta/test/stats.8120.json b/metricbeat/module/kibana/stats/_meta/test/stats.8120.json new file mode 100644 index 00000000000..5cde8984ec5 --- /dev/null +++ b/metricbeat/module/kibana/stats/_meta/test/stats.8120.json @@ -0,0 +1,172 @@ +{ + "kibana":{ + "uuid":"5b2de169-2785-441b-ae8c-186a1936b17d", + "name":"Shaunaks-MBP-2", + "index":".kibana", + "host":"localhost", + "transport_address":"localhost:5601", + "version":"7.0.0-alpha1", + "snapshot":false, + "status":"green" + }, + "last_updated":"2018-07-18T00:32:00.948Z", + "collection_interval_ms":5000, + "process":{ + "memory":{ + "heap":{ + "total_bytes":223391744, + "used_bytes":198413592, + "size_limit":1501560832 + }, + "resident_set_size_bytes":347242496, + "array_buffers_bytes": 6472439291, + "external_bytes": 953799759 + }, + "event_loop_delay":0.25226891040802, + "event_loop_utilization": { + "active": 629.1224170000005, + "idle": 359.23554199999995, + "utilization": 0.6365329598160299 + }, + "pid":46426, + "uptime_ms":1753889 + }, + "os":{ + "load":{ + "1m":3.50634765625, + "5m":3.76904296875, + "15m":3.54833984375 + }, + "memory":{ + "total_bytes":17179869184, + "free_bytes":31711232, + "used_bytes":17148157952 + }, + "uptime_ms":2187246000, + "cpuacct": { + "control_group": "cgroup", + "usage_nanos": 56789873 + }, + "cgroup_memory": { + "current_bytes": 97647654, + "swap_current_bytes": 40765796 + } + }, + "elasticsearch_client": { + "protocol": "http", + "connected_nodes": 2, + "nodes_with_active_sockets": 2, + "nodes_with_idle_sockets": 1, + "total_active_sockets": 14, + "total_idle_sockets": 42, + "total_queued_requests": 0, + "most_active_node_sockets": 10, + "average_active_sockets_per_node": 7, + "most_idle_node_sockets": 42, + "average_idle_sockets_per_node": 21 + }, + "response_times":{ + "max_ms":0 + }, + "requests":{ + "total":0, + "disconnects":0, + "status_codes":{ + + } + }, + "concurrent_connections":3, + "usage":{ + "kibana":{ + "index":".kibana", + "dashboard":{ + "total":0 + }, + "visualization":{ + "total":0 + }, + "search":{ + "total":0 + }, + "index_pattern":{ + "total":0 + }, + "graph_workspace":{ + "total":0 + }, + "timelion_sheet":{ + "total":0 + } + }, + "reporting":{ + "available":true, + "enabled":true, + "browser_type":"phantom", + "all":0, + "csv":{ + "available":true, + "total":0 + }, + "printable_pdf":{ + "available":true, + "total":0, + "app":{ + "visualization":0, + "dashboard":0 + }, + "layout":{ + "print":0, + "preserve_layout":0 + } + }, + "status":{ + + }, + "last_day":{ + "all":0, + "csv":{ + "available":true, + "total":0 + }, + "printable_pdf":{ + "available":true, + "total":0, + "app":{ + "visualization":0, + "dashboard":0 + }, + "layout":{ + "print":0, + "preserve_layout":0 + } + }, + "status":{ + + } + }, + "last_7_days":{ + "all":0, + "csv":{ + "available":true, + "total":0 + }, + "printable_pdf":{ + "available":true, + "total":0, + "app":{ + "visualization":0, + "dashboard":0 + }, + "layout":{ + "print":0, + "preserve_layout":0 + } + }, + "status":{ + + } + } + } + }, + "cluster_uuid":"NkfU5AinRnyFnqBD36zhEw" + } diff --git a/metricbeat/module/kibana/stats/data.go b/metricbeat/module/kibana/stats/data.go index 2b0ca12ef20..9adc6b7802c 100644 --- a/metricbeat/module/kibana/stats/data.go +++ b/metricbeat/module/kibana/stats/data.go @@ -46,6 +46,14 @@ var ( "distroRelease": c.Str("distro_release", s.Optional), "platform": c.Str("platform", s.Optional), "platformRelease": c.Str("platform_release", s.Optional), + "cpuacct": c.Dict("cpuacct", s.Schema{ + "control_group": c.Str("control_group"), + "usage_nanos": c.Int("usage_nanos"), + }, c.DictOptional), + "cgroup_memory": c.Dict("cgroup_memory", s.Schema{ + "current_in_bytes": c.Int("current_bytes"), + "swap_current_in_bytes": c.Int("swap_current_bytes"), + }, c.DictOptional), }), "kibana": c.Ifc("kibana"), "elasticsearch_client": c.Dict("elasticsearch_client", s.Schema{ @@ -77,6 +85,12 @@ var ( "resident_set_size": s.Object{ "bytes": c.Int("resident_set_size_bytes"), }, + "array_buffers": s.Object{ + "bytes": c.Int("array_buffers_bytes", s.Optional), + }, + "external": s.Object{ + "bytes": c.Int("external_bytes", s.Optional), + }, "heap": c.Dict("heap", s.Schema{ "total": s.Object{ "bytes": c.Int("total_bytes"), From 7f3924725371eab054eee2b9cdbe52cf893b455e Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 29 Nov 2023 15:10:56 +0100 Subject: [PATCH 2/2] update changelog --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c849160d04f..6127abf96de 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -244,7 +244,7 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d - Align on the algorithm used to transform Prometheus histograms into Elasticsearch histograms {pull}36647[36647] - Enhance GCP billing with detailed tables identification, additional fields, and optimized data handling. {pull}36902[36902] - Add a `/inputs/` route to the HTTP monitoring endpoint that exposes metrics for each metricset instance. {pull}36971[36971] - +- Add new memory/cgroup metrics to Kibana module {pull}37232[37232] *Osquerybeat*