From cd2eabebc874c8b2a629f2cbf516d2fa9483e4fa Mon Sep 17 00:00:00 2001 From: chrismark Date: Tue, 22 Sep 2020 13:05:22 +0300 Subject: [PATCH 1/8] Handle multiple upstreams in ingress-controller Signed-off-by: chrismark --- filebeat/docs/fields.asciidoc | 72 ++-- filebeat/module/nginx/fields.go | 2 +- .../nginx/ingress_controller/_meta/fields.yml | 36 +- .../ingress_controller/ingest/pipeline.yml | 60 ++- .../nginx/ingress_controller/test/test.log | 1 + .../test/test.log-expected.json | 389 ++++++++++++++---- 6 files changed, 437 insertions(+), 123 deletions(-) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index b4888ec8c5e..a2a4e1616fa 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -105135,54 +105135,50 @@ type: array -- -*`nginx.ingress_controller.http.request.length`*:: +*`nginx.ingress_controller.upstream_ip_list`*:: + -- -The request length (including request line, header, and request body) +An array of the upstream IP addresses. It is a list because it is common that several upstream servers were contacted during request processing. -type: long - -format: bytes +type: array -- -*`nginx.ingress_controller.http.request.time`*:: +*`nginx.ingress_controller.upstream.response.length_list`*:: + -- -Time elapsed since the first bytes were read from the client - +An array of upstream response lengths. It is a list because it is common that several upstream servers were contacted during request processing. -type: double -format: duration +type: array -- -*`nginx.ingress_controller.upstream.name`*:: +*`nginx.ingress_controller.upstream.response.time_list`*:: + -- -The name of the upstream. +An array of upstream response durations. It is a list because it is common that several upstream servers were contacted during request processing. -type: keyword +type: array -- -*`nginx.ingress_controller.upstream.alternative_name`*:: +*`nginx.ingress_controller.upstream.response.status_code_list`*:: + -- -The name of the alternative upstream. +An array of upstream response status codes. It is a list because it is common that several upstream servers were contacted during request processing. -type: keyword +type: array -- -*`nginx.ingress_controller.upstream.response.length`*:: +*`nginx.ingress_controller.http.request.length`*:: + -- -The length of the response obtained from the upstream server +The request length (including request line, header, and request body) type: long @@ -105191,10 +105187,10 @@ format: bytes -- -*`nginx.ingress_controller.upstream.response.time`*:: +*`nginx.ingress_controller.http.request.time`*:: + -- -The time spent on receiving the response from the upstream server as seconds with millisecond resolution +Time elapsed since the first bytes were read from the client type: double @@ -105203,43 +105199,57 @@ format: duration -- -*`nginx.ingress_controller.upstream.response.status_code`*:: +*`nginx.ingress_controller.upstream.name`*:: + -- -The status code of the response obtained from the upstream server +The name of the upstream. -type: long +type: keyword -- -*`nginx.ingress_controller.http.request.id`*:: +*`nginx.ingress_controller.upstream.alternative_name`*:: + -- -The randomly generated ID of the request +The name of the alternative upstream. type: keyword -- -*`nginx.ingress_controller.upstream.ip`*:: +*`nginx.ingress_controller.upstream.response.length`*:: + -- -The IP address of the upstream server. If several servers were contacted during request processing, their addresses are separated by commas. +The length of the response obtained from the upstream -type: ip +type: long + +format: bytes -- -*`nginx.ingress_controller.upstream.port`*:: +*`nginx.ingress_controller.upstream.response.time`*:: + -- -The port of the upstream server. +The time spent on receiving the response from the upstream as seconds with millisecond resolution -type: long +type: double + +format: duration + +-- + +*`nginx.ingress_controller.http.request.id`*:: ++ +-- +The randomly generated ID of the request + + +type: keyword -- diff --git a/filebeat/module/nginx/fields.go b/filebeat/module/nginx/fields.go index 2f9e50ceb60..c44a9896fff 100644 --- a/filebeat/module/nginx/fields.go +++ b/filebeat/module/nginx/fields.go @@ -32,5 +32,5 @@ func init() { // AssetNginx returns asset data. // This is the base64 encoded gzipped contents of module/nginx. func AssetNginx() string { - return "eJzsWM9u48YPvucpiP1dfgUSPYAPBYotFsihRVH00Jt3rKFkIqOhSo6c+u2LkeQ/kUe25GS3PUinRDK/7+OQwxnyCV5wvwJfkv/7ASBQcLiCT7/G/z89AFjUXKgOxH4FPz4AAPzCtnEIBQvURpR8CWGL0JqA4xIKcqjZA4BuWcI6Z19QuYIgDT4AFITO6qqFegJvKjzRxyfsa1xBKdzU/ZuEhvh8aYGgEK7GBMTnnO+c0+Q5qh5fp4ivkMfnM/tgyGtP0a7ISUiHH/UcpaTknEsSrDjgmuq1Iw1vfnKQZ0TMfvDlisT4/OQ7K+CiZ4Dn38BYK6iKmsFzAFIwEElhg7lpFIHalzlXFXsIDORz11h8hA0qWdTW09wR+qFQOIN/fEPVxWqLxqIoOHpB+Prn0xeWVyMWbfzra3aB9jsaB8qN5K1wUhDUwII26vrafcmoPjNNru6G7X6t6EO22QfU9PI6MsMvtQnbFWxDqDNBrdkrZhErCVNRKaaLRJ/vl0IaRVnHP2dKiHZZwm4KZ4Vhy/Y+n/9qUEOWRJjkrri5jorLWKgkb4amUwij7PUORYn9PR6nTacwH/JjnbOdG923CabBhEZTONN0FCiC8p54j2BMoTflZVWYktzr1nBu6Mf32LiOYZWHkcp8DmlxR/kwGtddS7rX4aQ28jUvh2pGzGdqeacITpHMlMCaFY1zqYI4T8oYwnw975WSzt97tIwjzdoNJTIN0/3+jZCzD+TRh/eseH9sl8jZTbypC59z44Ps16Scqpx3SbuJOFWc47z92ftFXUGaKkawJPYfFL/rYJODR2H/UQl1BWrmCn1cKt0GHJN2kIQiLN+uYWnhZ/UrOXuPeWjdSt8tHftyXrfy+YgJZNEHKgjlxs3e4Q7n3jEdl1nKbsr9ph7xdpytFo7dYHZpOYUv3M0XtoLGZnexVqhqyrnX2LTVrdQmX8YWcR2PAmHn8Jvk+XPH0uf7iWvp1f8rvfr/DpnwdBYdrTGngvI+HqM937F1cejLsJ1akAqWyoQVpNr5G+H6Y4vQk0JHCv/vFp98efpCHh/7pXwE4+3xy4bt/ofb/gQamRVYbjZu+Ongj20kdU245RJVCOhMrWhByefYZk9BEvXGJYJXlOi2safZWyK5jp1YrUHQVKlbdefFC+5fWYYlasLSR8S4U6KCI811EcYFFG8C7XB8BPMhgs6Ypoo7dv/fNYH7xO1lHzQAb2INxbMgH3SCouwGg4FxX75r8m4RIl8sGT4AexDMkXaHGfXRuTGfwCgo5uytwiuFLVTkHHVvojW75kLVuOvjQ5x7b0fRwQ4VIuoHBe1NsRm5a9y9J8R4y5XbQ4kexQS08PzzSXhLen09R1rXi9cTxJxOuGHd6Ncng+cCFHco8SBrX/UFLx5IJo/ybSPn5b2/a5EvHyMiydkhagRBsTad35t9e0gbvVEHapb0jeK+hIlwY+5ev1gvI/NlZL6MzJeR+TIyX0bm1/UsI/NlZL6MzJeR+b88Mv8nAAD///5cnQ8=" + return "eJzsWc1u4zYQvucpBttLCyR6AB8KFFsskEOLouihNy9NjqRBKI46pJz67QtK/lFkypacn+4C0imRPN/3zQ8pjuYBnnC3AleQ+/cOIFCwuIJPv8f/P90BGPRaqA7EbgU/3wEA/MamsQg5C9RKPLkCQonQmoDlAnKy6LM7AF+yhLVml1OxgiAN3gHkhNb4VQv1AE5VeKKPV9jVuIJCuKn3dxIa4vWlBYJcuBoTEK8+X59TaY3eH2+niC+Qx+szu6DI+T1FG5GTkA4/6jlKScnpSxKsOOCa6rUlH1785CBPiajd4MkFifH6xXVWwPmeAR7/AGWMoPfoM3gMQB4URFLYoFaNR6D2puaqYgeBgZy2jcF72KAng771VFtCNxQKPfj7F1RdrkpUBsWDpSeEr38/fGF5VmLQxL++Zmdof6Ky4LkR3QonD4I+sKCJur52TzKqe6bJ6G7Y7NYeXcg2u4A+HV5LavikVqFcQRlCnQn6mp3HLGIlYSoqRHWZ2Nf7uZDGo6zjnzMlRLssYTeFs8JQsrnN538a9CFLIkxyV+xcR8VmLFSQU0PTKYRR9nqL4ondLR6nTacwH+pjrdnMze7LAvNBhcancKbpyFEE5TX5HsGYQq+K811hSnGvW8O5qR9fY+M6hrs8jOzMfUiDW9LDbFx2Leleh5NayJe8HKoZMZ+p5ZUiOEUyUwL7LG+sTW2I86SMIczX81op6fq9Rcs40qzVUCDTsNxvXwiaXSCHLrwm4vvXdoGcXcWbGnjNjQuyW5Pn1M55k7SriFPFWdbtz14v6gLSVDGCBbF7o/xdBpucPAq7tyqoC1AzI/R2pXQdcEzaQRKKsLxfw9LCz+pXNDuHOrRupc+Wll0xr1v5fMQEMugC5YRy5WRvcYtzz5iWiyxlN+V8U494O85WC8duMDu3nMIXbuYLpaAy2U2sFXqvirnH2LTVtdImV8QWcR1fBcLW4rvU+WPHsq/3E9fSq38rvfoPh0p46GXH16gpJ73PR/q8X/sgqKr3TEiM4oFnflpKFcDjFkXZE4pHid3mGe0zCrYFqnRAA6YRcgXsW7PD6iZXZBeDcWonLboilO8WmaM/B0LoCL+DyASq8APjYppuH/wOItP7BPGBAepYIbJ+czF68X2kq/Cpp56cpVJhBalvhlfi9VeJR4UdKfzY7fB97ZYc3u/363tQzhyfbNjsfrruT1wISW8MNxs7fHTw51DPM12iCgGtqj0a8OQ0tptrThL1xhB1mYqHl9MH/sQb7Kx6Rz+rPuHumWV4DpoQ+og43P2vLCFlA4pTgbY4/p33TQT1mKaKG7wTPqiA94W7l31c7byJBzXsJfmgc8b2/WFVWyJEvnggcQHYgaBG2h4mYEevzpwB5cGjZmc8PFMooSJrqbsTzdg2Z3KSi3SkEbi5lkQ5w5XdQYEORcU98fHXU5Za0su91zJVWaYqy1RlmaosU5VlqnJZzzJVWaYqy1Rlmar8z1OV/wIAAP//LIFLNg==" } diff --git a/filebeat/module/nginx/ingress_controller/_meta/fields.yml b/filebeat/module/nginx/ingress_controller/_meta/fields.yml index 2c467e3856a..41520f8a29f 100644 --- a/filebeat/module/nginx/ingress_controller/_meta/fields.yml +++ b/filebeat/module/nginx/ingress_controller/_meta/fields.yml @@ -11,6 +11,26 @@ Real source IP is restored to `source.ip`. # ingress-controller specific fields + - name: upstream_ip_list + type: array + description: > + An array of the upstream IP addresses. It is a list because it is common that several upstream servers + were contacted during request processing. + - name: upstream.response.length_list + type: array + description: > + An array of upstream response lengths. It is a list because it is common that several upstream servers + were contacted during request processing. + - name: upstream.response.time_list + type: array + description: > + An array of upstream response durations. It is a list because it is common that several upstream servers + were contacted during request processing. + - name: upstream.response.status_code_list + type: array + description: > + An array of upstream response status codes. It is a list because it is common that several upstream servers + were contacted during request processing. - name: http.request.length type: long format: bytes @@ -33,28 +53,16 @@ type: long format: bytes description: > - The length of the response obtained from the upstream server + The length of the response obtained from the upstream - name: upstream.response.time type: double format: duration description: > - The time spent on receiving the response from the upstream server as seconds with millisecond resolution - - name: upstream.response.status_code - type: long - description: > - The status code of the response obtained from the upstream server + The time spent on receiving the response from the upstream as seconds with millisecond resolution - name: http.request.id type: keyword description: > The randomly generated ID of the request - - name: upstream.ip - type: ip - description: > - The IP address of the upstream server. If several servers were contacted during request processing, their addresses are separated by commas. - - name: upstream.port - type: long - description: > - The port of the upstream server. - name: body_sent.bytes type: alias diff --git a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml index c9f4a5860c7..83f090d5e92 100644 --- a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml +++ b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml @@ -12,14 +12,18 @@ processors: %{NUMBER:http.response.status_code:long} %{NUMBER:http.response.body.bytes:long} "(-|%{DATA:http.request.referrer})" "(-|%{DATA:user_agent.original})" %{NUMBER:nginx.ingress_controller.http.request.length:long} %{NUMBER:nginx.ingress_controller.http.request.time:double} \[%{DATA:nginx.ingress_controller.upstream.name}\] - \[%{DATA:nginx.ingress_controller.upstream.alternative_name}\] (%{UPSTREAM_ADDRESS}|-) - (%{NUMBER:nginx.ingress_controller.upstream.response.length:long}|-) (%{NUMBER:nginx.ingress_controller.upstream.response.time:double}|-) - (%{NUMBER:nginx.ingress_controller.upstream.response.status_code:long}|-) %{GREEDYDATA:nginx.ingress_controller.http.request.id} + \[%{DATA:nginx.ingress_controller.upstream.alternative_name}\] (%{UPSTREAM_ADDRESS_LIST:nginx.ingress_controller.upstream_ip_list}|-) + (%{UPSTREAM_RESPONSE_LENGTH_LIST:nginx.ingress_controller.upstream.response.length_list}|-) (%{UPSTREAM_RESPONSE_TIME_LIST:nginx.ingress_controller.upstream.response.time_list}|-) + (%{UPSTREAM_RESPONSE_STATUS_CODE_LIST:nginx.ingress_controller.upstream.response.status_code_list}|-) %{GREEDYDATA:nginx.ingress_controller.http.request.id} pattern_definitions: NGINX_HOST: (?:%{IP:destination.ip}|%{NGINX_NOTSEPARATOR:destination.domain})(:%{NUMBER:destination.port})? NGINX_NOTSEPARATOR: "[^\t ,:]+" NGINX_ADDRESS_LIST: (?:%{IP}|%{WORD})("?,?\s*(?:%{IP}|%{WORD}))* UPSTREAM_ADDRESS: '%{IP:nginx.ingress_controller.upstream.ip}(:%{NUMBER:nginx.ingress_controller.upstream.port})?' + UPSTREAM_ADDRESS_LIST: (?:%{IP}(:%{NUMBER})?)("?,?\s*(?:%{IP}(:%{NUMBER})?))* + UPSTREAM_RESPONSE_LENGTH_LIST: (?:%{NUMBER})("?,?\s*(?:%{NUMBER}))* + UPSTREAM_RESPONSE_TIME_LIST: (?:%{NUMBER})("?,?\s*(?:%{NUMBER}))* + UPSTREAM_RESPONSE_STATUS_CODE_LIST: (?:%{NUMBER})("?,?\s*(?:%{NUMBER}))* ignore_missing: true - grok: field: nginx.ingress_controller.info @@ -33,6 +37,22 @@ processors: field: nginx.ingress_controller.remote_ip_list separator: '"?,?\s+' ignore_missing: true +- split: + field: nginx.ingress_controller.upstream_ip_list + separator: '"?,?\s+' + ignore_missing: true +- split: + field: nginx.ingress_controller.upstream.response.length_list + separator: '"?,?\s+' + ignore_missing: true +- split: + field: nginx.ingress_controller.upstream.response.time_list + separator: '"?,?\s+' + ignore_missing: true +- split: + field: nginx.ingress_controller.upstream.response.status_code_list + separator: '"?,?\s+' + ignore_missing: true - split: field: nginx.ingress_controller.origin separator: '"?,?\s+' @@ -41,6 +61,40 @@ processors: field: source.address if: ctx.source?.address == null value: "" +- script: + if: ctx.nginx?.ingress_controller?.upstream?.response?.length_list != null && ctx.nginx.ingress_controller.upstream.response.length_list.length > 0 + lang: painless + source: >- + try { + if (ctx.nginx.ingress_controller.upstream.response.length_list.length == null) { + return; + } + int length = 0; + for (def item : ctx.nginx.ingress_controller.upstream.response.length_list) { + length = length + Integer.parseInt(item); + } + ctx.nginx.ingress_controller.upstream.response.length = length; + } + catch (Exception e) { + ctx.nginx.ingress_controller.upstream.response.length = null; + } +- script: + if: ctx.nginx?.ingress_controller?.upstream?.response?.time_list != null && ctx.nginx.ingress_controller.upstream.response.time_list.length > 0 + lang: painless + source: >- + try { + if (ctx.nginx.ingress_controller.upstream.response.time_list.length == null) { + return; + } + float res_time = 0; + for (def item : ctx.nginx.ingress_controller.upstream.response.time_list) { + res_time = res_time + Float.parseFloat(item); + } + ctx.nginx.ingress_controller.upstream.response.time = res_time; + } + catch (Exception e) { + ctx.nginx.ingress_controller.upstream.response.time = null; + } - script: if: ctx.nginx?.ingress_controller?.remote_ip_list != null && ctx.nginx.ingress_controller.remote_ip_list.length > 0 lang: painless diff --git a/filebeat/module/nginx/ingress_controller/test/test.log b/filebeat/module/nginx/ingress_controller/test/test.log index 862c03a4af2..c8ba580f64d 100644 --- a/filebeat/module/nginx/ingress_controller/test/test.log +++ b/filebeat/module/nginx/ingress_controller/test/test.log @@ -20,3 +20,4 @@ 192.168.64.1 - - [07/Feb/2020:12:02:38 +0000] "GET /v2 HTTP/1.1" 200 61 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0" 343 0.000 [default-web2-8080] [] 172.17.0.6:8080 61 0.001 200 ba91c30454893c121879396b0a78be79 192.168.64.1 - - [07/Feb/2020:12:02:38 +0000] "GET /favicon.ico HTTP/1.1" 200 59 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0" 262 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 98c81aa2d50c67f6fb1fa16d5ce62f8f 192.168.64.1 - - [07/Feb/2020:12:02:42 +0000] "GET /v2/some HTTP/1.1" 200 61 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0" 348 0.001 [default-web2-8080] [] 172.17.0.6:8080 61 0.000 200 835136ae24486dbb4156dcbe21f5d402 +192.168.64.14 - - [07/Feb/2020:12:02:42 +0000] "GET /v2/some HTTP/1.1" 200 61 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0" 348 0.001 [default-web2-8080] [] 172.17.0.6:8080, 172.17.0.7:8080 61, 100 0.100, 0.004 200, 203 835136ae24486dbb4156dcbe21f5d402 diff --git a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json index 4bf393a5906..a6913a48d20 100644 --- a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json +++ b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json @@ -26,12 +26,21 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", - "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", - "nginx.ingress_controller.upstream.port": "8080", "nginx.ingress_controller.upstream.response.length": 59, - "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.length_list": [ + "59" + ], + "nginx.ingress_controller.upstream.response.status_code_list": [ + "200" + ], "nginx.ingress_controller.upstream.response.time": 0.0, + "nginx.ingress_controller.upstream.response.time_list": [ + "0.000" + ], + "nginx.ingress_controller.upstream_ip_list": [ + "172.17.0.5:8080" + ], "related.ip": [ "192.168.64.1" ], @@ -71,12 +80,21 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", - "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", - "nginx.ingress_controller.upstream.port": "8080", "nginx.ingress_controller.upstream.response.length": 59, - "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.length_list": [ + "59" + ], + "nginx.ingress_controller.upstream.response.status_code_list": [ + "200" + ], "nginx.ingress_controller.upstream.response.time": 0.0, + "nginx.ingress_controller.upstream.response.time_list": [ + "0.000" + ], + "nginx.ingress_controller.upstream_ip_list": [ + "172.17.0.5:8080" + ], "related.ip": [ "192.168.64.1" ], @@ -116,12 +134,21 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", - "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", - "nginx.ingress_controller.upstream.port": "8080", "nginx.ingress_controller.upstream.response.length": 59, - "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.length_list": [ + "59" + ], + "nginx.ingress_controller.upstream.response.status_code_list": [ + "200" + ], "nginx.ingress_controller.upstream.response.time": 0.001, + "nginx.ingress_controller.upstream.response.time_list": [ + "0.001" + ], + "nginx.ingress_controller.upstream_ip_list": [ + "172.17.0.5:8080" + ], "related.ip": [ "192.168.64.1" ], @@ -161,12 +188,21 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", - "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", - "nginx.ingress_controller.upstream.port": "8080", "nginx.ingress_controller.upstream.response.length": 59, - "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.length_list": [ + "59" + ], + "nginx.ingress_controller.upstream.response.status_code_list": [ + "200" + ], "nginx.ingress_controller.upstream.response.time": 0.0, + "nginx.ingress_controller.upstream.response.time_list": [ + "0.000" + ], + "nginx.ingress_controller.upstream_ip_list": [ + "172.17.0.5:8080" + ], "related.ip": [ "192.168.64.1" ], @@ -278,12 +314,21 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", - "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", - "nginx.ingress_controller.upstream.port": "8080", "nginx.ingress_controller.upstream.response.length": 59, - "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.length_list": [ + "59" + ], + "nginx.ingress_controller.upstream.response.status_code_list": [ + "200" + ], "nginx.ingress_controller.upstream.response.time": 0.0, + "nginx.ingress_controller.upstream.response.time_list": [ + "0.000" + ], + "nginx.ingress_controller.upstream_ip_list": [ + "172.17.0.5:8080" + ], "related.ip": [ "192.168.64.1" ], @@ -323,12 +368,21 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", - "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", - "nginx.ingress_controller.upstream.port": "8080", "nginx.ingress_controller.upstream.response.length": 59, - "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.length_list": [ + "59" + ], + "nginx.ingress_controller.upstream.response.status_code_list": [ + "200" + ], "nginx.ingress_controller.upstream.response.time": 0.0, + "nginx.ingress_controller.upstream.response.time_list": [ + "0.000" + ], + "nginx.ingress_controller.upstream_ip_list": [ + "172.17.0.5:8080" + ], "related.ip": [ "192.168.64.1" ], @@ -336,7 +390,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", - "user_agent.device.name": "Mac", + "user_agent.device.name": "Other", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", "user_agent.os.full": "Mac OS X 10.14.6", @@ -372,12 +426,21 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", - "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", - "nginx.ingress_controller.upstream.port": "8080", "nginx.ingress_controller.upstream.response.length": 59, - "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.length_list": [ + "59" + ], + "nginx.ingress_controller.upstream.response.status_code_list": [ + "200" + ], "nginx.ingress_controller.upstream.response.time": 0.0, + "nginx.ingress_controller.upstream.response.time_list": [ + "0.000" + ], + "nginx.ingress_controller.upstream_ip_list": [ + "172.17.0.5:8080" + ], "related.ip": [ "192.168.64.1" ], @@ -385,7 +448,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/favicon.ico", - "user_agent.device.name": "Mac", + "user_agent.device.name": "Other", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", "user_agent.os.full": "Mac OS X 10.14.6", @@ -420,12 +483,21 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", - "nginx.ingress_controller.upstream.ip": "172.17.0.6", "nginx.ingress_controller.upstream.name": "default-web2-8080", - "nginx.ingress_controller.upstream.port": "8080", "nginx.ingress_controller.upstream.response.length": 61, - "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.length_list": [ + "61" + ], + "nginx.ingress_controller.upstream.response.status_code_list": [ + "200" + ], "nginx.ingress_controller.upstream.response.time": 0.001, + "nginx.ingress_controller.upstream.response.time_list": [ + "0.001" + ], + "nginx.ingress_controller.upstream_ip_list": [ + "172.17.0.6:8080" + ], "related.ip": [ "192.168.64.1" ], @@ -433,7 +505,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/v2", - "user_agent.device.name": "Mac", + "user_agent.device.name": "Other", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", "user_agent.os.full": "Mac OS X 10.14.6", @@ -469,12 +541,21 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", - "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", - "nginx.ingress_controller.upstream.port": "8080", "nginx.ingress_controller.upstream.response.length": 59, - "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.length_list": [ + "59" + ], + "nginx.ingress_controller.upstream.response.status_code_list": [ + "200" + ], "nginx.ingress_controller.upstream.response.time": 0.002, + "nginx.ingress_controller.upstream.response.time_list": [ + "0.002" + ], + "nginx.ingress_controller.upstream_ip_list": [ + "172.17.0.5:8080" + ], "related.ip": [ "192.168.64.1" ], @@ -482,7 +563,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/favicon.ico", - "user_agent.device.name": "Mac", + "user_agent.device.name": "Other", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", "user_agent.os.full": "Mac OS X 10.14.6", @@ -517,12 +598,21 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", - "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", - "nginx.ingress_controller.upstream.port": "8080", "nginx.ingress_controller.upstream.response.length": 59, - "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.length_list": [ + "59" + ], + "nginx.ingress_controller.upstream.response.status_code_list": [ + "200" + ], "nginx.ingress_controller.upstream.response.time": 0.001, + "nginx.ingress_controller.upstream.response.time_list": [ + "0.001" + ], + "nginx.ingress_controller.upstream_ip_list": [ + "172.17.0.5:8080" + ], "related.ip": [ "192.168.64.1" ], @@ -530,7 +620,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", - "user_agent.device.name": "Mac", + "user_agent.device.name": "Other", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", "user_agent.os.full": "Mac OS X 10.14.6", @@ -566,12 +656,21 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", - "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", - "nginx.ingress_controller.upstream.port": "8080", "nginx.ingress_controller.upstream.response.length": 59, - "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.length_list": [ + "59" + ], + "nginx.ingress_controller.upstream.response.status_code_list": [ + "200" + ], "nginx.ingress_controller.upstream.response.time": 0.001, + "nginx.ingress_controller.upstream.response.time_list": [ + "0.001" + ], + "nginx.ingress_controller.upstream_ip_list": [ + "172.17.0.5:8080" + ], "related.ip": [ "192.168.64.1" ], @@ -579,7 +678,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/favicon.ico", - "user_agent.device.name": "Mac", + "user_agent.device.name": "Other", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", "user_agent.os.full": "Mac OS X 10.14.6", @@ -614,12 +713,21 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", - "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", - "nginx.ingress_controller.upstream.port": "8080", "nginx.ingress_controller.upstream.response.length": 59, - "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.length_list": [ + "59" + ], + "nginx.ingress_controller.upstream.response.status_code_list": [ + "200" + ], "nginx.ingress_controller.upstream.response.time": 0.002, + "nginx.ingress_controller.upstream.response.time_list": [ + "0.002" + ], + "nginx.ingress_controller.upstream_ip_list": [ + "172.17.0.5:8080" + ], "related.ip": [ "192.168.64.1" ], @@ -627,7 +735,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", - "user_agent.device.name": "Mac", + "user_agent.device.name": "Other", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", "user_agent.os.full": "Mac OS X 10.14.6", @@ -662,12 +770,21 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", - "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", - "nginx.ingress_controller.upstream.port": "8080", "nginx.ingress_controller.upstream.response.length": 59, - "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.length_list": [ + "59" + ], + "nginx.ingress_controller.upstream.response.status_code_list": [ + "200" + ], "nginx.ingress_controller.upstream.response.time": 0.001, + "nginx.ingress_controller.upstream.response.time_list": [ + "0.001" + ], + "nginx.ingress_controller.upstream_ip_list": [ + "172.17.0.5:8080" + ], "related.ip": [ "192.168.64.1" ], @@ -675,7 +792,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/", - "user_agent.device.name": "Mac", + "user_agent.device.name": "Other", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", "user_agent.os.full": "Mac OS X 10.14.6", @@ -711,12 +828,21 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", - "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", - "nginx.ingress_controller.upstream.port": "8080", "nginx.ingress_controller.upstream.response.length": 59, - "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.length_list": [ + "59" + ], + "nginx.ingress_controller.upstream.response.status_code_list": [ + "200" + ], "nginx.ingress_controller.upstream.response.time": 0.002, + "nginx.ingress_controller.upstream.response.time_list": [ + "0.002" + ], + "nginx.ingress_controller.upstream_ip_list": [ + "172.17.0.5:8080" + ], "related.ip": [ "192.168.64.1" ], @@ -724,7 +850,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/favicon.ico", - "user_agent.device.name": "Mac", + "user_agent.device.name": "Other", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", "user_agent.os.full": "Mac OS X 10.14.6", @@ -759,12 +885,21 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", - "nginx.ingress_controller.upstream.ip": "172.17.0.6", "nginx.ingress_controller.upstream.name": "default-web2-8080", - "nginx.ingress_controller.upstream.port": "8080", "nginx.ingress_controller.upstream.response.length": 61, - "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.length_list": [ + "61" + ], + "nginx.ingress_controller.upstream.response.status_code_list": [ + "200" + ], "nginx.ingress_controller.upstream.response.time": 0.002, + "nginx.ingress_controller.upstream.response.time_list": [ + "0.002" + ], + "nginx.ingress_controller.upstream_ip_list": [ + "172.17.0.6:8080" + ], "related.ip": [ "192.168.64.1" ], @@ -772,7 +907,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/v2", - "user_agent.device.name": "Mac", + "user_agent.device.name": "Other", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", "user_agent.os.full": "Mac OS X 10.14.6", @@ -808,12 +943,21 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", - "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", - "nginx.ingress_controller.upstream.port": "8080", "nginx.ingress_controller.upstream.response.length": 59, - "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.length_list": [ + "59" + ], + "nginx.ingress_controller.upstream.response.status_code_list": [ + "200" + ], "nginx.ingress_controller.upstream.response.time": 0.0, + "nginx.ingress_controller.upstream.response.time_list": [ + "0.000" + ], + "nginx.ingress_controller.upstream_ip_list": [ + "172.17.0.5:8080" + ], "related.ip": [ "192.168.64.1" ], @@ -821,7 +965,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/favicon.ico", - "user_agent.device.name": "Mac", + "user_agent.device.name": "Other", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", "user_agent.os.full": "Mac OS X 10.14.6", @@ -856,12 +1000,21 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", - "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", - "nginx.ingress_controller.upstream.port": "8080", "nginx.ingress_controller.upstream.response.length": 59, - "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.length_list": [ + "59" + ], + "nginx.ingress_controller.upstream.response.status_code_list": [ + "200" + ], "nginx.ingress_controller.upstream.response.time": 0.001, + "nginx.ingress_controller.upstream.response.time_list": [ + "0.001" + ], + "nginx.ingress_controller.upstream_ip_list": [ + "172.17.0.5:8080" + ], "related.ip": [ "192.168.64.1" ], @@ -901,12 +1054,21 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", - "nginx.ingress_controller.upstream.ip": "172.17.0.6", "nginx.ingress_controller.upstream.name": "default-web2-8080", - "nginx.ingress_controller.upstream.port": "8080", "nginx.ingress_controller.upstream.response.length": 61, - "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.length_list": [ + "61" + ], + "nginx.ingress_controller.upstream.response.status_code_list": [ + "200" + ], "nginx.ingress_controller.upstream.response.time": 0.001, + "nginx.ingress_controller.upstream.response.time_list": [ + "0.001" + ], + "nginx.ingress_controller.upstream_ip_list": [ + "172.17.0.6:8080" + ], "related.ip": [ "192.168.64.1" ], @@ -914,7 +1076,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/v2", - "user_agent.device.name": "Mac", + "user_agent.device.name": "Other", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", "user_agent.os.full": "Mac OS X 10.14", @@ -949,12 +1111,21 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", - "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", - "nginx.ingress_controller.upstream.port": "8080", "nginx.ingress_controller.upstream.response.length": 59, - "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.length_list": [ + "59" + ], + "nginx.ingress_controller.upstream.response.status_code_list": [ + "200" + ], "nginx.ingress_controller.upstream.response.time": 0.0, + "nginx.ingress_controller.upstream.response.time_list": [ + "0.000" + ], + "nginx.ingress_controller.upstream_ip_list": [ + "172.17.0.5:8080" + ], "related.ip": [ "192.168.64.1" ], @@ -962,7 +1133,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/favicon.ico", - "user_agent.device.name": "Mac", + "user_agent.device.name": "Other", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", "user_agent.os.full": "Mac OS X 10.14", @@ -997,12 +1168,21 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", - "nginx.ingress_controller.upstream.ip": "172.17.0.6", "nginx.ingress_controller.upstream.name": "default-web2-8080", - "nginx.ingress_controller.upstream.port": "8080", "nginx.ingress_controller.upstream.response.length": 61, - "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.length_list": [ + "61" + ], + "nginx.ingress_controller.upstream.response.status_code_list": [ + "200" + ], "nginx.ingress_controller.upstream.response.time": 0.0, + "nginx.ingress_controller.upstream.response.time_list": [ + "0.000" + ], + "nginx.ingress_controller.upstream_ip_list": [ + "172.17.0.6:8080" + ], "related.ip": [ "192.168.64.1" ], @@ -1010,7 +1190,68 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/v2/some", - "user_agent.device.name": "Mac", + "user_agent.device.name": "Other", + "user_agent.name": "Firefox", + "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", + "user_agent.os.full": "Mac OS X 10.14", + "user_agent.os.name": "Mac OS X", + "user_agent.os.version": "10.14", + "user_agent.version": "72.0." + }, + { + "@timestamp": "2020-02-07T12:02:42.000Z", + "event.category": [ + "web" + ], + "event.dataset": "nginx.ingress_controller", + "event.kind": "event", + "event.module": "nginx", + "event.outcome": "success", + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "ingress_controller", + "http.request.method": "GET", + "http.response.body.bytes": 61, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 5730, + "nginx.ingress_controller.http.request.id": "835136ae24486dbb4156dcbe21f5d402", + "nginx.ingress_controller.http.request.length": 348, + "nginx.ingress_controller.http.request.time": 0.001, + "nginx.ingress_controller.remote_ip_list": [ + "192.168.64.14" + ], + "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.name": "default-web2-8080", + "nginx.ingress_controller.upstream.response.length": 161, + "nginx.ingress_controller.upstream.response.length_list": [ + "61", + "100" + ], + "nginx.ingress_controller.upstream.response.status_code_list": [ + "200", + "203" + ], + "nginx.ingress_controller.upstream.response.time": 0.104, + "nginx.ingress_controller.upstream.response.time_list": [ + "0.100", + "0.004" + ], + "nginx.ingress_controller.upstream_ip_list": [ + "172.17.0.6:8080", + "172.17.0.7:8080" + ], + "related.ip": [ + "192.168.64.14" + ], + "service.type": "nginx", + "source.address": "192.168.64.14", + "source.ip": "192.168.64.14", + "url.original": "/v2/some", + "user_agent.device.name": "Other", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", "user_agent.os.full": "Mac OS X 10.14", From 6452ee900010cddad62eef75e6cb988439bb8572 Mon Sep 17 00:00:00 2001 From: chrismark Date: Tue, 22 Sep 2020 14:41:56 +0300 Subject: [PATCH 2/8] Add final status_code and remote Signed-off-by: chrismark --- filebeat/docs/fields.asciidoc | 34 +++++++++- filebeat/module/nginx/fields.go | 2 +- .../nginx/ingress_controller/_meta/fields.yml | 21 ++++++- .../ingress_controller/ingest/pipeline.yml | 41 ++++++++++++ .../test/test.log-expected.json | 63 +++++++++++++++++++ 5 files changed, 156 insertions(+), 5 deletions(-) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index a2a4e1616fa..115b8383179 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -105222,7 +105222,7 @@ type: keyword *`nginx.ingress_controller.upstream.response.length`*:: + -- -The length of the response obtained from the upstream +The length of the response obtained from the upstream server. If several servers were contacted during request process, the summary of the multiple response lengths is stored. type: long @@ -105234,7 +105234,7 @@ format: bytes *`nginx.ingress_controller.upstream.response.time`*:: + -- -The time spent on receiving the response from the upstream as seconds with millisecond resolution +The time spent on receiving the response from the upstream as seconds with millisecond resolution. If several servers were contacted during request process, the summary of the multiple response times is stored. type: double @@ -105243,6 +105243,36 @@ format: duration -- +*`nginx.ingress_controller.upstream.response.status_code`*:: ++ +-- +The status code of the response obtained from the upstream server. If several servers were contacted during request process, only the status code of the response from the last one is stored in this field. + + +type: long + +-- + +*`nginx.ingress_controller.upstream.ip`*:: ++ +-- +The IP address of the upstream server. If several servers were contacted during request process, only the last one is stored in this field. + + +type: ip + +-- + +*`nginx.ingress_controller.upstream.port`*:: ++ +-- +The port of the upstream server. If several servers were contacted during request process, only the last one is stored in this field. + + +type: long + +-- + *`nginx.ingress_controller.http.request.id`*:: + -- diff --git a/filebeat/module/nginx/fields.go b/filebeat/module/nginx/fields.go index c44a9896fff..5c6f56a2e7f 100644 --- a/filebeat/module/nginx/fields.go +++ b/filebeat/module/nginx/fields.go @@ -32,5 +32,5 @@ func init() { // AssetNginx returns asset data. // This is the base64 encoded gzipped contents of module/nginx. func AssetNginx() string { - return "eJzsWc1u4zYQvucpBttLCyR6AB8KFFsskEOLouihNy9NjqRBKI46pJz67QtK/lFkypacn+4C0imRPN/3zQ8pjuYBnnC3AleQ+/cOIFCwuIJPv8f/P90BGPRaqA7EbgU/3wEA/MamsQg5C9RKPLkCQonQmoDlAnKy6LM7AF+yhLVml1OxgiAN3gHkhNb4VQv1AE5VeKKPV9jVuIJCuKn3dxIa4vWlBYJcuBoTEK8+X59TaY3eH2+niC+Qx+szu6DI+T1FG5GTkA4/6jlKScnpSxKsOOCa6rUlH1785CBPiajd4MkFifH6xXVWwPmeAR7/AGWMoPfoM3gMQB4URFLYoFaNR6D2puaqYgeBgZy2jcF72KAng771VFtCNxQKPfj7F1RdrkpUBsWDpSeEr38/fGF5VmLQxL++Zmdof6Ky4LkR3QonD4I+sKCJur52TzKqe6bJ6G7Y7NYeXcg2u4A+HV5LavikVqFcQRlCnQn6mp3HLGIlYSoqRHWZ2Nf7uZDGo6zjnzMlRLssYTeFs8JQsrnN538a9CFLIkxyV+xcR8VmLFSQU0PTKYRR9nqL4ondLR6nTacwH+pjrdnMze7LAvNBhcancKbpyFEE5TX5HsGYQq+K811hSnGvW8O5qR9fY+M6hrs8jOzMfUiDW9LDbFx2Leleh5NayJe8HKoZMZ+p5ZUiOEUyUwL7LG+sTW2I86SMIczX81op6fq9Rcs40qzVUCDTsNxvXwiaXSCHLrwm4vvXdoGcXcWbGnjNjQuyW5Pn1M55k7SriFPFWdbtz14v6gLSVDGCBbF7o/xdBpucPAq7tyqoC1AzI/R2pXQdcEzaQRKKsLxfw9LCz+pXNDuHOrRupc+Wll0xr1v5fMQEMugC5YRy5WRvcYtzz5iWiyxlN+V8U494O85WC8duMDu3nMIXbuYLpaAy2U2sFXqvirnH2LTVtdImV8QWcR1fBcLW4rvU+WPHsq/3E9fSq38rvfoPh0p46GXH16gpJ73PR/q8X/sgqKr3TEiM4oFnflpKFcDjFkXZE4pHid3mGe0zCrYFqnRAA6YRcgXsW7PD6iZXZBeDcWonLboilO8WmaM/B0LoCL+DyASq8APjYppuH/wOItP7BPGBAepYIbJ+czF68X2kq/Cpp56cpVJhBalvhlfi9VeJR4UdKfzY7fB97ZYc3u/363tQzhyfbNjsfrruT1wISW8MNxs7fHTw51DPM12iCgGtqj0a8OQ0tptrThL1xhB1mYqHl9MH/sQb7Kx6Rz+rPuHumWV4DpoQ+og43P2vLCFlA4pTgbY4/p33TQT1mKaKG7wTPqiA94W7l31c7byJBzXsJfmgc8b2/WFVWyJEvnggcQHYgaBG2h4mYEevzpwB5cGjZmc8PFMooSJrqbsTzdg2Z3KSi3SkEbi5lkQ5w5XdQYEORcU98fHXU5Za0su91zJVWaYqy1RlmaosU5VlqnJZzzJVWaYqy1Rlmar8z1OV/wIAAP//LIFLNg==" + return "eJzsWcFu4zYQvecrBttLCyT6AB8KFFsskEOLouihNy8tjuTBUhx1SCX13xeUZFmRKFuSkzQLWKdEMue9eTNDDskH+IaHDdic7L93AJ68wQ18+j38/+kOQKNLhUpPbDfw8x0AwG+sK4OQsUCpxJHNwe8R6iFgOIeMDLrkDsDtWfw2ZZtRvgEvFd4BZIRGu01t6gGsKvAEHx5/KHEDuXBVtm8iHMLzpTYEmXAxRSA8fbw+pkpTdK57HQM+Ax6ez2y9IutaiFqRE5HGfuDTUYnR6VMSLNjjlsqtIedf/ORIT4mow+DLGYrh+cU2o4CzFgEe/wCltaBz6BJ49EAOFARQ2GGqKodA9cuUi4IteAayqak03sMOHWl0taepIbRDotAzf/8CqonVHpVGcWDoG8LXvx++sDwr0ajDX1+TkbU/URlwXElaEycHgs6zoA68vjZfEip7Q6Pq7lgftg6tT3YHjy4uryE1/FIqv9/A3vsyEXQlW4dJsBU1U1AuqolEm+9jIpVD2YY/F1II45LIuDmYBfo963U+/1Oh80nUwix3xSx1VEzCQjlZNRw6BzDQ3j6hOGK7xuP40DnIx/zYpqyXRvdlgjmvfOVidubxyFAE5Zp4T9iYA6/y8awwJ7m39cCloZ+usWkew1keJmbmvkmNT5QOo3Hetah7jZ1YIZ/zcshmYvhCLleS4BjIQgrskqwyJjYhLqMyZWE5n2upxPN3DZdpS4uqIUemYbqvL4SUrSeL1l+jeLts58jJRXtzhU+5sl4OW3IcmzlXUbtocS45w2n9s+tJnbE0l4xgTmxfKX7njc0OHvnDayXUGVMLFXq9VLpscIrakRKKsLzdhqU2v2i/krK1mPrarXhvadjmy3YrnzubQBqtp4xQLnT2Bp9waY9pOE9i4+b0N+WEt9NopXDYDSbjkXPw/Go8vxdUOlmFWqBzKl/axsZHXUptsnnYIm7DUiBsDL5Jnj82KG2+n7Bue/WPslf/4ZgJD73ouBJTyiht4xHv90vnBVXxlgEJKh5xlodlrzw4fEJR5mTFoYTd5gj2GQXrBFWpRw26ErI5tFuzY3WTzZOzYpy2kwZt7vdvpkznzxEQGsDvQBlPBb6jLrpq5sHvQJneEcQ7CtSgQkD9cBq9OB9pMnxu15OxFMpvIHZmeEGvv/bYMWxA4cdmhu9zN2Txvp2v70FZ3X3ZsT78dNmfUAhRbzRXOzP8dPTnmM8LXaICAY0qHWpwZFOsJ9eMJPANEjWRCs3L6YA/soKNsnfyWPUbHp5Zhn3QDOmDxeHsf6GElPEoVnl6wulz3lch1EOaS26wJrxTAreJ29Luqp13oVHDXpAHJZzAY9aVd1vV86r4fsQjmHdVUSjplvOiMp5Kg6OFK8wwTeOyZCF5t/rZIwS80BpZD2xBMEV6Ot7Fdd6MZVUOHKZstYNn8nsoyBhq3oRhbKoAOu7sVodhnuzBmxWiTx+Tr91/Bml7a9BbZuwIeyQdW3No9DvDqCNilAu5gCcZgcICSe1G6IKoEyeEo9czBDw1x6O++fXLuhPpSv9LlniPsy6JgrmP7/2LFmDimGH1SiXKai7MAXK0KCq49fjrKX9r0PMnO7c729ud7e3O9nZne7uzvd3Znudzu7O93dne7mxvd7b/853tfwEAAP//oqGn0A==" } diff --git a/filebeat/module/nginx/ingress_controller/_meta/fields.yml b/filebeat/module/nginx/ingress_controller/_meta/fields.yml index 41520f8a29f..0c76c1e1b2f 100644 --- a/filebeat/module/nginx/ingress_controller/_meta/fields.yml +++ b/filebeat/module/nginx/ingress_controller/_meta/fields.yml @@ -53,12 +53,29 @@ type: long format: bytes description: > - The length of the response obtained from the upstream + The length of the response obtained from the upstream server. If several servers were contacted during request process, + the summary of the multiple response lengths is stored. - name: upstream.response.time type: double format: duration description: > - The time spent on receiving the response from the upstream as seconds with millisecond resolution + The time spent on receiving the response from the upstream as seconds with millisecond resolution. + If several servers were contacted during request process, the summary of the multiple response times is stored. + - name: upstream.response.status_code + type: long + description: > + The status code of the response obtained from the upstream server. If several servers were contacted during + request process, only the status code of the response from the last one is stored in this field. + - name: upstream.ip + type: ip + description: > + The IP address of the upstream server. If several servers were contacted during request process, + only the last one is stored in this field. + - name: upstream.port + type: long + description: > + The port of the upstream server. If several servers were contacted during request process, + only the last one is stored in this field. - name: http.request.id type: keyword description: > diff --git a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml index 83f090d5e92..50999172b72 100644 --- a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml +++ b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml @@ -95,6 +95,47 @@ processors: catch (Exception e) { ctx.nginx.ingress_controller.upstream.response.time = null; } +- script: + if: ctx.nginx?.ingress_controller?.upstream?.response?.status_code_list != null && ctx.nginx.ingress_controller.upstream.response.status_code_list.length > 0 + lang: painless + source: >- + try { + if (ctx.nginx.ingress_controller.upstream.response.status_code_list.length == null) { + return; + } + int last_status_code; + for (def item : ctx.nginx.ingress_controller.upstream.response.status_code_list) { + last_status_code = Integer.parseInt(item); + } + ctx.nginx.ingress_controller.upstream.response.status_code = last_status_code; + } + catch (Exception e) { + ctx.nginx.ingress_controller.upstream.response.time = null; + } +- script: + if: ctx.nginx?.ingress_controller?.upstream_ip_list != null && ctx.nginx.ingress_controller.upstream_ip_list.length > 0 + lang: painless + source: >- + try { + if (ctx.nginx.ingress_controller.upstream_ip_list.length == null) { + return; + } + def last_upstream = ""; + for (def item : ctx.nginx.ingress_controller.upstream_ip_list) { + last_upstream = item; + } + StringTokenizer tok = new StringTokenizer(last_upstream, ":"); + if (tok.countTokens()>1) { + ctx.nginx.ingress_controller.upstream.ip = tok.nextToken(); + ctx.nginx.ingress_controller.upstream.port = Integer.parseInt(tok.nextToken()); + } else { + ctx.nginx.ingress_controller.upstream.ip = last_upstream; + } + } + catch (Exception e) { + ctx.nginx.ingress_controller.upstream.ip = null; + ctx.nginx.ingress_controller.upstream.port = null; + } - script: if: ctx.nginx?.ingress_controller?.remote_ip_list != null && ctx.nginx.ingress_controller.remote_ip_list.length > 0 lang: painless diff --git a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json index a6913a48d20..d89e8eb71c6 100644 --- a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json +++ b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json @@ -26,11 +26,14 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", + "nginx.ingress_controller.upstream.port": 8080, "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], + "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], @@ -80,11 +83,14 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", + "nginx.ingress_controller.upstream.port": 8080, "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], + "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], @@ -134,11 +140,14 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", + "nginx.ingress_controller.upstream.port": 8080, "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], + "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], @@ -188,11 +197,14 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", + "nginx.ingress_controller.upstream.port": 8080, "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], + "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], @@ -314,11 +326,14 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", + "nginx.ingress_controller.upstream.port": 8080, "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], + "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], @@ -368,11 +383,14 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", + "nginx.ingress_controller.upstream.port": 8080, "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], + "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], @@ -426,11 +444,14 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", + "nginx.ingress_controller.upstream.port": 8080, "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], + "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], @@ -483,11 +504,14 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.6", "nginx.ingress_controller.upstream.name": "default-web2-8080", + "nginx.ingress_controller.upstream.port": 8080, "nginx.ingress_controller.upstream.response.length": 61, "nginx.ingress_controller.upstream.response.length_list": [ "61" ], + "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], @@ -541,11 +565,14 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", + "nginx.ingress_controller.upstream.port": 8080, "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], + "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], @@ -598,11 +625,14 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", + "nginx.ingress_controller.upstream.port": 8080, "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], + "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], @@ -656,11 +686,14 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", + "nginx.ingress_controller.upstream.port": 8080, "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], + "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], @@ -713,11 +746,14 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", + "nginx.ingress_controller.upstream.port": 8080, "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], + "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], @@ -770,11 +806,14 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", + "nginx.ingress_controller.upstream.port": 8080, "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], + "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], @@ -828,11 +867,14 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", + "nginx.ingress_controller.upstream.port": 8080, "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], + "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], @@ -885,11 +927,14 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.6", "nginx.ingress_controller.upstream.name": "default-web2-8080", + "nginx.ingress_controller.upstream.port": 8080, "nginx.ingress_controller.upstream.response.length": 61, "nginx.ingress_controller.upstream.response.length_list": [ "61" ], + "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], @@ -943,11 +988,14 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", + "nginx.ingress_controller.upstream.port": 8080, "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], + "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], @@ -1000,11 +1048,14 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", + "nginx.ingress_controller.upstream.port": 8080, "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], + "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], @@ -1054,11 +1105,14 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.6", "nginx.ingress_controller.upstream.name": "default-web2-8080", + "nginx.ingress_controller.upstream.port": 8080, "nginx.ingress_controller.upstream.response.length": 61, "nginx.ingress_controller.upstream.response.length_list": [ "61" ], + "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], @@ -1111,11 +1165,14 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", + "nginx.ingress_controller.upstream.port": 8080, "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], + "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], @@ -1168,11 +1225,14 @@ "192.168.64.1" ], "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.6", "nginx.ingress_controller.upstream.name": "default-web2-8080", + "nginx.ingress_controller.upstream.port": 8080, "nginx.ingress_controller.upstream.response.length": 61, "nginx.ingress_controller.upstream.response.length_list": [ "61" ], + "nginx.ingress_controller.upstream.response.status_code": 200, "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], @@ -1225,12 +1285,15 @@ "192.168.64.14" ], "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.7", "nginx.ingress_controller.upstream.name": "default-web2-8080", + "nginx.ingress_controller.upstream.port": 8080, "nginx.ingress_controller.upstream.response.length": 161, "nginx.ingress_controller.upstream.response.length_list": [ "61", "100" ], + "nginx.ingress_controller.upstream.response.status_code": 203, "nginx.ingress_controller.upstream.response.status_code_list": [ "200", "203" From 3e9aefd33614e6c3cf96ad850f94110dd5c00be9 Mon Sep 17 00:00:00 2001 From: chrismark Date: Tue, 22 Sep 2020 16:18:41 +0300 Subject: [PATCH 3/8] fix tests Signed-off-by: chrismark --- .../test/test.log-expected.json | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json index d89e8eb71c6..3298b9ead6c 100644 --- a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json +++ b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json @@ -51,7 +51,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products", - "user_agent.device.name": "Other", + "user_agent.device.name": "Mac", "user_agent.name": "curl", "user_agent.original": "curl/7.54.0", "user_agent.version": "7.54.0" @@ -108,7 +108,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", - "user_agent.device.name": "Other", + "user_agent.device.name": "Mac", "user_agent.name": "curl", "user_agent.original": "curl/7.54.0", "user_agent.version": "7.54.0" @@ -165,7 +165,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", - "user_agent.device.name": "Other", + "user_agent.device.name": "Mac", "user_agent.name": "curl", "user_agent.original": "curl/7.54.0", "user_agent.version": "7.54.0" @@ -222,7 +222,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", - "user_agent.device.name": "Other", + "user_agent.device.name": "Mac", "user_agent.name": "curl", "user_agent.original": "curl/7.54.0", "user_agent.version": "7.54.0" @@ -351,7 +351,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", - "user_agent.device.name": "Other", + "user_agent.device.name": "Mac", "user_agent.name": "Wget", "user_agent.original": "Wget/1.20.3 (darwin18.6.0)", "user_agent.version": "1.20.3" @@ -408,7 +408,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", - "user_agent.device.name": "Other", + "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", "user_agent.os.full": "Mac OS X 10.14.6", @@ -469,7 +469,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/favicon.ico", - "user_agent.device.name": "Other", + "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", "user_agent.os.full": "Mac OS X 10.14.6", @@ -529,7 +529,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/v2", - "user_agent.device.name": "Other", + "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", "user_agent.os.full": "Mac OS X 10.14.6", @@ -590,7 +590,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/favicon.ico", - "user_agent.device.name": "Other", + "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", "user_agent.os.full": "Mac OS X 10.14.6", @@ -650,7 +650,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", - "user_agent.device.name": "Other", + "user_agent.device.name": "Mac", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", "user_agent.os.full": "Mac OS X 10.14.6", @@ -711,7 +711,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/favicon.ico", - "user_agent.device.name": "Other", + "user_agent.device.name": "Mac", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", "user_agent.os.full": "Mac OS X 10.14.6", @@ -771,7 +771,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", - "user_agent.device.name": "Other", + "user_agent.device.name": "Mac", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", "user_agent.os.full": "Mac OS X 10.14.6", @@ -831,7 +831,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/", - "user_agent.device.name": "Other", + "user_agent.device.name": "Mac", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", "user_agent.os.full": "Mac OS X 10.14.6", @@ -892,7 +892,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/favicon.ico", - "user_agent.device.name": "Other", + "user_agent.device.name": "Mac", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", "user_agent.os.full": "Mac OS X 10.14.6", @@ -952,7 +952,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/v2", - "user_agent.device.name": "Other", + "user_agent.device.name": "Mac", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", "user_agent.os.full": "Mac OS X 10.14.6", @@ -1013,7 +1013,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/favicon.ico", - "user_agent.device.name": "Other", + "user_agent.device.name": "Mac", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", "user_agent.os.full": "Mac OS X 10.14.6", @@ -1073,7 +1073,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42?address=delhi+technological+university", - "user_agent.device.name": "Other", + "user_agent.device.name": "Mac", "user_agent.name": "Python Requests", "user_agent.original": "python-requests/2.22.0", "user_agent.version": "2.22" @@ -1130,7 +1130,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/v2", - "user_agent.device.name": "Other", + "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", "user_agent.os.full": "Mac OS X 10.14", @@ -1190,7 +1190,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/favicon.ico", - "user_agent.device.name": "Other", + "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", "user_agent.os.full": "Mac OS X 10.14", @@ -1250,7 +1250,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/v2/some", - "user_agent.device.name": "Other", + "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", "user_agent.os.full": "Mac OS X 10.14", @@ -1314,7 +1314,7 @@ "source.address": "192.168.64.14", "source.ip": "192.168.64.14", "url.original": "/v2/some", - "user_agent.device.name": "Other", + "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", "user_agent.os.full": "Mac OS X 10.14", @@ -1322,4 +1322,4 @@ "user_agent.os.version": "10.14", "user_agent.version": "72.0." } -] \ No newline at end of file +] From c8fe66d25f60341f76afd5e1c2d62423b581ad06 Mon Sep 17 00:00:00 2001 From: chrismark Date: Tue, 22 Sep 2020 16:19:38 +0300 Subject: [PATCH 4/8] Add changelog Signed-off-by: chrismark --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index ea288f1c2f3..3c92cf3ed79 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -264,6 +264,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Remove wrongly mapped `tls.client.server_name` from `fortinet/firewall` fileset. {pull}20983[20983] - Fix an error updating file size being logged when EOF is reached. {pull}21048[21048] - Fix error when processing AWS Cloudtrail Digest logs. {pull}21086[21086] {issue}20943[20943] +- Handle multiple upstreams in ingress-controller. {pull}21215[21215] *Heartbeat* From 53c9a145cee1ee057d21815f2e0560bd038cc723 Mon Sep 17 00:00:00 2001 From: chrismark Date: Tue, 22 Sep 2020 16:54:44 +0300 Subject: [PATCH 5/8] Fix field naming Signed-off-by: chrismark --- .../Filebeat-nginx-ingress-overview.json | 4 +- .../nginx/ingress_controller/_meta/fields.yml | 4 +- .../ingress_controller/ingest/pipeline.yml | 8 +- .../test/test.log-expected.json | 84 +++++++++---------- 4 files changed, 50 insertions(+), 50 deletions(-) diff --git a/filebeat/module/nginx/_meta/kibana/7/dashboard/Filebeat-nginx-ingress-overview.json b/filebeat/module/nginx/_meta/kibana/7/dashboard/Filebeat-nginx-ingress-overview.json index 3f618adf2ec..7fb61b9a6ff 100644 --- a/filebeat/module/nginx/_meta/kibana/7/dashboard/Filebeat-nginx-ingress-overview.json +++ b/filebeat/module/nginx/_meta/kibana/7/dashboard/Filebeat-nginx-ingress-overview.json @@ -917,7 +917,7 @@ "line_width": 1, "metrics": [ { - "field": "nginx.ingress_controller.upstream.response.time", + "field": "nginx.ingress_controller.upstream.response.time_total", "id": "61ca57f2-469d-11e7-af02-69e470af7417", "type": "sum" } @@ -1004,7 +1004,7 @@ "line_width": 1, "metrics": [ { - "field": "nginx.ingress_controller.upstream.response.length", + "field": "nginx.ingress_controller.upstream.response.length_total", "id": "61ca57f2-469d-11e7-af02-69e470af7417", "type": "sum", "values": [ diff --git a/filebeat/module/nginx/ingress_controller/_meta/fields.yml b/filebeat/module/nginx/ingress_controller/_meta/fields.yml index 0c76c1e1b2f..44717f1de12 100644 --- a/filebeat/module/nginx/ingress_controller/_meta/fields.yml +++ b/filebeat/module/nginx/ingress_controller/_meta/fields.yml @@ -49,13 +49,13 @@ type: keyword description: > The name of the alternative upstream. - - name: upstream.response.length + - name: upstream.response.length_total type: long format: bytes description: > The length of the response obtained from the upstream server. If several servers were contacted during request process, the summary of the multiple response lengths is stored. - - name: upstream.response.time + - name: upstream.response.time_total type: double format: duration description: > diff --git a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml index 50999172b72..07fe24614ff 100644 --- a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml +++ b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml @@ -73,10 +73,10 @@ processors: for (def item : ctx.nginx.ingress_controller.upstream.response.length_list) { length = length + Integer.parseInt(item); } - ctx.nginx.ingress_controller.upstream.response.length = length; + ctx.nginx.ingress_controller.upstream.response.length_total = length; } catch (Exception e) { - ctx.nginx.ingress_controller.upstream.response.length = null; + ctx.nginx.ingress_controller.upstream.response.length_total = null; } - script: if: ctx.nginx?.ingress_controller?.upstream?.response?.time_list != null && ctx.nginx.ingress_controller.upstream.response.time_list.length > 0 @@ -90,10 +90,10 @@ processors: for (def item : ctx.nginx.ingress_controller.upstream.response.time_list) { res_time = res_time + Float.parseFloat(item); } - ctx.nginx.ingress_controller.upstream.response.time = res_time; + ctx.nginx.ingress_controller.upstream.response.time_total = res_time; } catch (Exception e) { - ctx.nginx.ingress_controller.upstream.response.time = null; + ctx.nginx.ingress_controller.upstream.response.time_total = null; } - script: if: ctx.nginx?.ingress_controller?.upstream?.response?.status_code_list != null && ctx.nginx.ingress_controller.upstream.response.status_code_list.length > 0 diff --git a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json index 3298b9ead6c..0f52b337892 100644 --- a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json +++ b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json @@ -29,7 +29,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length": 59, + "nginx.ingress_controller.upstream.response.length_total": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], @@ -37,7 +37,7 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time": 0.0, + "nginx.ingress_controller.upstream.response.time_total": 0.0, "nginx.ingress_controller.upstream.response.time_list": [ "0.000" ], @@ -86,7 +86,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length": 59, + "nginx.ingress_controller.upstream.response.length_total": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], @@ -94,7 +94,7 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time": 0.0, + "nginx.ingress_controller.upstream.response.time_total": 0.0, "nginx.ingress_controller.upstream.response.time_list": [ "0.000" ], @@ -143,7 +143,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length": 59, + "nginx.ingress_controller.upstream.response.length_total": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], @@ -151,7 +151,7 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time": 0.001, + "nginx.ingress_controller.upstream.response.time_total": 0.001, "nginx.ingress_controller.upstream.response.time_list": [ "0.001" ], @@ -200,7 +200,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length": 59, + "nginx.ingress_controller.upstream.response.length_total": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], @@ -208,7 +208,7 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time": 0.0, + "nginx.ingress_controller.upstream.response.time_total": 0.0, "nginx.ingress_controller.upstream.response.time_list": [ "0.000" ], @@ -329,7 +329,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length": 59, + "nginx.ingress_controller.upstream.response.length_total": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], @@ -337,7 +337,7 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time": 0.0, + "nginx.ingress_controller.upstream.response.time_total": 0.0, "nginx.ingress_controller.upstream.response.time_list": [ "0.000" ], @@ -386,7 +386,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length": 59, + "nginx.ingress_controller.upstream.response.length_total": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], @@ -394,7 +394,7 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time": 0.0, + "nginx.ingress_controller.upstream.response.time_total": 0.0, "nginx.ingress_controller.upstream.response.time_list": [ "0.000" ], @@ -447,7 +447,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length": 59, + "nginx.ingress_controller.upstream.response.length_total": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], @@ -455,7 +455,7 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time": 0.0, + "nginx.ingress_controller.upstream.response.time_total": 0.0, "nginx.ingress_controller.upstream.response.time_list": [ "0.000" ], @@ -507,7 +507,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.6", "nginx.ingress_controller.upstream.name": "default-web2-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length": 61, + "nginx.ingress_controller.upstream.response.length_total": 61, "nginx.ingress_controller.upstream.response.length_list": [ "61" ], @@ -515,7 +515,7 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time": 0.001, + "nginx.ingress_controller.upstream.response.time_total": 0.001, "nginx.ingress_controller.upstream.response.time_list": [ "0.001" ], @@ -568,7 +568,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length": 59, + "nginx.ingress_controller.upstream.response.length_total": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], @@ -576,7 +576,7 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time": 0.002, + "nginx.ingress_controller.upstream.response.time_total": 0.002, "nginx.ingress_controller.upstream.response.time_list": [ "0.002" ], @@ -628,7 +628,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length": 59, + "nginx.ingress_controller.upstream.response.length_total": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], @@ -636,7 +636,7 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time": 0.001, + "nginx.ingress_controller.upstream.response.time_total": 0.001, "nginx.ingress_controller.upstream.response.time_list": [ "0.001" ], @@ -689,7 +689,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length": 59, + "nginx.ingress_controller.upstream.response.length_total": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], @@ -697,7 +697,7 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time": 0.001, + "nginx.ingress_controller.upstream.response.time_total": 0.001, "nginx.ingress_controller.upstream.response.time_list": [ "0.001" ], @@ -749,7 +749,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length": 59, + "nginx.ingress_controller.upstream.response.length_total": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], @@ -757,7 +757,7 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time": 0.002, + "nginx.ingress_controller.upstream.response.time_total": 0.002, "nginx.ingress_controller.upstream.response.time_list": [ "0.002" ], @@ -809,7 +809,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length": 59, + "nginx.ingress_controller.upstream.response.length_total": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], @@ -817,7 +817,7 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time": 0.001, + "nginx.ingress_controller.upstream.response.time_total": 0.001, "nginx.ingress_controller.upstream.response.time_list": [ "0.001" ], @@ -870,7 +870,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length": 59, + "nginx.ingress_controller.upstream.response.length_total": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], @@ -878,7 +878,7 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time": 0.002, + "nginx.ingress_controller.upstream.response.time_total": 0.002, "nginx.ingress_controller.upstream.response.time_list": [ "0.002" ], @@ -930,7 +930,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.6", "nginx.ingress_controller.upstream.name": "default-web2-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length": 61, + "nginx.ingress_controller.upstream.response.length_total": 61, "nginx.ingress_controller.upstream.response.length_list": [ "61" ], @@ -938,7 +938,7 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time": 0.002, + "nginx.ingress_controller.upstream.response.time_total": 0.002, "nginx.ingress_controller.upstream.response.time_list": [ "0.002" ], @@ -991,7 +991,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length": 59, + "nginx.ingress_controller.upstream.response.length_total": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], @@ -999,7 +999,7 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time": 0.0, + "nginx.ingress_controller.upstream.response.time_total": 0.0, "nginx.ingress_controller.upstream.response.time_list": [ "0.000" ], @@ -1051,7 +1051,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length": 59, + "nginx.ingress_controller.upstream.response.length_total": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], @@ -1059,7 +1059,7 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time": 0.001, + "nginx.ingress_controller.upstream.response.time_total": 0.001, "nginx.ingress_controller.upstream.response.time_list": [ "0.001" ], @@ -1108,7 +1108,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.6", "nginx.ingress_controller.upstream.name": "default-web2-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length": 61, + "nginx.ingress_controller.upstream.response.length_total": 61, "nginx.ingress_controller.upstream.response.length_list": [ "61" ], @@ -1116,7 +1116,7 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time": 0.001, + "nginx.ingress_controller.upstream.response.time_total": 0.001, "nginx.ingress_controller.upstream.response.time_list": [ "0.001" ], @@ -1168,7 +1168,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length": 59, + "nginx.ingress_controller.upstream.response.length_total": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], @@ -1176,7 +1176,7 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time": 0.0, + "nginx.ingress_controller.upstream.response.time_total": 0.0, "nginx.ingress_controller.upstream.response.time_list": [ "0.000" ], @@ -1228,7 +1228,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.6", "nginx.ingress_controller.upstream.name": "default-web2-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length": 61, + "nginx.ingress_controller.upstream.response.length_total": 61, "nginx.ingress_controller.upstream.response.length_list": [ "61" ], @@ -1236,7 +1236,7 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time": 0.0, + "nginx.ingress_controller.upstream.response.time_total": 0.0, "nginx.ingress_controller.upstream.response.time_list": [ "0.000" ], @@ -1288,7 +1288,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.7", "nginx.ingress_controller.upstream.name": "default-web2-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length": 161, + "nginx.ingress_controller.upstream.response.length_total": 161, "nginx.ingress_controller.upstream.response.length_list": [ "61", "100" @@ -1298,7 +1298,7 @@ "200", "203" ], - "nginx.ingress_controller.upstream.response.time": 0.104, + "nginx.ingress_controller.upstream.response.time_total": 0.104, "nginx.ingress_controller.upstream.response.time_list": [ "0.100", "0.004" From 1656658d99e8a668e5292356b36660dd30b64fce Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 23 Sep 2020 12:21:01 +0300 Subject: [PATCH 6/8] Review fixes Signed-off-by: chrismark --- filebeat/docs/fields.asciidoc | 12 +- .../Filebeat-nginx-ingress-overview.json | 4 +- filebeat/module/nginx/fields.go | 2 +- .../nginx/ingress_controller/_meta/fields.yml | 16 +-- .../ingress_controller/ingest/pipeline.yml | 23 ++-- .../test/test.log-expected.json | 126 +++++++++--------- 6 files changed, 91 insertions(+), 92 deletions(-) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 115b8383179..8a145ff8724 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -105135,13 +105135,13 @@ type: array -- -*`nginx.ingress_controller.upstream_ip_list`*:: +*`nginx.ingress_controller.upstream_address_list`*:: + -- -An array of the upstream IP addresses. It is a list because it is common that several upstream servers were contacted during request processing. +An array of the upstream addresses. It is a list because it is common that several upstream servers were contacted during request processing. -type: array +type: keyword -- @@ -105151,7 +105151,7 @@ type: array An array of upstream response lengths. It is a list because it is common that several upstream servers were contacted during request processing. -type: array +type: keyword -- @@ -105161,7 +105161,7 @@ type: array An array of upstream response durations. It is a list because it is common that several upstream servers were contacted during request processing. -type: array +type: keyword -- @@ -105171,7 +105171,7 @@ type: array An array of upstream response status codes. It is a list because it is common that several upstream servers were contacted during request processing. -type: array +type: keyword -- diff --git a/filebeat/module/nginx/_meta/kibana/7/dashboard/Filebeat-nginx-ingress-overview.json b/filebeat/module/nginx/_meta/kibana/7/dashboard/Filebeat-nginx-ingress-overview.json index 7fb61b9a6ff..3f618adf2ec 100644 --- a/filebeat/module/nginx/_meta/kibana/7/dashboard/Filebeat-nginx-ingress-overview.json +++ b/filebeat/module/nginx/_meta/kibana/7/dashboard/Filebeat-nginx-ingress-overview.json @@ -917,7 +917,7 @@ "line_width": 1, "metrics": [ { - "field": "nginx.ingress_controller.upstream.response.time_total", + "field": "nginx.ingress_controller.upstream.response.time", "id": "61ca57f2-469d-11e7-af02-69e470af7417", "type": "sum" } @@ -1004,7 +1004,7 @@ "line_width": 1, "metrics": [ { - "field": "nginx.ingress_controller.upstream.response.length_total", + "field": "nginx.ingress_controller.upstream.response.length", "id": "61ca57f2-469d-11e7-af02-69e470af7417", "type": "sum", "values": [ diff --git a/filebeat/module/nginx/fields.go b/filebeat/module/nginx/fields.go index 5c6f56a2e7f..4e622f85db6 100644 --- a/filebeat/module/nginx/fields.go +++ b/filebeat/module/nginx/fields.go @@ -32,5 +32,5 @@ func init() { // AssetNginx returns asset data. // This is the base64 encoded gzipped contents of module/nginx. func AssetNginx() string { - return "eJzsWcFu4zYQvecrBttLCyT6AB8KFFsskEOLouihNy8tjuTBUhx1SCX13xeUZFmRKFuSkzQLWKdEMue9eTNDDskH+IaHDdic7L93AJ68wQ18+j38/+kOQKNLhUpPbDfw8x0AwG+sK4OQsUCpxJHNwe8R6iFgOIeMDLrkDsDtWfw2ZZtRvgEvFd4BZIRGu01t6gGsKvAEHx5/KHEDuXBVtm8iHMLzpTYEmXAxRSA8fbw+pkpTdK57HQM+Ax6ez2y9IutaiFqRE5HGfuDTUYnR6VMSLNjjlsqtIedf/ORIT4mow+DLGYrh+cU2o4CzFgEe/wCltaBz6BJ49EAOFARQ2GGqKodA9cuUi4IteAayqak03sMOHWl0taepIbRDotAzf/8CqonVHpVGcWDoG8LXvx++sDwr0ajDX1+TkbU/URlwXElaEycHgs6zoA68vjZfEip7Q6Pq7lgftg6tT3YHjy4uryE1/FIqv9/A3vsyEXQlW4dJsBU1U1AuqolEm+9jIpVD2YY/F1II45LIuDmYBfo963U+/1Oh80nUwix3xSx1VEzCQjlZNRw6BzDQ3j6hOGK7xuP40DnIx/zYpqyXRvdlgjmvfOVidubxyFAE5Zp4T9iYA6/y8awwJ7m39cCloZ+usWkew1keJmbmvkmNT5QOo3Hetah7jZ1YIZ/zcshmYvhCLleS4BjIQgrskqwyJjYhLqMyZWE5n2upxPN3DZdpS4uqIUemYbqvL4SUrSeL1l+jeLts58jJRXtzhU+5sl4OW3IcmzlXUbtocS45w2n9s+tJnbE0l4xgTmxfKX7njc0OHvnDayXUGVMLFXq9VLpscIrakRKKsLzdhqU2v2i/krK1mPrarXhvadjmy3YrnzubQBqtp4xQLnT2Bp9waY9pOE9i4+b0N+WEt9NopXDYDSbjkXPw/Go8vxdUOlmFWqBzKl/axsZHXUptsnnYIm7DUiBsDL5Jnj82KG2+n7Bue/WPslf/4ZgJD73ouBJTyiht4xHv90vnBVXxlgEJKh5xlodlrzw4fEJR5mTFoYTd5gj2GQXrBFWpRw26ErI5tFuzY3WTzZOzYpy2kwZt7vdvpkznzxEQGsDvQBlPBb6jLrpq5sHvQJneEcQ7CtSgQkD9cBq9OB9pMnxu15OxFMpvIHZmeEGvv/bYMWxA4cdmhu9zN2Txvp2v70FZ3X3ZsT78dNmfUAhRbzRXOzP8dPTnmM8LXaICAY0qHWpwZFOsJ9eMJPANEjWRCs3L6YA/soKNsnfyWPUbHp5Zhn3QDOmDxeHsf6GElPEoVnl6wulz3lch1EOaS26wJrxTAreJ29Luqp13oVHDXpAHJZzAY9aVd1vV86r4fsQjmHdVUSjplvOiMp5Kg6OFK8wwTeOyZCF5t/rZIwS80BpZD2xBMEV6Ot7Fdd6MZVUOHKZstYNn8nsoyBhq3oRhbKoAOu7sVodhnuzBmxWiTx+Tr91/Bml7a9BbZuwIeyQdW3No9DvDqCNilAu5gCcZgcICSe1G6IKoEyeEo9czBDw1x6O++fXLuhPpSv9LlniPsy6JgrmP7/2LFmDimGH1SiXKai7MAXK0KCq49fjrKX9r0PMnO7c729ud7e3O9nZne7uzvd3Znudzu7O93dne7mxvd7b/853tfwEAAP//oqGn0A==" + return "eJzsWcFu4zYQvecrBttLCyT6AB8KFFsskEOLouihNy8tjuTBUhx1SCX13xeUZFmRKFuSkzQLWKdEMue9eTNDDskH+IaHDdic7L93AJ68wQ18+j38/+kOQKNLhUpPbDfw8x0AwG+sK4OQsUCpxJHNwe8R6iFgOIeMDLrkDsDtWfw2ZZtRvgEvFd4BZIRGu01t6gGsKvAEHx5/KHEDuXBVtm8iHMLzpTYEmXAxRSA8fbw+pkpTdK57HQM+Ax6ez2y9IutaiFqRE5HGfuDTUYnR6VMSLNjjlsqtIedf/ORIT4mow+DLGYrh+cU2o4CzFgEe/wCltaBz6BJ49EAOFARQ2GGqKodA9cuUi4IteAayqak03sMOHWl0taepIbRDotAzf/8CqonVHpVGcWDoG8LXvx++sDwr0ajDX1+TkbU/URlwXElaEycHgs6zoA68vjZfEip7Q6Pq7lgftg6tT3YHjy4uryE1/FIqv9/A3vsyEXQlW4dJsBU1U1AuqolEm+9jIpVD2YY/F1II45LIuDmYBfo963U+/1Oh80nUwix3xSx1VEzCQjlZNRw6BzDQ3j6hOGK7xuP40DnIx/zYpqyXRvdlgjmvfOVidubxyFAE5Zp4T9iYA6/y8awwJ7m39cCloZ+usWkew1keJmbmvkmNT5QOo3Hetah7jZ1YIZ/zcshmYvhCLleS4BjIQgrskqwyJjYhLqMyZWE5n2upxPN3DZdpS4uqIUemYbqvL4SUrSeL1l+jeLts58jJRXtzhU+5sl4OW3IcmzlXUbtocS45w2n9s+tJnbE0l4xgTmxfKX7njc0OHvnDayXUGVMLFXq9VLpscIrakRKKsLzdhqU2v2i/krK1mPrarXhvadjmy3YrnzubQBqtp4xQLnT2Bp9waY9pOE9i4+b0N+WEt9NopXDYDSbjkXPw/Go8vxdUOlmFWqBzKl/axsZHXUptsnnYIm7DUiBsDL5Jnj82KG2+n7Bue/WPslf/4ZgJD73ouBJTyiht4xHv90vnBVWxbR2Yjso3PDyzDAtiQVyCmEe4haHZKw8On1CUOZlwKGHHOcJ8RsE6SVXqUYOuhGwO7fbsWOFk8+SsIKctpUGb+/0bCtN5dISEBvI70MZTge+qjK6a+fA70KZ3FPGuEjW4EHA/nEovTkqaLJ/b/2QshfIbiJ0eXtDrrz12DBtQ+LGZ6/vcDVm8b2fue1BWd192rA8/XfYnFEPUG83Vzgw/Hf05ZvRCl6hAQKNKhxoc2RTr+TUjCXyDRE2kQhtzOuqPrGWj/J08YF2VqkH6YHG4AFwoImU8ilWennD6xPdVCPWQ5pIbrAzvlMBt4ra0u2rnXWjZsBfkQQkn8Jh15d1W9bwqvh/xCOZdVRRKuhW9qIyn0uBo8QozTNPCLFlM3q1+9ggBLzRJ1gNbEEyRno63cp03Y1mVA4cpW+3gmfweCjKGmjdhGJsqgI57vNVhmCd78GaF6NMH5mt3okHa3hr0lhk7wh5Jx9YcGv3OMOqIGOVCLuBJRqCwQFK7Jbog6sRZ4ej1DAFPW4pR6/z6Zd2JdKX/JUu8y1mXRMHcx/f+RQswceCweqUSZTUX5gA5WhQV3Hr89ZS/Nej5M57b7e3t9vZ2e3u7vb3d3t5ub8/zud3e3m5vb7e3t9vb//n29r8AAAD//ydZrLw=" } diff --git a/filebeat/module/nginx/ingress_controller/_meta/fields.yml b/filebeat/module/nginx/ingress_controller/_meta/fields.yml index 44717f1de12..30bc1f5ad9e 100644 --- a/filebeat/module/nginx/ingress_controller/_meta/fields.yml +++ b/filebeat/module/nginx/ingress_controller/_meta/fields.yml @@ -11,23 +11,23 @@ Real source IP is restored to `source.ip`. # ingress-controller specific fields - - name: upstream_ip_list - type: array + - name: upstream_address_list + type: keyword description: > - An array of the upstream IP addresses. It is a list because it is common that several upstream servers + An array of the upstream addresses. It is a list because it is common that several upstream servers were contacted during request processing. - name: upstream.response.length_list - type: array + type: keyword description: > An array of upstream response lengths. It is a list because it is common that several upstream servers were contacted during request processing. - name: upstream.response.time_list - type: array + type: keyword description: > An array of upstream response durations. It is a list because it is common that several upstream servers were contacted during request processing. - name: upstream.response.status_code_list - type: array + type: keyword description: > An array of upstream response status codes. It is a list because it is common that several upstream servers were contacted during request processing. @@ -49,13 +49,13 @@ type: keyword description: > The name of the alternative upstream. - - name: upstream.response.length_total + - name: upstream.response.length type: long format: bytes description: > The length of the response obtained from the upstream server. If several servers were contacted during request process, the summary of the multiple response lengths is stored. - - name: upstream.response.time_total + - name: upstream.response.time type: double format: duration description: > diff --git a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml index 07fe24614ff..0eca28c6084 100644 --- a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml +++ b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml @@ -12,14 +12,13 @@ processors: %{NUMBER:http.response.status_code:long} %{NUMBER:http.response.body.bytes:long} "(-|%{DATA:http.request.referrer})" "(-|%{DATA:user_agent.original})" %{NUMBER:nginx.ingress_controller.http.request.length:long} %{NUMBER:nginx.ingress_controller.http.request.time:double} \[%{DATA:nginx.ingress_controller.upstream.name}\] - \[%{DATA:nginx.ingress_controller.upstream.alternative_name}\] (%{UPSTREAM_ADDRESS_LIST:nginx.ingress_controller.upstream_ip_list}|-) + \[%{DATA:nginx.ingress_controller.upstream.alternative_name}\] (%{UPSTREAM_ADDRESS_LIST:nginx.ingress_controller.upstream_address_list}|-) (%{UPSTREAM_RESPONSE_LENGTH_LIST:nginx.ingress_controller.upstream.response.length_list}|-) (%{UPSTREAM_RESPONSE_TIME_LIST:nginx.ingress_controller.upstream.response.time_list}|-) (%{UPSTREAM_RESPONSE_STATUS_CODE_LIST:nginx.ingress_controller.upstream.response.status_code_list}|-) %{GREEDYDATA:nginx.ingress_controller.http.request.id} pattern_definitions: NGINX_HOST: (?:%{IP:destination.ip}|%{NGINX_NOTSEPARATOR:destination.domain})(:%{NUMBER:destination.port})? NGINX_NOTSEPARATOR: "[^\t ,:]+" NGINX_ADDRESS_LIST: (?:%{IP}|%{WORD})("?,?\s*(?:%{IP}|%{WORD}))* - UPSTREAM_ADDRESS: '%{IP:nginx.ingress_controller.upstream.ip}(:%{NUMBER:nginx.ingress_controller.upstream.port})?' UPSTREAM_ADDRESS_LIST: (?:%{IP}(:%{NUMBER})?)("?,?\s*(?:%{IP}(:%{NUMBER})?))* UPSTREAM_RESPONSE_LENGTH_LIST: (?:%{NUMBER})("?,?\s*(?:%{NUMBER}))* UPSTREAM_RESPONSE_TIME_LIST: (?:%{NUMBER})("?,?\s*(?:%{NUMBER}))* @@ -38,7 +37,7 @@ processors: separator: '"?,?\s+' ignore_missing: true - split: - field: nginx.ingress_controller.upstream_ip_list + field: nginx.ingress_controller.upstream_address_list separator: '"?,?\s+' ignore_missing: true - split: @@ -69,14 +68,14 @@ processors: if (ctx.nginx.ingress_controller.upstream.response.length_list.length == null) { return; } - int length = 0; + int last_length = 0; for (def item : ctx.nginx.ingress_controller.upstream.response.length_list) { - length = length + Integer.parseInt(item); + last_length = Integer.parseInt(item); } - ctx.nginx.ingress_controller.upstream.response.length_total = length; + ctx.nginx.ingress_controller.upstream.response.length = last_length; } catch (Exception e) { - ctx.nginx.ingress_controller.upstream.response.length_total = null; + ctx.nginx.ingress_controller.upstream.response.length = null; } - script: if: ctx.nginx?.ingress_controller?.upstream?.response?.time_list != null && ctx.nginx.ingress_controller.upstream.response.time_list.length > 0 @@ -90,10 +89,10 @@ processors: for (def item : ctx.nginx.ingress_controller.upstream.response.time_list) { res_time = res_time + Float.parseFloat(item); } - ctx.nginx.ingress_controller.upstream.response.time_total = res_time; + ctx.nginx.ingress_controller.upstream.response.time = res_time; } catch (Exception e) { - ctx.nginx.ingress_controller.upstream.response.time_total = null; + ctx.nginx.ingress_controller.upstream.response.time = null; } - script: if: ctx.nginx?.ingress_controller?.upstream?.response?.status_code_list != null && ctx.nginx.ingress_controller.upstream.response.status_code_list.length > 0 @@ -113,15 +112,15 @@ processors: ctx.nginx.ingress_controller.upstream.response.time = null; } - script: - if: ctx.nginx?.ingress_controller?.upstream_ip_list != null && ctx.nginx.ingress_controller.upstream_ip_list.length > 0 + if: ctx.nginx?.ingress_controller?.upstream_address_list != null && ctx.nginx.ingress_controller.upstream_address_list.length > 0 lang: painless source: >- try { - if (ctx.nginx.ingress_controller.upstream_ip_list.length == null) { + if (ctx.nginx.ingress_controller.upstream_address_list.length == null) { return; } def last_upstream = ""; - for (def item : ctx.nginx.ingress_controller.upstream_ip_list) { + for (def item : ctx.nginx.ingress_controller.upstream_address_list) { last_upstream = item; } StringTokenizer tok = new StringTokenizer(last_upstream, ":"); diff --git a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json index 0f52b337892..767aaa6b227 100644 --- a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json +++ b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json @@ -29,7 +29,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length_total": 59, + "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], @@ -37,11 +37,11 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time_total": 0.0, + "nginx.ingress_controller.upstream.response.time": 0.0, "nginx.ingress_controller.upstream.response.time_list": [ "0.000" ], - "nginx.ingress_controller.upstream_ip_list": [ + "nginx.ingress_controller.upstream_address_list": [ "172.17.0.5:8080" ], "related.ip": [ @@ -86,7 +86,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length_total": 59, + "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], @@ -94,11 +94,11 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time_total": 0.0, + "nginx.ingress_controller.upstream.response.time": 0.0, "nginx.ingress_controller.upstream.response.time_list": [ "0.000" ], - "nginx.ingress_controller.upstream_ip_list": [ + "nginx.ingress_controller.upstream_address_list": [ "172.17.0.5:8080" ], "related.ip": [ @@ -143,7 +143,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length_total": 59, + "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], @@ -151,11 +151,11 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time_total": 0.001, + "nginx.ingress_controller.upstream.response.time": 0.001, "nginx.ingress_controller.upstream.response.time_list": [ "0.001" ], - "nginx.ingress_controller.upstream_ip_list": [ + "nginx.ingress_controller.upstream_address_list": [ "172.17.0.5:8080" ], "related.ip": [ @@ -200,7 +200,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length_total": 59, + "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], @@ -208,11 +208,11 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time_total": 0.0, + "nginx.ingress_controller.upstream.response.time": 0.0, "nginx.ingress_controller.upstream.response.time_list": [ "0.000" ], - "nginx.ingress_controller.upstream_ip_list": [ + "nginx.ingress_controller.upstream_address_list": [ "172.17.0.5:8080" ], "related.ip": [ @@ -329,7 +329,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length_total": 59, + "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], @@ -337,11 +337,11 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time_total": 0.0, + "nginx.ingress_controller.upstream.response.time": 0.0, "nginx.ingress_controller.upstream.response.time_list": [ "0.000" ], - "nginx.ingress_controller.upstream_ip_list": [ + "nginx.ingress_controller.upstream_address_list": [ "172.17.0.5:8080" ], "related.ip": [ @@ -386,7 +386,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length_total": 59, + "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], @@ -394,11 +394,11 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time_total": 0.0, + "nginx.ingress_controller.upstream.response.time": 0.0, "nginx.ingress_controller.upstream.response.time_list": [ "0.000" ], - "nginx.ingress_controller.upstream_ip_list": [ + "nginx.ingress_controller.upstream_address_list": [ "172.17.0.5:8080" ], "related.ip": [ @@ -447,7 +447,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length_total": 59, + "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], @@ -455,11 +455,11 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time_total": 0.0, + "nginx.ingress_controller.upstream.response.time": 0.0, "nginx.ingress_controller.upstream.response.time_list": [ "0.000" ], - "nginx.ingress_controller.upstream_ip_list": [ + "nginx.ingress_controller.upstream_address_list": [ "172.17.0.5:8080" ], "related.ip": [ @@ -507,7 +507,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.6", "nginx.ingress_controller.upstream.name": "default-web2-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length_total": 61, + "nginx.ingress_controller.upstream.response.length": 61, "nginx.ingress_controller.upstream.response.length_list": [ "61" ], @@ -515,11 +515,11 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time_total": 0.001, + "nginx.ingress_controller.upstream.response.time": 0.001, "nginx.ingress_controller.upstream.response.time_list": [ "0.001" ], - "nginx.ingress_controller.upstream_ip_list": [ + "nginx.ingress_controller.upstream_address_list": [ "172.17.0.6:8080" ], "related.ip": [ @@ -568,7 +568,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length_total": 59, + "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], @@ -576,11 +576,11 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time_total": 0.002, + "nginx.ingress_controller.upstream.response.time": 0.002, "nginx.ingress_controller.upstream.response.time_list": [ "0.002" ], - "nginx.ingress_controller.upstream_ip_list": [ + "nginx.ingress_controller.upstream_address_list": [ "172.17.0.5:8080" ], "related.ip": [ @@ -628,7 +628,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length_total": 59, + "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], @@ -636,11 +636,11 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time_total": 0.001, + "nginx.ingress_controller.upstream.response.time": 0.001, "nginx.ingress_controller.upstream.response.time_list": [ "0.001" ], - "nginx.ingress_controller.upstream_ip_list": [ + "nginx.ingress_controller.upstream_address_list": [ "172.17.0.5:8080" ], "related.ip": [ @@ -689,7 +689,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length_total": 59, + "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], @@ -697,11 +697,11 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time_total": 0.001, + "nginx.ingress_controller.upstream.response.time": 0.001, "nginx.ingress_controller.upstream.response.time_list": [ "0.001" ], - "nginx.ingress_controller.upstream_ip_list": [ + "nginx.ingress_controller.upstream_address_list": [ "172.17.0.5:8080" ], "related.ip": [ @@ -749,7 +749,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length_total": 59, + "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], @@ -757,11 +757,11 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time_total": 0.002, + "nginx.ingress_controller.upstream.response.time": 0.002, "nginx.ingress_controller.upstream.response.time_list": [ "0.002" ], - "nginx.ingress_controller.upstream_ip_list": [ + "nginx.ingress_controller.upstream_address_list": [ "172.17.0.5:8080" ], "related.ip": [ @@ -809,7 +809,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length_total": 59, + "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], @@ -817,11 +817,11 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time_total": 0.001, + "nginx.ingress_controller.upstream.response.time": 0.001, "nginx.ingress_controller.upstream.response.time_list": [ "0.001" ], - "nginx.ingress_controller.upstream_ip_list": [ + "nginx.ingress_controller.upstream_address_list": [ "172.17.0.5:8080" ], "related.ip": [ @@ -870,7 +870,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length_total": 59, + "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], @@ -878,11 +878,11 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time_total": 0.002, + "nginx.ingress_controller.upstream.response.time": 0.002, "nginx.ingress_controller.upstream.response.time_list": [ "0.002" ], - "nginx.ingress_controller.upstream_ip_list": [ + "nginx.ingress_controller.upstream_address_list": [ "172.17.0.5:8080" ], "related.ip": [ @@ -930,7 +930,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.6", "nginx.ingress_controller.upstream.name": "default-web2-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length_total": 61, + "nginx.ingress_controller.upstream.response.length": 61, "nginx.ingress_controller.upstream.response.length_list": [ "61" ], @@ -938,11 +938,11 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time_total": 0.002, + "nginx.ingress_controller.upstream.response.time": 0.002, "nginx.ingress_controller.upstream.response.time_list": [ "0.002" ], - "nginx.ingress_controller.upstream_ip_list": [ + "nginx.ingress_controller.upstream_address_list": [ "172.17.0.6:8080" ], "related.ip": [ @@ -991,7 +991,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length_total": 59, + "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], @@ -999,11 +999,11 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time_total": 0.0, + "nginx.ingress_controller.upstream.response.time": 0.0, "nginx.ingress_controller.upstream.response.time_list": [ "0.000" ], - "nginx.ingress_controller.upstream_ip_list": [ + "nginx.ingress_controller.upstream_address_list": [ "172.17.0.5:8080" ], "related.ip": [ @@ -1051,7 +1051,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length_total": 59, + "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], @@ -1059,11 +1059,11 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time_total": 0.001, + "nginx.ingress_controller.upstream.response.time": 0.001, "nginx.ingress_controller.upstream.response.time_list": [ "0.001" ], - "nginx.ingress_controller.upstream_ip_list": [ + "nginx.ingress_controller.upstream_address_list": [ "172.17.0.5:8080" ], "related.ip": [ @@ -1108,7 +1108,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.6", "nginx.ingress_controller.upstream.name": "default-web2-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length_total": 61, + "nginx.ingress_controller.upstream.response.length": 61, "nginx.ingress_controller.upstream.response.length_list": [ "61" ], @@ -1116,11 +1116,11 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time_total": 0.001, + "nginx.ingress_controller.upstream.response.time": 0.001, "nginx.ingress_controller.upstream.response.time_list": [ "0.001" ], - "nginx.ingress_controller.upstream_ip_list": [ + "nginx.ingress_controller.upstream_address_list": [ "172.17.0.6:8080" ], "related.ip": [ @@ -1168,7 +1168,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.5", "nginx.ingress_controller.upstream.name": "default-web-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length_total": 59, + "nginx.ingress_controller.upstream.response.length": 59, "nginx.ingress_controller.upstream.response.length_list": [ "59" ], @@ -1176,11 +1176,11 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time_total": 0.0, + "nginx.ingress_controller.upstream.response.time": 0.0, "nginx.ingress_controller.upstream.response.time_list": [ "0.000" ], - "nginx.ingress_controller.upstream_ip_list": [ + "nginx.ingress_controller.upstream_address_list": [ "172.17.0.5:8080" ], "related.ip": [ @@ -1228,7 +1228,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.6", "nginx.ingress_controller.upstream.name": "default-web2-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length_total": 61, + "nginx.ingress_controller.upstream.response.length": 61, "nginx.ingress_controller.upstream.response.length_list": [ "61" ], @@ -1236,11 +1236,11 @@ "nginx.ingress_controller.upstream.response.status_code_list": [ "200" ], - "nginx.ingress_controller.upstream.response.time_total": 0.0, + "nginx.ingress_controller.upstream.response.time": 0.0, "nginx.ingress_controller.upstream.response.time_list": [ "0.000" ], - "nginx.ingress_controller.upstream_ip_list": [ + "nginx.ingress_controller.upstream_address_list": [ "172.17.0.6:8080" ], "related.ip": [ @@ -1288,7 +1288,7 @@ "nginx.ingress_controller.upstream.ip": "172.17.0.7", "nginx.ingress_controller.upstream.name": "default-web2-8080", "nginx.ingress_controller.upstream.port": 8080, - "nginx.ingress_controller.upstream.response.length_total": 161, + "nginx.ingress_controller.upstream.response.length": 100, "nginx.ingress_controller.upstream.response.length_list": [ "61", "100" @@ -1298,12 +1298,12 @@ "200", "203" ], - "nginx.ingress_controller.upstream.response.time_total": 0.104, + "nginx.ingress_controller.upstream.response.time": 0.104, "nginx.ingress_controller.upstream.response.time_list": [ "0.100", "0.004" ], - "nginx.ingress_controller.upstream_ip_list": [ + "nginx.ingress_controller.upstream_address_list": [ "172.17.0.6:8080", "172.17.0.7:8080" ], From d22498622942cd1840811df47aad4115864b1ffc Mon Sep 17 00:00:00 2001 From: chrismark Date: Mon, 28 Sep 2020 12:30:51 +0300 Subject: [PATCH 7/8] Fix tests Signed-off-by: chrismark --- .../ingress_controller/test/test.log-expected.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json index 767aaa6b227..34af9173b9c 100644 --- a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json +++ b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json @@ -51,7 +51,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products", - "user_agent.device.name": "Mac", + "user_agent.device.name": "Other", "user_agent.name": "curl", "user_agent.original": "curl/7.54.0", "user_agent.version": "7.54.0" @@ -108,7 +108,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", - "user_agent.device.name": "Mac", + "user_agent.device.name": "Other", "user_agent.name": "curl", "user_agent.original": "curl/7.54.0", "user_agent.version": "7.54.0" @@ -165,7 +165,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", - "user_agent.device.name": "Mac", + "user_agent.device.name": "Other", "user_agent.name": "curl", "user_agent.original": "curl/7.54.0", "user_agent.version": "7.54.0" @@ -222,7 +222,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", - "user_agent.device.name": "Mac", + "user_agent.device.name": "Other", "user_agent.name": "curl", "user_agent.original": "curl/7.54.0", "user_agent.version": "7.54.0" @@ -351,7 +351,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", - "user_agent.device.name": "Mac", + "user_agent.device.name": "Other", "user_agent.name": "Wget", "user_agent.original": "Wget/1.20.3 (darwin18.6.0)", "user_agent.version": "1.20.3" From 539d0721f42bb103da41064aa780084ca7f6a730 Mon Sep 17 00:00:00 2001 From: chrismark Date: Tue, 29 Sep 2020 10:56:16 +0300 Subject: [PATCH 8/8] Fix test Signed-off-by: chrismark --- .../module/nginx/ingress_controller/test/test.log-expected.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json index 34af9173b9c..e8b09bc1abd 100644 --- a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json +++ b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json @@ -1073,7 +1073,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42?address=delhi+technological+university", - "user_agent.device.name": "Mac", + "user_agent.device.name": "Other", "user_agent.name": "Python Requests", "user_agent.original": "python-requests/2.22.0", "user_agent.version": "2.22"