diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index a166cef96fc..7ca3a56c1f1 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -243,6 +243,7 @@ automatic splitting at root level, if root level element is an array. {pull}3415 - Register MIME handlers for CSV types in CEL input. {pull}34934[34934] - Add MySQL authentication message parsing and `related.ip` and `related.user` fields {pull}34810[34810] - Mention `mito` CEL tool in CEL input docs. {pull}34959[34959] +- Add nginx ingress_controller parsing if one of upstreams fails to return response {pull}34787[34787] *Auditbeat* diff --git a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml index 10d722a98a3..051a07eb631 100644 --- a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml +++ b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml @@ -17,16 +17,16 @@ processors: "(-|%{DATA:http.request.referrer})" "(-|%{DATA:user_agent.original})" %{NUMBER:nginx.ingress_controller.http.request.length:long} %{NUMBER:nginx.ingress_controller.http.request.time:double} \[%{DATA:nginx.ingress_controller.upstream.name}\] \[%{DATA:nginx.ingress_controller.upstream.alternative_name}\] (%{UPSTREAM_ADDRESS_LIST:nginx.ingress_controller.upstream_address_list}|-) - (%{UPSTREAM_RESPONSE_LENGTH_LIST:nginx.ingress_controller.upstream.response.length_list}|-) (%{UPSTREAM_RESPONSE_TIME_LIST:nginx.ingress_controller.upstream.response.time_list}|-) - (%{UPSTREAM_RESPONSE_STATUS_CODE_LIST:nginx.ingress_controller.upstream.response.status_code_list}|-) %{GREEDYDATA:nginx.ingress_controller.http.request.id} + (%{UPSTREAM_RESPONSE_LENGTH_LIST:nginx.ingress_controller.upstream.response.length_list}|-) (-|%{UPSTREAM_RESPONSE_TIME_LIST:nginx.ingress_controller.upstream.response.time_list}) + (-|%{UPSTREAM_RESPONSE_STATUS_CODE_LIST:nginx.ingress_controller.upstream.response.status_code_list}) %{GREEDYDATA:nginx.ingress_controller.http.request.id} pattern_definitions: NGINX_HOST: (?:%{IP:destination.ip}|%{NGINX_NOTSEPARATOR:destination.domain})(:%{NUMBER:destination.port})? NGINX_NOTSEPARATOR: "[^\t ,:]+" NGINX_ADDRESS_LIST: (?:%{IP}|%{WORD})("?,?\s*(?:%{IP}|%{WORD}))* UPSTREAM_ADDRESS_LIST: (unix:%{NOTSPACE}|(?:%{IP}(:%{NUMBER})?)("?,?\s*(?:%{IP}(:%{NUMBER})?))*) UPSTREAM_RESPONSE_LENGTH_LIST: (?:%{NUMBER})("?,?\s*(?:%{NUMBER}))* - UPSTREAM_RESPONSE_TIME_LIST: (?:%{NUMBER})("?,?\s*(?:%{NUMBER}))* - UPSTREAM_RESPONSE_STATUS_CODE_LIST: (?:%{NUMBER})("?,?\s*(?:%{NUMBER}))* + UPSTREAM_RESPONSE_TIME_LIST: (%{NUMBER}|(-|(?:%{NUMBER})),\s+(-|(?:%{NUMBER}))(\"?,?\s*(-|(?:%{NUMBER})))*) + UPSTREAM_RESPONSE_STATUS_CODE_LIST: (%{NUMBER}|(-|(?:%{NUMBER})),\s+(-|(?:%{NUMBER}))(\"?,?\s*(-|(?:%{NUMBER})))*) IP: (?:\[?%{IPV6}\]?|%{IPV4}) ignore_missing: true - grok: @@ -104,7 +104,9 @@ processors: } float res_time = 0; for (def item : ctx.nginx.ingress_controller.upstream.response.time_list) { - res_time = res_time + Float.parseFloat(item); + if (item != '-') { + res_time = res_time + Float.parseFloat(item); + } } ctx.nginx.ingress_controller.upstream.response.time = res_time; } @@ -121,7 +123,9 @@ processors: } int last_status_code; for (def item : ctx.nginx.ingress_controller.upstream.response.status_code_list) { - last_status_code = Integer.parseInt(item); + if (item != '-') { + last_status_code = Integer.parseInt(item); + } } ctx.nginx.ingress_controller.upstream.response.status_code = last_status_code; } diff --git a/filebeat/module/nginx/ingress_controller/test/test.log b/filebeat/module/nginx/ingress_controller/test/test.log index 3e9090fe2ec..2b2e350c495 100644 --- a/filebeat/module/nginx/ingress_controller/test/test.log +++ b/filebeat/module/nginx/ingress_controller/test/test.log @@ -26,3 +26,4 @@ 2a02:cf40::4e36 - - [24/Aug/2022:18:05:41 +0000] "GET /favicon.ico HTTP/2.0" 502 552 "https://localhost:8080/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36" 27 0.000 [localhost-8080] [] [2a02:cf40::7]:5000, [2a02:cf40::4e36]:5000 0, 0 0.000, 0.000 502, 502 3db73c6c673c4256ade033a6ce08c2ab 2a02:cf40::4e36 - - [24/Aug/2022:18:05:41 +0000] "GET /favicon.ico HTTP/2.0" 502 552 "https://localhost:8080/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36" 27 0.000 [localhost-8080] [] [2a02:cf40::7]:5000, [2a02:cf40::::::::4e36]:500000000 0, 0 0.000, 0.000 502, 502 3db73c6c673c4256ade033a6ce08c2ab 192.168.64.1 - - [07/Feb/2020:11:56:54 +0000] "GET /products/42 HTTP/1.1" 200 59 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15" 369 0.002 [default-web-8080] [] unix:/var/run/php-fpm.sock 59 0.002 200 0f76ea730f282d5759018eb756b23b14 +192.168.64.14 - - [07/Feb/2020:12:02:42 +0000] "GET /test/test.mp4 HTTP/1.1" 200 61 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0" 348 0.001 [default-web2-8080] [] 172.17.0.6:8080, 172.17.0.7:8080 - 0.100, - 200, - 835136ae24486dbb4156dcbe21f5d402 diff --git a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json index 75a611dbb90..c9f86053b7e 100644 --- a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json +++ b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json @@ -1763,5 +1763,69 @@ "user_agent.os.name": "Mac OS X", "user_agent.os.version": "10.14.6", "user_agent.version": "13.0.5" + }, + { + "@timestamp": "2020-02-07T12:02:42.000Z", + "event.category": [ + "web" + ], + "event.dataset": "nginx.ingress_controller", + "event.kind": "event", + "event.module": "nginx", + "event.original": "192.168.64.14 - - [07/Feb/2020:12:02:42 +0000] \"GET /test/test.mp4 HTTP/1.1\" 200 61 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0\" 348 0.001 [default-web2-8080] [] 172.17.0.6:8080, 172.17.0.7:8080 - 0.100, - 200, - 835136ae24486dbb4156dcbe21f5d402", + "event.outcome": "success", + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "ingress_controller", + "http.request.id": "835136ae24486dbb4156dcbe21f5d402", + "http.request.method": "GET", + "http.response.body.bytes": 61, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 7742, + "nginx.ingress_controller.http.request.id": "835136ae24486dbb4156dcbe21f5d402", + "nginx.ingress_controller.http.request.length": 348, + "nginx.ingress_controller.http.request.time": 0.001, + "nginx.ingress_controller.remote_ip_list": [ + "192.168.64.14" + ], + "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.7", + "nginx.ingress_controller.upstream.name": "default-web2-8080", + "nginx.ingress_controller.upstream.port": 8080, + "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.status_code_list": [ + "-", + "200" + ], + "nginx.ingress_controller.upstream.response.time": 0.1, + "nginx.ingress_controller.upstream.response.time_list": [ + "-", + "0.100" + ], + "nginx.ingress_controller.upstream_address_list": [ + "172.17.0.6:8080", + "172.17.0.7:8080" + ], + "related.ip": [ + "172.17.0.7", + "192.168.64.14" + ], + "service.type": "nginx", + "source.address": "192.168.64.14", + "source.ip": "192.168.64.14", + "url.extension": "mp4", + "url.original": "/test/test.mp4", + "url.path": "/test/test.mp4", + "user_agent.device.name": "Mac", + "user_agent.name": "Firefox", + "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", + "user_agent.os.full": "Mac OS X 10.14", + "user_agent.os.name": "Mac OS X", + "user_agent.os.version": "10.14", + "user_agent.version": "72.0." } ]