Handle multiple upstreams in ingress-controller

CHANGELOG.next.asciidoc

@@ -264,6 +264,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove wrongly mapped `tls.client.server_name` from `fortinet/firewall` fileset. {pull}20983[20983]
 - Fix an error updating file size being logged when EOF is reached. {pull}21048[21048]
 - Fix error when processing AWS Cloudtrail Digest logs. {pull}21086[21086] {issue}20943[20943]
+- Handle multiple upstreams in ingress-controller. {pull}21215[21215]
 
 *Heartbeat*
 

filebeat/docs/fields.asciidoc

@@ -105131,6 +105131,46 @@ Contains fields for the Ingress Nginx controller access logs.
 An array of remote IP addresses. It is a list because it is common to include, besides the client IP address, IP addresses from headers like `X-Forwarded-For`. Real source IP is restored to `source.ip`.
 
 
+type: array
+
+--
+
+*`nginx.ingress_controller.upstream_ip_list`*::
++
+--
+An array of the upstream IP addresses. It is a list because it is common that several upstream servers were contacted during request processing.
+
+
+type: array
+
+--
+
+*`nginx.ingress_controller.upstream.response.length_list`*::
++
+--
+An array of upstream response lengths. It is a list because it is common that several upstream servers were contacted during request processing.
+
+
+type: array
+
+--
+
+*`nginx.ingress_controller.upstream.response.time_list`*::
++
+--
+An array of upstream response durations. It is a list because it is common that several upstream servers were contacted during request processing.
+
+
+type: array
+
+--
+
+*`nginx.ingress_controller.upstream.response.status_code_list`*::
++
+--
+An array of upstream response status codes. It is a list because it is common that several upstream servers were contacted during request processing.
+
+
 type: array
 
 --
@@ -105182,7 +105222,7 @@ type: keyword
 *`nginx.ingress_controller.upstream.response.length`*::
 +
 --
-The length of the response obtained from the upstream server
+The length of the response obtained from the upstream server. If several servers were contacted during request process, the summary of the multiple response lengths is stored.
 
 
 type: long
@@ -105194,7 +105234,7 @@ format: bytes
 *`nginx.ingress_controller.upstream.response.time`*::
 +
 --
-The time spent on receiving the response from the upstream server as seconds with millisecond resolution
+The time spent on receiving the response from the upstream as seconds with millisecond resolution. If several servers were contacted during request process, the summary of the multiple response times is stored.
 
 
 type: double
@@ -105206,40 +105246,40 @@ format: duration
 *`nginx.ingress_controller.upstream.response.status_code`*::
 +
 --
-The status code of the response obtained from the upstream server
+The status code of the response obtained from the upstream server. If several servers were contacted during request process, only the status code of the response from the last one is stored in this field.
 
 
 type: long
 
 --
 
-*`nginx.ingress_controller.http.request.id`*::
+*`nginx.ingress_controller.upstream.ip`*::
 +
 --
-The randomly generated ID of the request
+The IP address of the upstream server. If several servers were contacted during request process, only the last one is stored in this field.
 
 
-type: keyword
+type: ip
 
 --
 
-*`nginx.ingress_controller.upstream.ip`*::
+*`nginx.ingress_controller.upstream.port`*::
 +
 --
-The IP address of the upstream server. If several servers were contacted during request processing, their addresses are separated by commas.
+The port of the upstream server. If several servers were contacted during request process, only the last one is stored in this field.
 
 
-type: ip
+type: long
 
 --
 
-*`nginx.ingress_controller.upstream.port`*::
+*`nginx.ingress_controller.http.request.id`*::
 +
 --
-The port of the upstream server.
+The randomly generated ID of the request
 
 
-type: long
+type: keyword
 
 --
 

filebeat/module/nginx/_meta/kibana/7/dashboard/Filebeat-nginx-ingress-overview.json

@@ -917,7 +917,7 @@
                 "line_width": 1,
                 "metrics": [
                   {
-                    "field": "nginx.ingress_controller.upstream.response.time",
+                    "field": "nginx.ingress_controller.upstream.response.time_total",
                     "id": "61ca57f2-469d-11e7-af02-69e470af7417",
                     "type": "sum"
                   }
@@ -1004,7 +1004,7 @@
                 "line_width": 1,
                 "metrics": [
                   {
-                    "field": "nginx.ingress_controller.upstream.response.length",
+                    "field": "nginx.ingress_controller.upstream.response.length_total",
                     "id": "61ca57f2-469d-11e7-af02-69e470af7417",
                     "type": "sum",
                     "values": [

filebeat/module/nginx/ingress_controller/_meta/fields.yml

@@ -11,6 +11,26 @@
         Real source IP is restored to `source.ip`.
 
     # ingress-controller specific fields
+    - name: upstream_ip_list
+      type: array
+      description: >
+        An array of the upstream IP addresses. It is a list because it is common that several upstream servers
+        were contacted during request processing.
+    - name: upstream.response.length_list
+      type: array
+      description: >
+        An array of upstream response lengths. It is a list because it is common that several upstream servers
+        were contacted during request processing.
+    - name: upstream.response.time_list
+      type: array
+      description: >
+        An array of upstream response durations. It is a list because it is common that several upstream servers
+        were contacted during request processing.
+    - name: upstream.response.status_code_list
+      type: array
+      description: >
+        An array of upstream response status codes. It is a list because it is common that several upstream servers
+        were contacted during request processing.
     - name: http.request.length
       type: long
       format: bytes
@@ -29,32 +49,37 @@
       type: keyword
       description: >
         The name of the alternative upstream.
-    - name: upstream.response.length
+    - name: upstream.response.length_total
       type: long
       format: bytes
       description: >
-        The length of the response obtained from the upstream server
-    - name: upstream.response.time
+        The length of the response obtained from the upstream server. If several servers were contacted during request process,
+        the summary of the multiple response lengths is stored.
+    - name: upstream.response.time_total
       type: double
       format: duration
       description: >
-        The time spent on receiving the response from the upstream server as seconds with millisecond resolution
+        The time spent on receiving the response from the upstream as seconds with millisecond resolution.
+        If several servers were contacted during request process, the summary of the multiple response times is stored.
     - name: upstream.response.status_code
       type: long
       description: >
-        The status code of the response obtained from the upstream server
-    - name: http.request.id
-      type: keyword
-      description: >
-        The randomly generated ID of the request
+        The status code of the response obtained from the upstream server. If several servers were contacted during
+        request process, only the status code of the response from the last one is stored in this field.
     - name: upstream.ip
       type: ip
       description: >
-        The IP address of the upstream server. If several servers were contacted during request processing, their addresses are separated by commas.
+        The IP address of the upstream server. If several servers were contacted during request process,
+        only the last one is stored in this field.
     - name: upstream.port
       type: long
       description: >
-        The port of the upstream server.
+        The port of the upstream server. If several servers were contacted during request process,
+        only the last one is stored in this field.
+    - name: http.request.id
+      type: keyword
+      description: >
+        The randomly generated ID of the request
 
     - name: body_sent.bytes
       type: alias

filebeat/module/nginx/ingress_controller/ingest/pipeline.yml

@@ -12,14 +12,18 @@ processors:
       %{NUMBER:http.response.status_code:long} %{NUMBER:http.response.body.bytes:long}
       "(-|%{DATA:http.request.referrer})" "(-|%{DATA:user_agent.original})" %{NUMBER:nginx.ingress_controller.http.request.length:long}
       %{NUMBER:nginx.ingress_controller.http.request.time:double} \[%{DATA:nginx.ingress_controller.upstream.name}\]
-      \[%{DATA:nginx.ingress_controller.upstream.alternative_name}\] (%{UPSTREAM_ADDRESS}|-)
-      (%{NUMBER:nginx.ingress_controller.upstream.response.length:long}|-) (%{NUMBER:nginx.ingress_controller.upstream.response.time:double}|-)
-      (%{NUMBER:nginx.ingress_controller.upstream.response.status_code:long}|-) %{GREEDYDATA:nginx.ingress_controller.http.request.id}
+      \[%{DATA:nginx.ingress_controller.upstream.alternative_name}\] (%{UPSTREAM_ADDRESS_LIST:nginx.ingress_controller.upstream_ip_list}|-)
+      (%{UPSTREAM_RESPONSE_LENGTH_LIST:nginx.ingress_controller.upstream.response.length_list}|-) (%{UPSTREAM_RESPONSE_TIME_LIST:nginx.ingress_controller.upstream.response.time_list}|-)
+      (%{UPSTREAM_RESPONSE_STATUS_CODE_LIST:nginx.ingress_controller.upstream.response.status_code_list}|-) %{GREEDYDATA:nginx.ingress_controller.http.request.id}
     pattern_definitions:
       NGINX_HOST: (?:%{IP:destination.ip}|%{NGINX_NOTSEPARATOR:destination.domain})(:%{NUMBER:destination.port})?
       NGINX_NOTSEPARATOR: "[^\t ,:]+"
       NGINX_ADDRESS_LIST: (?:%{IP}|%{WORD})("?,?\s*(?:%{IP}|%{WORD}))*
       UPSTREAM_ADDRESS: '%{IP:nginx.ingress_controller.upstream.ip}(:%{NUMBER:nginx.ingress_controller.upstream.port})?'
+      UPSTREAM_ADDRESS_LIST: (?:%{IP}(:%{NUMBER})?)("?,?\s*(?:%{IP}(:%{NUMBER})?))*
+      UPSTREAM_RESPONSE_LENGTH_LIST: (?:%{NUMBER})("?,?\s*(?:%{NUMBER}))*
+      UPSTREAM_RESPONSE_TIME_LIST: (?:%{NUMBER})("?,?\s*(?:%{NUMBER}))*
+      UPSTREAM_RESPONSE_STATUS_CODE_LIST: (?:%{NUMBER})("?,?\s*(?:%{NUMBER}))*
     ignore_missing: true
 - grok:
     field: nginx.ingress_controller.info
@@ -33,6 +37,22 @@ processors:
     field: nginx.ingress_controller.remote_ip_list
     separator: '"?,?\s+'
     ignore_missing: true
+- split:
+    field: nginx.ingress_controller.upstream_ip_list
+    separator: '"?,?\s+'
+    ignore_missing: true
+- split:
+    field: nginx.ingress_controller.upstream.response.length_list
+    separator: '"?,?\s+'
+    ignore_missing: true
+- split:
+    field: nginx.ingress_controller.upstream.response.time_list
+    separator: '"?,?\s+'
+    ignore_missing: true
+- split:
+    field: nginx.ingress_controller.upstream.response.status_code_list
+    separator: '"?,?\s+'
+    ignore_missing: true
 - split:
     field: nginx.ingress_controller.origin
     separator: '"?,?\s+'
@@ -41,6 +61,81 @@ processors:
     field: source.address
     if: ctx.source?.address == null
     value: ""
+- script:
+    if: ctx.nginx?.ingress_controller?.upstream?.response?.length_list != null && ctx.nginx.ingress_controller.upstream.response.length_list.length > 0
+    lang: painless
+    source: >-
+      try {
+        if (ctx.nginx.ingress_controller.upstream.response.length_list.length == null) {
+          return;
+        }
+        int length = 0;
+        for (def item : ctx.nginx.ingress_controller.upstream.response.length_list) {
+          length = length + Integer.parseInt(item);
+        }
+        ctx.nginx.ingress_controller.upstream.response.length_total = length;
+      }
+      catch (Exception e) {
+        ctx.nginx.ingress_controller.upstream.response.length_total = null;
+      }
+- script:
+    if: ctx.nginx?.ingress_controller?.upstream?.response?.time_list != null && ctx.nginx.ingress_controller.upstream.response.time_list.length > 0
+    lang: painless
+    source: >-
+      try {
+        if (ctx.nginx.ingress_controller.upstream.response.time_list.length == null) {
+          return;
+        }
+        float res_time = 0;
+        for (def item : ctx.nginx.ingress_controller.upstream.response.time_list) {
+          res_time = res_time + Float.parseFloat(item);
+        }
+        ctx.nginx.ingress_controller.upstream.response.time_total = res_time;
+      }
+      catch (Exception e) {
+        ctx.nginx.ingress_controller.upstream.response.time_total = null;
+      }
+- script:
+    if: ctx.nginx?.ingress_controller?.upstream?.response?.status_code_list != null && ctx.nginx.ingress_controller.upstream.response.status_code_list.length > 0
+    lang: painless
+    source: >-
+      try {
+        if (ctx.nginx.ingress_controller.upstream.response.status_code_list.length == null) {
+          return;
+        }
+        int last_status_code;
+        for (def item : ctx.nginx.ingress_controller.upstream.response.status_code_list) {
+          last_status_code = Integer.parseInt(item);
+        }
+        ctx.nginx.ingress_controller.upstream.response.status_code = last_status_code;
+      }
+      catch (Exception e) {
+        ctx.nginx.ingress_controller.upstream.response.time = null;
+      }
+- script:
+    if: ctx.nginx?.ingress_controller?.upstream_ip_list != null && ctx.nginx.ingress_controller.upstream_ip_list.length > 0
+    lang: painless
+    source: >-
+      try {
+        if (ctx.nginx.ingress_controller.upstream_ip_list.length == null) {
+          return;
+        }
+        def last_upstream = "";
+        for (def item : ctx.nginx.ingress_controller.upstream_ip_list) {
+          last_upstream = item;
+        }
+        StringTokenizer tok = new StringTokenizer(last_upstream, ":");
+        if (tok.countTokens()>1) {
+          ctx.nginx.ingress_controller.upstream.ip = tok.nextToken();
+          ctx.nginx.ingress_controller.upstream.port = Integer.parseInt(tok.nextToken());
+        } else {
+          ctx.nginx.ingress_controller.upstream.ip = last_upstream;
+        }
+      }
+      catch (Exception e) {
+        ctx.nginx.ingress_controller.upstream.ip = null;
+        ctx.nginx.ingress_controller.upstream.port = null;
+      }
 - script:
     if: ctx.nginx?.ingress_controller?.remote_ip_list != null && ctx.nginx.ingress_controller.remote_ip_list.length > 0
     lang: painless

filebeat/module/nginx/ingress_controller/test/test.log

@@ -20,3 +20,4 @@
 192.168.64.1 - - [07/Feb/2020:12:02:38 +0000] "GET /v2 HTTP/1.1" 200 61 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0" 343 0.000 [default-web2-8080] [] 172.17.0.6:8080 61 0.001 200 ba91c30454893c121879396b0a78be79
 192.168.64.1 - - [07/Feb/2020:12:02:38 +0000] "GET /favicon.ico HTTP/1.1" 200 59 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0" 262 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 98c81aa2d50c67f6fb1fa16d5ce62f8f
 192.168.64.1 - - [07/Feb/2020:12:02:42 +0000] "GET /v2/some HTTP/1.1" 200 61 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0" 348 0.001 [default-web2-8080] [] 172.17.0.6:8080 61 0.000 200 835136ae24486dbb4156dcbe21f5d402
+192.168.64.14 - - [07/Feb/2020:12:02:42 +0000] "GET /v2/some HTTP/1.1" 200 61 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0" 348 0.001 [default-web2-8080] [] 172.17.0.6:8080, 172.17.0.7:8080 61, 100 0.100, 0.004 200, 203 835136ae24486dbb4156dcbe21f5d402