[Metricbeat] Add collecting Application ELB and Network ELB into elb metricset

CHANGELOG.next.asciidoc

@@ -476,6 +476,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Release CoreDNS module as GA. {pull}14308[14308]
 - Release CouchDB module as GA. {pull}14300[14300]
 - Add `elasticsearch/enrich` metricset. {pull}14243[14243] {issue}14221[14221]
+- Add support for Application ELB and Network ELB. {pull}14123[14123] {issue}13538[13538] {issue}13539[13539]
 - Add `connection.state` field for RabbitMQ module. {pull}13981[13981]
 - Add more TCP states to Metricbeat system socket_summary. {pull}14347[14347]
 

x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go

@@ -450,7 +450,7 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatchiface.ClientAPI, svcRes
 			m.Logger().Info(errors.Wrap(err, "getResourcesTags failed, skipping region "+regionName))
 		}
 
-		if tagsFilter != nil && resourceTagMap == nil {
+		if len(tagsFilter) != 0 && resourceTagMap == nil {
 			continue
 		}
 
@@ -472,7 +472,7 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatchiface.ClientAPI, svcRes
 					labels := strings.Split(*output.Label, labelSeperator)
 					if len(labels) != 5 {
 						// if there is no tag in labels but there is a tagsFilter, then no event should be reported.
-						if tagsFilter != nil {
+						if len(tagsFilter) != 0 {
 							continue
 						}
 						eventNew := aws.InitEvent(regionName, m.AccountName, m.AccountID)
@@ -483,7 +483,7 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatchiface.ClientAPI, svcRes
 
 					identifierValue := labels[identifierValueIdx]
 					tags := resourceTagMap[identifierValue]
-					if tagsFilter != nil && tags == nil {
+					if len(tagsFilter) != 0 && tags == nil {
 						continue
 					}
 

x-pack/metricbeat/module/aws/elb/_meta/data.json

@@ -3,20 +3,27 @@
     "aws": {
         "cloudwatch": {
             "dimensions": {
-                "AvailabilityZone": "us-east-1c"
+                "LoadBalancerName": "tf-lb-20171218185946539900000004"
             },
             "namespace": "AWS/ELB"
         },
         "metrics": {
-            "BackendConnectionErrors": {
-                "sum": 10
+            "EstimatedALBActiveConnectionCount": {
+                "avg": 2.6
             },
-            "HealthyHostCount": {
-                "max": 2
+            "EstimatedALBConsumedLCUs": {
+                "avg": 0.000015000000000000002
             },
-            "UnHealthyHostCount": {
-                "max": 1
+            "EstimatedALBNewConnectionCount": {
+                "avg": 0
+            },
+            "EstimatedProcessedBytes": {
+                "avg": 237.6
             }
+        },
+        "tags": {
+            "Name": "logstash",
+            "owner": "logstash"
         }
     },
     "cloud": {

x-pack/metricbeat/module/aws/elb/_meta/data_alb.json

@@ -0,0 +1,39 @@
+{
+    "@timestamp": "2017-10-12T08:05:34.853Z",
+    "aws": {
+        "cloudwatch": {
+            "dimensions": {
+                "LoadBalancer": "app/filebeat-aws-elb-test-lb/aaa75bb026cc4c54"
+            },
+            "namespace": "AWS/ApplicationELB"
+        },
+        "metrics": {
+            "ConsumedLCUs": {
+                "avg": 0
+            },
+            "RequestCount": {
+                "sum": 0
+            }
+        }
+    },
+    "cloud": {
+        "account": {
+            "id": "627959692251",
+            "name": "elastic-test"
+        },
+        "provider": "aws",
+        "region": "ap-southeast-1"
+    },
+    "event": {
+        "dataset": "aws.elb",
+        "duration": 115000,
+        "module": "aws"
+    },
+    "metricset": {
+        "name": "elb",
+        "period": 10000
+    },
+    "service": {
+        "type": "aws"
+    }
+}

x-pack/metricbeat/module/aws/elb/_meta/data_nlb.json

@@ -0,0 +1,54 @@
+{
+    "@timestamp": "2017-10-12T08:05:34.853Z",
+    "aws": {
+        "cloudwatch": {
+            "dimensions": {
+                "LoadBalancer": "net/filebeat-aws-elb-test-tcp-lb/d194184ba08f017f"
+            },
+            "namespace": "AWS/NetworkELB"
+        },
+        "metrics": {
+            "ActiveFlowCount": {
+                "avg": 0
+            },
+            "ConsumedLCUs": {
+                "avg": 0
+            },
+            "NewFlowCount": {
+                "sum": 0
+            },
+            "ProcessedBytes": {
+                "sum": 336
+            },
+            "TCP_Client_Reset_Count": {
+                "sum": 0
+            },
+            "TCP_ELB_Reset_Count": {
+                "sum": 0
+            },
+            "TCP_Target_Reset_Count": {
+                "sum": 0
+            }
+        }
+    },
+    "cloud": {
+        "account": {
+            "id": "627959692251",
+            "name": "elastic-test"
+        },
+        "provider": "aws",
+        "region": "ap-southeast-1"
+    },
+    "event": {
+        "dataset": "aws.elb",
+        "duration": 115000,
+        "module": "aws"
+    },
+    "metricset": {
+        "name": "elb",
+        "period": 10000
+    },
+    "service": {
+        "type": "aws"
+    }
+}

x-pack/metricbeat/module/aws/elb/_meta/docs.asciidoc

@@ -17,7 +17,7 @@ iam:ListAccountAliases
 [float]
 === Dashboard
 
-The aws elb metricset comes with a predefined dashboard. For example:
+The aws elb metricset comes with a predefined dashboard for classic ELB. For example:
 
 image::./images/metricbeat-aws-elb-overview.png[]
 
@@ -39,10 +39,12 @@ image::./images/metricbeat-aws-elb-overview.png[]
 
 [float]
 === Metrics
-Elastic Load Balancing publishes data points to Amazon CloudWatch for your load
-balancers and back-end instances. Please see more details for each metric in
-https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/elb-cloudwatch-metrics.html[elb-cloudwatch-metric].
+elb metricset collects Cloudwatch metrics from https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/elb-cloudwatch-metrics.html[classic ELB],
+https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-cloudwatch-metrics.html[application ELB] and
+https://docs.aws.amazon.com/elasticloadbalancing/latest/network/load-balancer-cloudwatch-metrics.html[network ELB].
 
+[float]
+==== Metrics for Classic ELB
 |===
 |Metric Name|Statistic Method
 |BackendConnectionErrors | Sum
@@ -63,3 +65,50 @@ https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/elb-cloudwatch-m
 |EstimatedALBNewConnectionCount | Average
 |EstimatedProcessedBytes | Average
 |===
+
+[float]
+==== Metrics for Application ELB
+|===
+|Metric Name|Statistic Method
+|ActiveConnectionCount | Sum
+|ClientTLSNegotiationErrorCount | Sum
+|ConsumedLCUs | Average
+|HTTP_Fixed_Response_Count | Sum
+|HTTP_Redirect_Count | Sum
+|HTTP_Redirect_Url_Limit_Exceeded_Count | Sum
+|HTTPCode_ELB_3XX_Count | Sum
+|HTTPCode_ELB_4XX_Count | Sum
+|HTTPCode_ELB_5XX_Count | Sum
+|HTTPCode_ELB_500_Count | Sum
+|HTTPCode_ELB_502_Count | Sum
+|HTTPCode_ELB_503_Count | Sum
+|HTTPCode_ELB_504_Count | Sum
+|IPv6ProcessedBytes | Sum
+|IPv6RequestCount | Sum
+|NewConnectionCount | Sum
+|ProcessedBytes | Sum
+|RejectedConnectionCount | Sum
+|RequestCount | Sum
+|RuleEvaluations | Sum
+|===
+
+[float]
+==== Metrics for Network ELB
+|===
+|Metric Name|Statistic Method
+|ActiveFlowCount | Average
+|ActiveFlowCount_TLS | Average
+|ClientTLSNegotiationErrorCount | Sum
+|ConsumedLCUs | Average
+|HealthyHostCount | Maximum
+|NewFlowCount | Sum
+|NewFlowCount_TLS | Sum
+|ProcessedBytes | Sum
+|ProcessedBytes_TLS | Sum
+|TargetTLSNegotiationErrorCount | Sum
+|TCP_Client_Reset_Count | Sum
+|TCP_ELB_Reset_Count | Sum
+|TCP_Target_Reset_Count | Sum
+|UnHealthyHostCount | Average
+|EstimatedALBConsumedLCUs | Maximum
+|===

x-pack/metricbeat/module/aws/elb/elb_integration_test.go

@@ -7,18 +7,40 @@
 package elb
 
 import (
+	"fmt"
 	"testing"
 
+	"github.com/elastic/beats/libbeat/common"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
 	"github.com/elastic/beats/x-pack/metricbeat/module/aws/mtest"
 )
 
 func TestData(t *testing.T) {
+	namespaceIs := func(namespace string) func(e common.MapStr) bool {
+		return func(e common.MapStr) bool {
+			v, err := e.GetValue("aws.cloudwatch.namespace")
+			return err == nil && v == namespace
+		}
+	}
+
+	dataFiles := []struct {
+		namespace string
+		path      string
+	}{
+		{"AWS/ELB", "./_meta/data.json"},
+		{"AWS/ApplicationELB", "./_meta/data_alb.json"},
+		{"AWS/NetworkELB", "./_meta/data_nlb.json"},
+	}
+
 	config, info := mtest.GetConfigForTest("elb", "300s")
 	if info != "" {
 		t.Skip("Skipping TestData: " + info)
 	}
 
-	metricSet := mbtest.NewFetcher(t, config)
-	metricSet.WriteEvents(t, "/")
+	for _, df := range dataFiles {
+		metricSet := mbtest.NewFetcher(t, config)
+		t.Run(fmt.Sprintf("namespace: %s", df.namespace), func(t *testing.T) {
+			metricSet.WriteEventsCond(t, df.path, namespaceIs(df.namespace))
+		})
+	}
 }

x-pack/metricbeat/module/aws/elb/manifest.yml

@@ -16,3 +16,31 @@ input:
         statistic: ["Average"]
         name: ["Latency", "EstimatedALBActiveConnectionCount", "EstimatedALBConsumedLCUs", "EstimatedALBNewConnectionCount", "EstimatedProcessedBytes"]
         tags.resource_type_filter: elasticloadbalancing
+      - namespace: AWS/ApplicationELB
+        statistic: ["Sum"]
+        name: ["ActiveConnectionCount", "ClientTLSNegotiationErrorCount", "HTTP_Fixed_Response_Count",
+               "HTTP_Redirect_Count", "HTTP_Redirect_Url_Limit_Exceeded_Count",
+               "HTTPCode_ELB_3XX_Count", "HTTPCode_ELB_4XX_Count", "HTTPCode_ELB_5XX_Count",
+               "HTTPCode_ELB_500_Count", "HTTPCode_ELB_502_Count", "HTTPCode_ELB_503_Count",
+               "HTTPCode_ELB_504_Count", "IPv6ProcessedBytes", "IPv6RequestCount",
+               "NewConnectionCount", "ProcessedBytes", "RejectedConnectionCount",
+               "RequestCount", "RuleEvaluations"]
+        tags.resource_type_filter: elasticloadbalancing
+      - namespace: AWS/ApplicationELB
+        statistic: ["Average"]
+        name: ["ConsumedLCUs"]
+        tags.resource_type_filter: elasticloadbalancing
+      - namespace: AWS/NetworkELB
+        statistic: ["Average"]
+        name: ["ActiveFlowCount", "ActiveFlowCount_TLS", "ConsumedLCUs"]
+        tags.resource_type_filter: elasticloadbalancing
+      - namespace: AWS/NetworkELB
+        statistic: ["Sum"]
+        name: ["ClientTLSNegotiationErrorCount", "NewFlowCount", "NewFlowCount_TLS",
+               "ProcessedBytes", "ProcessedBytes_TLS", "TargetTLSNegotiationErrorCount",
+               "TCP_Client_Reset_Count", "TCP_ELB_Reset_Count", "TCP_Target_Reset_Count"]
+        tags.resource_type_filter: elasticloadbalancing
+      - namespace: AWS/NetworkELB
+        statistic: ["Maximum"]
+        name: ["HealthyHostCount", "UnHealthyHostCount"]
+        tags.resource_type_filter: elasticloadbalancing