elastic · jsoriano · Sep 23, 2020 · Sep 22, 2020 · Sep 23, 2020
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -504,6 +504,7 @@ field. You can revert this change by configuring tags for the module and omittin
 - Added experimental dataset `fortinet/fortimanager`. {pull}20820[20820]
 - Added experimental dataset `juniper/netscreen`. {pull}20820[20820]
 - Added experimental dataset `sophos/utm`. {pull}20820[20820]
+- Add Cloud Foundry tags in related events. {pull}21177[21177]
 
 *Auditbeat*
 

diff --git a/x-pack/libbeat/common/cloudfoundry/events.go b/x-pack/libbeat/common/cloudfoundry/events.go
@@ -497,10 +497,19 @@ func baseMap(evt Event) common.MapStr {
 				"timestamp": evt.Timestamp(),
 			},
 			"envelope": envelopMap(evt),
+			"tags":     dedotedTags(evt.Tags()),
 		},
 	}
 }
 
+func dedotedTags(tags map[string]string) common.MapStr {
+	result := common.MapStr{}
+	for name, value := range tags {
+		result[common.DeDot(name)] = value
+	}
+	return result
+}
+
 func baseMapWithApp(evt EventWithAppID) common.MapStr {
 	base := baseMap(evt)
 	appID := evt.AppGuid()

diff --git a/x-pack/libbeat/common/cloudfoundry/events_test.go b/x-pack/libbeat/common/cloudfoundry/events_test.go
@@ -89,6 +89,9 @@ func TestEventTypeHttpAccess(t *testing.T) {
 			"app": common.MapStr{
 				"id": "f47ac10b-58cc-4372-a567-0e02b2c3d479",
 			},
+			"tags": common.MapStr{
+				"tag": "value",
+			},
 		},
 		"http": common.MapStr{
 			"response": common.MapStr{
@@ -164,6 +167,9 @@ func TestEventTypeLog(t *testing.T) {
 			"app": common.MapStr{
 				"id": "f47ac10b-58cc-4372-a567-0e02b2c3d479",
 			},
+			"tags": common.MapStr{
+				"tag": "value",
+			},
 		},
 		"message": "log message",
 		"stream":  "stdout",
@@ -212,6 +218,9 @@ func TestEventCounter(t *testing.T) {
 				"job":        "job",
 				"index":      "index",
 			},
+			"tags": common.MapStr{
+				"tag": "value",
+			},
 		},
 	}, evt.ToFields())
 }
@@ -258,6 +267,9 @@ func TestEventValueMetric(t *testing.T) {
 				"job":        "job",
 				"index":      "index",
 			},
+			"tags": common.MapStr{
+				"tag": "value",
+			},
 		},
 	}, evt.ToFields())
 }
@@ -322,6 +334,9 @@ func TestEventContainerMetric(t *testing.T) {
 			"app": common.MapStr{
 				"id": "f47ac10b-58cc-4372-a567-0e02b2c3d479",
 			},
+			"tags": common.MapStr{
+				"tag": "value",
+			},
 		},
 	}, evt.ToFields())
 }
@@ -366,6 +381,9 @@ func TestEventError(t *testing.T) {
 				"job":        "job",
 				"index":      "index",
 			},
+			"tags": common.MapStr{
+				"tag": "value",
+			},
 		},
 		"message": "message",
 		"code":    int32(100),

diff --git a/x-pack/libbeat/processors/add_cloudfoundry_metadata/_meta/fields.yml b/x-pack/libbeat/processors/add_cloudfoundry_metadata/_meta/fields.yml
@@ -37,3 +37,9 @@
           type: keyword
           description: >
             Cloud Foundry organization name
+
+        - name: tags.*
+          type: object
+          object_type: keyword
+          description: >
+            Cloud Foundry tags.
diff --git a/x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json b/x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json
@@ -1,35 +1,32 @@
 {
-    "@timestamp": "2020-02-27T18:09:35.583Z",
+    "@timestamp": "2017-10-12T08:05:34.853Z",
     "cloudfoundry": {
+        "app": {
+            "id": "ba41cfb8-8658-47d3-a021-2d1a6321aa42"
+        },
         "container": {
+            "cpu.pct": 0.22287691157883535,
+            "disk.bytes": 224665600,
+            "disk.quota.bytes": 268435456,
             "instance_index": 0,
-            "cpu.pct": 0.18815706013687805,
-            "memory.bytes": 23320985,
-            "memory.quota.bytes": 134217728,
-            "disk.bytes": 186187776,
-            "timestamp": "2020-02-27T18:09:35.583Z",
-            "type": "container",
-            "disk.quota.bytes": 2122136037
+            "memory.bytes": 175937174,
+            "memory.quota.bytes": 536870912,
+            "timestamp": "2020-09-22T17:17:47.928768995+02:00"
         },
         "envelope": {
-            "origin": "rep",
-            "deployment": "cf",
-            "ip": "10.144.0.13",
-            "job": "diego-cell",
-            "index": "3a0034ac-59ef-43e2-8b06-b14b32fe62f0"
+            "deployment": "cf-6b7aee31c8d07637ad78",
+            "index": "cd6c6b2c-797b-49a9-b1a2-a3b52da80965",
+            "ip": "192.168.16.32",
+            "job": "diego_cell",
+            "origin": "rep"
         },
-        "app": {
-            "id": "8cfc6857-3576-45a9-b1cd-890b28aa9c4a"
-        }
-    },
-    "event": {
-        "dataset": "cloudfoundry.container",
-        "module": "cloudfoundry"
-    },
-    "metricset": {
-        "name": "container"
+        "tags": {
+            "product": "Pivotal Application Service",
+            "source_id": "ba41cfb8-8658-47d3-a021-2d1a6321aa42"
+        },
+        "type": "container"
     },
     "service": {
         "type": "cloudfoundry"
     }
-}
+}
diff --git a/x-pack/metricbeat/module/cloudfoundry/container/container_integration_test.go b/x-pack/metricbeat/module/cloudfoundry/container/container_integration_test.go
@@ -49,5 +49,6 @@ func TestData(t *testing.T) {
 	require.NotEmpty(t, events)
 
 	beatEvent := mbtest.StandardizeEvent(ms, events[0])
+	mtest.CleanFields(beatEvent)
 	mbtest.WriteEventToDataJSON(t, beatEvent, "")
 }
diff --git a/x-pack/metricbeat/module/cloudfoundry/counter/_meta/data.json b/x-pack/metricbeat/module/cloudfoundry/counter/_meta/data.json
@@ -1,29 +1,27 @@
 {
-    "@timestamp": "2020-02-24T18:49:02.674Z",
+    "@timestamp": "2017-10-12T08:05:34.853Z",
     "cloudfoundry": {
         "counter": {
-            "total": 0,
-            "name": "promhttp_metric_handler_errors_total",
-            "delta": 0,
-            "timestamp": "2020-02-24T18:49:02.674Z",
-            "type": "counter"
+            "delta": 594,
+            "name": "egress",
+            "timestamp": "2020-09-22T17:17:41.065425923+02:00",
+            "total": 1563170851
         },
         "envelope": {
-            "origin": "loggregator_forwarder_agent",
-            "deployment": "cf",
-            "ip": "10.144.0.13",
-            "job": "diego-cell",
-            "index": "1e58e943-a498-4339-9d5f-2885bf223db9"
-        }
-    },
-    "event": {
-        "dataset": "cloudfoundry.counter",
-        "module": "cloudfoundry"
-    },
-    "metricset": {
-        "name": "counter"
+            "deployment": "cf-6b7aee31c8d07637ad78",
+            "index": "15e472dc-d37c-4e88-a4ff-96bd1ee80e57",
+            "ip": "192.168.16.57",
+            "job": "doppler",
+            "origin": "loggregator.doppler"
+        },
+        "tags": {
+            "metric_version": "2.0",
+            "product": "Pivotal Application Service",
+            "source_id": "doppler"
+        },
+        "type": "counter"
     },
     "service": {
         "type": "cloudfoundry"
     }
-}
+}
diff --git a/x-pack/metricbeat/module/cloudfoundry/counter/counter_integration_test.go b/x-pack/metricbeat/module/cloudfoundry/counter/counter_integration_test.go
@@ -49,5 +49,6 @@ func TestData(t *testing.T) {
 	require.NotEmpty(t, events)
 
 	beatEvent := mbtest.StandardizeEvent(ms, events[0])
+	mtest.CleanFields(beatEvent)
 	mbtest.WriteEventToDataJSON(t, beatEvent, "")
 }
diff --git a/x-pack/metricbeat/module/cloudfoundry/mtest/config.go b/x-pack/metricbeat/module/cloudfoundry/mtest/config.go
@@ -7,6 +7,7 @@ package mtest
 import (
 	"testing"
 
+	"github.com/elastic/beats/v7/libbeat/beat"
 	cftest "github.com/elastic/beats/v7/x-pack/libbeat/common/cloudfoundry/test"
 )
 
@@ -19,3 +20,8 @@ func GetConfig(t *testing.T, metricset string) map[string]interface{} {
 
 	return config
 }
+
+// CleanFields removes fields that can contain data of real deployments
+func CleanFields(e beat.Event) {
+	e.Fields.Delete("cloudfoundry.tags.system_domain")
+}
diff --git a/x-pack/metricbeat/module/cloudfoundry/value/_meta/data.json b/x-pack/metricbeat/module/cloudfoundry/value/_meta/data.json
@@ -1,29 +1,27 @@
 {
-    "@timestamp": "2020-02-27T18:20:48.379Z",
+    "@timestamp": "2017-10-12T08:05:34.853Z",
     "cloudfoundry": {
-        "value": {
-            "unit": "",
-            "value": 5.125735912782789e-05,
-            "timestamp": "2020-02-27T18:20:48.379Z",
-            "type": "value",
-            "name": "go_memstats_gc_cpu_fraction"
-        },
         "envelope": {
-            "origin": "loggregator_forwarder_agent",
-            "deployment": "cf",
-            "ip": "10.144.0.11",
-            "job": "scheduler",
-            "index": "d6b7bb59-ac57-4ed5-a0a7-0b23e19d9f8f"
+            "deployment": "cf-6b7aee31c8d07637ad78",
+            "index": "8895cf68-2e4f-4c50-9842-e73febd65c2d",
+            "ip": "192.168.16.31",
+            "job": "diego_cell",
+            "origin": "rep"
+        },
+        "tags": {
+            "instance_id": "8895cf68-2e4f-4c50-9842-e73febd65c2d",
+            "product": "Pivotal Application Service",
+            "source_id": "rep"
+        },
+        "type": "value",
+        "value": {
+            "name": "memoryStats.numBytesAllocatedStack",
+            "timestamp": "2020-09-22T17:17:40.257407125+02:00",
+            "unit": "Bytes",
+            "value": 3899392
         }
     },
-    "event": {
-        "dataset": "cloudfoundry.value",
-        "module": "cloudfoundry"
-    },
-    "metricset": {
-        "name": "value"
-    },
     "service": {
         "type": "cloudfoundry"
     }
-}
+}
diff --git a/x-pack/metricbeat/module/cloudfoundry/value/value_integration_test.go b/x-pack/metricbeat/module/cloudfoundry/value/value_integration_test.go
@@ -49,5 +49,6 @@ func TestData(t *testing.T) {
 	require.NotEmpty(t, events)
 
 	beatEvent := mbtest.StandardizeEvent(ms, events[0])
+	mtest.CleanFields(beatEvent)
 	mbtest.WriteEventToDataJSON(t, beatEvent, "")
 }