[beat] Set cluster UUID in state metricset correctly (#37419)

* [beat] Set cluster UUID in state metricset correctly * Add test * Fix file ending with new line * Fix spacing error
elastic · Dec 13, 2023 · ef1bce5 · ef1bce5
1 parent fe9b7ce
commit ef1bce5
Show file tree

Hide file tree

Showing 6 changed files with 283 additions and 7 deletions.
diff --git a/metricbeat/module/beat/state/_meta/test/uuid_es_output.json b/metricbeat/module/beat/state/_meta/test/uuid_es_output.json
@@ -0,0 +1,43 @@
+{
+    "beat": {
+        "name": "Shaunaks-MBP-2"
+    },
+    "host": {
+        "architecture": "x86_64",
+        "hostname": "Shaunaks-MBP-2",
+        "id": "EF6274EA-462F-5316-A14A-850E7BFD8126",
+        "os": {
+            "build": "18F132",
+            "family": "darwin",
+            "kernel": "18.6.0",
+            "name": "Mac OS X",
+            "platform": "darwin",
+            "version": "10.14.5"
+        }
+    },
+    "management": {
+        "enabled": false
+    },
+    "module": {
+        "count": 3,
+        "names": [
+            "system"
+        ]
+    },
+    "output": {
+        "name": "elasticsearch"
+    },
+    "outputs": {
+        "elasticsearch": {
+            "cluster_uuid": "uuid_from_es_output"
+        }
+    },
+    "queue": {
+        "name": "mem"
+    },
+    "service": {
+        "id": "1f0c187b-f2ef-4950-b9cc-dd6864b9191a",
+        "name": "metricbeat",
+        "version": "8.0.0"
+    }
+}
diff --git a/metricbeat/module/beat/state/_meta/test/uuid_es_output_pre_connect.json b/metricbeat/module/beat/state/_meta/test/uuid_es_output_pre_connect.json
@@ -0,0 +1,43 @@
+{
+    "beat": {
+        "name": "Shaunaks-MBP-2"
+    },
+    "host": {
+        "architecture": "x86_64",
+        "hostname": "Shaunaks-MBP-2",
+        "id": "EF6274EA-462F-5316-A14A-850E7BFD8126",
+        "os": {
+            "build": "18F132",
+            "family": "darwin",
+            "kernel": "18.6.0",
+            "name": "Mac OS X",
+            "platform": "darwin",
+            "version": "10.14.5"
+        }
+    },
+    "management": {
+        "enabled": false
+    },
+    "module": {
+        "count": 3,
+        "names": [
+            "system"
+        ]
+    },
+    "output": {
+        "name": "elasticsearch"
+    },
+    "outputs": {
+        "elasticsearch": {
+            "cluster_uuid": ""
+        }
+    },
+    "queue": {
+        "name": "mem"
+    },
+    "service": {
+        "id": "1f0c187b-f2ef-4950-b9cc-dd6864b9191a",
+        "name": "metricbeat",
+        "version": "8.0.0"
+    }
+}
diff --git a/metricbeat/module/beat/state/_meta/test/uuid_monitoring_config.json b/metricbeat/module/beat/state/_meta/test/uuid_monitoring_config.json
@@ -0,0 +1,52 @@
+{
+    "beat": {
+        "name": "MacBook-Pro"
+    },
+    "host": {
+        "architecture": "arm64",
+        "hostname": "MacBook-Pro",
+        "id": "B0C1E948-241E-53FD-A6A3-0D0F352403AF",
+        "os": {
+            "build": "23B92",
+            "family": "darwin",
+            "kernel": "23.1.0",
+            "name": "macOS",
+            "platform": "darwin",
+            "version": "14.1.2"
+        }
+    },
+    "input": {
+        "count": 1,
+        "names": [
+            "log"
+        ]
+    },
+    "management": {
+        "enabled": false
+    },
+    "module": {
+        "count": 0,
+        "names": []
+    },
+    "monitoring": {
+        "cluster_uuid": "uuid_from_monitoring_config"
+    },
+    "output": {
+        "batch_size": 2048,
+        "clients": 1,
+        "name": "logstash"
+    },
+    "outputs": {
+        "elasticsearch": {
+            "cluster_uuid": ""
+        }
+    },
+    "queue": {
+        "name": "mem"
+    },
+    "service": {
+        "id": "8d6af17b-cf55-4029-bd22-64cd538acbd0",
+        "name": "filebeat",
+        "version": "8.13.0"
+    }
+}
diff --git a/metricbeat/module/beat/state/_meta/test/uuid_no_monitoring_config.json b/metricbeat/module/beat/state/_meta/test/uuid_no_monitoring_config.json
@@ -0,0 +1,52 @@
+{
+    "beat": {
+        "name": "MacBook-Pro"
+    },
+    "host": {
+        "architecture": "arm64",
+        "hostname": "MacBook-Pro",
+        "id": "B0C1E948-241E-53FD-A6A3-0D0F352403AF",
+        "os": {
+            "build": "23B92",
+            "family": "darwin",
+            "kernel": "23.1.0",
+            "name": "macOS",
+            "platform": "darwin",
+            "version": "14.1.2"
+        }
+    },
+    "input": {
+        "count": 1,
+        "names": [
+            "log"
+        ]
+    },
+    "management": {
+        "enabled": false
+    },
+    "module": {
+        "count": 0,
+        "names": []
+    },
+    "monitoring": {
+        "cluster_uuid": ""
+    },
+    "output": {
+        "batch_size": 2048,
+        "clients": 1,
+        "name": "logstash"
+    },
+    "outputs": {
+        "elasticsearch": {
+            "cluster_uuid": ""
+        }
+    },
+    "queue": {
+        "name": "mem"
+    },
+    "service": {
+        "id": "8d6af17b-cf55-4029-bd22-64cd538acbd0",
+        "name": "filebeat",
+        "version": "8.13.0"
+    }
+}
diff --git a/metricbeat/module/beat/state/data.go b/metricbeat/module/beat/state/data.go
@@ -77,22 +77,19 @@ func eventMapping(r mb.ReporterV2, info beat.Info, content []byte, isXpack bool)
 		return fmt.Errorf("failure parsing Beat's State API response: %w", err)
 	}
 
-	event.MetricSetFields, _ = schema.Apply(data)
-
 	clusterUUID := getMonitoringClusterUUID(data)
 	if clusterUUID == "" {
 		if isOutputES(data) {
 			clusterUUID = getClusterUUID(data)
-			if clusterUUID != "" {
-				event.ModuleFields.Put("elasticsearch.cluster.id", clusterUUID)
 
-				if event.MetricSetFields != nil {
-					event.MetricSetFields.Put("cluster.uuid", clusterUUID)
-				}
+			if clusterUUID == "" {
+				return nil
 			}
 		}
 	}
 
+	event.ModuleFields.Put("elasticsearch.cluster.id", clusterUUID)
+
 	event.MetricSetFields, _ = schema.Apply(data)
 
 	if event.MetricSetFields != nil {

diff --git a/metricbeat/module/beat/state/data_test.go b/metricbeat/module/beat/state/data_test.go
@@ -53,3 +53,92 @@ func TestEventMapping(t *testing.T) {
 		require.Equal(t, 0, len(reporter.GetErrors()), f)
 	}
 }
+
+func TestUuidFromEsOutput(t *testing.T) {
+	reporter := &mbtest.CapturingReporterV2{}
+
+	info := beat.Info{
+		UUID: "1234",
+		Beat: "testbeat",
+	}
+
+	input, err := ioutil.ReadFile("./_meta/test/uuid_es_output.json")
+	require.NoError(t, err)
+
+	err = eventMapping(reporter, info, input, true)
+	require.NoError(t, err)
+	require.True(t, len(reporter.GetEvents()) >= 1)
+	require.Equal(t, 0, len(reporter.GetErrors()))
+
+	event := reporter.GetEvents()[0]
+
+	uuid, err := event.ModuleFields.GetValue("elasticsearch.cluster.id")
+	require.NoError(t, err)
+
+	require.Equal(t, "uuid_from_es_output", uuid)
+}
+
+func TestNoEventIfEsOutputButNoUuidYet(t *testing.T) {
+	reporter := &mbtest.CapturingReporterV2{}
+
+	info := beat.Info{
+		UUID: "1234",
+		Beat: "testbeat",
+	}
+
+	input, err := ioutil.ReadFile("./_meta/test/uuid_es_output_pre_connect.json")
+	require.NoError(t, err)
+
+	err = eventMapping(reporter, info, input, true)
+	require.NoError(t, err)
+	require.Equal(t, 0, len(reporter.GetEvents()))
+	require.Equal(t, 0, len(reporter.GetErrors()))
+}
+
+func TestUuidFromMonitoringConfig(t *testing.T) {
+	reporter := &mbtest.CapturingReporterV2{}
+
+	info := beat.Info{
+		UUID: "1234",
+		Beat: "testbeat",
+	}
+
+	input, err := ioutil.ReadFile("./_meta/test/uuid_monitoring_config.json")
+	require.NoError(t, err)
+
+	err = eventMapping(reporter, info, input, true)
+	require.NoError(t, err)
+	require.True(t, len(reporter.GetEvents()) >= 1)
+	require.Equal(t, 0, len(reporter.GetErrors()))
+
+	event := reporter.GetEvents()[0]
+
+	uuid, err := event.ModuleFields.GetValue("elasticsearch.cluster.id")
+	require.NoError(t, err)
+
+	require.Equal(t, "uuid_from_monitoring_config", uuid)
+}
+
+func TestNoUuidInMonitoringConfig(t *testing.T) {
+	reporter := &mbtest.CapturingReporterV2{}
+
+	info := beat.Info{
+		UUID: "1234",
+		Beat: "testbeat",
+	}
+
+	input, err := ioutil.ReadFile("./_meta/test/uuid_no_monitoring_config.json")
+	require.NoError(t, err)
+
+	err = eventMapping(reporter, info, input, true)
+	require.NoError(t, err)
+	require.True(t, len(reporter.GetEvents()) >= 1)
+	require.Equal(t, 0, len(reporter.GetErrors()))
+
+	event := reporter.GetEvents()[0]
+
+	uuid, err := event.ModuleFields.GetValue("elasticsearch.cluster.id")
+	require.NoError(t, err)
+
+	require.Equal(t, "", uuid)
+}