feat: add tags to the Volume and Cluster dashboards

cmd/collectors/rest/plugins/cluster/cluster.go

@@ -0,0 +1,93 @@
+package cluster
+
+import (
+	"github.com/netapp/harvest/v2/cmd/poller/plugin"
+	"github.com/netapp/harvest/v2/pkg/conf"
+	"github.com/netapp/harvest/v2/pkg/matrix"
+	"github.com/netapp/harvest/v2/pkg/slogx"
+	"github.com/netapp/harvest/v2/pkg/tree/node"
+	"github.com/netapp/harvest/v2/pkg/util"
+	"log/slog"
+	"strings"
+)
+
+type Cluster struct {
+	*plugin.AbstractPlugin
+	tags *matrix.Matrix
+}
+
+func New(p *plugin.AbstractPlugin) plugin.Plugin {
+	return &Cluster{AbstractPlugin: p}
+}
+
+func (c *Cluster) Init(_ conf.Remote) error {
+	var err error
+
+	if err := c.InitAbc(); err != nil {
+		return err
+	}
+
+	c.tags = matrix.New(c.Parent+".Cluster", "cluster", "cluster")
+	exportOptions := node.NewS("export_options")
+	instanceKeys := exportOptions.NewChildS("instance_keys", "")
+	instanceKeys.NewChildS("", "tag")
+	c.tags.SetExportOptions(exportOptions)
+	_, err = c.tags.NewMetricFloat64("tags", "tags")
+	if err != nil {
+		c.SLogger.Error("add metric", slogx.Err(err))
+		return err
+	}
+
+	return nil
+}
+
+func (c *Cluster) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) {
+	data := dataMap[c.Object]
+
+	// Based on the tags array, cluster_tags instances/metrics would be created
+	c.handleTags(data)
+
+	return []*matrix.Matrix{c.tags}, nil, nil
+}
+
+func (c *Cluster) handleTags(data *matrix.Matrix) {
+	var (
+		tagInstance *matrix.Instance
+		err         error
+	)
+
+	// Purge and reset data
+	c.tags.PurgeInstances()
+	c.tags.Reset()
+
+	// Set all global labels
+	c.tags.SetGlobalLabels(data.GetGlobalLabels())
+
+	// Based on the tags array, cluster_tags instances/metrics would be created.
+	for _, cluster := range data.GetInstances() {
+		if tags := cluster.GetLabel("tags"); tags != "" {
+			for _, tag := range strings.Split(tags, ",") {
+				tagInstanceKey := data.GetGlobalLabels()["cluster"] + tag
+				if tagInstance, err = c.tags.NewInstance(tagInstanceKey); err != nil {
+					c.SLogger.Error(
+						"Failed to create tag instance",
+						slogx.Err(err),
+						slog.String("tagInstanceKey", tagInstanceKey),
+					)
+					return
+				}
+
+				tagInstance.SetLabel("tag", tag)
+
+				m := c.tags.GetMetric("tags")
+				// populate numeric data
+				value := 1.0
+				if err = m.SetValueFloat64(tagInstance, value); err != nil {
+					c.SLogger.Error("Failed to parse value", slogx.Err(err), slog.Float64("value", value))
+				} else {
+					c.SLogger.Debug("added value", slog.Float64("value", value))
+				}
+			}
+		}
+	}
+}

cmd/collectors/rest/plugins/volume/volume.go

@@ -18,6 +18,7 @@ import (
 	"github.com/tidwall/gjson"
 	"log/slog"
 	"strconv"
+	"strings"
 	"time"
 )
 
@@ -28,8 +29,10 @@ type Volume struct {
 	*plugin.AbstractPlugin
 	currentVal            int
 	client                *rest.Client
-	aggrsMap              map[string]bool // aggregate-name -> exist map
+	aggrsMap              map[string]bool      // aggregate-name -> exist map
+	volTagMap             map[string]volumeTag // volume-key -> volumeTag map
 	arw                   *matrix.Matrix
+	tags                  *matrix.Matrix
 	includeConstituents   bool
 	isArwSupportedVersion bool
 }
@@ -45,6 +48,12 @@ type volumeInfo struct {
 	isDestinationCloud       string
 }
 
+type volumeTag struct {
+	vol  string
+	svm  string
+	tags string
+}
+
 func New(p *plugin.AbstractPlugin) plugin.Plugin {
 	return &Volume{AbstractPlugin: p}
 }
@@ -58,6 +67,7 @@ func (v *Volume) Init(remote conf.Remote) error {
 	}
 
 	v.aggrsMap = make(map[string]bool)
+	v.volTagMap = make(map[string]volumeTag)
 
 	// Assigned the value to currentVal so that plugin would be invoked first time to populate cache.
 	v.currentVal = v.SetPluginInterval()
@@ -87,6 +97,19 @@ func (v *Volume) Init(remote conf.Remote) error {
 		return err
 	}
 
+	v.tags = matrix.New(v.Parent+".Volume", "volume", "volume")
+	exportOptions = node.NewS("export_options")
+	instanceKeys = exportOptions.NewChildS("instance_keys", "")
+	instanceKeys.NewChildS("", "tag")
+	instanceKeys.NewChildS("", "svm")
+	instanceKeys.NewChildS("", "volume")
+	v.tags.SetExportOptions(exportOptions)
+	_, err = v.tags.NewMetricFloat64("tags", "tags")
+	if err != nil {
+		v.SLogger.Error("add metric", slogx.Err(err))
+		return err
+	}
+
 	// Read template to decide inclusion of flexgroup constituents
 	v.includeConstituents = collectors.ReadPluginKey(v.Params, "include_constituents")
 	// ARW feature is supported from 9.10 onwards, If we ask this field in Rest call in plugin, then it will be failed.
@@ -121,28 +144,40 @@ func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util
 	if err != nil {
 		v.SLogger.Error("Failed to collect volume info data", slogx.Err(err))
 	} else {
-		// update volume instance labels
+		// update volume instance labels and populate volTagMap
 		v.updateVolumeLabels(data, volumeMap)
 	}
 
 	// parse anti_ransomware_start_time, antiRansomwareState for all volumes and export at cluster level
 	v.handleARWProtection(data)
 
+	// Based on the tags array in volTagMap, volume_tags instances/metrics would be created
+	v.handleTags(data.GetGlobalLabels())
+
 	v.currentVal++
-	return []*matrix.Matrix{v.arw}, v.client.Metadata, nil
+	return []*matrix.Matrix{v.arw, v.tags}, v.client.Metadata, nil
 }
 
 func (v *Volume) updateVolumeLabels(data *matrix.Matrix, volumeMap map[string]volumeInfo) {
 	var err error
 
+	// clear volTagMap
+	clear(v.volTagMap)
+
 	cloneSplitEstimateMetric := data.GetMetric("clone_split_estimate")
 	if cloneSplitEstimateMetric == nil {
 		if cloneSplitEstimateMetric, err = data.NewMetricFloat64("clone_split_estimate"); err != nil {
 			v.SLogger.Error("error while creating clone split estimate metric", slogx.Err(err))
 			return
 		}
 	}
-	for _, volume := range data.GetInstances() {
+	for vKey, volume := range data.GetInstances() {
+		// update volumeTagMap used to update tag details
+		svm := volume.GetLabel("svm")
+		vol := volume.GetLabel("volume")
+		tags := volume.GetLabel("tags")
+		v.volTagMap[vKey] = volumeTag{vol: vol, svm: svm, tags: tags}
+
 		if !volume.IsExportable() {
 			continue
 		}
@@ -344,3 +379,49 @@ func (v *Volume) updateAggrMap(disks []gjson.Result) {
 		}
 	}
 }
+
+func (v *Volume) handleTags(globalLabels map[string]string) {
+	var (
+		tagInstance *matrix.Instance
+		err         error
+	)
+
+	// Purge and reset data
+	v.tags.PurgeInstances()
+	v.tags.Reset()
+
+	// Set all global labels
+	v.tags.SetGlobalLabels(globalLabels)
+
+	// Based on the tags array, volume_tags instances/metrics would be created.
+	for _, volume := range v.volTagMap {
+		svm := volume.svm
+		vol := volume.vol
+		if tags := volume.tags; tags != "" {
+			for _, tag := range strings.Split(tags, ",") {
+				tagInstanceKey := globalLabels["cluster"] + svm + vol + tag
+				if tagInstance, err = v.tags.NewInstance(tagInstanceKey); err != nil {
+					v.SLogger.Error(
+						"Failed to create tag instance",
+						slogx.Err(err),
+						slog.String("tagInstanceKey", tagInstanceKey),
+					)
+					return
+				}
+
+				tagInstance.SetLabel("tag", tag)
+				tagInstance.SetLabel("volume", vol)
+				tagInstance.SetLabel("svm", svm)
+
+				m := v.tags.GetMetric("tags")
+				// populate numeric data
+				value := 1.0
+				if err = m.SetValueFloat64(tagInstance, value); err != nil {
+					v.SLogger.Error("Failed to parse value", slogx.Err(err), slog.Float64("value", value))
+				} else {
+					v.SLogger.Debug("added value", slog.Float64("value", value))
+				}
+			}
+		}
+	}
+}

cmd/collectors/rest/rest.go

@@ -6,6 +6,7 @@ import (
 	"github.com/netapp/harvest/v2/cmd/collectors"
 	"github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/aggregate"
 	"github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/certificate"
+	"github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/cluster"
 	"github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/disk"
 	"github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/health"
 	"github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/metroclustercheck"
@@ -484,6 +485,8 @@ func (r *Rest) LoadPlugin(kind string, abc *plugin.AbstractPlugin) plugin.Plugin
 	switch kind {
 	case "Aggregate":
 		return aggregate.New(abc)
+	case "Cluster":
+		return cluster.New(abc)
 	case "Disk":
 		return disk.New(abc)
 	case "Health":

cmd/collectors/restperf/plugins/volume/volume.go

@@ -79,7 +79,7 @@ func (v *Volume) fetchVolumes() map[string]string {
 
 	href := rest.NewHrefBuilder().
 		APIPath(query).
-		Fields([]string{"volume", "volume_style_extended"}).
+		Fields([]string{"volume", "vserver", "volume_style_extended"}).
 		Filter([]string{"is_constituent=*"}).
 		MaxRecords(collectors.DefaultBatchSize).
 		Build()
@@ -101,7 +101,8 @@ func (v *Volume) fetchVolumes() map[string]string {
 		}
 		styleExtended := volume.Get("volume_style_extended").String()
 		name := volume.Get("volume").String()
-		volumesMap[name] = styleExtended
+		svm := volume.Get("vserver").String()
+		volumesMap[svm+name] = styleExtended
 	}
 
 	return volumesMap

cmd/collectors/restperf/plugins/volume/volume_test.go

@@ -37,26 +37,26 @@ func runVolumeTest(t *testing.T, createVolume func(params *node.Node) plugin.Plu
 	instance1.SetLabel("volume", "RahulTest__0001")
 	instance1.SetLabel("svm", "svm1")
 	instance1.SetLabel("aggr", "aggr1")
-	volumesMap["RahulTest__0001"] = "flexgroup_constituent"
+	volumesMap["svm1"+"RahulTest__0001"] = "flexgroup_constituent"
 
 	instance2, _ := data.NewInstance("RahulTest__0002")
 	instance2.SetLabel("volume", "RahulTest__0002")
 	instance2.SetLabel("svm", "svm1")
 	instance2.SetLabel("aggr", "aggr2")
-	volumesMap["RahulTest__0002"] = "flexgroup_constituent"
+	volumesMap["svm1"+"RahulTest__0002"] = "flexgroup_constituent"
 
 	instance3, _ := data.NewInstance("RahulTest__0003")
 	instance3.SetLabel("volume", "RahulTest__0003")
 	instance3.SetLabel("svm", "svm1")
 	instance3.SetLabel("aggr", "aggr3")
-	volumesMap["RahulTest__0003"] = "flexgroup_constituent"
+	volumesMap["svm1"+"RahulTest__0003"] = "flexgroup_constituent"
 
 	// Create a simple volume instance
 	simpleInstance, _ := data.NewInstance("SimpleVolume")
 	simpleInstance.SetLabel("volume", "SimpleVolume")
 	simpleInstance.SetLabel("svm", "svm1")
 	simpleInstance.SetLabel("aggr", "aggr4")
-	volumesMap["SimpleVolume"] = "flexvol"
+	volumesMap["svm1"+"SimpleVolume"] = "flexvol"
 
 	// Create latency and ops metrics
 	latencyMetric, _ := data.NewMetricFloat64("read_latency")

cmd/collectors/volume.go

@@ -40,10 +40,11 @@ func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string
 
 	for _, i := range data.GetInstances() {
 		volName := i.GetLabel("volume")
-		switch volumesMap[volName] {
+		svmName := i.GetLabel("svm")
+		switch volumesMap[svmName+volName] {
 		case "flexgroup_constituent":
 			match := flexgroupRegex.FindStringSubmatch(volName)
-			key := i.GetLabel("svm") + "." + match[1]
+			key := svmName + "." + match[1]
 			if cache.GetInstance(key) == nil {
 				fg, _ := cache.NewInstance(key)
 				fg.SetLabels(maps.Clone(i.GetLabels()))
@@ -72,7 +73,7 @@ func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string
 			i.SetExportable(includeConstituents)
 		case "flexvol":
 			i.SetLabel(style, "flexvol")
-			key := i.GetLabel("svm") + "." + volName
+			key := svmName + "." + volName
 			flexvolInstance, err := volumeAggrmetric.NewInstance(key)
 			if err != nil {
 				logger.Error("Failed to create new instance", slogx.Err(err), slog.String("key", key))
@@ -91,11 +92,12 @@ func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string
 	recordFGFalse := make(map[string]*set.Set)
 	for _, i := range data.GetInstances() {
 		volName := i.GetLabel("volume")
-		if volumesMap[volName] != "flexgroup_constituent" {
+		svmName := i.GetLabel("svm")
+		if volumesMap[svmName+volName] != "flexgroup_constituent" {
 			continue
 		}
 		match := flexgroupRegex.FindStringSubmatch(volName)
-		key := i.GetLabel("svm") + "." + match[1]
+		key := svmName + "." + match[1]
 
 		flexgroupInstance := volumeAggrmetric.GetInstance(key)
 		if flexgroupInstance != nil {

cmd/collectors/zapiperf/plugins/volume/volume.go

@@ -92,6 +92,7 @@ func (v *Volume) fetchVolumes() map[string]string {
 	volumeAttributes := node.NewXMLS("desired-attributes")
 	volumeIDAttributes := node.NewXMLS("volume-id-attributes")
 	volumeIDAttributes.NewChildS("name", "")
+	volumeIDAttributes.NewChildS("owning-vserver-name", "")
 	volumeIDAttributes.NewChildS("style-extended", "")
 	volumeAttributes.AddChild(volumeIDAttributes)
 	desired.AddChild(volumeAttributes)
@@ -119,7 +120,8 @@ func (v *Volume) fetchVolumes() map[string]string {
 		for _, volume := range volumes {
 			styleExtended := volume.GetChildS("volume-id-attributes").GetChildContentS("style-extended")
 			name := volume.GetChildS("volume-id-attributes").GetChildContentS("name")
-			volumesMap[name] = styleExtended
+			svm := volume.GetChildS("volume-id-attributes").GetChildContentS("owning-vserver-name")
+			volumesMap[svm+name] = styleExtended
 		}
 	}
 

cmd/tools/generate/counter.go

@@ -172,6 +172,7 @@ var (
 		"_labels",
 		"volume_arw_status",
 		"ALERTS",
+		"_tags",
 	}
 
 	// Exclude extra metrics for ZAPI

cmd/tools/grafana/dashboard_test.go

@@ -699,6 +699,7 @@ func checkVariablesHaveAll(t *testing.T, path string, data []byte) {
 	}
 	exceptionForAllValues := map[string]bool{
 		"cmode/security.json": true,
+		"cmode/cluster.json":  true,
 	}
 
 	if exceptionToAll[ShortPath(path)] {