fix: handled flexgroup based on volume config call

cmd/collectors/restperf/plugins/volume/volume.go

@@ -3,22 +3,29 @@ package volume
 import (
 	"github.com/netapp/harvest/v2/cmd/collectors"
 	"github.com/netapp/harvest/v2/cmd/poller/plugin"
+	"github.com/netapp/harvest/v2/cmd/tools/rest"
+	"github.com/netapp/harvest/v2/pkg/conf"
 	"github.com/netapp/harvest/v2/pkg/matrix"
 	"github.com/netapp/harvest/v2/pkg/util"
+	"log/slog"
+	"time"
 )
 
 type Volume struct {
 	*plugin.AbstractPlugin
+	currentVal          int
 	styleType           string
 	includeConstituents bool
+	client              *rest.Client
+	volumesMap          map[string]string // volume-name -> volume-extended-style map
 }
 
 func New(p *plugin.AbstractPlugin) plugin.Plugin {
 	return &Volume{AbstractPlugin: p}
 }
 
 func (v *Volume) Init() error {
-
+	var err error
 	if err := v.InitAbc(); err != nil {
 		return err
 	}
@@ -29,15 +36,73 @@ func (v *Volume) Init() error {
 		v.styleType = "type"
 	}
 
+	v.volumesMap = make(map[string]string)
+
+	// Assigned the value to currentVal so that plugin would be invoked first time to populate cache.
+	v.currentVal = v.SetPluginInterval()
+
 	// Read template to decide inclusion of flexgroup constituents
 	v.includeConstituents = collectors.ReadPluginKey(v.Params, "include_constituents")
-	return nil
+
+	if v.Options.IsTest {
+		v.client = &rest.Client{Metadata: &util.Metadata{}}
+		return nil
+	}
+
+	timeout, _ := time.ParseDuration(rest.DefaultTimeout)
+	if v.client, err = rest.New(conf.ZapiPoller(v.ParentParams), timeout, v.Auth); err != nil {
+		v.SLogger.Error("connecting", slog.Any("err", err))
+		return err
+	}
+
+	return v.client.Init(5)
 }
 
 func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) {
 	data := dataMap[v.Object]
 	style := v.styleType
 	opsKeyPrefix := "temp_"
+	if v.currentVal >= v.PluginInvocationRate {
+		v.currentVal = 0
+		// Clean volumesMap map
+		clear(v.volumesMap)
+		v.volumesMap = v.fetchVolumes()
+	}
+
+	v.currentVal++
+	return collectors.ProcessFlexGroupData(v.SLogger, data, style, v.includeConstituents, opsKeyPrefix, v.volumesMap)
+}
+
+func (v *Volume) fetchVolumes() map[string]string {
+	volumesMap := make(map[string]string)
+	query := "api/private/cli/volume"
+
+	href := rest.NewHrefBuilder().
+		APIPath(query).
+		Fields([]string{"volume", "volume_style_extended"}).
+		Filter([]string{"is_constituent=*"}).
+		MaxRecords(collectors.DefaultBatchSize).
+		Build()
+
+	records, err := rest.FetchAll(v.client, href)
+	if err != nil {
+		v.SLogger.Error("Failed to fetch data", slog.Any("err", err), slog.String("href", href))
+		return nil
+	}
+
+	if len(records) == 0 {
+		return nil
+	}
+
+	for _, volume := range records {
+		if !volume.IsObject() {
+			v.SLogger.Warn("volume is not object, skipping", slog.String("type", volume.Type.String()))
+			continue
+		}
+		styleExtended := volume.Get("volume_style_extended").String()
+		name := volume.Get("volume").String()
+		volumesMap[name] = styleExtended
+	}
 
-	return collectors.ProcessFlexGroupData(v.SLogger, data, style, v.includeConstituents, opsKeyPrefix)
+	return volumesMap
 }

cmd/collectors/restperf/plugins/volume/volume_test.go

@@ -1,21 +1,29 @@
 package volume_test
 
 import (
+	"github.com/netapp/harvest/v2/cmd/collectors"
 	volume2 "github.com/netapp/harvest/v2/cmd/collectors/restperf/plugins/volume"
 	"github.com/netapp/harvest/v2/cmd/collectors/zapiperf/plugins/volume"
+	"github.com/netapp/harvest/v2/cmd/poller/options"
 	"log/slog"
+	"strconv"
 	"testing"
 
 	"github.com/netapp/harvest/v2/cmd/poller/plugin"
 	"github.com/netapp/harvest/v2/pkg/matrix"
 	"github.com/netapp/harvest/v2/pkg/tree/node"
 )
 
+const OpsKeyPrefix = "temp_"
+const StyleType = "style"
+const PollerName = "test"
+
 // Common test logic for RestPerf/ZapiPerf Volume plugin
 func runVolumeTest(t *testing.T, createVolume func(params *node.Node) plugin.Plugin, includeConstituents string, expectedCount int, setMetricNaN bool) {
 	params := node.NewS("Volume")
 	params.NewChildS("include_constituents", includeConstituents)
 	v := createVolume(params)
+	volumesMap := make(map[string]string)
 
 	// Initialize the plugin
 	if err := v.Init(); err != nil {
@@ -28,22 +36,26 @@ func runVolumeTest(t *testing.T, createVolume func(params *node.Node) plugin.Plu
 	instance1.SetLabel("volume", "RahulTest__0001")
 	instance1.SetLabel("svm", "svm1")
 	instance1.SetLabel("aggr", "aggr1")
+	volumesMap["RahulTest__0001"] = "flexgroup_constituent"
 
 	instance2, _ := data.NewInstance("RahulTest__0002")
 	instance2.SetLabel("volume", "RahulTest__0002")
 	instance2.SetLabel("svm", "svm1")
 	instance2.SetLabel("aggr", "aggr2")
+	volumesMap["RahulTest__0002"] = "flexgroup_constituent"
 
 	instance3, _ := data.NewInstance("RahulTest__0003")
 	instance3.SetLabel("volume", "RahulTest__0003")
 	instance3.SetLabel("svm", "svm1")
 	instance3.SetLabel("aggr", "aggr3")
+	volumesMap["RahulTest__0003"] = "flexgroup_constituent"
 
 	// Create a simple volume instance
 	simpleInstance, _ := data.NewInstance("SimpleVolume")
 	simpleInstance.SetLabel("volume", "SimpleVolume")
 	simpleInstance.SetLabel("svm", "svm1")
 	simpleInstance.SetLabel("aggr", "aggr4")
+	volumesMap["SimpleVolume"] = "flexvol"
 
 	// Create latency and ops metrics
 	latencyMetric, _ := data.NewMetricFloat64("read_latency")
@@ -73,12 +85,9 @@ func runVolumeTest(t *testing.T, createVolume func(params *node.Node) plugin.Plu
 	_ = latencyMetric.SetValueFloat64(simpleInstance, 50)
 	_ = opsMetric.SetValueFloat64(simpleInstance, 5)
 
-	dataMap := map[string]*matrix.Matrix{
-		"volume": data,
-	}
-
 	// Run the plugin
-	output, _, err := v.Run(dataMap)
+	boolValue, _ := strconv.ParseBool(includeConstituents)
+	output, _, err := collectors.ProcessFlexGroupData(slog.Default(), data, StyleType, boolValue, OpsKeyPrefix, volumesMap)
 	if err != nil {
 		t.Fatalf("Run method failed: %v", err)
 	}
@@ -230,13 +239,17 @@ func TestRunForAllImplementations(t *testing.T) {
 }
 
 func createRestVolume(params *node.Node) plugin.Plugin {
-	v := &volume2.Volume{AbstractPlugin: plugin.New("volume", nil, params, nil, "volume", nil)}
+	opts := options.New(options.WithConfPath("testdata/conf"))
+	opts.IsTest = true
+	v := &volume2.Volume{AbstractPlugin: plugin.New("volume", opts, params, nil, "volume", nil)}
 	v.SLogger = slog.Default()
 	return v
 }
 
 func createZapiVolume(params *node.Node) plugin.Plugin {
-	v := &volume.Volume{AbstractPlugin: plugin.New("volume", nil, params, nil, "volume", nil)}
+	opts := options.New(options.WithConfPath("testdata/conf"))
+	opts.IsTest = true
+	v := &volume.Volume{AbstractPlugin: plugin.New("volume", opts, params, nil, "volume", nil)}
 	v.SLogger = slog.Default()
 	return v
 }

cmd/collectors/volume.go

@@ -13,9 +13,14 @@ import (
 
 var flexgroupRegex = regexp.MustCompile(`^(.*)__(\d{4})$`)
 
-func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string, includeConstituents bool, opsKeyPrefix string) ([]*matrix.Matrix, *util.Metadata, error) {
+func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string, includeConstituents bool, opsKeyPrefix string, volumesMap map[string]string) ([]*matrix.Matrix, *util.Metadata, error) {
 	var err error
 
+	if volumesMap == nil {
+		logger.Info("volumes config data not found")
+		return nil, nil, nil
+	}
+
 	fgAggrMap := make(map[string]*set.Set)
 	flexgroupAggrsMap := make(map[string]*set.Set)
 
@@ -33,7 +38,10 @@ func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string
 	cache.UUID += ".Volume"
 
 	for _, i := range data.GetInstances() {
-		if match := flexgroupRegex.FindStringSubmatch(i.GetLabel("volume")); len(match) == 3 {
+		volName := i.GetLabel("volume")
+		switch volumesMap[volName] {
+		case "flexgroup_constituent":
+			match := flexgroupRegex.FindStringSubmatch(volName)
 			key := i.GetLabel("svm") + "." + match[1]
 			if cache.GetInstance(key) == nil {
 				fg, _ := cache.NewInstance(key)
@@ -61,9 +69,9 @@ func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string
 			flexgroupAggrsMap[key].Add(i.GetLabel("aggr"))
 			i.SetLabel(style, "flexgroup_constituent")
 			i.SetExportable(includeConstituents)
-		} else {
+		case "flexvol":
 			i.SetLabel(style, "flexvol")
-			key := i.GetLabel("svm") + "." + i.GetLabel("volume")
+			key := i.GetLabel("svm") + "." + volName
 			flexvolInstance, err := volumeAggrmetric.NewInstance(key)
 			if err != nil {
 				logger.Error("Failed to create new instance", slog.Any("err", err), slog.String("key", key))
@@ -81,10 +89,11 @@ func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string
 
 	recordFGFalse := make(map[string]*set.Set)
 	for _, i := range data.GetInstances() {
-		match := flexgroupRegex.FindStringSubmatch(i.GetLabel("volume"))
-		if len(match) != 3 {
+		volName := i.GetLabel("volume")
+		if volumesMap[volName] != "flexgroup_constituent" {
 			continue
 		}
+		match := flexgroupRegex.FindStringSubmatch(volName)
 		key := i.GetLabel("svm") + "." + match[1]
 
 		flexgroupInstance := volumeAggrmetric.GetInstance(key)

cmd/collectors/zapiperf/plugins/volume/volume.go

@@ -7,22 +7,31 @@ package volume
 import (
 	"github.com/netapp/harvest/v2/cmd/collectors"
 	"github.com/netapp/harvest/v2/cmd/poller/plugin"
+	"github.com/netapp/harvest/v2/pkg/api/ontapi/zapi"
+	"github.com/netapp/harvest/v2/pkg/conf"
 	"github.com/netapp/harvest/v2/pkg/matrix"
+	"github.com/netapp/harvest/v2/pkg/tree/node"
 	"github.com/netapp/harvest/v2/pkg/util"
+	"log/slog"
 )
 
+const batchSize = "500"
+
 type Volume struct {
 	*plugin.AbstractPlugin
+	currentVal          int
 	styleType           string
 	includeConstituents bool
+	client              *zapi.Client
+	volumesMap          map[string]string // volume-name -> volume-extended-style map
 }
 
 func New(p *plugin.AbstractPlugin) plugin.Plugin {
 	return &Volume{AbstractPlugin: p}
 }
 
 func (v *Volume) Init() error {
-
+	var err error
 	if err := v.InitAbc(); err != nil {
 		return err
 	}
@@ -33,15 +42,86 @@ func (v *Volume) Init() error {
 		v.styleType = "type"
 	}
 
+	if v.Options.IsTest {
+		return nil
+	}
+
+	v.volumesMap = make(map[string]string)
+
+	// Assigned the value to currentVal so that plugin would be invoked first time to populate cache.
+	v.currentVal = v.SetPluginInterval()
+
 	// Read template to decide inclusion of flexgroup constituents
 	v.includeConstituents = collectors.ReadPluginKey(v.Params, "include_constituents")
-	return nil
+
+	if v.client, err = zapi.New(conf.ZapiPoller(v.ParentParams), v.Auth); err != nil {
+		v.SLogger.Error("connecting", slog.Any("err", err))
+		return err
+	}
+	return v.client.Init(5)
 }
 
 func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) {
 	data := dataMap[v.Object]
 	style := v.styleType
 	opsKeyPrefix := "temp_"
+	if v.currentVal >= v.PluginInvocationRate {
+		v.currentVal = 0
+		// Clean volumesMap map
+		clear(v.volumesMap)
+		v.volumesMap = v.fetchVolumes()
+	}
+
+	v.currentVal++
+	return collectors.ProcessFlexGroupData(v.SLogger, data, style, v.includeConstituents, opsKeyPrefix, v.volumesMap)
+}
+
+func (v *Volume) fetchVolumes() map[string]string {
+	var (
+		result     *node.Node
+		volumes    []*node.Node
+		volumesMap map[string]string
+	)
+
+	volumesMap = make(map[string]string)
+	query := "volume-get-iter"
+	tag := "initial"
+	request := node.NewXMLS(query)
+	request.NewChildS("max-records", batchSize)
+	desired := node.NewXMLS("desired-attributes")
+	volumeAttributes := node.NewXMLS("desired-attributes")
+	volumeIDAttributes := node.NewXMLS("volume-id-attributes")
+	volumeIDAttributes.NewChildS("name", "")
+	volumeIDAttributes.NewChildS("style-extended", "")
+	volumeAttributes.AddChild(volumeIDAttributes)
+	desired.AddChild(volumeAttributes)
+	request.AddChild(desired)
+
+	for {
+		responseData, err := v.client.InvokeBatchRequest(request, tag, "")
+		if err != nil {
+			return nil
+		}
+		result = responseData.Result
+		tag = responseData.Tag
+
+		if result == nil {
+			break
+		}
+
+		if x := result.GetChildS("attributes-list"); x != nil {
+			volumes = x.GetChildren()
+		}
+		if len(volumes) == 0 {
+			return nil
+		}
+
+		for _, volume := range volumes {
+			styleExtended := volume.GetChildS("volume-id-attributes").GetChildContentS("style-extended")
+			name := volume.GetChildS("volume-id-attributes").GetChildContentS("name")
+			volumesMap[name] = styleExtended
+		}
+	}
 
-	return collectors.ProcessFlexGroupData(v.SLogger, data, style, v.includeConstituents, opsKeyPrefix)
+	return volumesMap
 }

cmd/poller/plugin/plugin.go

@@ -167,15 +167,18 @@ func (p *AbstractPlugin) SetPluginInterval() int {
 }
 
 func GetInterval(param *node.Node, defaultInterval time.Duration) float64 {
-	schedule := param.GetChildS("schedule")
-	if schedule != nil {
-		dataInterval := schedule.GetChildContentS("data")
-		if dataInterval != "" {
-			if durationVal, err := time.ParseDuration(dataInterval); err == nil {
-				return durationVal.Seconds()
+	if param != nil {
+		schedule := param.GetChildS("schedule")
+		if schedule != nil {
+			dataInterval := schedule.GetChildContentS("data")
+			if dataInterval != "" {
+				if durationVal, err := time.ParseDuration(dataInterval); err == nil {
+					return durationVal.Seconds()
+				}
 			}
 		}
 	}
+
 	return defaultInterval.Seconds()
 }