Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: handled user/group quota in HistoricalLabels #3060

Merged
merged 3 commits into from
Jul 23, 2024
Merged

feat: handled user/group quota in HistoricalLabels #3060

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
03e9be9
feat: handled user/group quota in HistoricalLabels
Hardikl Jul 22, 2024
d9ed0f8
feat: handled user/group quota in HistoricalLabels for zapi/rest
Hardikl Jul 23, 2024
d2f8dfc
feat: handled review comments
Hardikl Jul 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
156 changes: 44 additions & 112 deletions cmd/collectors/rest/plugins/qtree/qtree.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ import (
"github.com/netapp/harvest/v2/pkg/tree/node"
"github.com/netapp/harvest/v2/pkg/util"
"github.com/tidwall/gjson"
"strconv"
"strings"
"time"
)
Expand Down Expand Up @@ -106,7 +105,6 @@ func (q *Qtree) Init() error {
}

instanceKeys.NewChildS("", "type")
instanceKeys.NewChildS("", "index")
instanceKeys.NewChildS("", "unit")

q.data.SetExportOptions(exportOptions)
Expand Down Expand Up @@ -175,15 +173,9 @@ func (q *Qtree) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.
}

quotaCount := 0
cluster := data.GetGlobalLabels()["cluster"]

if q.historicalLabels {
// In 22.05, populate metrics with qtree prefix and old labels
err = q.handlingHistoricalMetrics(result, data, cluster, &quotaCount, &numMetrics)
} else {
// Populate metrics with quota prefix and current labels
err = q.handlingQuotaMetrics(result, &quotaCount, &numMetrics)
}
// Populate metrics with quota prefix
err = q.handlingQuotaMetrics(result, data, &quotaCount, &numMetrics)

if err != nil {
return nil, nil, err
Expand All @@ -207,11 +199,11 @@ func (q *Qtree) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.
return []*matrix.Matrix{q.data}, q.client.Metadata, nil
}

func (q *Qtree) handlingHistoricalMetrics(result []gjson.Result, data *matrix.Matrix, cluster string, quotaIndex *int, numMetrics *int) error {
func (q *Qtree) handlingQuotaMetrics(result []gjson.Result, data *matrix.Matrix, quotaCount *int, numMetrics *int) error {
for _, quota := range result {
var tree string
var quotaInstance *matrix.Instance
var err error
var qtreeInstance *matrix.Instance
var value float64

if !quota.IsObject() {
q.Logger.Error().Str("type", quota.Type.String()).Msg("Quota is not an object, skipping")
Expand All @@ -221,115 +213,39 @@ func (q *Qtree) handlingHistoricalMetrics(result []gjson.Result, data *matrix.Ma
if quota.Get("qtree.name").Exists() {
tree = quota.Get("qtree.name").String()
}
quotaType := quota.Get("type").String()
volume := quota.Get("volume.name").String()
vserver := quota.Get("svm.name").String()
qIndex := quota.Get("index").String()
quotaType := quota.Get("type").String()

// If quota-type is not a tree, then skip
if quotaType != "tree" {
continue
}

// Ex. InstanceKey: vserver1vol1qtree15989279
quotaInstanceKey := vserver + volume + tree + qIndex
uName := quota.Get("users.0.name").String()
uid := quota.Get("users.0.id").String()
group := quota.Get("group.name").String()
*quotaCount++

if quotaInstance = q.data.GetInstance(quotaInstanceKey); quotaInstance == nil {
if quotaInstance, err = q.data.NewInstance(quotaInstanceKey); err != nil {
q.Logger.Error().Stack().Err(err).Str("quotaInstanceKey", quotaInstanceKey).Msg("Failed to create quota instance")
return err
// In 22.05, populate metrics with qtree labels
if q.historicalLabels {
// qtree instancekey would be qtree, svm and volume(sorted keys)
qtreeInstance = data.GetInstance(tree + vserver + volume)
if qtreeInstance == nil {
q.Logger.Warn().
Str("tree", tree).
Str("volume", volume).
Str("vserver", vserver).
Msg("No instance matching tree.vserver.volume")
continue
}
}

// qtree instancekey would be qtree, svm and volume(sorted keys)
qtreeInstance := data.GetInstance(tree + vserver + volume)
if qtreeInstance == nil {
q.Logger.Warn().
Str("tree", tree).
Str("volume", volume).
Str("vserver", vserver).
Msg("No instance matching tree.vserver.volume")
continue
}

if !qtreeInstance.IsExportable() {
continue
}

for _, label := range q.data.GetExportOptions().GetChildS("instance_keys").GetAllChildContentS() {
if value := qtreeInstance.GetLabel(label); value != "" {
quotaInstance.SetLabel(label, value)
if !qtreeInstance.IsExportable() {
continue
}
}

// set labels
quotaInstance.SetLabel("type", quotaType)
quotaInstance.SetLabel("qtree", tree)
quotaInstance.SetLabel("volume", volume)
quotaInstance.SetLabel("svm", vserver)
quotaInstance.SetLabel("index", cluster+"_"+strconv.Itoa(*quotaIndex))

// If the Qtree is the volume itself, then qtree label is empty, so copy the volume name to qtree.
if tree == "" {
quotaInstance.SetLabel("qtree", volume)
}

*quotaIndex++
for attribute, m := range q.data.GetMetrics() {
value := 0.0

if attrValue := quota.Get(attribute); attrValue.Exists() {
// space limits are in bytes, converted to kilobytes
if attribute == "space.hard_limit" || attribute == "space.soft_limit" {
value = attrValue.Float() / 1024
quotaInstance.SetLabel("unit", "Kbyte")
if attribute == "space.soft_limit" {
t := q.data.GetMetric("threshold")
if err = t.SetValueFloat64(quotaInstance, value); err != nil {
q.Logger.Error().Err(err).Str("attribute", attribute).Float64("value", value).Msg("Failed to parse value")
} else {
*numMetrics++
}
}
} else {
value = attrValue.Float()
}
}

// populate numeric data
if err = m.SetValueFloat64(quotaInstance, value); err != nil {
q.Logger.Error().Stack().Err(err).Str("attribute", attribute).Float64("value", value).Msg("Failed to parse value")
// In 22.05, populate metrics value with 0
if q.historicalLabels {
value = 0.0
} else {
*numMetrics++
// set -1 for unlimited
value = -1.0
}
}
}
return nil
}

func (q *Qtree) handlingQuotaMetrics(result []gjson.Result, quotaCount *int, numMetrics *int) error {
for _, quota := range result {
var tree string

if !quota.IsObject() {
q.Logger.Error().Str("type", quota.Type.String()).Msg("Quota is not an object, skipping")
return errs.New(errs.ErrNoInstance, "quota is not an object")
}

if quota.Get("qtree.name").Exists() {
tree = quota.Get("qtree.name").String()
}
quotaType := quota.Get("type").String()
volume := quota.Get("volume.name").String()
vserver := quota.Get("svm.name").String()
uName := quota.Get("users.0.name").String()
uid := quota.Get("users.0.id").String()
group := quota.Get("group.name").String()
*quotaCount++

for attribute, m := range q.data.GetMetrics() {
// set -1 for unlimited
value := -1.0

quotaInstanceKey := vserver + "." + volume + "." + tree + "." + group + "." + uName + "." + attribute

Expand All @@ -338,12 +254,27 @@ func (q *Qtree) handlingQuotaMetrics(result []gjson.Result, quotaCount *int, num
q.Logger.Debug().Msgf("add (%s) instance: %v", attribute, err)
return err
}

// In 22.05, populate metrics with qtree labels
if q.historicalLabels {
for _, label := range q.data.GetExportOptions().GetChildS("instance_keys").GetAllChildContentS() {
if val := qtreeInstance.GetLabel(label); val != "" {
quotaInstance.SetLabel(label, val)
}
}
}

// set labels
quotaInstance.SetLabel("type", quotaType)
quotaInstance.SetLabel("qtree", tree)
quotaInstance.SetLabel("volume", volume)
quotaInstance.SetLabel("svm", vserver)

// If the Qtree is the volume itself, then qtree label is empty, so copy the volume name to qtree.
Hardikl marked this conversation as resolved.
Show resolved Hide resolved
if tree == "" {
quotaInstance.SetLabel("qtree", volume)
}

if quotaType == "user" {
if uName != "" {
quotaInstance.SetLabel("user", uName)
Expand All @@ -357,6 +288,7 @@ func (q *Qtree) handlingQuotaMetrics(result []gjson.Result, quotaCount *int, num
quotaInstance.SetLabel("group", uid)
}
}

if attrValue := quota.Get(attribute); attrValue.Exists() {
// space limits are in bytes, converted to kilobytes to match ZAPI
if attribute == "space.hard_limit" || attribute == "space.soft_limit" || attribute == "space.used.total" {
Expand Down
46 changes: 38 additions & 8 deletions cmd/collectors/rest/plugins/qtree/qtree_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"github.com/netapp/harvest/v2/cmd/poller/plugin"
"github.com/netapp/harvest/v2/pkg/logging"
"github.com/netapp/harvest/v2/pkg/matrix"
"github.com/netapp/harvest/v2/pkg/tree/node"
"github.com/tidwall/gjson"
"os"
"testing"
Expand All @@ -19,27 +20,56 @@ func NewQtree() *Qtree {
}

func TestHandlingQuotaMetrics(t *testing.T) {
q := NewQtree()

jsonResponse, err := os.ReadFile("testdata/quota.json")
if err != nil {
t.Fatalf("Failed to read JSON response from file: %v", err)
}

result := gjson.Get(string(jsonResponse), "records").Array()

// Case 1: with historicalLabels = false
q1 := NewQtree()
q1.historicalLabels = false
testLabels(t, q1, result, nil, "astra_300"+"."+"trident_qtree_pool_trident_TIXRBILLKA"+"."+"trident_pvc_19913841_a29f_4a54_8bc0_a3c1c4155826"+"."+""+"."+""+"."+"space.hard_limit", 3, 6, 5)
Hardikl marked this conversation as resolved.
Show resolved Hide resolved

// Case 2: with historicalLabels = true
q2 := NewQtree()
data := matrix.New(q2.Parent+".Qtree", "qtree", "qtree")
qtreeInstance, _ := data.NewInstance("" + "abcde" + "abcd_root")
qtreeInstance.SetLabel("export_policy", "default")
qtreeInstance.SetLabel("oplocks", "enabled")
qtreeInstance.SetLabel("security_style", "unix")
qtreeInstance.SetLabel("status", "normal")

exportOptions := node.NewS("export_options")
instanceKeys := exportOptions.NewChildS("instance_keys", "")
// apply all instance keys, instance labels from qtree.yaml to all quota metrics
keys := []string{"export_policy", "oplocks", "security_style", "status"}
for _, key := range keys {
instanceKeys.NewChildS("", key)
}
q2.data.SetExportOptions(exportOptions)
q2.historicalLabels = true
testLabels(t, q2, result, data, "abcde"+"."+"abcd_root"+"."+""+"."+""+"."+"root"+"."+"space.used.total", 3, 4, 10)
Hardikl marked this conversation as resolved.
Show resolved Hide resolved
}

func testLabels(t *testing.T, q *Qtree, quotas []gjson.Result, data *matrix.Matrix, quotaInstanceKey string, expectedQuotaCount int, expectedQuotaMetricCount int, expectedQuotaLabels int) {
quotaCount := 0
numMetrics := 0

err = q.handlingQuotaMetrics(result, &quotaCount, &numMetrics)
err := q.handlingQuotaMetrics(quotas, data, &quotaCount, &numMetrics)
if err != nil {
t.Errorf("handlingQuotaMetrics returned an error: %v", err)
}

if quotaCount != 3 {
t.Errorf("quotaCount = %d; want 3", quotaCount)
if quotaCount != expectedQuotaCount {
t.Errorf("quotaCount = %d; want %d", quotaCount, expectedQuotaCount)
}
if numMetrics != 6 {
t.Errorf("numMetrics = %d; want 6", numMetrics)
if numMetrics != expectedQuotaMetricCount {
t.Errorf("numMetrics = %d; want %d", numMetrics, expectedQuotaMetricCount)
}

quotaInstance := q.data.GetInstance(quotaInstanceKey)
if len(quotaInstance.GetLabels()) != expectedQuotaLabels {
t.Errorf("labels = %d; want %d", len(quotaInstance.GetLabels()), expectedQuotaLabels)
}
}
Loading