fix: Zapi Rest Parity

cmd/collectors/rest/plugins/qtree/qtree.go

@@ -50,7 +50,7 @@ func (q *Qtree) Init() error {
 		"files.used.hard_limit_percent => files_used_pct_file_limit",
 		"files.used.soft_limit_percent => files_used_pct_soft_file_limit",
 		"files.soft_limit => soft_file_limit",
-		// "threshold",   # deprecated
+		"threshold => threshold",
 	}
 
 	if err := q.InitAbc(); err != nil {
@@ -281,6 +281,14 @@ func (q *Qtree) handlingHistoricalMetrics(result []gjson.Result, data *matrix.Ma
 				if attribute == "space.hard_limit" || attribute == "space.soft_limit" {
 					value = attrValue.Float() / 1024
 					quotaInstance.SetLabel("unit", "Kbyte")
+					if attribute == "space.soft_limit" {
+						t := q.data.GetMetric("threshold")
+						if err = t.SetValueFloat64(quotaInstance, value); err != nil {
+							q.Logger.Error().Stack().Err(err).Str("attribute", attribute).Float64("value", value).Msg("Failed to parse value")
+						} else {
+							*numMetrics++
+						}
+					}
 				} else {
 					value = attrValue.Float()
 				}
@@ -352,6 +360,14 @@ func (q *Qtree) handlingQuotaMetrics(result []gjson.Result, quotaCount *int, num
 				if attribute == "space.hard_limit" || attribute == "space.soft_limit" || attribute == "space.used.total" {
 					value = attrValue.Float() / 1024
 					quotaInstance.SetLabel("unit", "Kbyte")
+					if attribute == "space.soft_limit" {
+						t := q.data.GetMetric("threshold")
+						if err = t.SetValueFloat64(quotaInstance, value); err != nil {
+							q.Logger.Error().Stack().Err(err).Str("attribute", attribute).Float64("value", value).Msg("Failed to parse value")
+						} else {
+							*numMetrics++
+						}
+					}
 				} else {
 					value = attrValue.Float()
 				}

cmd/tools/generate/counter.go

@@ -337,12 +337,13 @@ func processRestConfigCounters(path string) map[string]Counter {
 // processZAPIPerfCounters process ZapiPerf counters
 func processZAPIPerfCounters(path string, client *zapi.Client) map[string]Counter {
 	var (
-		counters           = make(map[string]Counter)
-		request, response  *node.Node
-		zapiUnitMap        = make(map[string]string)
-		zapiTypeMap        = make(map[string]string)
-		zapiDescMap        = make(map[string]string)
-		zapiBaseCounterMap = make(map[string]string)
+		counters                = make(map[string]Counter)
+		request, response       *node.Node
+		zapiUnitMap             = make(map[string]string)
+		zapiTypeMap             = make(map[string]string)
+		zapiDescMap             = make(map[string]string)
+		zapiBaseCounterMap      = make(map[string]string)
+		zapiDeprecateCounterMap = make(map[string]string)
 	)
 	t, err := tree.ImportYaml(path)
 	if t == nil || err != nil {
@@ -395,10 +396,17 @@ func processZAPIPerfCounters(path string, client *zapi.Client) map[string]Counte
 					ty = oty
 				}
 			}
+
 			zapiUnitMap[name] = counter.GetChildContentS("unit")
 			zapiDescMap[name] = updateDescription(counter.GetChildContentS("desc"))
 			zapiTypeMap[name] = ty
 			zapiBaseCounterMap[name] = counter.GetChildContentS("base-counter")
+
+			if counter.GetChildContentS("is-deprecated") == "true" {
+				if r := counter.GetChildContentS("replaced-by"); r != "" {
+					zapiDeprecateCounterMap[name] = r
+				}
+			}
 		}
 	}
 
@@ -432,6 +440,27 @@ func processZAPIPerfCounters(path string, client *zapi.Client) map[string]Counte
 					}
 					counters[harvestName] = co
 
+					// handle deprecate counters
+					if rName, ok := zapiDeprecateCounterMap[name]; ok {
+						hName := model.Object + "_" + rName
+						ro := Counter{
+							Name:        hName,
+							Description: zapiDescMap[rName],
+							APIs: []MetricDef{
+								{
+									API:          "ZAPI",
+									Endpoint:     "perf-object-get-instances" + " " + model.Query,
+									Template:     path,
+									ONTAPCounter: rName,
+									Unit:         zapiUnitMap[rName],
+									Type:         zapiTypeMap[rName],
+									BaseCounter:  zapiBaseCounterMap[rName],
+								},
+							},
+						}
+						counters[hName] = ro
+					}
+
 					// If the template has any MultiplierMetrics, add them
 					for _, metric := range model.MultiplierMetrics {
 						mc := co

cmd/tools/generate/counter.yaml

@@ -1063,3 +1063,19 @@ counters:
 
   - Name: volume_total_data
     Description: This metric represents the total amount of data that has been read from and written to a specific volume.
+
+  - Name: node_nvmf_data_sent
+    Description: NVMe/FC kilobytes (KB) sent per second.
+
+  - Name: node_nvmf_ops
+    Description: NVMe/FC operations per second.
+
+  - Name: node_nvmf_data_recv
+    Description: NVMe/FC kilobytes (KB) received per second.
+
+  - Name: quota_threshold
+    APIs:
+      - API: REST
+        Endpoint: NA
+        ONTAPCounter: Harvest generated
+        Template: conf/rest/9.12.0/qtree.yaml

conf/restperf/9.12.0/iscsi_lif.yaml

@@ -11,7 +11,7 @@ counters:
   - average_other_latency  => avg_other_latency
   - average_read_latency   => avg_read_latency
   - average_write_latency  => avg_write_latency
-  - cmd_transferred
+  - cmd_transferred        => cmd_transfered    # for backward compatibility with Zapi
   - iscsi_other_ops
   - iscsi_read_ops
   - iscsi_write_ops

conf/restperf/9.12.0/system_node.yaml

@@ -22,15 +22,22 @@ counters:
   - network_data_received              => net_data_recv
   - network_data_sent                  => net_data_sent
   - nfs_ops
-  - nvme_fc_data_received              => nvmf_data_recv
-  - nvme_fc_data_sent                  => nvmf_data_sent
-  - nvme_fc_ops                        => nvmf_ops
+  - nvme_fc_data_received              => nvme_fc_data_recv
+  - nvme_fc_data_sent                  => nvme_fc_data_sent
+  - nvme_fc_ops                        => nvme_fc_ops
   - ssd_data_read
   - ssd_data_written
   - total_data
   - total_latency
   - total_ops
 
+plugins:
+  - MetricAgent:
+      compute_metric:
+        - nvmf_data_recv MULTIPLY nvme_fc_data_received 1     # Added for backward compatibility with ZAPI
+        - nvmf_data_sent MULTIPLY nvme_fc_data_sent 1         # Added for backward compatibility with ZAPI
+        - nvmf_ops MULTIPLY nvme_fc_ops 1                     # Added for backward compatibility with ZAPI
+
 export_options:
   instance_keys:
     - node

docs/ontap-metrics.md

@@ -7,7 +7,7 @@ These can be generated on demand by running `bin/harvest grafana metrics`. See
 - More information about ONTAP REST performance counters can be found [here](https://docs.netapp.com/us-en/ontap-pcmap-9121/index.html).
 
 ```
-Creation Date : 2024-May-17
+Creation Date : 2024-May-29
 ONTAP Version: 9.13.1
 ```
 ## Understanding the structure
@@ -3046,20 +3046,12 @@ Average latency for write operations
 
 ### iscsi_lif_cmd_transfered
 
-Command transfered by this iSCSI conn
-
-| API    | Endpoint | Metric | Template |
-|--------|----------|--------|---------|
-| ZAPI | `perf-object-get-instances iscsi_lif` | `cmd_transfered`<br><span class="key">Unit:</span> none<br><span class="key">Type:</span> rate<br><span class="key">Base:</span>  | conf/zapiperf/cdot/9.8.0/iscsi_lif.yaml | 
-
-
-### iscsi_lif_cmd_transferred
-
 Command transferred by this iSCSI connection
 
 | API    | Endpoint | Metric | Template |
 |--------|----------|--------|---------|
 | REST | `api/cluster/counter/tables/iscsi_lif` | `cmd_transferred`<br><span class="key">Unit:</span> none<br><span class="key">Type:</span> rate<br><span class="key">Base:</span>  | conf/restperf/9.12.0/iscsi_lif.yaml | 
+| ZAPI | `perf-object-get-instances iscsi_lif` | `cmd_transfered`<br><span class="key">Unit:</span> none<br><span class="key">Type:</span> rate<br><span class="key">Base:</span>  | conf/zapiperf/cdot/9.8.0/iscsi_lif.yaml | 
 
 
 ### iscsi_lif_iscsi_other_ops
@@ -6795,33 +6787,63 @@ Total number of Write procedure requests. It is the total number of write succes
 | ZAPI | `perf-object-get-instances nfsv4:node` | `write_total`<br><span class="key">Unit:</span> none<br><span class="key">Type:</span> rate<br><span class="key">Base:</span>  | conf/zapiperf/cdot/9.8.0/nfsv4_node.yaml | 
 
 
-### node_nvmf_data_recv
+### node_nvme_fc_data_recv
 
 NVMe/FC kilobytes (KB) received per second
 
 | API    | Endpoint | Metric | Template |
 |--------|----------|--------|---------|
 | REST | `api/cluster/counter/tables/system:node` | `nvme_fc_data_received`<br><span class="key">Unit:</span> kb_per_sec<br><span class="key">Type:</span> rate<br><span class="key">Base:</span>  | conf/restperf/9.12.0/system_node.yaml | 
-| ZAPI | `perf-object-get-instances system:node` | `nvmf_data_recv`<br><span class="key">Unit:</span> kb_per_sec<br><span class="key">Type:</span> rate<br><span class="key">Base:</span>  | conf/zapiperf/cdot/9.8.0/system_node.yaml | 
+| ZAPI | `perf-object-get-instances system:node` | `nvme_fc_data_recv`<br><span class="key">Unit:</span> kb_per_sec<br><span class="key">Type:</span> rate<br><span class="key">Base:</span>  | conf/zapiperf/cdot/9.8.0/system_node.yaml | 
 
 
-### node_nvmf_data_sent
+### node_nvme_fc_data_sent
 
 NVMe/FC kilobytes (KB) sent per second
 
 | API    | Endpoint | Metric | Template |
 |--------|----------|--------|---------|
 | REST | `api/cluster/counter/tables/system:node` | `nvme_fc_data_sent`<br><span class="key">Unit:</span> kb_per_sec<br><span class="key">Type:</span> rate<br><span class="key">Base:</span>  | conf/restperf/9.12.0/system_node.yaml | 
-| ZAPI | `perf-object-get-instances system:node` | `nvmf_data_sent`<br><span class="key">Unit:</span> kb_per_sec<br><span class="key">Type:</span> rate<br><span class="key">Base:</span>  | conf/zapiperf/cdot/9.8.0/system_node.yaml | 
+| ZAPI | `perf-object-get-instances system:node` | `nvme_fc_data_sent`<br><span class="key">Unit:</span> kb_per_sec<br><span class="key">Type:</span> rate<br><span class="key">Base:</span>  | conf/zapiperf/cdot/9.8.0/system_node.yaml | 
 
 
-### node_nvmf_ops
+### node_nvme_fc_ops
 
 NVMe/FC operations per second
 
 | API    | Endpoint | Metric | Template |
 |--------|----------|--------|---------|
 | REST | `api/cluster/counter/tables/system:node` | `nvme_fc_ops`<br><span class="key">Unit:</span> per_sec<br><span class="key">Type:</span> rate<br><span class="key">Base:</span>  | conf/restperf/9.12.0/system_node.yaml | 
+| ZAPI | `perf-object-get-instances system:node` | `nvme_fc_ops`<br><span class="key">Unit:</span> per_sec<br><span class="key">Type:</span> rate<br><span class="key">Base:</span>  | conf/zapiperf/cdot/9.8.0/system_node.yaml | 
+
+
+### node_nvmf_data_recv
+
+NVMe/FC kilobytes (KB) received per second.
+
+| API    | Endpoint | Metric | Template |
+|--------|----------|--------|---------|
+| REST | `api/cluster/counter/tables/system:node` | `nvme_fc_data_received, 1`<br><span class="key">Unit:</span> <br><span class="key">Type:</span> <br><span class="key">Base:</span>  | conf/restperf/9.12.0/system_node.yaml | 
+| ZAPI | `perf-object-get-instances system:node` | `nvmf_data_recv`<br><span class="key">Unit:</span> kb_per_sec<br><span class="key">Type:</span> rate<br><span class="key">Base:</span>  | conf/zapiperf/cdot/9.8.0/system_node.yaml | 
+
+
+### node_nvmf_data_sent
+
+NVMe/FC kilobytes (KB) sent per second.
+
+| API    | Endpoint | Metric | Template |
+|--------|----------|--------|---------|
+| REST | `api/cluster/counter/tables/system:node` | `nvme_fc_data_sent, 1`<br><span class="key">Unit:</span> <br><span class="key">Type:</span> <br><span class="key">Base:</span>  | conf/restperf/9.12.0/system_node.yaml | 
+| ZAPI | `perf-object-get-instances system:node` | `nvmf_data_sent`<br><span class="key">Unit:</span> kb_per_sec<br><span class="key">Type:</span> rate<br><span class="key">Base:</span>  | conf/zapiperf/cdot/9.8.0/system_node.yaml | 
+
+
+### node_nvmf_ops
+
+NVMe/FC operations per second.
+
+| API    | Endpoint | Metric | Template |
+|--------|----------|--------|---------|
+| REST | `api/cluster/counter/tables/system:node` | `nvme_fc_ops, 1`<br><span class="key">Unit:</span> <br><span class="key">Type:</span> <br><span class="key">Base:</span>  | conf/restperf/9.12.0/system_node.yaml | 
 | ZAPI | `perf-object-get-instances system:node` | `nvmf_ops`<br><span class="key">Unit:</span> per_sec<br><span class="key">Type:</span> rate<br><span class="key">Base:</span>  | conf/zapiperf/cdot/9.8.0/system_node.yaml | 
 
 
@@ -9547,6 +9569,7 @@ Disk space threshold, in kilobytes, for the quota target. The value is -1 if the
 | API    | Endpoint | Metric | Template |
 |--------|----------|--------|---------|
 | ZAPI | `quota-report-iter` | `threshold` | conf/zapi/cdot/9.8.0/qtree.yaml |
+| REST | `NA` | `Harvest generated` | conf/rest/9.12.0/qtree.yaml |
 
 
 ### raid_disk_busy

integration/go.mod

@@ -6,8 +6,8 @@ replace github.com/netapp/harvest/v2 => ../
 
 require (
 	github.com/carlmjohnson/requests v0.23.5
-	github.com/netapp/harvest/v2 v2.0.0-20240509061649-efa3ed2b64e0
-	github.com/rs/zerolog v1.32.0
+	github.com/netapp/harvest/v2 v2.0.0-20240529062525-2ab393d07921
+	github.com/rs/zerolog v1.33.0
 	github.com/tidwall/gjson v1.17.1
 	golang.org/x/text v0.15.0
 )
@@ -20,7 +20,7 @@ require (
 	github.com/go-openapi/swag v0.23.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
-	github.com/lufia/plan9stats v0.0.0-20240408141607-282e7b5d6b74 // indirect
+	github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
@@ -36,7 +36,7 @@ require (
 	github.com/tklauser/go-sysconf v0.3.14 // indirect
 	github.com/tklauser/numcpus v0.8.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
-	golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect
+	golang.org/x/exp v0.0.0-20240529005216-23cca8864a10 // indirect
 	golang.org/x/net v0.25.0 // indirect
 	golang.org/x/sys v0.20.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect

integration/go.sum

@@ -30,8 +30,8 @@ github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3x
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
-github.com/lufia/plan9stats v0.0.0-20240408141607-282e7b5d6b74 h1:1KuuSOy4ZNgW0KA2oYIngXVFhQcXxhLqCVK7cBcldkk=
-github.com/lufia/plan9stats v0.0.0-20240408141607-282e7b5d6b74/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k=
+github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae h1:dIZY4ULFcto4tAFlj1FYZl8ztUZ13bdq+PLY+NOfbyI=
+github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
@@ -54,8 +54,8 @@ github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUc
 github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
 github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
-github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0=
-github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
+github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8=
+github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/shirou/gopsutil/v3 v3.24.4 h1:dEHgzZXt4LMNm+oYELpzl9YCqV65Yr/6SfrvgRBtXeU=
 github.com/shirou/gopsutil/v3 v3.24.4/go.mod h1:lTd2mdiOspcqLgAnr9/nGi71NkeMpWKdmhuxm9GusH8=
@@ -91,8 +91,8 @@ github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYg
 github.com/tklauser/numcpus v0.8.0/go.mod h1:ZJZlAY+dmR4eut8epnzf0u/VwodKmryxR8txiloSqBE=
 github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
 github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
-golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM=
-golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc=
+golang.org/x/exp v0.0.0-20240529005216-23cca8864a10 h1:vpzMC/iZhYFAjJzHU0Cfuq+w1vLLsF2vLkDrPjzKYck=
+golang.org/x/exp v0.0.0-20240529005216-23cca8864a10/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc=
 golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=