feat: upgrade ZAPI conversations to REST when ZAPIs are suspended or … (

#2200)

* feat: upgrade ZAPI conversations to REST when ZAPIs are suspended or disabled

* feat: error handling changes

---------

Co-authored-by: Rahul Gupta <[email protected]>

cmd/poller/poller.go

@@ -53,6 +53,7 @@ import (
 	"github.com/netapp/harvest/v2/pkg/matrix"
 	"github.com/netapp/harvest/v2/pkg/tree/node"
 	"github.com/netapp/harvest/v2/pkg/util"
+	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v3"
 	"io"
@@ -1129,11 +1130,10 @@ func (p *Poller) createClient() {
 
 // upgradeCollector checks if the collector c should be upgraded to a REST collector.
 // ZAPI collectors should be upgraded to REST collectors when the cluster no longer speaks Zapi
-// If any error happens during the REST query, the upgrade is aborted and the original collector c returned.
 func (p *Poller) upgradeCollector(c conf.Collector) conf.Collector {
 	// If REST is desired, use REST
-	// If ZAPI is desired, check that the cluster speaks ZAPI and if so, use ZAPI otherwise use REST
-	// EMS and StorageGRID are will be ignored
+	// If ZAPI is desired, check that the cluster speaks ZAPI and if so, use ZAPI, otherwise use REST
+	// EMS and StorageGRID are ignored
 
 	if !strings.HasPrefix(c.Name, "Zapi") {
 		return c
@@ -1142,17 +1142,34 @@ func (p *Poller) upgradeCollector(c conf.Collector) conf.Collector {
 	return p.negotiateAPI(c, p.doZAPIsExist)
 }
 
-// clusterVersion should be of the form 9.12.1
+// Harvest will upgrade ZAPI conversations to REST in two cases:
+//   - if ONTAP returns a ZAPI error with errno=61253
+//   - if ONTAP returns an HTTP status code of 400
 func (p *Poller) negotiateAPI(c conf.Collector, checkZAPIs func() error) conf.Collector {
+	var switchToRest bool
 	err := checkZAPIs()
-	// if there is an error talking to ONTAP, assume that is because ZAPIs no longer exist and use REST
+
 	if err != nil {
-		logger.Warn().Err(err).Str("collector", c.Name).Msg("ZAPI EOA. Use REST")
-		upgradeCollector := strings.ReplaceAll(c.Name, "Zapi", "Rest")
-		return conf.Collector{
-			Name:      upgradeCollector,
-			Templates: c.Templates,
+		var he errs.HarvestError
+		if errors.As(err, &he) {
+			if he.ErrNum == errs.ErrNumZAPISuspended {
+				logger.Warn().Str("collector", c.Name).Msg("ZAPIs suspended. Use REST")
+				switchToRest = true
+			}
+
+			if he.StatusCode == 400 {
+				logger.Warn().Str("collector", c.Name).Msg("ZAPIs EOA. Use REST")
+				switchToRest = true
+			}
+		}
+		if switchToRest {
+			upgradeCollector := strings.ReplaceAll(c.Name, "Zapi", "Rest")
+			return conf.Collector{
+				Name:      upgradeCollector,
+				Templates: c.Templates,
+			}
 		}
+		log.Error().Err(err).Str("collector", c.Name).Msg("Failed to negotiateAPI")
 	}
 
 	return c

pkg/api/ontapi/zapi/client.go

@@ -514,7 +514,9 @@ func (c *Client) invoke(withTimers bool) (*node.Node, time.Duration, time.Durati
 		start             time.Time
 		responseT, parseT time.Duration
 		body              []byte
-		status, reason    string
+		status            string
+		reason            string
+		errNum            string
 		found             bool
 		err               error
 	)
@@ -545,9 +547,9 @@ func (c *Client) invoke(withTimers bool) (*node.Node, time.Duration, time.Durati
 
 	if response.StatusCode != 200 {
 		if response.StatusCode == 401 {
-			return result, responseT, parseT, errs.New(errs.ErrAuthFailed, response.Status)
+			return result, responseT, parseT, errs.New(errs.ErrAuthFailed, response.Status, errs.WithStatus(response.StatusCode))
 		}
-		return result, responseT, parseT, errs.New(errs.ErrAPIResponse, response.Status)
+		return result, responseT, parseT, errs.New(errs.ErrAPIResponse, response.Status, errs.WithStatus(response.StatusCode))
 	}
 
 	// read response body
@@ -581,7 +583,8 @@ func (c *Client) invoke(withTimers bool) (*node.Node, time.Duration, time.Durati
 		if reason == "" {
 			reason = "no reason"
 		}
-		err = errs.New(errs.ErrAPIRequestRejected, reason)
+		errNum, _ = result.GetAttrValueS("errno")
+		err = errs.New(errs.ErrAPIRequestRejected, reason, errs.WithErrorNum(errNum))
 		return result, responseT, parseT, err
 	}
 

pkg/errs/errors.go

@@ -31,9 +31,29 @@ const (
 	ErrWrongTemplate      = harvestError("wrong template")
 )
 
+const (
+	ErrNumZAPISuspended = "61253"
+)
+
 type HarvestError struct {
-	Message string
-	Inner   error
+	Message    string
+	Inner      error
+	ErrNum     string
+	StatusCode int
+}
+
+type Option func(*HarvestError)
+
+func WithStatus(statusCode int) Option {
+	return func(e *HarvestError) {
+		e.StatusCode = statusCode
+	}
+}
+
+func WithErrorNum(errNum string) Option {
+	return func(e *HarvestError) {
+		e.ErrNum = errNum
+	}
 }
 
 func (e HarvestError) Error() string {
@@ -43,13 +63,20 @@ func (e HarvestError) Error() string {
 	if e.Message == "" {
 		return e.Inner.Error()
 	}
-	return fmt.Sprintf("%s => %s", e.Inner.Error(), e.Message)
+	if e.ErrNum == "" && e.StatusCode == 0 {
+		return fmt.Sprintf("%s => %s", e.Inner.Error(), e.Message)
+	}
+	return fmt.Sprintf(`%s => %s errNum="%s" statusCode="%d"`, e.Inner.Error(), e.Message, e.ErrNum, e.StatusCode)
 }
 
 func (e HarvestError) Unwrap() error {
 	return e.Inner
 }
 
-func New(err error, message string) error {
-	return HarvestError{Message: message, Inner: err}
+func New(innerError error, message string, opts ...Option) error {
+	err := HarvestError{Message: message, Inner: innerError}
+	for _, opt := range opts {
+		opt(&err)
+	}
+	return err
 }