reset consul_autopilot_healthy to NaN on leadership state

Signed-off-by: FFMMM <[email protected]>

.changelog/11231.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+telemetry: fixes a bug with Prometheus consul_autopilot_healthy metric where 0 is reported instead of NaN on servers.
+```

agent/consul/autopilot.go

@@ -52,6 +52,12 @@ func (d *AutopilotDelegate) NotifyState(state *autopilot.State) {
 		} else {
 			metrics.SetGauge([]string{"autopilot", "healthy"}, 0)
 		}
+	} else {
+
+		// if we are not a leader, emit NaN per
+		// https://www.consul.io/docs/agent/telemetry#autopilot
+		metrics.SetGauge([]string{"autopilot", "healthy"}, float32(math.NaN()))
+
 	}
 }
 

agent/metrics_test.go

@@ -7,14 +7,14 @@ import (
 	"testing"
 )
 
-// TestMetrics_ConsulAutopilotHealthy_Prometheus adds testing around
+// TestHTTPHandlers_AgentMetrics_ConsulAutopilotHealthy_Prometheus adds testing around
 // the published autopilot metrics on https://www.consul.io/docs/agent/telemetry#autopilot
-func TestMetrics_ConsulAutopilotHealthy_Prometheus(t *testing.T) {
+func TestHTTPHandlers_AgentMetrics_ConsulAutopilotHealthy_Prometheus(t *testing.T) {
 	checkForShortTesting(t)
 	t.Parallel()
 
 	// configure agent to emit Prometheus metrics
-	hcl :=`
+	hcl := `
 	telemetry = {
 		prometheus_retention_time = "5s",
 		disable_hostname = true
@@ -24,7 +24,6 @@ func TestMetrics_ConsulAutopilotHealthy_Prometheus(t *testing.T) {
 	a := StartTestAgent(t, TestAgent{HCL: hcl, NoWaitForStartup: true})
 	defer a.Shutdown()
 
-
 	req, err := http.NewRequest("GET", "/v1/agent/metrics?format=prometheus", nil)
 	if err != nil {
 		t.Fatalf("Failed to generate new http request. err: %v", err)
@@ -49,4 +48,4 @@ func checkForShortTesting(t *testing.T) {
 	if testing.Short() {
 		t.Skip("too slow for testing.Short")
 	}
-}
+}

agent/testagent.go

@@ -5,7 +5,6 @@ import (
 	"context"
 	"crypto/x509"
 	"fmt"
-	"github.com/armon/go-metrics"
 	"io"
 	"math/rand"
 	"net"
@@ -17,6 +16,7 @@ import (
 	"text/template"
 	"time"
 
+	metrics "github.com/armon/go-metrics"
 	"github.com/hashicorp/go-hclog"
 	uuid "github.com/hashicorp/go-uuid"
 
@@ -83,8 +83,7 @@ type TestAgent struct {
 	// It is valid after Start().
 	*Agent
 
-	// NoWaitForStartup, if false, will wait for the agent to be up or for leader
-	// election to have happened before finishing the start process for an Agent
+	// NoWaitForStartup, if false, will start the anti entropy sync and waitForUp() (see *waitForUp()).
 	NoWaitForStartup bool
 }
 
@@ -232,7 +231,6 @@ func (a *TestAgent) Start(t *testing.T) error {
 		// Start the anti-entropy syncer
 		a.Agent.StartSync()
 
-
 		if err := a.waitForUp(); err != nil {
 			a.Shutdown()
 			a.Agent = nil