add metrics test for consul_autopilot_healthy and plumbing for testagent

Signed-off-by: FFMMM <[email protected]>

agent/metrics_test.go

@@ -0,0 +1,52 @@
+package agent
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+// TestMetrics_ConsulAutopilotHealthy_Prometheus adds testing around
+// the published autopilot metrics on https://www.consul.io/docs/agent/telemetry#autopilot
+func TestMetrics_ConsulAutopilotHealthy_Prometheus(t *testing.T) {
+	checkForShortTesting(t)
+	t.Parallel()
+
+	// configure agent to emit Prometheus metrics
+	hcl :=`
+	telemetry = {
+		prometheus_retention_time = "5s",
+		disable_hostname = true
+	}
+	`
+
+	a := StartTestAgent(t, TestAgent{HCL: hcl, NoWaitForStartup: true})
+	defer a.Shutdown()
+
+
+	req, err := http.NewRequest("GET", "/v1/agent/metrics?format=prometheus", nil)
+	if err != nil {
+		t.Fatalf("Failed to generate new http request. err: %v", err)
+	}
+
+	respRec := httptest.NewRecorder()
+	_, err = a.srv.AgentMetrics(respRec, req)
+	if err != nil {
+		t.Fatalf("Failed to serve agent metrics. err: %v", err)
+	}
+
+	t.Run("Check consul_autopilot_healthy metric value on startup", func(t *testing.T) {
+		target := "consul_autopilot_healthy NaN"
+		keyValue := strings.Split(target, " ")
+		if !strings.Contains(respRec.Body.String(), target) {
+			t.Fatalf("Could not find the metric \"%s\" with value \"%s\" in the /v1/agent/metrics response", keyValue[0], keyValue[1])
+		}
+	})
+}
+
+func checkForShortTesting(t *testing.T) {
+	if testing.Short() {
+		t.Skip("too slow for testing.Short")
+	}
+}

agent/testagent.go

@@ -5,17 +5,18 @@ import (
 	"context"
 	"crypto/x509"
 	"fmt"
+	"github.com/armon/go-metrics"
 	"io"
 	"math/rand"
 	"net"
 	"net/http/httptest"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"testing"
 	"text/template"
 	"time"
 
-	metrics "github.com/armon/go-metrics"
 	"github.com/hashicorp/go-hclog"
 	uuid "github.com/hashicorp/go-uuid"
 
@@ -81,6 +82,10 @@ type TestAgent struct {
 	// Agent is the embedded consul agent.
 	// It is valid after Start().
 	*Agent
+
+	// NoWaitForStartup, if false, will wait for the agent to be up or for leader
+	// election to have happened before finishing the start process for an Agent
+	NoWaitForStartup bool
 }
 
 // NewTestAgent returns a started agent with the given configuration. It fails
@@ -185,7 +190,12 @@ func (a *TestAgent) Start(t *testing.T) error {
 		}
 		result, err := config.Load(opts)
 		if result.RuntimeConfig != nil {
-			result.RuntimeConfig.Telemetry.Disable = true
+			// If "telemetry" settings are present, then we probably want
+			// to actually emit consul specific metrics
+			// otherwise, disable telemetry
+			if !strings.Contains(a.HCL, "telemetry") {
+				result.RuntimeConfig.Telemetry.Disable = true
+			}
 		}
 		return result, err
 	}
@@ -195,7 +205,11 @@ func (a *TestAgent) Start(t *testing.T) error {
 	}
 
 	bd.Logger = logger
-	bd.MetricsHandler = metrics.NewInmemSink(1*time.Second, time.Minute)
+	// if we are not testing telemetry things, let's use a "mock" sink for metrics
+	if !strings.Contains(a.HCL, "telemetry") {
+		bd.MetricsHandler = metrics.NewInmemSink(1*time.Second, time.Minute)
+	}
+
 	a.Config = bd.RuntimeConfig
 
 	agent, err := New(bd)
@@ -212,19 +226,22 @@ func (a *TestAgent) Start(t *testing.T) error {
 	}
 
 	a.Agent = agent
+	a.srv = a.Agent.httpHandlers
 
-	// Start the anti-entropy syncer
-	a.Agent.StartSync()
+	if !a.NoWaitForStartup {
+		// Start the anti-entropy syncer
+		a.Agent.StartSync()
 
-	a.srv = a.Agent.httpHandlers
 
-	if err := a.waitForUp(); err != nil {
-		a.Shutdown()
-		a.Agent = nil
-		return fmt.Errorf("error waiting for test agent to start: %w", err)
+		if err := a.waitForUp(); err != nil {
+			a.Shutdown()
+			a.Agent = nil
+			return fmt.Errorf("error waiting for test agent to start: %w", err)
+		}
+
+		a.dns = a.dnsServers[0]
 	}
 
-	a.dns = a.dnsServers[0]
 	return nil
 }