-
Notifications
You must be signed in to change notification settings - Fork 323
/
metrics_test.go
156 lines (126 loc) · 7.23 KB
/
metrics_test.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
package metrics
import (
"context"
"fmt"
"testing"
"time"
"github.com/hashicorp/consul/sdk/testutil/retry"
"github.com/hashicorp/consul-k8s/acceptance/framework/consul"
"github.com/hashicorp/consul-k8s/acceptance/framework/environment"
"github.com/hashicorp/consul-k8s/acceptance/framework/helpers"
"github.com/hashicorp/consul-k8s/acceptance/framework/k8s"
"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
"github.com/stretchr/testify/require"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
const StaticClientName = "static-client"
// Test that prometheus metrics, when enabled, are accessible from the
// endpoints that have been exposed on the server, client and gateways.
func TestComponentMetrics(t *testing.T) {
env := suite.Environment()
cfg := suite.Config()
ctx := env.DefaultContext(t)
ns := ctx.KubectlOptions(t).Namespace
helmValues := map[string]string{
"global.datacenter": "dc1",
"global.metrics.enabled": "true",
"global.metrics.enableAgentMetrics": "true",
// Agents have been removed but there could potentially be customers that are still running them. We
// are using client.enabled to cover that scenario and to make sure agent metrics still works with
// consul-dataplane.
"client.enabled": "true",
"connectInject.enabled": "true",
"controller.enabled": "true",
"meshGateway.enabled": "true",
"meshGateway.replicas": "1",
"meshGateway.service.type": "ClusterIP",
"ingressGateways.enabled": "true",
"ingressGateways.gateways[0].name": "ingress-gateway",
"ingressGateways.gateways[0].replicas": "1",
"terminatingGateways.enabled": "true",
"terminatingGateways.gateways[0].name": "terminating-gateway",
"terminatingGateways.gateways[0].replicas": "1",
// Reduce CPU resource requests because tests were running into CPU scheduling
// limits and because we're not really testing performance.
"controller.resources.requests.cpu": "50m",
"ingressGateways.defaults.resources.requests.cpu": "50m",
"terminatingGateways.defaults.resources.requests.cpu": "50m",
"meshGateway.resources.requests.cpu": "50m",
}
releaseName := helpers.RandomName()
// Install the consul cluster in the default kubernetes ctx.
consulCluster := consul.NewHelmCluster(t, helmValues, ctx, cfg, releaseName)
consulCluster.Create(t)
// Create the static-client deployment so we can use it for in-cluster calls to metrics endpoints.
// This simulates queries that would be made by a prometheus server that runs externally to the consul
// components in the cluster.
logger.Log(t, "creating static-client")
k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/bases/static-client")
// Server Metrics
metricsOutput, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "exec", "deploy/"+StaticClientName, "--", "curl", "--silent", "--show-error", fmt.Sprintf("http://%s:8500/v1/agent/metrics?format=prometheus", fmt.Sprintf("%s-consul-server.%s.svc", releaseName, ns)))
require.NoError(t, err)
require.Contains(t, metricsOutput, `consul_acl_ResolveToken{quantile="0.5"}`)
// Client Metrics
metricsOutput, err = k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "exec", "deploy/"+StaticClientName, "--", "sh", "-c", "curl --silent --show-error http://$HOST_IP:8500/v1/agent/metrics?format=prometheus")
require.NoError(t, err)
require.Contains(t, metricsOutput, `consul_acl_ResolveToken{quantile="0.5"}`)
logger.Log(t, "ingress gateway metrics")
assertGatewayMetricsEnabled(t, ctx, ns, "ingress-gateway", `envoy_cluster_assignment_stale{local_cluster="ingress-gateway",consul_source_service="ingress-gateway"`)
logger.Log(t, "terminating gateway metrics")
assertGatewayMetricsEnabled(t, ctx, ns, "terminating-gateway", `envoy_cluster_assignment_stale{local_cluster="terminating-gateway",consul_source_service="terminating-gateway"`)
logger.Log(t, "mesh gateway metrics")
assertGatewayMetricsEnabled(t, ctx, ns, "mesh-gateway", `envoy_cluster_assignment_stale{local_cluster="mesh-gateway",consul_source_service="mesh-gateway"`)
}
// Test that merged service and envoy metrics are accessible from the
// endpoints that have been exposed on the service.
func TestAppMetrics(t *testing.T) {
env := suite.Environment()
cfg := suite.Config()
ctx := env.DefaultContext(t)
ns := ctx.KubectlOptions(t).Namespace
helmValues := map[string]string{
"global.datacenter": "dc1",
"global.metrics.enabled": "true",
// todo (agentless): remove once we have consul-dataplane image with these changes.
"global.imageConsulDataplane": "hashicorppreview/consul-dataplane:1.0-dev",
"connectInject.enabled": "true",
"connectInject.metrics.defaultEnableMerging": "true",
}
releaseName := helpers.RandomName()
// Install the consul cluster in the default kubernetes ctx.
consulCluster := consul.NewHelmCluster(t, helmValues, ctx, cfg, releaseName)
consulCluster.Create(t)
// Deploy service that will emit app and envoy metrics at merged metrics endpoint
logger.Log(t, "creating static-metrics-app")
k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/bases/static-metrics-app")
// Create the static-client deployment so we can use it for in-cluster calls to metrics endpoints.
// This simulates queries that would be made by a prometheus server that runs externally to the consul
// components in the cluster.
logger.Log(t, "creating static-client")
k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/bases/static-client")
// Merged App Metrics
podList, err := ctx.KubernetesClient(t).CoreV1().Pods(ns).List(context.Background(), metav1.ListOptions{LabelSelector: "app=static-metrics-app"})
require.NoError(t, err)
require.Len(t, podList.Items, 1)
podIP := podList.Items[0].Status.PodIP
// Retry because sometimes the merged metrics server takes a couple hundred milliseconds
// to start.
retry.RunWith(&retry.Counter{Count: 3, Wait: 1 * time.Second}, t, func(r *retry.R) {
metricsOutput, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "exec", "deploy/"+StaticClientName, "--", "curl", "--silent", "--show-error", fmt.Sprintf("http://%s:20200/metrics", podIP))
require.NoError(r, err)
// This assertion represents the metrics from the envoy sidecar.
require.Contains(r, metricsOutput, `envoy_cluster_assignment_stale{local_cluster="server",consul_source_service="server"`)
// This assertion represents the metrics from the application.
require.Contains(r, metricsOutput, `service_started_total 1`)
})
}
func assertGatewayMetricsEnabled(t *testing.T, ctx environment.TestContext, ns, label, metricsAssertion string) {
pods, err := ctx.KubernetesClient(t).CoreV1().Pods(ns).List(context.Background(), metav1.ListOptions{LabelSelector: fmt.Sprintf("component=%s", label)})
require.NoError(t, err)
for _, pod := range pods.Items {
podIP := pod.Status.PodIP
metricsOutput, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "exec", "deploy/"+StaticClientName, "--", "curl", "--silent", "--show-error", fmt.Sprintf("http://%s:20200/metrics", podIP))
require.NoError(t, err)
require.Contains(t, metricsOutput, metricsAssertion)
}
}