Skip to content

Commit

Permalink
update v2 tests
Browse files Browse the repository at this point in the history 
* add some new test cases
* update mock server to handle queries / formulas validation
* update no data tests to reflect reality
* stop all values being the same. it makes it difficult to know find which test case failed. move meaning from comments into the metric name.
* stop using deprecated ioutil

Signed-off-by: mitchell amihod <[email protected]>
  • Loading branch information
@meeech
meeech committed Oct 12, 2023
1 parent febeef3 commit 250b99e
Show file tree
Hide file tree
Showing 2 changed files with 109 additions and 70 deletions.
176 changes: 108 additions & 68 deletions metricproviders/datadog/datadogV2_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package datadog
import (
"encoding/json"
"io"
"io/ioutil"
"net/http"
"net/http/httptest"
"os"
Expand All @@ -19,30 +18,48 @@ import (
kubetesting "k8s.io/client-go/testing"
)

func TestRunSuiteV2(t *testing.T) {

const expectedApiKey = "0123456789abcdef0123456789abcdef"
const expectedAppKey = "0123456789abcdef0123456789abcdef01234567"

unixNow = func() int64 { return 1599076435 }

ddProviderIntervalDefault := v1alpha1.MetricProvider{
func newQueryDefaultProvider() v1alpha1.MetricProvider {
return v1alpha1.MetricProvider{
Datadog: &v1alpha1.DatadogMetric{
Interval: "5m",
Query: "avg:kubernetes.cpu.user.total{*}",
ApiVersion: "v2",
},
}
}

ddProviderInterval10m := v1alpha1.MetricProvider{
func newQueriesDefaultProvider() v1alpha1.MetricProvider {
return v1alpha1.MetricProvider{
Datadog: &v1alpha1.DatadogMetric{
Interval: "5m",
Queries: map[string]string{
"a": "avg:error_requests{*}",
"b": "avg:total_requests{*}",
},
Formula: "a/b",
ApiVersion: "v2",
},
}
}

func newQueryProviderInterval10m() v1alpha1.MetricProvider {
return v1alpha1.MetricProvider{
Datadog: &v1alpha1.DatadogMetric{
Query: "avg:kubernetes.cpu.user.total{*}",
Interval: "10m",
ApiVersion: "v2",
},
}
}

func TestRunSuiteV2(t *testing.T) {
const expectedApiKey = "0123456789abcdef0123456789abcdef"
const expectedAppKey = "0123456789abcdef0123456789abcdef01234567"

unixNow = func() int64 { return 1599076435 }

// Test Cases
var tests = []struct {
tests := []struct {
serverURL string
webServerStatus int
webServerResponse string
Expand All @@ -53,181 +70,188 @@ func TestRunSuiteV2(t *testing.T) {
expectedErrorMessage string
useEnvVarForKeys bool
}{
// When last value of time series matches condition then succeed.
{
webServerStatus: 200,
webServerResponse: `{"data": {"attributes": {"values": [[0.0020008318672513122, 0.0003332881882246533]], "times": [1598867910000, 1598867925000]}}}`,
webServerResponse: `{"data": {"attributes": {"columns": [ {"values": [0.0006332881882246533]}]}}}`,
metric: v1alpha1.Metric{
Name: "foo",
Name: "simple scalar query",
SuccessCondition: "result < 0.001",
FailureCondition: "result >= 0.001",
Provider: ddProviderInterval10m,
Provider: newQueryProviderInterval10m(),
},
expectedIntervalSeconds: 600,
expectedValue: "0.0003332881882246533",
expectedValue: "0.0006332881882246533",
expectedPhase: v1alpha1.AnalysisPhaseSuccessful,
useEnvVarForKeys: false,
},
// Same test as above, but derive DD keys from env var instead of k8s secret
{
webServerStatus: 200,
webServerResponse: `{"data": {"attributes": {"values": [[0.0020008318672513122, 0.0003332881882246533]], "times": [1598867910000, 1598867925000]}}}`,
webServerResponse: `{"data": {"attributes": {"columns": [ {"values": [0.0003332881882246533]}]}}}`,
metric: v1alpha1.Metric{
Name: "foo",
Name: "keys from env vars",
SuccessCondition: "result < 0.001",
FailureCondition: "result >= 0.001",
Provider: ddProviderInterval10m,
Provider: newQueryProviderInterval10m(),
},
expectedIntervalSeconds: 600,
expectedValue: "0.0003332881882246533",
expectedPhase: v1alpha1.AnalysisPhaseSuccessful,
useEnvVarForKeys: true,
},
// When last value of time series does not match condition then fail.
{
webServerStatus: 200,
webServerResponse: `{"data": {"attributes": {"values": [[0.0020008318672513122, 0.006121378742186943]], "times": [1598867910000, 1598867925000]}}}`,
webServerResponse: `{"data": {"attributes": {"columns": [ {"values": [0.006121374442186943]}]}}}`,
metric: v1alpha1.Metric{
Name: "foo",
Name: "value does not match condition then fail",
SuccessCondition: "result < 0.001",
FailureCondition: "result >= 0.001",
Provider: ddProviderIntervalDefault,
Provider: newQueryDefaultProvider(),
},
expectedIntervalSeconds: 300,
expectedValue: "0.006121378742186943",
expectedValue: "0.006121374442186943",
expectedPhase: v1alpha1.AnalysisPhaseFailed,
useEnvVarForKeys: false,
},
// Error if the request is invalid
{
webServerStatus: 400,
webServerResponse: `{"status":"error","error":"error messsage"}`,
metric: v1alpha1.Metric{
Name: "foo",
Name: "error for invalid request",
SuccessCondition: "result < 0.001",
FailureCondition: "result >= 0.001",
Provider: ddProviderIntervalDefault,
Provider: newQueryDefaultProvider(),
},
expectedIntervalSeconds: 300,
expectedPhase: v1alpha1.AnalysisPhaseError,
expectedErrorMessage: "received non 2xx response code: 400 {\"status\":\"error\",\"error\":\"error messsage\"}",
useEnvVarForKeys: false,
},
// Error if there is an authentication issue
{
webServerStatus: 401,
webServerResponse: `{"errors": ["No authenticated user."]}`,
metric: v1alpha1.Metric{
Name: "foo",
Name: "no authenticated user",
SuccessCondition: "result < 0.001",
FailureCondition: "result >= 0.001",
Provider: ddProviderIntervalDefault,
Provider: newQueryDefaultProvider(),
},
expectedIntervalSeconds: 300,
expectedPhase: v1alpha1.AnalysisPhaseError,
expectedErrorMessage: "received authentication error response code: 401 {\"errors\": [\"No authenticated user.\"]}",
useEnvVarForKeys: false,
},

// Expect success with default() and data
{
webServerStatus: 200,
webServerResponse: `{"data": {"attributes": {"values": [[0.0020008318672513122, 0.006121378742186943]], "times": [1598867910000, 1598867925000]}}}`,
webServerResponse: `{"data": {"attributes": {"columns": [ {"values": [0.006121378742186943]}]}}}`,
metric: v1alpha1.Metric{
Name: "foo",
Name: "success with default and data",
SuccessCondition: "default(result, 0) < 0.05",
Provider: ddProviderIntervalDefault,
Provider: newQueryDefaultProvider(),
},
expectedIntervalSeconds: 300,
expectedValue: "0.006121378742186943",
expectedPhase: v1alpha1.AnalysisPhaseSuccessful,
useEnvVarForKeys: false,
},

// Expect error with no default() and no data
{
webServerStatus: 200,
webServerResponse: `{"data": {"attributes": {"values": [], "times": []}}}`,
webServerResponse: `{"data": {"attributes": {"columns": [ {"values": []}]}}}`,
metric: v1alpha1.Metric{
Name: "foo",
Name: "error with no default and no data",
SuccessCondition: "result < 0.05",
Provider: ddProviderIntervalDefault,
Provider: newQueryDefaultProvider(),
},
expectedIntervalSeconds: 300,
expectedPhase: v1alpha1.AnalysisPhaseError,
expectedErrorMessage: `invalid operation: < (mismatched types <nil> and float64)`,
useEnvVarForKeys: false,
},

// Expect success with default() and no data
{
webServerStatus: 200,
webServerResponse: `{"data": {"attributes": {"values": [], "times": []}}}`,
webServerResponse: `{"data": {"attributes": {"columns": [ {"values": []}]}}}`,
metric: v1alpha1.Metric{
Name: "foo",
Name: "success with default and no data",
SuccessCondition: "default(result, 0) < 0.05",
Provider: ddProviderIntervalDefault,
Provider: newQueryDefaultProvider(),
},
expectedIntervalSeconds: 300,
expectedValue: `{"Values":[],"Times":[]}`,
expectedValue: `[]`,
expectedPhase: v1alpha1.AnalysisPhaseSuccessful,
useEnvVarForKeys: false,
},

// Expect failure with bad default() and no data
{
webServerStatus: 200,
webServerResponse: `{"data": {"attributes": {"values": [], "times": []}}}`,
webServerResponse: `{"data": {"attributes": {"columns": [ {"values": []}]}}}`,
metric: v1alpha1.Metric{
Name: "foo",
Name: "fail when bad default and no data",
SuccessCondition: "default(result, 1) < 0.05",
Provider: ddProviderIntervalDefault,
Provider: newQueryDefaultProvider(),
},
expectedIntervalSeconds: 300,
expectedValue: `{"Values":[],"Times":[]}`,
expectedValue: `[]`,
expectedPhase: v1alpha1.AnalysisPhaseFailed,
useEnvVarForKeys: false,
},

// Expect success with bad default() and good data
{
webServerStatus: 200,
webServerResponse: `{"data": {"attributes": {"values": [[0.0020008318672513122, 0.006121378742186943]], "times": [1598867910000, 1598867925000]}}}`,
webServerResponse: `{"data": {"attributes": {"columns": [ {"values": [0.006721378742186999]}]}}}`,
metric: v1alpha1.Metric{
Name: "foo",
Name: "success bad default and good data",
SuccessCondition: "default(result, 1) < 0.05",
Provider: ddProviderIntervalDefault,
Provider: newQueryDefaultProvider(),
},
expectedIntervalSeconds: 300,
expectedValue: `0.006121378742186943`,
expectedValue: `0.006721378742186999`,
expectedPhase: v1alpha1.AnalysisPhaseSuccessful,
useEnvVarForKeys: false,
},

// Error if datadog returns non-array values
{
webServerStatus: 200,
webServerResponse: `{"data": {"attributes": {"values": "invalid", "times": "invalid"}}}`,
webServerResponse: `{"data": {"attributes": {"columns": [{"values": "invalid"}]}}}`,
metric: v1alpha1.Metric{
Name: "foo",
Name: "error when bad values from dd",
SuccessCondition: "result < 0.001",
FailureCondition: "result >= 0.001",
Provider: ddProviderIntervalDefault,
Provider: newQueryDefaultProvider(),
},
expectedIntervalSeconds: 300,
expectedPhase: v1alpha1.AnalysisPhaseError,
expectedErrorMessage: "Could not parse JSON body: json: cannot unmarshal string into Go struct field .Data.Attributes.Values of type [][]float64",
expectedErrorMessage: "Could not parse JSON body: json: cannot unmarshal string into Go struct field .Data.Attributes.Columns.Values of type []float64",
useEnvVarForKeys: false,
},

// Error if server address is faulty
{
serverURL: "://wrong.schema",
metric: v1alpha1.Metric{},
serverURL: "://wrong.schema",
metric: v1alpha1.Metric{
Provider: newQueryProviderInterval10m(),
},
expectedPhase: v1alpha1.AnalysisPhaseError,
expectedErrorMessage: "parse \"://wrong.schema\": missing protocol scheme",
useEnvVarForKeys: false,
},

// Queries + Formula
// Expect success with default() and data
{
webServerStatus: 200,
webServerResponse: `{"data": {"attributes": {"columns": [ {"values": [0.0006444881882246533]}]}}}`,
metric: v1alpha1.Metric{
Name: "expect success queries and formula",
SuccessCondition: "default(result, 0) < 0.05",
Provider: newQueriesDefaultProvider(),
},
expectedIntervalSeconds: 300,
expectedValue: "0.0006444881882246533",
expectedPhase: v1alpha1.AnalysisPhaseSuccessful,
useEnvVarForKeys: false,
},
}

// Run
Expand All @@ -238,9 +262,8 @@ func TestRunSuiteV2(t *testing.T) {
if serverURL == "" {
// Server setup with response
server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {

//Check query variables
bodyBytes, err := ioutil.ReadAll(req.Body)
// Check query variables
bodyBytes, err := io.ReadAll(req.Body)
if err != nil {
t.Errorf("\nreceived no bytes in request: %v", err)
}
Expand All @@ -251,12 +274,29 @@ func TestRunSuiteV2(t *testing.T) {
t.Errorf("\nCould not parse JSON request body: %v", err)
}

// Keep the simple check behaviour if there is no Queries passed in from the analysis run
usesQuery := len(test.metric.Provider.Datadog.Queries) == 0
usesFormula := test.metric.Provider.Datadog.Formula != ""

actualFormulas := reqBody.Data.Attributes.Formulas
actualQuery := reqBody.Data.Attributes.Queries[0]["query"]
actualQueries := reqBody.Data.Attributes.Queries
actualFrom := reqBody.Data.Attributes.From
actualTo := reqBody.Data.Attributes.To

if actualQuery != "avg:kubernetes.cpu.user.total{*}" {
t.Errorf("\nquery expected avg:kubernetes.cpu.user.total{*} but got %s", actualQuery)
if usesQuery {
if actualQuery != "avg:kubernetes.cpu.user.total{*}" {
t.Errorf("\nquery expected avg:kubernetes.cpu.user.total{*} but got %s", actualQuery)
}
} else {
// Check queries has expected number of queries
if len(actualQueries) != len(test.metric.Provider.Datadog.Queries) {
t.Errorf("\nExpected %d queries but received %d", len(test.metric.Provider.Datadog.Queries), len(reqBody.Data.Attributes.Queries))
}

if usesFormula && len(actualFormulas) == 0 {
t.Errorf("\nExpected formula but no Formulas in request: %+v", actualFormulas)
}
}

if actualFrom != (unixNow()-test.expectedIntervalSeconds)*1000 {
Expand All @@ -271,7 +311,7 @@ func TestRunSuiteV2(t *testing.T) {
t.Errorf("\nfailed to parse to: %v", err)
}

//Check headers
// Check headers
if req.Header.Get("Content-Type") != "application/json" {
t.Errorf("\nContent-Type header expected to be application/json but got %s", req.Header.Get("Content-Type"))
}
Expand Down Expand Up @@ -326,7 +366,7 @@ func TestRunSuiteV2(t *testing.T) {
return true, tokenSecret, nil
})

provider, _ := NewDatadogProvider(*logCtx, fakeClient)
provider, _ := NewDatadogProvider(*logCtx, fakeClient, test.metric)

metricsMetadata := provider.GetMetadata(test.metric)
assert.Nil(t, metricsMetadata)
Expand Down
3 changes: 1 addition & 2 deletions metricproviders/datadog/datadog_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,15 +43,14 @@ func TestDatadogSpecDefaults(t *testing.T) {

t.Run("apiVersion: Validate enum exists to restrict apiVersion to 2 options", func(t *testing.T) {
versionEnums := ddSpec.Properties["apiVersion"].Enum

assert.Equal(t, 2, len(versionEnums), "Expecting 2 enum options")
assert.Equal(t, "\"v1\"", string(versionEnums[0].Raw), "\"v1\" expected, got %s", string(versionEnums[0].Raw))
assert.Equal(t, "\"v2\"", string(versionEnums[1].Raw), "\"v2\" is missing, got %s", string(versionEnums[1].Raw))
})

t.Run("interval: Validate default is 5m", func(t *testing.T) {
defaultInterval := string(ddSpec.Properties["interval"].Default.Raw)
assert.Equal(t, "\"5m\"", defaultInterval, "Default version should be v1")
assert.Equal(t, "\"5m\"", defaultInterval, "Default interval should be "\"5m\" ")

Check failure on line 53 in metricproviders/datadog/datadog_test.go

View workflow job for this annotation

GitHub Actions / Build 

illegal character U+005C '\'

Check failure on line 53 in metricproviders/datadog/datadog_test.go

View workflow job for this annotation

GitHub Actions / Build 

illegal character U+005C '\'

Check failure on line 53 in metricproviders/datadog/datadog_test.go

View workflow job for this annotation

GitHub Actions / Lint Go code 

invalid character U+005C '\'

Check failure on line 53 in metricproviders/datadog/datadog_test.go

View workflow job for this annotation

GitHub Actions / Lint Go code 

syntax error: unexpected literal "5m\" " in argument list; possibly missing comma or ) (typecheck)

Check failure on line 53 in metricproviders/datadog/datadog_test.go

View workflow job for this annotation

GitHub Actions / Lint Go code 

illegal character U+005C '\' (typecheck)
})
}

Expand Down

0 comments on commit 250b99e

Please sign in to comment.