From 36da36ac1dae9c8e52757eb8145ec3a1764f25ff Mon Sep 17 00:00:00 2001 From: Shahzad Lone Date: Fri, 18 Nov 2022 12:46:12 -0500 Subject: [PATCH] pr: Add the explain tests refactor setup. --- tests/integration/explain/simple/utils.go | 11 + tests/integration/explain/utils.go | 416 ++++++++++++++++++++++ tests/integration/utils.go | 1 - 3 files changed, 427 insertions(+), 1 deletion(-) create mode 100644 tests/integration/explain/utils.go diff --git a/tests/integration/explain/simple/utils.go b/tests/integration/explain/simple/utils.go index fe6f8f38ce..c464420d69 100644 --- a/tests/integration/explain/simple/utils.go +++ b/tests/integration/explain/simple/utils.go @@ -14,6 +14,7 @@ import ( "testing" testUtils "github.com/sourcenetwork/defradb/tests/integration" + explainUtils "github.com/sourcenetwork/defradb/tests/integration/explain" ) type dataMap = map[string]any @@ -56,6 +57,7 @@ var bookAuthorGQLSchema = (` `) +// TODO: Remove after draft has received feedback. func executeTestCase(t *testing.T, test testUtils.QueryTestCase) { testUtils.ExecuteQueryTestCase( t, @@ -64,3 +66,12 @@ func executeTestCase(t *testing.T, test testUtils.QueryTestCase) { test, ) } + +func executeExplainTestCase(t *testing.T, test explainUtils.ExplainRequestTestCase) { + explainUtils.ExecuteExplainRequestTestCase( + t, + bookAuthorGQLSchema, + []string{"article", "book", "author", "authorContact", "contactAddress"}, + test, + ) +} diff --git a/tests/integration/explain/utils.go b/tests/integration/explain/utils.go new file mode 100644 index 0000000000..c0aff34394 --- /dev/null +++ b/tests/integration/explain/utils.go @@ -0,0 +1,416 @@ +// Copyright 2022 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package test_explain + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/sourcenetwork/defradb/client" + "github.com/sourcenetwork/defradb/logging" + + testUtils "github.com/sourcenetwork/defradb/tests/integration" +) + +var ( + log = logging.MustNewLogger("defra.tests.integration.explain") + + allPlanNodeNames = [...]string{ + "explain", // not a planNode but need it here as this is root of the explain graph + "averageNode", + "dagScanNode", + "countNode", + "createNode", + "deleteNode", + "groupNode", + "limitNode", + "parallelNode", + "orderNode", + "pipeNode", + "scanNode", + "multiScanNode", + "selectTopNode", + "selectNode", + "sumNode", + "topLevelNode", + "typeIndexJoin", + "typeJoinOne", + "typeJoinMany", + "updateNode", + "valuesNode", + } +) + +type PlanNodeTargetCase struct { + // Name of the plan node, whose attribute(s) we are targetting to be asserted. + TargetNodeName string + + // How many occurances of this target name to encounter, till the target (0 means match first). + Occurance uint + + // If set to 'true' will include the nested node(s), with their attribute(s) as well. + IncludeChildNodes bool + + // Expected value of the target node's attribute(s). + ExpectedAttributes any +} + +type ExplainRequestTestCase struct { + Description string + + // Has to be a valid explain request type (one of: 'simple', 'debug', 'execute', 'predict'). + Request string + + // Docs is a map from Collection Index, to a list + // of docs in stringified JSON format + Docs map[int][]string + + // The raw expected explain graph with everything (helpful for debugging purposes). + // Note: This is not always asserted (i.e. ignored from the comparison if not provided). + ExpectedFullGraph []map[string]any + + // Pattern is used to assert that the plan nodes are in the correct order (attributes are omitted). + // Note: - Explain requests of type 'debug' will only have Pattern (as they don't have attributes). + // - This is not always asserted (i.e. ignored from the comparison if not provided). + ExpectedPatterns []map[string]any + + // Every target helps assert an individual node somewhere in the explain graph (node's position is omitted). + // Each target assertion is only responsible to check if the node's attributes are correct. + // Note: This is not always asserted (i.e. ignored from the comparison if not provided). + ExpectedTargets []PlanNodeTargetCase + + // The expected error from the explain request. + ExpectedError string +} + +func ExecuteExplainRequestTestCase( + t *testing.T, + schema string, + collectionNames []string, + explainTest ExplainRequestTestCase, +) { + if testUtils.DetectDbChanges && testUtils.DetectDbChangesPreTestChecks(t, collectionNames, false) { + return + } + + ctx := context.Background() + dbs, err := testUtils.GetDatabases(ctx, t, false) + if testUtils.AssertError(t, explainTest.Description, err, explainTest.ExpectedError) { + return + } + assert.NotEmpty(t, dbs) + + for _, dbi := range dbs { + log.Info(ctx, explainTest.Description, logging.NewKV("Database", dbi.Name())) + + if testUtils.DetectDbChanges { + if testUtils.SetupOnly { + testUtils.SetupDatabase( + ctx, + t, + dbi, + schema, + collectionNames, + explainTest.Description, + explainTest.ExpectedError, + explainTest.Docs, + client.None[map[int]map[int][]string](), + ) + dbi.DB().Close(ctx) + return + } else { + dbi = testUtils.SetupDatabaseUsingTargetBranch(ctx, t, dbi, collectionNames) + } + } else { + testUtils.SetupDatabase( + ctx, + t, + dbi, + schema, + collectionNames, + explainTest.Description, + explainTest.ExpectedError, + explainTest.Docs, + client.None[map[int]map[int][]string](), + ) + } + + if explainTest.Request != "" { + result := dbi.DB().ExecQuery(ctx, explainTest.Request) + if assertExplainRequestTestCaseWithActualResult( + ctx, + t, + &result.GQL, + explainTest, + ) { + continue + } + + if explainTest.ExpectedError != "" { + assert.Fail(t, "Expected an error however none was raised.", explainTest.Description) + } + } + + dbi.DB().Close(ctx) + } +} + +func assertExplainRequestTestCaseWithActualResult( + ctx context.Context, + t *testing.T, + actualResult *client.GQLResult, + explainTest ExplainRequestTestCase, +) bool { + // 1) Check expected error matches actual error. + if testUtils.AssertErrors( + t, + explainTest.Description, + actualResult.Errors, + explainTest.ExpectedError, + ) { + return true + } + + // Note: if returned gql result is `nil` this panics (the panic seems useful while testing). + resultantData := actualResult.Data.([]map[string]any) + log.Info(ctx, "", logging.NewKV("FullExplainGraphResult", actualResult.Data)) + + // If no expected results are provided, then it's invalid use of this explain testing setup. + if explainTest.ExpectedFullGraph == nil && + explainTest.ExpectedPatterns == nil && + explainTest.ExpectedTargets == nil { + assert.Fail(t, "Atleast one expected explain parameter must be provided.", explainTest.Description) + } + + // 2) Check if the expected full explain graph (if provided) matches the actual full explain graph + // that is returned, if doesn't match we would like to still see a diff comparison (handy while debugging). + if lengthOfExpectedFullGraph := len(explainTest.ExpectedFullGraph); lengthOfExpectedFullGraph != 0 { + assert.Equal(t, lengthOfExpectedFullGraph, len(resultantData), explainTest.Description) + for index, actualResult := range resultantData { + if lengthOfExpectedFullGraph > index { + assert.Equal( + t, + explainTest.ExpectedFullGraph[index], + actualResult, + explainTest.Description, + ) + } + } + } + + // 3) Ensure the complete high-level pattern matches, inother words check that all the + // explain graph nodes are in the correct expected ordering. + if lengthOfExpectedPatterns := len(explainTest.ExpectedPatterns); lengthOfExpectedPatterns != 0 { + assert.Equal(t, lengthOfExpectedPatterns, len(resultantData), explainTest.Description) + for index, actualResult := range resultantData { + // Trim away all attributes (non-plan nodes) from the returned full explain graph result. + actualResultWithoutAttributes := trimExplainAttributes(t, explainTest.Description, actualResult) + assert.Equal( + t, + explainTest.ExpectedPatterns[index], + actualResultWithoutAttributes, + explainTest.Description, + ) + } + } + + // 4) Match the targeted node's attributes (subset assertions), with the expected attributes. + // Note: This does not check if the node is in correct location or not. + if lengthOfExpectedTargets := len(explainTest.ExpectedTargets); lengthOfExpectedTargets != 0 { + for _, target := range explainTest.ExpectedTargets { + assertExplainTargetCase(t, explainTest.Description, target, resultantData) + } + } + + return false +} + +func assertExplainTargetCase( + t *testing.T, + description string, + targetCase PlanNodeTargetCase, + actualResults []map[string]any, +) { + for _, actualResult := range actualResults { + foundActualTarget, _, isFound := findTargetNode( + targetCase.TargetNodeName, + targetCase.Occurance, + targetCase.IncludeChildNodes, + actualResult, + ) + + if !isFound { + assert.Fail( + t, + "Expected Target: "+targetCase.TargetNodeName+ + ", but target wasn't found in the graph.", + description, + ) + } + + assert.Equal( + t, + targetCase.ExpectedAttributes, + foundActualTarget, + description, + ) + } +} + +// findTargetNode returns true if the targetName is found in the explain graph af # of occurances. +// 0 means first occurance. The function also returns total occurances it encountered so far. The +// returned matches value should always be <= occurance argument. +func findTargetNode( + targetName string, + occurance uint, + includeChildNodes bool, + actualResult any, +) (any, uint, bool) { + var totalMatchedSoFar uint = 0 + + switch r := actualResult.(type) { + case map[string]any: + for key, value := range r { + if isPlanNode(key) { + if key == targetName { + totalMatchedSoFar++ + + if occurance == 0 { + if includeChildNodes { + return value, totalMatchedSoFar, true + } + return trimSubNodes(value), totalMatchedSoFar, true + } + + target, matches, found := findTargetNode( + targetName, + occurance-1, + includeChildNodes, + value, + ) + totalMatchedSoFar = totalMatchedSoFar + matches + return target, totalMatchedSoFar, found + } + // Not a match, traverse furthur. + target, matches, found := findTargetNode( + targetName, + occurance, + includeChildNodes, + value, + ) + totalMatchedSoFar = totalMatchedSoFar + matches + return target, totalMatchedSoFar, found + } + } + + case []any: + for _, item := range r { + target, matches, found := findTargetNode( + targetName, + occurance-totalMatchedSoFar, + includeChildNodes, + item, + ) + totalMatchedSoFar = totalMatchedSoFar + matches + + if found { + if includeChildNodes { + return target, totalMatchedSoFar, true + } + return trimSubNodes(target), totalMatchedSoFar, true + } + } + + default: + return nil, totalMatchedSoFar, false + } + + return nil, totalMatchedSoFar, false +} + +// trimSubNodes returns a graph where all the immediate sub nodes are trimmed (i.e. no nested subnodes remain). +func trimSubNodes(graph any) any { + checkGraph, ok := graph.(map[string]any) + trimGraph := copyMap(checkGraph) + + if !ok { + // Is a []any, in that case do nothing, for now. + return graph + } + + for key := range trimGraph { + if isPlanNode(key) { + delete(trimGraph, key) + } + } + + return trimGraph +} + +// trimExplainAttributes trims away all keys that aren't plan nodes within the explain graph. +func trimExplainAttributes( + t *testing.T, + description string, + actualResult map[string]any, +) map[string]any { + trimmedMap := copyMap(actualResult) + + for key, value := range trimmedMap { + if !isPlanNode(key) { + delete(trimmedMap, key) + continue + } + + switch v := value.(type) { + case map[string]any: + trimmedMap[key] = trimExplainAttributes(t, description, v) + + case []any: + trimmedArrayElements := []map[string]any{} + for _, valueItem := range v { + valueItemMap, ok := valueItem.(map[string]any) + if !ok { + assert.Fail(t, "Found a planNode of type 'array' with incompatible element type.", description) + } + trimmedArrayElements = append( + trimmedArrayElements, + trimExplainAttributes(t, description, valueItemMap), + ) + } + trimmedMap[key] = trimmedArrayElements + + default: + assert.Fail(t, "Unsupported explain graph key-value type encountered.", description) + } + } + + return trimmedMap +} + +// isPlanNode returns true if someName matches a plan node name. +func isPlanNode(someName string) bool { + for _, planNodeName := range allPlanNodeNames { + if planNodeName == someName { + return true + } + } + return false +} + +func copyMap(originalMap map[string]any) map[string]any { + newMap := make(map[string]any, len(originalMap)) + for oKey, oValue := range originalMap { + newMap[oKey] = oValue + } + return newMap +} diff --git a/tests/integration/utils.go b/tests/integration/utils.go index 19d628284e..ad33e422a9 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -337,7 +337,6 @@ func ExecuteQueryTestCase( collectionNames []string, test QueryTestCase, ) { - isTransactional := false if len(test.TransactionalQueries) > 0 { isTransactional = true