feat: Add ability to Explain the response plan. (sourcenetwork#385)

- RELATED ISSUES:
Resolves sourcenetwork#325,
Closes sourcenetwork#361, Closes sourcenetwork#362 and also Closes sourcenetwork#363 (Closing the `TEXT` structured ones because we have decided to only go with JSON approach to avoid the wonky looking client side formatting).
Fixes sourcenetwork#486 in commit [d940f7b](sourcenetwork@d940f7b)

- DESCRIPTION:
This PR lays all the ground work needed to have the opt-ability for planner nodes to subscribe to the explain interface if they would like to be exposed to the explaining of the plan graph.

Even though top-level plan **node** name will be explained after this PR for all explainable nodes, this PR doesn't implement all the explain attributes for all explainable nodes. Here are the planNodes whose attributes can be explained after this PR:
1) `selectNode` (attributes completed)
2) `selectTopNode` (attributes completed / has no attributes)
3) `createNode` (attributes completed)
4) `deleteNode` (attributes completed)
5) `scanNode` (some attributes completed)

 Request:
 ```
query @Explain {
  user {
    _key
    age
    name
  }
}
```

 Response:
 ```
{
  "data": [
    {
      "explain": {
        "selectTopNode": {
          "selectNode": {
            "filter": null,
            "scanNode": {
              "collectionID": "1",
              "collectionName": "user",
              "filter": null
            }
          }
        }
      }
    }
  ]
}
```

Makefile

@@ -20,10 +20,18 @@ cross-build:
 start: build
 	./build/defradb start
 
-.PHONY: dump
-dump: build
+.PHONY: dev\:start
+dev\:start: build
+	DEFRA_ENV=dev ./build/defradb start
+
+.PHONY: client\:dump
+client\:dump:
 	./build/defradb client dump
 
+.PHONY: client\:add-schema
+client\:add-schema:
+	./build/defradb client schema add -f cli/defradb/examples/bookauthpub.graphql
+
 .PHONY: deps\:lint
 deps\:lint:
 	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ${GOPATH}/bin latest

cli/defradb/examples/address.graphql

@@ -1,6 +1,6 @@
 type address {
-	street: String 
-	number: Int 
-	city: String
+    street: String
+    number: Int
+    city: String
     country: String
 }

cli/defradb/examples/bookauthpub.graphql

@@ -0,0 +1,20 @@
+type book {
+    name: String
+    rating: Float
+    author: author
+    publisher: publisher
+}
+
+type author {
+    name: String
+    age: Int
+    verified: Boolean
+    wrote: book @primary
+}
+
+type publisher {
+    name: String
+    address: String
+    favouritePageNumbers: [Int]
+    published: [book]
+}

cli/defradb/examples/booksmany.graphql

@@ -1,4 +1,3 @@
-
 type book {
     name: String
     rating: Float
@@ -10,4 +9,4 @@ type author {
     age: Int
     verified: Boolean
     published: [book]
-}
+}

cli/defradb/examples/booksone.graphql

@@ -8,5 +8,5 @@ type author {
     name: String
     age: Int
     verified: Boolean
-    published: book 
-}
+    published: book
+}

cli/defradb/examples/user.graphql

@@ -1,6 +1,6 @@
 type user {
-	name: String 
-	age: Int 
-	verified: Boolean 
-	points: Float
+    name: String
+    age: Int
+    verified: Boolean
+    points: Float
 }

client/descriptions.go

@@ -40,8 +40,8 @@ func (col CollectionDescription) GetField(name string) (FieldDescription, bool)
 	return FieldDescription{}, false
 }
 
-func (c CollectionDescription) GetPrimaryIndex() IndexDescription {
-	return c.Indexes[0]
+func (col CollectionDescription) GetPrimaryIndex() IndexDescription {
+	return col.Indexes[0]
 }
 
 // IndexDescription describes an Index on a Collection

db/container/container.go

@@ -52,8 +52,7 @@ func (c *DocumentContainer) AddDoc(doc map[string]interface{}) error {
 		return nil
 	}
 	// append to docs slice
-	copyDoc := copyMap(doc)
-	c.docs = append(c.docs, copyDoc)
+	c.docs = append(c.docs, copyMap(doc))
 	c.numDocs++
 	return nil
 }
@@ -70,9 +69,10 @@ func (c *DocumentContainer) Swap(i, j int) {
 	c.docs[j] = tmp
 }
 
-func (c *DocumentContainer) Close() {
+func (c *DocumentContainer) Close() error {
 	c.docs = nil
 	c.numDocs = 0
+	return nil
 }
 
 func copyMap(m map[string]interface{}) map[string]interface{} {

go.mod

@@ -49,6 +49,7 @@ require (
 require (
 	github.com/fatih/color v1.13.0
 	github.com/go-chi/chi/v5 v5.0.7
+	github.com/iancoleman/strcase v0.2.0
 	github.com/pkg/errors v0.9.1
 	golang.org/x/text v0.3.7
 )

go.sum

@@ -530,6 +530,8 @@ github.com/huin/goupnp v1.0.2/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSa
 github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ=
 github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y=
 github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o=
+github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0=
+github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/improbable-eng/grpc-web v0.14.1/go.mod h1:zEjGHa8DAlkoOXmswrNvhUGEYQA9UI7DhrGeHR1DMGU=

query/graphql/parser/mutation.go

@@ -15,9 +15,9 @@ import (
 	"errors"
 	"strings"
 
-	parserTypes "github.com/sourcenetwork/defradb/query/graphql/parser/types"
-
 	"github.com/graphql-go/graphql/language/ast"
+
+	parserTypes "github.com/sourcenetwork/defradb/query/graphql/parser/types"
 )
 
 type MutationType int
@@ -134,16 +134,21 @@ func (m Mutation) ToSelect() *Select {
 	}
 }
 
-// parseOperationDefinition parses the individual GraphQL
-// 'query' operations, which there may be multiple of.
+// parseMutationOperationDefinition parses the individual GraphQL
+// 'mutation' operations, which there may be multiple of.
 func parseMutationOperationDefinition(def *ast.OperationDefinition) (*OperationDefinition, error) {
+
 	qdef := &OperationDefinition{
 		Statement:  def,
 		Selections: make([]Selection, len(def.SelectionSet.Selections)),
 	}
+
 	if def.Name != nil {
 		qdef.Name = def.Name.Value
 	}
+
+	qdef.IsExplain = parseExplainDirective(def.Directives)
+
 	for i, selection := range qdef.Statement.SelectionSet.Selections {
 		switch node := selection.(type) {
 		case *ast.Field:

query/graphql/parser/query.go

@@ -40,8 +40,8 @@ func (q Query) GetStatement() ast.Node {
 type OperationDefinition struct {
 	Name       string
 	Selections []Selection
-
-	Statement *ast.OperationDefinition
+	Statement  *ast.OperationDefinition
+	IsExplain  bool
 }
 
 func (q OperationDefinition) GetStatement() ast.Node {
@@ -194,17 +194,19 @@ func ParseQuery(doc *ast.Document) (*Query, error) {
 		Queries:   make([]*OperationDefinition, 0),
 		Mutations: make([]*OperationDefinition, 0),
 	}
+
 	for _, def := range q.Statement.Definitions {
 		switch node := def.(type) {
 		case *ast.OperationDefinition:
 			if node.Operation == "query" {
-				// parse query or mutation operation definition
+				// parse query operation definition.
 				qdef, err := parseQueryOperationDefinition(node)
 				if err != nil {
 					return nil, err
 				}
 				q.Queries = append(q.Queries, qdef)
 			} else if node.Operation == "mutation" {
+				// parse mutation operation definition.
 				mdef, err := parseMutationOperationDefinition(node)
 				if err != nil {
 					return nil, err
@@ -219,25 +221,45 @@ func ParseQuery(doc *ast.Document) (*Query, error) {
 	return q, nil
 }
 
-// parseOperationDefinition parses the individual GraphQL
+// parseExplainDirective returns true if we parsed / detected the explain directive label
+// in this ast, and false otherwise.
+func parseExplainDirective(directives []*ast.Directive) bool {
+
+	// Iterate through all directives and ensure that the directive is at there.
+	// - Note: the location we don't need to worry about as the schema takes care of it, as when
+	//         request is made there will be a syntax error for directive usage at the wrong location,
+	//         unless we add another directive named `@explain` at another location (which we should not).
+	for _, directive := range directives {
+		// The arguments pased to the directive are at `directive.Arguments`.
+		if directive.Name.Value == parserTypes.ExplainLabel {
+			return true
+		}
+	}
+
+	return false
+}
+
+// parseQueryOperationDefinition parses the individual GraphQL
 // 'query' operations, which there may be multiple of.
 func parseQueryOperationDefinition(def *ast.OperationDefinition) (*OperationDefinition, error) {
+
 	qdef := &OperationDefinition{
 		Statement:  def,
 		Selections: make([]Selection, len(def.SelectionSet.Selections)),
 	}
+
 	if def.Name != nil {
 		qdef.Name = def.Name.Value
 	}
+
+	qdef.IsExplain = parseExplainDirective(def.Directives)
+
 	for i, selection := range qdef.Statement.SelectionSet.Selections {
 		var parsed Selection
 		var err error
 		switch node := selection.(type) {
 		case *ast.Field:
-			// which query type is this
-			// database API query
-			// object query
-			// etc
+			// which query type is this database API query object query etc.
 			_, exists := dbAPIQueryNames[node.Name.Value]
 			if exists {
 				// the query matches a reserved DB API query name

query/graphql/parser/types/types.go

@@ -70,6 +70,8 @@ const (
 	SumFieldName     = "_sum"
 	VersionFieldName = "_version"
 
+	ExplainLabel = "explain"
+
 	ASC  = SortDirection("ASC")
 	DESC = SortDirection("DESC")
 )

query/graphql/planner/average.go

@@ -43,6 +43,7 @@ func (n *averageNode) Init() error {
 	return n.plan.Init()
 }
 
+func (n *averageNode) Kind() string           { return "averageNode" }
 func (n *averageNode) Start() error           { return n.plan.Start() }
 func (n *averageNode) Spans(spans core.Spans) { n.plan.Spans(spans) }
 func (n *averageNode) Close() error           { return n.plan.Close() }
@@ -90,3 +91,9 @@ func (n *averageNode) Next() (bool, error) {
 }
 
 func (n *averageNode) SetPlan(p planNode) { n.plan = p }
+
+// Explain method returns a map containing all attributes of this node that
+// are to be explained, subscribes / opts-in this node to be an explainablePlanNode.
+func (n *averageNode) Explain() (map[string]interface{}, error) {
+	return map[string]interface{}{}, nil
+}

query/graphql/planner/commit.go

@@ -19,6 +19,36 @@ import (
 	"github.com/sourcenetwork/defradb/query/graphql/parser"
 )
 
+// commitSelectTopNode is a wrapper for the selectTopNode
+// in the case where the select is actually a CommitSelect
+type commitSelectTopNode struct {
+	p    *Planner
+	plan planNode
+}
+
+func (n *commitSelectTopNode) Kind() string { return "commitSelectTopNode" }
+
+func (n *commitSelectTopNode) Init() error { return n.plan.Init() }
+
+func (n *commitSelectTopNode) Start() error { return n.plan.Start() }
+
+func (n *commitSelectTopNode) Next() (bool, error) { return n.plan.Next() }
+
+func (n *commitSelectTopNode) Spans(spans core.Spans) { n.plan.Spans(spans) }
+
+func (n *commitSelectTopNode) Value() map[string]interface{} { return n.plan.Value() }
+
+func (n *commitSelectTopNode) Source() planNode { return n.plan }
+
+func (n *commitSelectTopNode) Close() error {
+	if n.plan == nil {
+		return nil
+	}
+	return n.plan.Close()
+}
+
+func (n *commitSelectTopNode) Append() bool { return true }
+
 type commitSelectNode struct {
 	documentIterator
 
@@ -29,6 +59,10 @@ type commitSelectNode struct {
 	subRenderInfo map[string]renderInfo
 }
 
+func (n *commitSelectNode) Kind() string {
+	return "commitSelectNode"
+}
+
 func (n *commitSelectNode) Init() error {
 	return n.source.Init()
 }
@@ -58,10 +92,11 @@ func (n *commitSelectNode) Source() planNode {
 	return n.source
 }
 
-// AppendNode implements appendNode
-// func (n *commitSelectNode) AppendNode() bool {
-// 	return true
-// }
+// Explain method returns a map containing all attributes of this node that
+// are to be explained, subscribes / opts-in this node to be an explainablePlanNode.
+func (n *commitSelectNode) Explain() (map[string]interface{}, error) {
+	return map[string]interface{}{}, nil
+}
 
 func (p *Planner) CommitSelect(parsed *parser.CommitSelect) (planNode, error) {
 	// check type of commit select (all, latest, one)
@@ -159,25 +194,3 @@ func (p *Planner) commitSelectAll(parsed *parser.CommitSelect) (*commitSelectNod
 
 	return commit, nil
 }
-
-// commitSelectTopNode is a wrapper for the selectTopNode
-// in the case where the select is actually a CommitSelect
-type commitSelectTopNode struct {
-	p    *Planner
-	plan planNode
-}
-
-func (n *commitSelectTopNode) Init() error                   { return n.plan.Init() }
-func (n *commitSelectTopNode) Start() error                  { return n.plan.Start() }
-func (n *commitSelectTopNode) Next() (bool, error)           { return n.plan.Next() }
-func (n *commitSelectTopNode) Spans(spans core.Spans)        { n.plan.Spans(spans) }
-func (n *commitSelectTopNode) Value() map[string]interface{} { return n.plan.Value() }
-func (n *commitSelectTopNode) Source() planNode              { return n.plan }
-func (n *commitSelectTopNode) Close() error {
-	if n.plan == nil {
-		return nil
-	}
-	return n.plan.Close()
-}
-
-func (n *commitSelectTopNode) Append() bool { return true }