From 53699421711aa1cc23179150ec63cc8463b2fca9 Mon Sep 17 00:00:00 2001 From: AndrewSisley Date: Thu, 13 Jan 2022 16:36:33 +0000 Subject: [PATCH] feat: Add count aggregate support (#102) * Add count aggregate support to object querys * Add count aggregate support to commit queries --- db/tests/query/all_commits/with_count_test.go | 43 ++++ .../with_count_limit_offset_test.go | 97 ++++++++ .../one_to_many/with_count_limit_test.go | 88 +++++++ db/tests/query/one_to_many/with_count_test.go | 104 ++++++++ db/tests/query/one_to_many_multiple/utils.go | 40 +++ .../one_to_many_multiple/with_count_test.go | 90 +++++++ .../query/simple/with_group_count_test.go | 230 ++++++++++++++++++ query/graphql/parser/commit.go | 9 + query/graphql/parser/query.go | 58 ++++- query/graphql/planner/count.go | 69 ++++++ query/graphql/planner/limit.go | 79 +++++- query/graphql/planner/operations.go | 4 +- query/graphql/planner/planner.go | 88 +++++-- query/graphql/planner/render.go | 30 ++- query/graphql/planner/select.go | 108 ++++++-- query/graphql/schema/generate.go | 39 +++ query/graphql/schema/generate_test.go | 56 +++++ query/graphql/schema/types/types.go | 15 ++ 18 files changed, 1181 insertions(+), 66 deletions(-) create mode 100644 db/tests/query/all_commits/with_count_test.go create mode 100644 db/tests/query/one_to_many/with_count_limit_offset_test.go create mode 100644 db/tests/query/one_to_many/with_count_limit_test.go create mode 100644 db/tests/query/one_to_many/with_count_test.go create mode 100644 db/tests/query/one_to_many_multiple/utils.go create mode 100644 db/tests/query/one_to_many_multiple/with_count_test.go create mode 100644 db/tests/query/simple/with_group_count_test.go create mode 100644 query/graphql/planner/count.go diff --git a/db/tests/query/all_commits/with_count_test.go b/db/tests/query/all_commits/with_count_test.go new file mode 100644 index 0000000000..58ea958da8 --- /dev/null +++ b/db/tests/query/all_commits/with_count_test.go @@ -0,0 +1,43 @@ +// Copyright 2020 Source Inc. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. +package all_commits + +import ( + "testing" + + testUtils "github.com/sourcenetwork/defradb/db/tests" +) + +func TestQueryAllCommitsSingleDAGWithLinkCount(t *testing.T) { + test := testUtils.QueryTestCase{ + Description: "Simple latest commits query", + Query: `query { + allCommits(dockey: "bae-52b9170d-b77a-5887-b877-cbdbb99b009f") { + cid + _count(field: links) + } + }`, + Docs: map[int][]string{ + 0: { + (`{ + "Name": "John", + "Age": 21 + }`)}, + }, + Results: []map[string]interface{}{ + { + "cid": "bafkreiercmxn6e3qryxvuped5pplg733c5fj6gjypj5wykk63ouvcfb25m", + "_count": 2, + }, + }, + } + + executeTestCase(t, test) +} diff --git a/db/tests/query/one_to_many/with_count_limit_offset_test.go b/db/tests/query/one_to_many/with_count_limit_offset_test.go new file mode 100644 index 0000000000..c630089489 --- /dev/null +++ b/db/tests/query/one_to_many/with_count_limit_offset_test.go @@ -0,0 +1,97 @@ +// Copyright 2020 Source Inc. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. +package one_to_many + +import ( + "testing" + + testUtils "github.com/sourcenetwork/defradb/db/tests" +) + +func TestQueryOneToManyWithCountAndLimitAndOffset(t *testing.T) { + test := testUtils.QueryTestCase{ + Description: "One-to-many relation query from many side with count and limit and offset", + Query: `query { + author { + name + _count(field: published) + published(limit: 2, offset: 1) { + name + } + } + }`, + Docs: map[int][]string{ + //books + 0: { + (`{ + "name": "Painted House", + "rating": 4.9, + "author_id": "bae-41598f0c-19bc-5da6-813b-e80f14a10df3" + }`), + (`{ + "name": "A Time for Mercy", + "rating": 4.5, + "author_id": "bae-41598f0c-19bc-5da6-813b-e80f14a10df3" + }`), + (`{ + "name": "The Firm", + "rating": 4.1, + "author_id": "bae-41598f0c-19bc-5da6-813b-e80f14a10df3" + }`), + (`{ + "name": "The Pelican Brief", + "rating": 4.0, + "author_id": "bae-41598f0c-19bc-5da6-813b-e80f14a10df3" + }`), + (`{ + "name": "Theif Lord", + "rating": 4.8, + "author_id": "bae-b769708d-f552-5c3d-a402-ccfd7ac7fb04" + }`), + }, + //authors + 1: { + // bae-41598f0c-19bc-5da6-813b-e80f14a10df3 + (`{ + "name": "John Grisham", + "age": 65, + "verified": true + }`), + // bae-b769708d-f552-5c3d-a402-ccfd7ac7fb04 + (`{ + "name": "Cornelia Funke", + "age": 62, + "verified": false + }`), + }, + }, + Results: []map[string]interface{}{ + { + "name": "John Grisham", + "_count": 4, + "published": []map[string]interface{}{ + { + "name": "The Pelican Brief", + }, + { + "name": "Painted House", + }, + }, + }, + { + "name": "Cornelia Funke", + "_count": 1, + "published": []map[string]interface{}{}, + }, + }, + } + + executeTestCase(t, test) +} diff --git a/db/tests/query/one_to_many/with_count_limit_test.go b/db/tests/query/one_to_many/with_count_limit_test.go new file mode 100644 index 0000000000..41cf4aed58 --- /dev/null +++ b/db/tests/query/one_to_many/with_count_limit_test.go @@ -0,0 +1,88 @@ +// Copyright 2020 Source Inc. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. +package one_to_many + +import ( + "testing" + + testUtils "github.com/sourcenetwork/defradb/db/tests" +) + +func TestQueryOneToManyWithCountAndLimit(t *testing.T) { + test := testUtils.QueryTestCase{ + Description: "One-to-many relation query from many side with count and limit", + Query: `query { + author { + name + _count(field: published) + published(limit: 1) { + name + } + } + }`, + Docs: map[int][]string{ + //books + 0: { // bae-fd541c25-229e-5280-b44b-e5c2af3e374d + (`{ + "name": "Painted House", + "rating": 4.9, + "author_id": "bae-41598f0c-19bc-5da6-813b-e80f14a10df3" + }`), + (`{ + "name": "A Time for Mercy", + "rating": 4.5, + "author_id": "bae-41598f0c-19bc-5da6-813b-e80f14a10df3" + }`), + (`{ + "name": "Theif Lord", + "rating": 4.8, + "author_id": "bae-b769708d-f552-5c3d-a402-ccfd7ac7fb04" + }`), + }, + //authors + 1: { + // bae-41598f0c-19bc-5da6-813b-e80f14a10df3 + (`{ + "name": "John Grisham", + "age": 65, + "verified": true + }`), + // bae-b769708d-f552-5c3d-a402-ccfd7ac7fb04 + (`{ + "name": "Cornelia Funke", + "age": 62, + "verified": false + }`), + }, + }, + Results: []map[string]interface{}{ + { + "name": "John Grisham", + "_count": 2, + "published": []map[string]interface{}{ + { + "name": "Painted House", + }, + }, + }, + { + "name": "Cornelia Funke", + "_count": 1, + "published": []map[string]interface{}{ + { + "name": "Theif Lord", + }, + }, + }, + }, + } + + executeTestCase(t, test) +} diff --git a/db/tests/query/one_to_many/with_count_test.go b/db/tests/query/one_to_many/with_count_test.go new file mode 100644 index 0000000000..12cbab4617 --- /dev/null +++ b/db/tests/query/one_to_many/with_count_test.go @@ -0,0 +1,104 @@ +// Copyright 2020 Source Inc. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. +package one_to_many + +import ( + "testing" + + testUtils "github.com/sourcenetwork/defradb/db/tests" +) + +func TestQueryOneToManyWithCount(t *testing.T) { + tests := []testUtils.QueryTestCase{ + { + Description: "One-to-many relation query from many side with count, no child records", + Query: `query { + author { + name + _count(field: published) + } + }`, + Docs: map[int][]string{ + //authors + 1: { + (`{ + "name": "John Grisham", + "age": 65, + "verified": true + }`), + }, + }, + Results: []map[string]interface{}{ + { + "name": "John Grisham", + "_count": 0, + }, + }, + }, + { + Description: "One-to-many relation query from many side with count", + Query: `query { + author { + name + _count(field: published) + } + }`, + Docs: map[int][]string{ + //books + 0: { // bae-fd541c25-229e-5280-b44b-e5c2af3e374d + (`{ + "name": "Painted House", + "rating": 4.9, + "author_id": "bae-41598f0c-19bc-5da6-813b-e80f14a10df3" + }`), + (`{ + "name": "A Time for Mercy", + "rating": 4.5, + "author_id": "bae-41598f0c-19bc-5da6-813b-e80f14a10df3" + }`), + (`{ + "name": "Theif Lord", + "rating": 4.8, + "author_id": "bae-b769708d-f552-5c3d-a402-ccfd7ac7fb04" + }`), + }, + //authors + 1: { + // bae-41598f0c-19bc-5da6-813b-e80f14a10df3 + (`{ + "name": "John Grisham", + "age": 65, + "verified": true + }`), + // bae-b769708d-f552-5c3d-a402-ccfd7ac7fb04 + (`{ + "name": "Cornelia Funke", + "age": 62, + "verified": false + }`), + }, + }, + Results: []map[string]interface{}{ + { + "name": "John Grisham", + "_count": 2, + }, + { + "name": "Cornelia Funke", + "_count": 1, + }, + }, + }, + } + + for _, test := range tests { + executeTestCase(t, test) + } +} diff --git a/db/tests/query/one_to_many_multiple/utils.go b/db/tests/query/one_to_many_multiple/utils.go new file mode 100644 index 0000000000..67f9d7368f --- /dev/null +++ b/db/tests/query/one_to_many_multiple/utils.go @@ -0,0 +1,40 @@ +// Copyright 2020 Source Inc. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. +package one_to_many_multiple + +import ( + "testing" + + testUtils "github.com/sourcenetwork/defradb/db/tests" +) + +var bookAuthorGQLSchema = (` + type article { + name: String + author: author + } + + type book { + name: String + author: author + } + + type author { + name: String + age: Int + verified: Boolean + books: [book] + articles: [article] + } +`) + +func executeTestCase(t *testing.T, test testUtils.QueryTestCase) { + testUtils.ExecuteQueryTestCase(t, bookAuthorGQLSchema, []string{"article", "book", "author"}, test) +} diff --git a/db/tests/query/one_to_many_multiple/with_count_test.go b/db/tests/query/one_to_many_multiple/with_count_test.go new file mode 100644 index 0000000000..11e5a471c8 --- /dev/null +++ b/db/tests/query/one_to_many_multiple/with_count_test.go @@ -0,0 +1,90 @@ +// Copyright 2020 Source Inc. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. +package one_to_many_multiple + +import ( + "testing" + + testUtils "github.com/sourcenetwork/defradb/db/tests" +) + +func TestQueryOneToManyMultipleWithCount(t *testing.T) { + test := testUtils.QueryTestCase{ + Description: "One-to-many relation query from many side with count", + Query: `query { + author { + name + numberOfBooks: _count(field: books) + numberOfArticles: _count(field: articles) + } + }`, + Docs: map[int][]string{ + //articles + 0: { + (`{ + "name": "After Guantánamo, Another Injustice", + "author_id": "bae-41598f0c-19bc-5da6-813b-e80f14a10df3" + }`), + (`{ + "name": "To my dear readers", + "author_id": "bae-b769708d-f552-5c3d-a402-ccfd7ac7fb04" + }`), + (`{ + "name": "Twinklestar's Favourite Xmas Cookie", + "author_id": "bae-b769708d-f552-5c3d-a402-ccfd7ac7fb04" + }`), + }, + //books + 1: { + (`{ + "name": "Painted House", + "author_id": "bae-41598f0c-19bc-5da6-813b-e80f14a10df3" + }`), + (`{ + "name": "A Time for Mercy", + "author_id": "bae-41598f0c-19bc-5da6-813b-e80f14a10df3" + }`), + (`{ + "name": "Theif Lord", + "author_id": "bae-b769708d-f552-5c3d-a402-ccfd7ac7fb04" + }`), + }, + //authors + 2: { + // bae-41598f0c-19bc-5da6-813b-e80f14a10df3 + (`{ + "name": "John Grisham", + "age": 65, + "verified": true + }`), + // bae-b769708d-f552-5c3d-a402-ccfd7ac7fb04 + (`{ + "name": "Cornelia Funke", + "age": 62, + "verified": false + }`), + }, + }, + Results: []map[string]interface{}{ + { + "name": "John Grisham", + "numberOfBooks": 2, + "numberOfArticles": 1, + }, + { + "name": "Cornelia Funke", + "numberOfBooks": 1, + "numberOfArticles": 2, + }, + }, + } + + executeTestCase(t, test) +} diff --git a/db/tests/query/simple/with_group_count_test.go b/db/tests/query/simple/with_group_count_test.go new file mode 100644 index 0000000000..6b91874cf9 --- /dev/null +++ b/db/tests/query/simple/with_group_count_test.go @@ -0,0 +1,230 @@ +// Copyright 2020 Source Inc. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. +package simple + +import ( + "testing" + + testUtils "github.com/sourcenetwork/defradb/db/tests" +) + +func TestQuerySimpleWithGroupByNumberWithoutRenderedGroupAndChildCount(t *testing.T) { + test := testUtils.QueryTestCase{ + Description: "Simple query with group by number, no children, count on non-rendered group", + Query: `query { + users(groupBy: [Age]) { + Age + _count(field: _group) + } + }`, + Docs: map[int][]string{ + 0: { + (`{ + "Name": "John", + "Age": 32 + }`), + (`{ + "Name": "Bob", + "Age": 32 + }`), + (`{ + "Name": "Alice", + "Age": 19 + }`)}, + }, + Results: []map[string]interface{}{ + { + "Age": uint64(32), + "_count": 2, + }, + { + "Age": uint64(19), + "_count": 1, + }, + }, + } + + executeTestCase(t, test) +} + +func TestQuerySimpleWithGroupByNumberWithRenderedGroupAndChildCount(t *testing.T) { + test := testUtils.QueryTestCase{ + Description: "Simple query with group by number, no children, count on rendered group", + Query: `query { + users(groupBy: [Age]) { + Age + _count(field: _group) + _group { + Name + } + } + }`, + Docs: map[int][]string{ + 0: { + (`{ + "Name": "John", + "Age": 32 + }`), + (`{ + "Name": "Bob", + "Age": 32 + }`), + (`{ + "Name": "Alice", + "Age": 19 + }`)}, + }, + Results: []map[string]interface{}{ + { + "Age": uint64(32), + "_count": 2, + "_group": []map[string]interface{}{ + { + "Name": "Bob", + }, + { + "Name": "John", + }, + }, + }, + { + "Age": uint64(19), + "_count": 1, + "_group": []map[string]interface{}{ + { + "Name": "Alice", + }, + }, + }, + }, + } + + executeTestCase(t, test) +} + +func TestQuerySimpleWithGroupByNumberWithUndefinedField(t *testing.T) { + test := testUtils.QueryTestCase{ + Description: "Simple query with group by number, count on undefined field", + Query: `query { + users(groupBy: [Age]) { + Age + _count + } + }`, + Docs: map[int][]string{ + 0: { + (`{ + "Name": "John", + "Age": 32 + }`), + (`{ + "Name": "Bob", + "Age": 32 + }`), + (`{ + "Name": "Alice", + "Age": 19 + }`)}, + }, + Results: []map[string]interface{}{ + { + "Age": uint64(32), + "_count": 0, + }, + { + "Age": uint64(19), + "_count": 0, + }, + }, + } + + executeTestCase(t, test) +} + +func TestQuerySimpleWithGroupByNumberWithoutRenderedGroupAndAliasesChildCount(t *testing.T) { + test := testUtils.QueryTestCase{ + Description: "Simple query with group by number, no children, aliased count on non-rendered group", + Query: `query { + users(groupBy: [Age]) { + Age + Count: _count(field: _group) + } + }`, + Docs: map[int][]string{ + 0: { + (`{ + "Name": "John", + "Age": 32 + }`), + (`{ + "Name": "Bob", + "Age": 32 + }`), + (`{ + "Name": "Alice", + "Age": 19 + }`)}, + }, + Results: []map[string]interface{}{ + { + "Age": uint64(32), + "Count": 2, + }, + { + "Age": uint64(19), + "Count": 1, + }, + }, + } + + executeTestCase(t, test) +} + +func TestQuerySimpleWithGroupByNumberWithoutRenderedGroupAndDuplicatedAliasedChildCounts(t *testing.T) { + test := testUtils.QueryTestCase{ + Description: "Simple query with group by number, no children, duplicated aliased count on non-rendered group", + Query: `query { + users(groupBy: [Age]) { + Age + Count1: _count(field: _group) + Count2: _count(field: _group) + } + }`, + Docs: map[int][]string{ + 0: { + (`{ + "Name": "John", + "Age": 32 + }`), + (`{ + "Name": "Bob", + "Age": 32 + }`), + (`{ + "Name": "Alice", + "Age": 19 + }`)}, + }, + Results: []map[string]interface{}{ + { + "Age": uint64(32), + "Count1": 2, + "Count2": 2, + }, + { + "Age": uint64(19), + "Count1": 1, + "Count2": 1, + }, + }, + } + + executeTestCase(t, test) +} diff --git a/query/graphql/parser/commit.go b/query/graphql/parser/commit.go index 43c91a9740..de35e50d88 100644 --- a/query/graphql/parser/commit.go +++ b/query/graphql/parser/commit.go @@ -45,6 +45,7 @@ type CommitSelect struct { Limit *Limit OrderBy *OrderBy + Counts []Count Fields []Selection @@ -71,12 +72,17 @@ func (c CommitSelect) GetSelections() []Selection { return c.Fields } +func (s *CommitSelect) AddCount(count Count) { + s.Counts = append(s.Counts, count) +} + func (c CommitSelect) ToSelect() *Select { return &Select{ Name: c.Name, Alias: c.Alias, Limit: c.Limit, OrderBy: c.OrderBy, + Counts: c.Counts, Statement: c.Statement, Fields: c.Fields, Root: CommitSelection, @@ -119,5 +125,8 @@ func parseCommitSelect(field *ast.Field) (*CommitSelect, error) { var err error commit.Fields, err = parseSelectFields(commit.GetRoot(), field.SelectionSet) + + parseCounts(commit) + return commit, err } diff --git a/query/graphql/parser/query.go b/query/graphql/parser/query.go index 8ab5159f83..195e441229 100644 --- a/query/graphql/parser/query.go +++ b/query/graphql/parser/query.go @@ -11,6 +11,7 @@ package parser import ( "errors" + "fmt" "strconv" "github.com/graphql-go/graphql/language/ast" @@ -26,6 +27,8 @@ const ( VersionFieldName = "_version" GroupFieldName = "_group" DocKeyFieldName = "_key" + CountFieldName = "_count" + HiddenFieldName = "_hidden" ) var dbAPIQueryNames = map[string]bool{ @@ -37,6 +40,8 @@ var dbAPIQueryNames = map[string]bool{ var ReservedFields = map[string]bool{ VersionFieldName: true, GroupFieldName: true, + CountFieldName: true, + HiddenFieldName: true, DocKeyFieldName: true, } @@ -73,7 +78,10 @@ type Selection interface { GetRoot() SelectionType } -// type +type baseSelect interface { + Selection + AddCount(count Count) +} // Select is a complex Field with strong typing // It used for sub types in a query. Includes @@ -94,6 +102,7 @@ type Select struct { Limit *Limit OrderBy *OrderBy GroupBy *GroupBy + Counts []Count Fields []Selection @@ -121,6 +130,10 @@ func (s Select) GetAlias() string { return s.Alias } +func (s *Select) AddCount(count Count) { + s.Counts = append(s.Counts, count) +} + // Field implements Selection type Field struct { Name string @@ -157,6 +170,11 @@ type GroupBy struct { Fields []string } +type Count struct { + Name string + Field string +} + type SortDirection string const ( @@ -365,6 +383,15 @@ func parseSelect(rootType SelectionType, field *ast.Field) (*Select, error) { // parse field selections var err error slct.Fields, err = parseSelectFields(slct.Root, field.SelectionSet) + if err != nil { + return nil, err + } + + err = parseCounts(slct) + if err != nil { + return nil, err + } + return slct, err } @@ -417,3 +444,32 @@ func parseAPIQuery(field *ast.Field) (Selection, error) { return nil, errors.New("Unknown query") } } + +// Parses requested _count(s), creating a virtual name (alias) if an alias is not provided to allow for multiple _count +// fields. Strongly consider refactoring this as more aggregates get added. +func parseCounts(slct baseSelect) error { + for i, field := range slct.GetSelections() { + if field.GetName() == CountFieldName { + virtualName := fmt.Sprintf("count%v", i) + f := field.(*Field) + if f.Alias == "" { + f.Alias = f.Name + } + f.Name = virtualName + var fieldName string + fieldStatement, statementIsField := field.GetStatement().(*ast.Field) + if !statementIsField { + return fmt.Errorf("Unexpected error: could not cast field statement to field.") + } + + if len(fieldStatement.Arguments) == 0 { + fieldName = "" + } else { + fieldName = fieldStatement.Arguments[0].Value.GetValue().(string) + } + slct.AddCount(Count{Name: virtualName, Field: fieldName}) + } + } + + return nil +} diff --git a/query/graphql/planner/count.go b/query/graphql/planner/count.go new file mode 100644 index 0000000000..133b2575b3 --- /dev/null +++ b/query/graphql/planner/count.go @@ -0,0 +1,69 @@ +// Copyright 2020 Source Inc. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. +package planner + +// Consider moving this file into an `aggregate` sub-package to keep them organized, +// or moving all aggregates to within an do-all `aggregate` node when adding the next few +// aggregates in. + +import ( + "reflect" + + "github.com/sourcenetwork/defradb/core" + "github.com/sourcenetwork/defradb/query/graphql/parser" +) + +type countNode struct { + p *Planner + plan planNode + + sourceProperty string + virtualFieldId string +} + +func (p *Planner) Count(c *parser.Count, virtualFieldId string) (*countNode, error) { + return &countNode{ + p: p, + sourceProperty: c.Field, + virtualFieldId: virtualFieldId, + }, nil +} + +func (n *countNode) Init() error { + return n.plan.Init() +} + +func (n *countNode) Start() error { return n.plan.Start() } +func (n *countNode) Spans(spans core.Spans) { n.plan.Spans(spans) } +func (n *countNode) Close() error { return n.plan.Close() } +func (n *countNode) Source() planNode { return n.plan } + +func (n *countNode) Values() map[string]interface{} { + value := n.plan.Values() + + // Can just scan for now, can be replaced later by something fancier if needed + var count int + if property, hasProperty := value[n.sourceProperty]; hasProperty { + v := reflect.ValueOf(property) + switch v.Kind() { + // v.Len will panic if v is not one of these types, we don't want it to panic + case reflect.Array, reflect.Chan, reflect.Map, reflect.Slice, reflect.String: + count = v.Len() + } + } + + value[n.virtualFieldId] = count + + return value +} + +func (n *countNode) Next() (bool, error) { + return n.plan.Next() +} diff --git a/query/graphql/planner/limit.go b/query/graphql/planner/limit.go index c7ab8e34b4..3e660dc35d 100644 --- a/query/graphql/planner/limit.go +++ b/query/graphql/planner/limit.go @@ -14,9 +14,9 @@ import ( "github.com/sourcenetwork/defradb/query/graphql/parser" ) -// limit the results +// Limit the results, yielding only what the limit/offset permits // @todo: Handle cursor -type limitNode struct { +type hardLimitNode struct { p *Planner plan planNode @@ -25,13 +25,13 @@ type limitNode struct { rowIndex int64 } -// Limit creates a new limitNode initalized from +// HardLimit creates a new hardLimitNode initalized from // the parser.Limit object. -func (p *Planner) Limit(n *parser.Limit) (*limitNode, error) { +func (p *Planner) HardLimit(n *parser.Limit) (*hardLimitNode, error) { if n == nil { return nil, nil // nothing to do } - return &limitNode{ + return &hardLimitNode{ p: p, limit: n.Limit, offset: n.Offset, @@ -39,17 +39,17 @@ func (p *Planner) Limit(n *parser.Limit) (*limitNode, error) { }, nil } -func (n *limitNode) Init() error { +func (n *hardLimitNode) Init() error { n.rowIndex = 0 return n.plan.Init() } -func (n *limitNode) Start() error { return n.plan.Start() } -func (n *limitNode) Spans(spans core.Spans) { n.plan.Spans(spans) } -func (n *limitNode) Close() error { return n.plan.Close() } -func (n *limitNode) Values() map[string]interface{} { return n.plan.Values() } +func (n *hardLimitNode) Start() error { return n.plan.Start() } +func (n *hardLimitNode) Spans(spans core.Spans) { n.plan.Spans(spans) } +func (n *hardLimitNode) Close() error { return n.plan.Close() } +func (n *hardLimitNode) Values() map[string]interface{} { return n.plan.Values() } -func (n *limitNode) Next() (bool, error) { +func (n *hardLimitNode) Next() (bool, error) { // check if we're passed the limit if n.rowIndex-n.offset >= n.limit { return false, nil @@ -71,4 +71,59 @@ func (n *limitNode) Next() (bool, error) { return true, nil } -func (n *limitNode) Source() planNode { return n.plan } +func (n *hardLimitNode) Source() planNode { return n.plan } + +// limit the results, flagging any records outside the bounds of limit/offset with +// with a 'hidden' flag blocking rendering. Used if consumers of the results require +// the full dataset. +type renderLimitNode struct { + p *Planner + plan planNode + + limit int64 + offset int64 + rowIndex int64 +} + +// RenderLimit creates a new renderLimitNode initalized from +// the parser.Limit object. +func (p *Planner) RenderLimit(n *parser.Limit) (*renderLimitNode, error) { + if n == nil { + return nil, nil // nothing to do + } + return &renderLimitNode{ + p: p, + limit: n.Limit, + offset: n.Offset, + rowIndex: 0, + }, nil +} + +func (n *renderLimitNode) Init() error { + n.rowIndex = 0 + return n.plan.Init() +} + +func (n *renderLimitNode) Start() error { return n.plan.Start() } +func (n *renderLimitNode) Spans(spans core.Spans) { n.plan.Spans(spans) } +func (n *renderLimitNode) Close() error { return n.plan.Close() } +func (n *renderLimitNode) Values() map[string]interface{} { + value := n.plan.Values() + + if n.rowIndex-n.offset > n.limit || n.rowIndex <= n.offset { + value[parser.HiddenFieldName] = struct{}{} + } + + return value +} + +func (n *renderLimitNode) Next() (bool, error) { + if next, err := n.plan.Next(); !next { + return false, err + } + + n.rowIndex++ + return true, nil +} + +func (n *renderLimitNode) Source() planNode { return n.plan } diff --git a/query/graphql/planner/operations.go b/query/graphql/planner/operations.go index 8b8d5f1620..d739f399c0 100644 --- a/query/graphql/planner/operations.go +++ b/query/graphql/planner/operations.go @@ -12,7 +12,8 @@ package planner var ( _ planNode = (*scanNode)(nil) _ planNode = (*headsetScanNode)(nil) - _ planNode = (*limitNode)(nil) + _ planNode = (*hardLimitNode)(nil) + _ planNode = (*renderLimitNode)(nil) _ planNode = (*groupNode)(nil) _ planNode = (*pipeNode)(nil) _ planNode = (*selectNode)(nil) @@ -22,6 +23,7 @@ var ( _ planNode = (*typeIndexJoin)(nil) _ planNode = (*typeJoinOne)(nil) _ planNode = (*typeJoinMany)(nil) + _ planNode = (*countNode)(nil) ) // type joinNode struct { diff --git a/query/graphql/planner/planner.go b/query/graphql/planner/planner.go index 3ad21a8d8d..e051f59431 100644 --- a/query/graphql/planner/planner.go +++ b/query/graphql/planner/planner.go @@ -158,36 +158,36 @@ func (p *Planner) makePlan(stmt parser.Statement) (planNode, error) { // plan optimization. Includes plan expansion and wiring func (p *Planner) optimizePlan(plan planNode) error { - err := p.expandPlan(plan) + err := p.expandPlan(plan, nil) return err } // full plan graph expansion and optimization -func (p *Planner) expandPlan(plan planNode) error { +func (p *Planner) expandPlan(plan planNode, parentPlan *selectTopNode) error { switch n := plan.(type) { case *selectTopNode: - return p.expandSelectTopNodePlan(n) + return p.expandSelectTopNodePlan(n, parentPlan) case *commitSelectTopNode: - return p.expandPlan(n.plan) + return p.expandPlan(n.plan, parentPlan) case *selectNode: - return p.expandPlan(n.source) + return p.expandPlan(n.source, parentPlan) case *typeIndexJoin: - return p.expandTypeIndexJoinPlan(n) + return p.expandTypeIndexJoinPlan(n, parentPlan) case *groupNode: // We only care about expanding the child source here, it is assumed that the parent source // is expanded elsewhere/already - return p.expandPlan(n.dataSource.childSource) + return p.expandPlan(n.dataSource.childSource, parentPlan) case MultiNode: - return p.expandMultiNode(n) + return p.expandMultiNode(n, parentPlan) case *updateNode: - return p.expandPlan(n.results) + return p.expandPlan(n.results, parentPlan) default: return nil } } -func (p *Planner) expandSelectTopNodePlan(plan *selectTopNode) error { - if err := p.expandPlan(plan.source); err != nil { +func (p *Planner) expandSelectTopNodePlan(plan *selectTopNode, parentPlan *selectTopNode) error { + if err := p.expandPlan(plan.source, plan); err != nil { return err } @@ -203,10 +203,10 @@ func (p *Planner) expandSelectTopNodePlan(plan *selectTopNode) error { plan.plan = plan.group } - // wire up the render plan - if plan.render != nil { - plan.render.plan = plan.plan - plan.plan = plan.render + // consider extracting this out to an `expandAggregatePlan` when adding more aggregates + for _, countPlan := range plan.countPlans { + countPlan.plan = plan.plan + plan.plan = countPlan } // if order @@ -216,16 +216,21 @@ func (p *Planner) expandSelectTopNodePlan(plan *selectTopNode) error { } if plan.limit != nil { - plan.limit.plan = plan.plan - plan.plan = plan.limit + p.expandLimitPlan(plan, parentPlan) + } + + // wire up the render plan + if plan.render != nil { + plan.render.plan = plan.plan + plan.plan = plan.render } return nil } -func (p *Planner) expandMultiNode(plan MultiNode) error { +func (p *Planner) expandMultiNode(plan MultiNode, parentPlan *selectTopNode) error { for _, child := range plan.Children() { - if err := p.expandPlan(child); err != nil { + if err := p.expandPlan(child, parentPlan); err != nil { return err } } @@ -237,12 +242,12 @@ func (p *Planner) expandMultiNode(plan MultiNode) error { // return p.expandPlan(plan.source) // } -func (p *Planner) expandTypeIndexJoinPlan(plan *typeIndexJoin) error { +func (p *Planner) expandTypeIndexJoinPlan(plan *typeIndexJoin, parentPlan *selectTopNode) error { switch node := plan.joinPlan.(type) { case *typeJoinOne: - return p.expandPlan(node.subType) + return p.expandPlan(node.subType, parentPlan) case *typeJoinMany: - return p.expandPlan(node.subType) + return p.expandPlan(node.subType, parentPlan) } return errors.New("Unknown type index join plan") } @@ -279,7 +284,44 @@ func (p *Planner) expandGroupNodePlan(plan *selectTopNode) error { return err } - return p.expandPlan(childSource) + return p.expandPlan(childSource, plan) +} + +func (p *Planner) expandLimitPlan(plan *selectTopNode, parentPlan *selectTopNode) error { + switch l := plan.limit.(type) { + case *hardLimitNode: + if l == nil { + return nil + } + + // if this is a child node, and the parent select has an aggregate then we need to + // replace the hard limit with a render limit to allow the full set of child records + // to be aggregated + if parentPlan != nil && len(parentPlan.countPlans) > 0 { + renderLimit, err := p.RenderLimit(&parser.Limit{ + Offset: l.offset, + Limit: l.limit, + }) + if err != nil { + return err + } + plan.limit = renderLimit + + renderLimit.plan = plan.plan + plan.plan = plan.limit + } else { + l.plan = plan.plan + plan.plan = plan.limit + } + case *renderLimitNode: + if l == nil { + return nil + } + + l.plan = plan.plan + plan.plan = plan.limit + } + return nil } // walkAndReplace walks through the provided plan, and searches for an instance diff --git a/query/graphql/planner/render.go b/query/graphql/planner/render.go index 24a09f003f..f67801f8f7 100644 --- a/query/graphql/planner/render.go +++ b/query/graphql/planner/render.go @@ -78,9 +78,24 @@ func buildRenderInfo(parsed parser.Selection) renderInfo { return info } -func (n *renderNode) Init() error { return n.plan.Init() } -func (n *renderNode) Start() error { return n.plan.Start() } -func (n *renderNode) Next() (bool, error) { return n.plan.Next() } +func (n *renderNode) Init() error { return n.plan.Init() } +func (n *renderNode) Start() error { return n.plan.Start() } +func (n *renderNode) Next() (bool, error) { + hasNext, err := n.plan.Next() + if err != nil || !hasNext { + return hasNext, err + } + + doc := n.plan.Values() + if doc == nil { + return n.Next() + } + + if _, isHidden := doc[parser.HiddenFieldName]; isHidden { + return n.Next() + } + return hasNext, err +} func (n *renderNode) Spans(spans core.Spans) { n.plan.Spans(spans) } func (n *renderNode) Close() error { return n.plan.Close() } func (n *renderNode) Source() planNode { return n.plan } @@ -109,6 +124,11 @@ func (r *renderInfo) render(src map[string]interface{}, destination map[string]i // If the current property is itself a map, we should render any properties of the child case map[string]interface{}: inner := map[string]interface{}{} + + if _, isHidden := v[parser.HiddenFieldName]; isHidden { + return + } + for _, child := range r.children { child.render(v, inner) } @@ -117,6 +137,10 @@ func (r *renderInfo) render(src map[string]interface{}, destination map[string]i case []map[string]interface{}: subdocs := make([]map[string]interface{}, 0) for _, subv := range v { + if _, isHidden := subv[parser.HiddenFieldName]; isHidden { + continue + } + inner := map[string]interface{}{} for _, child := range r.children { child.render(subv, inner) diff --git a/query/graphql/planner/select.go b/query/graphql/planner/select.go index 41d5a15bf2..326b302890 100644 --- a/query/graphql/planner/select.go +++ b/query/graphql/planner/select.go @@ -23,11 +23,12 @@ import ( // expansion // Executes the top level plan node. type selectTopNode struct { - source planNode - group *groupNode - sort *sortNode - limit *limitNode - render *renderNode + source planNode + group *groupNode + sort *sortNode + limit planNode + countPlans []*countNode + render *renderNode // top of the plan graph plan planNode @@ -204,16 +205,38 @@ func (n *selectNode) initFields(parsed *parser.Select) error { if subtype.Name == parser.GroupFieldName { n.groupSelect = subtype } else { - typeIndexJoin, err := n.p.makeTypeIndexJoin(n, n.origSource, subtype) - if err != nil { - return err - } - - // n.source = typeIndexJoin - if err := n.addSubPlan(field.GetName(), typeIndexJoin); err != nil { - return err - } + n.addTypeIndexJoin(subtype) + } + } + } + } + + // Handle aggregates of child collection that are not rendered + for _, count := range parsed.Counts { + if count.Field == "" { + continue + } + + hasChildProperty := false + for _, field := range parsed.Fields { + if count.Field == field.GetName() { + hasChildProperty = true + break + } + } + + // If the child item is not requested, then we have add in the necessary components to force the child records to be scanned through (they wont be rendered) + if !hasChildProperty { + if count.Field == parser.GroupFieldName { + // It doesn't really matter at the moment if multiple counts are requested and we overwrite the n.groupSelect property + n.groupSelect = &parser.Select{ + Name: parser.GroupFieldName, + } + } else if parsed.Root != parser.CommitSelection { + subtype := &parser.Select{ + Name: count.Field, } + n.addTypeIndexJoin(subtype) } } } @@ -221,6 +244,19 @@ func (n *selectNode) initFields(parsed *parser.Select) error { return nil } +func (n *selectNode) addTypeIndexJoin(subSelect *parser.Select) error { + typeIndexJoin, err := n.p.makeTypeIndexJoin(n, n.origSource, subSelect) + if err != nil { + return err + } + + if err := n.addSubPlan(subSelect.Name, typeIndexJoin); err != nil { + return err + } + + return nil +} + func (n *selectNode) Source() planNode { return n.source } // func appendSource() {} @@ -280,7 +316,7 @@ func (p *Planner) SelectFromSource(parsed *parser.Select, source planNode, fromC return nil, err } - limitPlan, err := p.Limit(limit) + limitPlan, err := p.HardLimit(limit) if err != nil { return nil, err } @@ -290,12 +326,22 @@ func (p *Planner) SelectFromSource(parsed *parser.Select, source planNode, fromC return nil, err } + countPlans := []*countNode{} + for _, countItem := range parsed.Counts { + countNode, err := p.Count(&countItem, countItem.Name) + if err != nil { + return nil, err + } + countPlans = append(countPlans, countNode) + } + top := &selectTopNode{ - source: s, - render: p.render(parsed), - limit: limitPlan, - sort: sortPlan, - group: groupPlan, + source: s, + render: p.render(parsed), + limit: limitPlan, + sort: sortPlan, + group: groupPlan, + countPlans: countPlans, } return top, nil } @@ -318,7 +364,7 @@ func (p *Planner) Select(parsed *parser.Select) (planNode, error) { return nil, err } - limitPlan, err := p.Limit(limit) + limitPlan, err := p.HardLimit(limit) if err != nil { return nil, err } @@ -328,12 +374,22 @@ func (p *Planner) Select(parsed *parser.Select) (planNode, error) { return nil, err } + countPlans := []*countNode{} + for _, countItem := range parsed.Counts { + countNode, err := p.Count(&countItem, countItem.Name) + if err != nil { + return nil, err + } + countPlans = append(countPlans, countNode) + } + top := &selectTopNode{ - source: s, - render: p.render(parsed), - limit: limitPlan, - sort: sortPlan, - group: groupPlan, + source: s, + render: p.render(parsed), + limit: limitPlan, + sort: sortPlan, + group: groupPlan, + countPlans: countPlans, } return top, nil } diff --git a/query/graphql/schema/generate.go b/query/graphql/schema/generate.go index 74cf9ddb9f..bc5c5d56c7 100644 --- a/query/graphql/schema/generate.go +++ b/query/graphql/schema/generate.go @@ -84,6 +84,12 @@ func (g *Generator) FromAST(document *ast.Document) ([]*gql.Object, error) { return nil, err } + g.genAggregateFields() + // resolve types + if err := g.manager.ResolveTypes(); err != nil { + return nil, err + } + // for each built type // generate query inputs queryType := g.manager.schema.QueryType() @@ -374,6 +380,39 @@ func getRelationshipName(field *ast.FieldDefinition, hostName gql.ObjectConfig, return genRelationName(hostName.Name, targetName.Name()) } +func (g *Generator) genAggregateFields() { + for _, t := range g.typeDefs { + countField := genCountFieldConfig(t) + t.AddFieldConfig(countField.Name, &countField) + } +} + +func genCountFieldConfig(obj *gql.Object) gql.Field { + inputCfg := gql.EnumConfig{ + Name: genTypeName(obj, "CountArg"), + Values: gql.EnumValueConfigMap{}, + } + + for _, field := range obj.Fields() { + // Only lists can be counted + if _, isList := field.Type.(*gql.List); !isList { + continue + } + inputCfg.Values[field.Name] = &gql.EnumValueConfig{Value: field.Name} + } + countType := gql.NewEnum(inputCfg) + + field := gql.Field{ + Name: parser.CountFieldName, + Type: gql.Int, + Args: gql.FieldConfigArgument{ + "field": newArgConfig(countType), + }, + } + + return field +} + // Given a parsed ast.Node object, lookup the type in the TypeMap and return if its there // otherwise return an error // ast.Node, can either be a ast.Named type, a ast.List, or a ast.NonNull. diff --git a/query/graphql/schema/generate_test.go b/query/graphql/schema/generate_test.go index c356ec4b49..6848452005 100644 --- a/query/graphql/schema/generate_test.go +++ b/query/graphql/schema/generate_test.go @@ -60,6 +60,10 @@ func Test_Generator_buildTypesFromAST_SingleScalarField(t *testing.T) { Name: "_group", Type: gql.NewList(g.manager.schema.TypeMap()["MyObject"]), }, + "_count": &gql.Field{ + Name: "_count", + Type: gql.Int, + }, "myField": &gql.Field{ Name: "myField", Type: gql.String, @@ -96,6 +100,10 @@ func Test_Generator_buildTypesFromAST_SingleNonNullScalarField(t *testing.T) { Name: "_group", Type: gql.NewList(g.manager.schema.TypeMap()["MyObject"]), }, + "_count": &gql.Field{ + Name: "_count", + Type: gql.Int, + }, "myField": &gql.Field{ Name: "myField", Type: gql.NewNonNull(gql.String), @@ -132,6 +140,10 @@ func Test_Generator_buildTypesFromAST_SingleListScalarField(t *testing.T) { Name: "_group", Type: gql.NewList(g.manager.schema.TypeMap()["MyObject"]), }, + "_count": &gql.Field{ + Name: "_count", + Type: gql.Int, + }, "myField": &gql.Field{ Name: "myField", Type: gql.NewList(gql.String), @@ -168,6 +180,10 @@ func Test_Generator_buildTypesFromAST_SingleListNonNullScalarField(t *testing.T) Name: "_group", Type: gql.NewList(g.manager.schema.TypeMap()["MyObject"]), }, + "_count": &gql.Field{ + Name: "_count", + Type: gql.Int, + }, "myField": &gql.Field{ Name: "myField", Type: gql.NewList(gql.NewNonNull(gql.String)), @@ -204,6 +220,10 @@ func Test_Generator_buildTypesFromAST_SingleNonNullListScalarField(t *testing.T) Name: "_group", Type: gql.NewList(g.manager.schema.TypeMap()["MyObject"]), }, + "_count": &gql.Field{ + Name: "_count", + Type: gql.Int, + }, "myField": &gql.Field{ Name: "myField", Type: gql.NewNonNull(gql.NewList(gql.String)), @@ -240,6 +260,10 @@ func Test_Generator_buildTypesFromAST_SingleNonNullListNonNullScalarField(t *tes Name: "_group", Type: gql.NewList(g.manager.schema.TypeMap()["MyObject"]), }, + "_count": &gql.Field{ + Name: "_count", + Type: gql.Int, + }, "myField": &gql.Field{ Name: "myField", Type: gql.NewNonNull(gql.NewList(gql.NewNonNull(gql.String))), @@ -281,6 +305,10 @@ func Test_Generator_buildTypesFromAST_MultiScalarField(t *testing.T) { Name: "_group", Type: gql.NewList(g.manager.schema.TypeMap()["MyObject"]), }, + "_count": &gql.Field{ + Name: "_count", + Type: gql.Int, + }, "myField": &gql.Field{ Name: "myField", Type: gql.String, @@ -341,6 +369,10 @@ func Test_Generator_buildTypesFromAST_MultiObjectSingleScalarField(t *testing.T) Name: "_group", Type: gql.NewList(g.manager.schema.TypeMap()["MyObject"]), }, + "_count": &gql.Field{ + Name: "_count", + Type: gql.Int, + }, "myField": &gql.Field{ Name: "myField", Type: gql.String, @@ -364,6 +396,10 @@ func Test_Generator_buildTypesFromAST_MultiObjectSingleScalarField(t *testing.T) Name: "_group", Type: gql.NewList(g.manager.schema.TypeMap()["OtherObject"]), }, + "_count": &gql.Field{ + Name: "_count", + Type: gql.Int, + }, "otherField": &gql.Field{ Name: "otherField", Type: gql.Boolean, @@ -406,6 +442,10 @@ func Test_Generator_buildTypesFromAST_MultiObjectMultiScalarField(t *testing.T) Name: "_group", Type: gql.NewList(g.manager.schema.TypeMap()["MyObject"]), }, + "_count": &gql.Field{ + Name: "_count", + Type: gql.Int, + }, "myField": &gql.Field{ Name: "myField", Type: gql.String, @@ -433,6 +473,10 @@ func Test_Generator_buildTypesFromAST_MultiObjectMultiScalarField(t *testing.T) Name: "_group", Type: gql.NewList(g.manager.schema.TypeMap()["OtherObject"]), }, + "_count": &gql.Field{ + Name: "_count", + Type: gql.Int, + }, "otherField": &gql.Field{ Name: "otherField", Type: gql.Boolean, @@ -466,6 +510,10 @@ func Test_Generator_buildTypesFromAST_MultiObjectSingleObjectField(t *testing.T) Name: "_group", Type: gql.NewList(g.manager.schema.TypeMap()["MyObject"]), }, + "_count": &gql.Field{ + Name: "_count", + Type: gql.Int, + }, "myField": &gql.Field{ Name: "myField", Type: gql.String, @@ -502,6 +550,10 @@ func Test_Generator_buildTypesFromAST_MultiObjectSingleObjectField(t *testing.T) Name: "_group", Type: gql.NewList(g.manager.schema.TypeMap()["OtherObject"]), }, + "_count": &gql.Field{ + Name: "_count", + Type: gql.Int, + }, "otherField": &gql.Field{ Name: "otherField", Type: myObj, @@ -556,6 +608,10 @@ func Test_Generator_buildTypesFromAST_MissingObject(t *testing.T) { Name: "_group", Type: gql.NewList(g.manager.schema.TypeMap()["OtherObject"]), }, + "_count": &gql.Field{ + Name: "_count", + Type: gql.Int, + }, "otherField": &gql.Field{ Name: "otherField", Type: myObj, diff --git a/query/graphql/schema/types/types.go b/query/graphql/schema/types/types.go index ea3bd826fc..8ce7f64c01 100644 --- a/query/graphql/schema/types/types.go +++ b/query/graphql/schema/types/types.go @@ -41,6 +41,13 @@ var ( }, }) + CommitCountFieldArg = gql.NewEnum(gql.EnumConfig{ + Name: "commitCountFieldArg", + Values: gql.EnumValueConfigMap{ + "links": &gql.EnumValueConfig{Value: "links"}, + }, + }) + // Commit represents an individual commit to a MerkleCRDT // type Commit { // Height: Int @@ -70,6 +77,14 @@ var ( "links": &gql.Field{ Type: gql.NewList(CommitLink), }, + "_count": &gql.Field{ + Type: gql.Int, + Args: gql.FieldConfigArgument{ + "field": &gql.ArgumentConfig{ + Type: CommitCountFieldArg, + }, + }, + }, // "tests": &gql.Field{ // Type: gql.NewList(gql.String), // },