From e443aa354b5238a212d31b9d8a6baeeb4c863922 Mon Sep 17 00:00:00 2001 From: Ezra Haleva Date: Tue, 14 May 2024 07:22:34 -0700 Subject: [PATCH 01/22] added MakeList and MakeSet aggregators --- .../Pipeline/Aggregate/AggregateOperator.cs | 2 + .../Aggregators/MakeListAggregator.cs | 83 +++++++++++++++++++ .../Aggregators/MakeSetAggregator.cs | 83 +++++++++++++++++++ .../Aggregators/SingleGroupAggregator.cs | 9 +- 4 files changed, 176 insertions(+), 1 deletion(-) create mode 100644 Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeListAggregator.cs create mode 100644 Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeSetAggregator.cs diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/AggregateOperator.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/AggregateOperator.cs index fdd52e516b..5d2020af47 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/AggregateOperator.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/AggregateOperator.cs @@ -9,6 +9,8 @@ internal enum AggregateOperator Average, Count, Max, + MakeList, + MakeSet, Min, Sum, } diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeListAggregator.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeListAggregator.cs new file mode 100644 index 0000000000..9bb7f9fa95 --- /dev/null +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeListAggregator.cs @@ -0,0 +1,83 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Query.Core.Pipeline.Aggregate.Aggregators +{ + using System; + using System.Collections.Generic; + using System.Globalization; + using System.Text; + using Microsoft.Azure.Cosmos.CosmosElements; + using Microsoft.Azure.Cosmos.CosmosElements.Numbers; + using Microsoft.Azure.Cosmos.Query.Core.Exceptions; + using Microsoft.Azure.Cosmos.Query.Core.Monads; + + internal sealed class MakeListAggregator : IAggregator + { + private readonly List globalList; + + private MakeListAggregator(CosmosArray initialList) + { + this.globalList = new List(); + foreach (CosmosElement setItem in initialList) + { + this.globalList.Add(setItem); + } + } + + public void Aggregate(CosmosElement localList) + { + if (!(localList is CosmosArray cosmosArray)) + { + throw new ArgumentException($"{nameof(localList)} must be an array."); + } + + foreach (CosmosElement listItem in cosmosArray) + { + this.globalList.Add(listItem); + } + } + + public CosmosElement GetResult() + { + CosmosElement[] cosmosElementArray = new CosmosElement[this.globalList.Count]; + this.globalList.CopyTo(cosmosElementArray); + return CosmosArray.Create(cosmosElementArray); + } + + public string GetContinuationToken() + { + return this.globalList.ToString(); + } + + public static TryCatch TryCreate(CosmosElement continuationToken) + { + CosmosArray partialList; + if (continuationToken != null) + { + if (!(continuationToken is CosmosArray cosmosPartialList)) + { + return TryCatch.FromException( + new MalformedContinuationTokenException($@"Invalid MakeList continuation token: ""{continuationToken}"".")); + } + + partialList = cosmosPartialList; + } + else + { + partialList = CosmosArray.Empty; + } + + return TryCatch.FromResult( + new MakeListAggregator(initialList: partialList)); + } + + public CosmosElement GetCosmosElementContinuationToken() + { + CosmosElement[] cosmosElementArray = new CosmosElement[this.globalList.Count]; + this.globalList.CopyTo(cosmosElementArray); + return CosmosArray.Create(cosmosElementArray); + } + } +} diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeSetAggregator.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeSetAggregator.cs new file mode 100644 index 0000000000..079dcb6967 --- /dev/null +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeSetAggregator.cs @@ -0,0 +1,83 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Query.Core.Pipeline.Aggregate.Aggregators +{ + using System; + using System.Collections.Generic; + using System.Globalization; + using System.Text; + using Microsoft.Azure.Cosmos.CosmosElements; + using Microsoft.Azure.Cosmos.CosmosElements.Numbers; + using Microsoft.Azure.Cosmos.Query.Core.Exceptions; + using Microsoft.Azure.Cosmos.Query.Core.Monads; + + internal sealed class MakeSetAggregator : IAggregator + { + private readonly HashSet globalSet; + + private MakeSetAggregator(CosmosArray initialSet) + { + this.globalSet = new HashSet(); + foreach (CosmosElement setItem in initialSet) + { + this.globalSet.Add(setItem); + } + } + + public void Aggregate(CosmosElement localSet) + { + if (!(localSet is CosmosArray cosmosArray)) + { + throw new ArgumentException($"{nameof(localSet)} must be an array."); + } + + foreach (CosmosElement setItem in cosmosArray) + { + this.globalSet.Add(setItem); + } + } + + public CosmosElement GetResult() + { + CosmosElement[] cosmosElementArray = new CosmosElement[this.globalSet.Count]; + this.globalSet.CopyTo(cosmosElementArray); + return CosmosArray.Create(cosmosElementArray); + } + + public string GetContinuationToken() + { + return this.globalSet.ToString(); + } + + public static TryCatch TryCreate(CosmosElement continuationToken) + { + CosmosArray partialSet; + if (continuationToken != null) + { + if (!(continuationToken is CosmosArray cosmosPartialSet)) + { + return TryCatch.FromException( + new MalformedContinuationTokenException($@"Invalid MakeSet continuation token: ""{continuationToken}"".")); + } + + partialSet = cosmosPartialSet; + } + else + { + partialSet = CosmosArray.Empty; + } + + return TryCatch.FromResult( + new MakeSetAggregator(initialSet: partialSet)); + } + + public CosmosElement GetCosmosElementContinuationToken() + { + CosmosElement[] cosmosElementArray = new CosmosElement[this.globalSet.Count]; + this.globalSet.CopyTo(cosmosElementArray); + return CosmosArray.Create(cosmosElementArray); + } + } +} diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/SingleGroupAggregator.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/SingleGroupAggregator.cs index 806b311462..bd10c436a3 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/SingleGroupAggregator.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/SingleGroupAggregator.cs @@ -370,6 +370,14 @@ public static TryCatch TryCreate( tryCreateAggregator = CountAggregator.TryCreate(continuationToken); break; + case AggregateOperator.MakeList: + tryCreateAggregator = MakeListAggregator.TryCreate(continuationToken); + break; + + case AggregateOperator.MakeSet: + tryCreateAggregator = MakeSetAggregator.TryCreate(continuationToken); + break; + case AggregateOperator.Max: tryCreateAggregator = MinMaxAggregator.TryCreateMaxAggregator(continuationToken); break; @@ -381,7 +389,6 @@ public static TryCatch TryCreate( case AggregateOperator.Sum: tryCreateAggregator = SumAggregator.TryCreate(continuationToken); break; - default: throw new ArgumentException($"Unknown {nameof(AggregateOperator)}: {aggregateOperator}."); } From db1872a58dfd4b34c07c59ba7e2a8d33923253ad Mon Sep 17 00:00:00 2001 From: Ezra Haleva Date: Fri, 24 May 2024 09:01:28 -0700 Subject: [PATCH 02/22] Added MakeList and MakeSet to AggregateQueryTests.cs --- ...ueryTests.AggregateMixedTypes_baseline.xml | 206 +++++++++++++++++- .../Query/AggregateQueryTests.cs | 126 ++++++++++- 2 files changed, 328 insertions(+), 4 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.AggregateMixedTypes_baseline.xml b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.AggregateMixedTypes_baseline.xml index e7f3e96c19..71f90623cc 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.AggregateMixedTypes_baseline.xml +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.AggregateMixedTypes_baseline.xml @@ -1,4 +1,4 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + makeListResult = inputDocuments + .Select(doc => + { + if (!doc.TryGetValue(aggregateTestArgs.PartitionKey, out CosmosElement cosmosElement)) + { + Assert.Fail("Failed to get partition key from document"); + } + + return cosmosElement; + }) + .ToList(); + + IReadOnlyList makeSetResult = makeListResult.Distinct().ToList(); + AggregateQueryArguments[] aggregateQueryArgumentsList = new AggregateQueryArguments[] { new AggregateQueryArguments( @@ -237,6 +314,14 @@ private static AggregateQueryArguments[] CreateAggregateQueryArguments( aggregateOperator: "COUNT", expectedValue: CosmosNumber64.Create(inputDocuments.Count()), predicate: "true"), + new AggregateQueryArguments( + aggregateOperator: "MAKELIST", + expectedValue: CosmosArray.Create(makeListResult), // construct cosmos array from linq result + predicate: "true"), + new AggregateQueryArguments( + aggregateOperator: "MAKESET", + expectedValue: CosmosArray.Create(makeSetResult), // construct cosmos array from linq result + predicate: "true"), new AggregateQueryArguments( aggregateOperator: "MAX", expectedValue: CosmosString.Create("xyz"), @@ -392,6 +477,32 @@ async Task ImplementationAsync( Assert.Fail($"Something went wrong with query: {query}, ex: {ex}"); } } + + string[] arrayAggregateQueries = new string[] + { + $"SELECT VALUE MAKELIST(c.{uniqueField}) FROM c WHERE c.{uniqueField} = {valueOfInterest}", + $"SELECT VALUE MAKESET(c.{uniqueField}) FROM c WHERE c.{uniqueField} = {valueOfInterest}", + }; + + foreach (string query in arrayAggregateQueries) + { + try + { + List items = await QueryTestsBase.RunQueryAsync( + container, + query, + new QueryRequestOptions() + { + MaxConcurrency = 10, + }); + + Assert.IsTrue((items.Single() is CosmosArray result) && result.Equals(CosmosArray.Create(CosmosNumber64.Create(valueOfInterest)))); + } + catch (Exception ex) + { + Assert.Fail($"Something went wrong with query: {query}, ex: {ex}"); + } + } } } @@ -527,7 +638,7 @@ private async Task TestQueryCrossPartitionAggregateFunctionsWithMixedTypesHelper args.UndefinedKey }; - string[] aggregateOperators = new string[] { "AVG", "MIN", "MAX", "SUM", "COUNT" }; + string[] aggregateOperators = new string[] { "AVG", "MIN", "MAKELIST", "MAKESET", "MAX", "SUM", "COUNT" }; string[] typeCheckFunctions = new string[] { "IS_ARRAY", "IS_BOOL", "IS_NULL", "IS_NUMBER", "IS_OBJECT", "IS_STRING", "IS_DEFINED", "IS_PRIMITIVE" }; List queries = new List(); foreach (string aggregateOperator in aggregateOperators) @@ -609,10 +720,19 @@ FROM c { Assert.AreEqual(1, items.Count); CosmosElement aggregateResult = items.First(); - if(aggregateResult is not CosmosUndefined) { - writer.WriteCData(items.Single().ToString()); + if ((formattedQuery.Contains("MAKELIST") || formattedQuery.Contains("MAKESET")) + && (aggregateResult is CosmosArray aggregateResultArray)) + { + CosmosElement[] normalizedAggregateResult = aggregateResultArray.ToArray(); + Array.Sort(normalizedAggregateResult); + writer.WriteCData(CosmosArray.Create(normalizedAggregateResult).ToString()); + } + else + { + writer.WriteCData(items.Single().ToString()); + } } } From b84e8e21da11cf20d8b234c95ef7d8eff8b20a61 Mon Sep 17 00:00:00 2001 From: Ezra Haleva Date: Mon, 3 Jun 2024 09:52:37 -0700 Subject: [PATCH 03/22] Adjust Aggregators --- .../Pipeline/Aggregate/Aggregators/MakeListAggregator.cs | 6 +----- .../Pipeline/Aggregate/Aggregators/MakeSetAggregator.cs | 1 + 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeListAggregator.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeListAggregator.cs index 9bb7f9fa95..993ae3bd02 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeListAggregator.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeListAggregator.cs @@ -6,6 +6,7 @@ namespace Microsoft.Azure.Cosmos.Query.Core.Pipeline.Aggregate.Aggregators { using System; using System.Collections.Generic; + using System.Diagnostics; using System.Globalization; using System.Text; using Microsoft.Azure.Cosmos.CosmosElements; @@ -46,11 +47,6 @@ public CosmosElement GetResult() return CosmosArray.Create(cosmosElementArray); } - public string GetContinuationToken() - { - return this.globalList.ToString(); - } - public static TryCatch TryCreate(CosmosElement continuationToken) { CosmosArray partialList; diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeSetAggregator.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeSetAggregator.cs index 079dcb6967..56aa9d6d17 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeSetAggregator.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeSetAggregator.cs @@ -6,6 +6,7 @@ namespace Microsoft.Azure.Cosmos.Query.Core.Pipeline.Aggregate.Aggregators { using System; using System.Collections.Generic; + using System.Diagnostics; using System.Globalization; using System.Text; using Microsoft.Azure.Cosmos.CosmosElements; From f42c7ec44fa1d41c2f2c6af061818f0bd28d1dd2 Mon Sep 17 00:00:00 2001 From: Ezra Haleva Date: Mon, 3 Jun 2024 09:53:48 -0700 Subject: [PATCH 04/22] Add Array Aggregate Continuation Token Test --- .../Query/AggregateQueryTests.cs | 95 ++++++++++++++++++- 1 file changed, 94 insertions(+), 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs index 3be160c6d8..ac1a1476a2 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs @@ -15,13 +15,106 @@ using Microsoft.Azure.Cosmos.Json; using Microsoft.Azure.Documents; using Microsoft.VisualStudio.TestTools.UnitTesting; + using Newtonsoft.Json; using Newtonsoft.Json.Linq; - using static System.Net.Mime.MediaTypeNames; [TestClass] [TestCategory("Query")] public sealed class AggregateCrossPartitionQueryTests : QueryTestsBase { + + [TestMethod] + public async Task TestArrayAggregatesContinuationToken() + { + + int seed = (int)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds; + uint numberOfDocuments = 17000; + + Random rand = new Random(seed); + List people = new List(); + + for (int i = 0; i < numberOfDocuments; i++) + { + // Generate random people + Person person = PersonGenerator.GetRandomPerson(rand); + for (int j = 0; j < rand.Next(0, 4); j++) + { + // Force an exact duplicate + people.Add(person); + } + } + + List documents = new List(); + // Shuffle them so they end up in different pages + people = people.OrderBy((person) => Guid.NewGuid()).ToList(); + foreach (Person person in people) + { + documents.Add(JsonConvert.SerializeObject(person)); + } + + await this.CreateIngestQueryDeleteAsync( + ConnectionModes.Direct | ConnectionModes.Gateway, + CollectionTypes.MultiPartition, + documents, + ImplementationAsync, + "/id"); + + async static Task ImplementationAsync(Container container, IReadOnlyList documents) + { + foreach (string[] queriesToCompare in new string[][] + { + //new string[]{ "SELECT VALUE c.age FROM c ORDER BY c.age", "SELECT VALUE MakeList(c.age) FROM c GROUP BY c.name" }, + new string[]{ "SELECT VALUE c.age FROM c ORDER BY c.age", "SELECT VALUE MakeList(c.age) FROM c" }, + new string[]{ "SELECT DISTINCT VALUE c.age FROM c ORDER BY c.age", "SELECT VALUE MakeSet(c.age) FROM c" }, + }) + { + string queryWithoutAggregate = queriesToCompare[0]; + List expectedDocuments = await QueryTestsBase.RunQueryCombinationsAsync( + container, + queryWithoutAggregate, + new QueryRequestOptions() + { + MaxConcurrency = 10, + MaxItemCount = 100, + }, + QueryDrainingMode.ContinuationToken | QueryDrainingMode.HoldState); + + CosmosElement[] normalizedExpectedResult = expectedDocuments.ToArray(); + Array.Sort(normalizedExpectedResult); + + CosmosArray normalizedExpectedCosmosArray = CosmosArray.Create(expectedDocuments); + + foreach (int pageSize in new int[] { 1, 10, 100 }) + { + string queryWithAggregate = queriesToCompare[1]; + List actualDocuments = await QueryTestsBase.RunQueryCombinationsAsync( + container, + queryWithAggregate, + new QueryRequestOptions() + { + MaxConcurrency = 10, + MaxItemCount = pageSize + }, + QueryDrainingMode.ContinuationToken | QueryDrainingMode.HoldState | QueryDrainingMode.CosmosElementContinuationToken); + + CosmosElement aggregateResult = actualDocuments.First(); + CosmosArray normalizedActualCosmosArray = null; + if(aggregateResult is CosmosArray actualCosmosArray) + { + CosmosElement[] normalizedActualArray = actualCosmosArray.ToArray(); + Array.Sort(normalizedActualArray); + normalizedActualCosmosArray = CosmosArray.Create(normalizedActualArray); + } + + Assert.AreEqual( + expected: normalizedExpectedCosmosArray, + actual: normalizedActualCosmosArray, + message: $"Documents didn't match for {queryWithAggregate} on a Partitioned container"); + } + } + } + } + [TestMethod] public async Task TestAggregateFunctionsAsync() { From 2cb739c29651c9eadaf741755d7f15e1286e6d88 Mon Sep 17 00:00:00 2001 From: Ezra Haleva Date: Mon, 3 Jun 2024 11:54:08 -0700 Subject: [PATCH 05/22] Added group by coverage for MakeList and MakeSet --- .../Query/GroupByQueryTests.cs | 122 +++++++++++++++++- 1 file changed, 115 insertions(+), 7 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/GroupByQueryTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/GroupByQueryTests.cs index f2257bbf34..52dbbafbca 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/GroupByQueryTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/GroupByQueryTests.cs @@ -2,6 +2,7 @@ { using System; using System.Collections.Generic; + using System.Collections.ObjectModel; using System.Linq; using System.Threading.Tasks; using Microsoft.Azure.Cosmos.CosmosElements; @@ -537,22 +538,129 @@ FROM c } } - // Test that continuation token is blocked + List<(string, List)> queryAndExpectedResultsListArrayAggregates = new List<(string, List)>() + { + ( + "SELECT c.name AS NameGroup, MAKELIST(c.age) AS list_age FROM c GROUP BY c.name", + documents + .GroupBy(document => document["name"]) + .Select(grouping => CosmosObject.Create( + new Dictionary() + { + { "NameGroup", grouping.Key }, + { "list_age", CosmosArray.Create( + grouping.Select(document => document["age"]) + )} + })) + .ToList() + ), + ( + "SELECT c.name AS NameGroup, MAKESET(c.age) AS list_age FROM c GROUP BY c.name", + documents + .GroupBy(document => document["name"]) + .Select(grouping => CosmosObject.Create( + new Dictionary() + { + { "NameGroup", grouping.Key }, + { "list_age", CosmosArray.Create( + grouping.Select(document => document["age"]).Distinct() + )} + })) + .ToList() + ) + }; + + foreach ((string query, List expectedResults) in queryAndExpectedResultsListArrayAggregates) { - try + foreach (int maxItemCount in new int[] { 1, 5, 10 }) { - List actual = await QueryTestsBase.QueryWithContinuationTokensAsync( + List actualWithoutContinuationTokens = await QueryTestsBase.QueryWithoutContinuationTokensAsync( + container, + query, + new QueryRequestOptions() + { + MaxConcurrency = 2, + MaxItemCount = maxItemCount, + MaxBufferedItemCount = 100, + }); + + this.NormalizeGroupByArrayAggregateResults(actualWithoutContinuationTokens); + HashSet actualWithoutContinuationTokensSet = new HashSet(actualWithoutContinuationTokens); + + List actualWithTryGetContinuationTokens = await QueryTestsBase.QueryWithCosmosElementContinuationTokenAsync( container, - "SELECT c.age FROM c GROUP BY c.age", + query, new QueryRequestOptions() { MaxConcurrency = 2, - MaxItemCount = 1 + MaxItemCount = maxItemCount, + MaxBufferedItemCount = 100, }); - Assert.Fail("Expected an error when trying to drain a GROUP BY query with continuation tokens."); + + this.NormalizeGroupByArrayAggregateResults(actualWithTryGetContinuationTokens); + HashSet actualWithTryGetContinuationTokensSet = new HashSet(actualWithTryGetContinuationTokens); + + Assert.IsTrue( + actualWithoutContinuationTokensSet.SetEquals(actualWithTryGetContinuationTokensSet), + $"Results did not match for query: {query} with maxItemCount: {maxItemCount}" + + $"ActualWithoutContinuationTokens: {JsonConvert.SerializeObject(actualWithoutContinuationTokensSet)}" + + $"ActualWithTryGetContinuationTokens: {JsonConvert.SerializeObject(actualWithTryGetContinuationTokensSet)}"); + + this.NormalizeGroupByArrayAggregateResults(expectedResults); + HashSet expectedSet = new HashSet(expectedResults); + + Assert.IsTrue( + actualWithoutContinuationTokensSet.SetEquals(expectedSet), + $"Results did not match for query: {query} with maxItemCount: {maxItemCount}" + + $"Actual {JsonConvert.SerializeObject(actualWithoutContinuationTokensSet)}" + + $"Expected: {JsonConvert.SerializeObject(expectedSet)}"); } - catch (Exception) + } + + // Test that continuation token is blocked + + try + { + List actual = await QueryTestsBase.QueryWithContinuationTokensAsync( + container, + "SELECT c.age FROM c GROUP BY c.age", + new QueryRequestOptions() + { + MaxConcurrency = 2, + MaxItemCount = 1 + }); + Assert.Fail("Expected an error when trying to drain a GROUP BY query with continuation tokens."); + } + catch (Exception) + { + } + } + + private void NormalizeGroupByArrayAggregateResults( + List results) + { + for (int i = 0; i < results.Count; i++) + { + if (results[i] is CosmosObject cosmosObject) { + IDictionary myDict = new Dictionary(); + + foreach (KeyValuePair kvp in cosmosObject) + { + if (kvp.Value is CosmosArray cosmosArray) + { + CosmosElement[] normalizedArray = cosmosArray.ToArray(); + Array.Sort(normalizedArray); + myDict.Add(kvp.Key, CosmosArray.Create(normalizedArray)); + } + else + { + myDict.Add(kvp.Key, kvp.Value); + } + } + + IReadOnlyDictionary newDict = new ReadOnlyDictionary(myDict); + results[i] = CosmosObject.Create(newDict); } } } From 7c863d42de2682e0e981caa32c795b76374ff242 Mon Sep 17 00:00:00 2001 From: Ezra Haleva Date: Tue, 4 Jun 2024 15:16:43 -0700 Subject: [PATCH 06/22] address comments --- .../Query/AggregateQueryTests.cs | 69 ++++++++----------- 1 file changed, 28 insertions(+), 41 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs index ac1a1476a2..5d64cc9714 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs @@ -2,7 +2,6 @@ { using System; using System.Collections.Generic; - using System.Collections.Immutable; using System.Globalization; using System.IO; using System.Linq; @@ -26,9 +25,8 @@ public sealed class AggregateCrossPartitionQueryTests : QueryTestsBase [TestMethod] public async Task TestArrayAggregatesContinuationToken() { - int seed = (int)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds; - uint numberOfDocuments = 17000; + uint numberOfDocuments = 100; Random rand = new Random(seed); List people = new List(); @@ -221,38 +219,32 @@ async Task NonOdeImplementationAsync( { Assert.AreEqual(Number64.ToDouble(expectedNumber.Value), Number64.ToDouble(actualNumber.Value), .01); } - else if (argument.AggregateOperator.Equals("MAKELIST") || argument.AggregateOperator.Equals("MAKESET")) + else { - if ((expected is CosmosArray expectedArray) && (actual is CosmosArray actualArray)) + if (argument.SortResult) { - CosmosElement[] normalizedExpected = expectedArray.ToArray(); - Array.Sort(normalizedExpected); - CosmosElement[] normalizedActual = actualArray.ToArray(); - Array.Sort(normalizedActual); - - Assert.IsTrue(normalizedExpected.Length == normalizedActual.Length, message); + // We need to sort the results for MakeList and MakeSet when comparing because these aggregates don't + // provide a guarantee of the order in which elements appear, and the order can change based on the + // order in which we access the logical partitions. + if ((expected is CosmosArray expectedArray) && (actual is CosmosArray actualArray)) + { + CosmosElement[] normalizedExpected = expectedArray.ToArray(); + Array.Sort(normalizedExpected); + CosmosElement[] normalizedActual = actualArray.ToArray(); + Array.Sort(normalizedActual); - bool expectedEqualsActual = true; - for( int i = 0; i < normalizedExpected.Length; i++ ) + CollectionAssert.AreEqual(normalizedExpected, normalizedActual); + } + else { - if (normalizedActual[i].CompareTo(normalizedExpected[i]) != 0) - { - expectedEqualsActual = false; - break; - } + Assert.AreEqual(expected, actual, message); } - - Assert.IsTrue(expectedEqualsActual, message); } else { Assert.AreEqual(expected, actual, message); } } - else - { - Assert.AreEqual(expected, actual, message); - } } } } @@ -302,8 +294,11 @@ async Task OdeImplementationAsync( { Assert.AreEqual(0, items.Count, message); } - else if (argument.AggregateOperator.Equals("MAKELIST") || argument.AggregateOperator.Equals("MAKESET")) + else if (argument.SortResult) { + // We need to sort the results for MakeList and MakeSet when comparing because these aggregates don't + // provide a guarantee of the order in which elements appear, and the order can change based on the + // order in which we access the logical partitions. Assert.AreEqual(1, items.Count, message); CosmosElement expected = argument.ExpectedValue; CosmosElement actual = items.Single(); @@ -315,19 +310,7 @@ async Task OdeImplementationAsync( CosmosElement[] normalizedActual = actualArray.ToArray(); Array.Sort(normalizedActual); - Assert.IsTrue(normalizedExpected.Length == normalizedActual.Length, message); - - bool expectedEqualsActual = true; - for (int i = 0; i < normalizedExpected.Length; i++) - { - if (normalizedActual[i].CompareTo(normalizedExpected[i]) != 0) - { - expectedEqualsActual = false; - break; - } - } - - Assert.IsTrue(expectedEqualsActual, message); + CollectionAssert.AreEqual(normalizedExpected, normalizedActual); } else { @@ -410,11 +393,13 @@ private static AggregateQueryArguments[] CreateAggregateQueryArguments( new AggregateQueryArguments( aggregateOperator: "MAKELIST", expectedValue: CosmosArray.Create(makeListResult), // construct cosmos array from linq result - predicate: "true"), + predicate: "true", + sortResult: true), new AggregateQueryArguments( aggregateOperator: "MAKESET", expectedValue: CosmosArray.Create(makeSetResult), // construct cosmos array from linq result - predicate: "true"), + predicate: "true", + sortResult: true), new AggregateQueryArguments( aggregateOperator: "MAX", expectedValue: CosmosString.Create("xyz"), @@ -464,16 +449,18 @@ public AggregateTestArgs( private readonly struct AggregateQueryArguments { - public AggregateQueryArguments(string aggregateOperator, CosmosElement expectedValue, string predicate) + public AggregateQueryArguments(string aggregateOperator, CosmosElement expectedValue, string predicate, bool sortResult=false) { this.AggregateOperator = aggregateOperator; this.ExpectedValue = expectedValue; this.Predicate = predicate; + this.SortResult = sortResult; } public string AggregateOperator { get; } public CosmosElement ExpectedValue { get; } public string Predicate { get; } + public bool SortResult { get; } public override string ToString() { From ff6637ce2a9c2a4a4f13c81f2ff2b0ef783ec036 Mon Sep 17 00:00:00 2001 From: Ezra Haleva Date: Tue, 4 Jun 2024 16:27:35 -0700 Subject: [PATCH 07/22] cleaning --- .../Query/AggregateQueryTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs index 5d64cc9714..4c7268ee35 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs @@ -392,12 +392,12 @@ private static AggregateQueryArguments[] CreateAggregateQueryArguments( predicate: "true"), new AggregateQueryArguments( aggregateOperator: "MAKELIST", - expectedValue: CosmosArray.Create(makeListResult), // construct cosmos array from linq result + expectedValue: CosmosArray.Create(makeListResult), predicate: "true", sortResult: true), new AggregateQueryArguments( aggregateOperator: "MAKESET", - expectedValue: CosmosArray.Create(makeSetResult), // construct cosmos array from linq result + expectedValue: CosmosArray.Create(makeSetResult), predicate: "true", sortResult: true), new AggregateQueryArguments( From a2c6d44938b293a1614eee41eaeb52e45e7dd88f Mon Sep 17 00:00:00 2001 From: Ezra Haleva Date: Wed, 5 Jun 2024 10:18:45 -0700 Subject: [PATCH 08/22] Refactored test to better detect when to ignore result order --- .../Query/AggregateQueryTests.cs | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs index 4c7268ee35..7ab0aec408 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs @@ -720,41 +720,46 @@ private async Task TestQueryCrossPartitionAggregateFunctionsWithMixedTypesHelper string[] aggregateOperators = new string[] { "AVG", "MIN", "MAKELIST", "MAKESET", "MAX", "SUM", "COUNT" }; string[] typeCheckFunctions = new string[] { "IS_ARRAY", "IS_BOOL", "IS_NULL", "IS_NUMBER", "IS_OBJECT", "IS_STRING", "IS_DEFINED", "IS_PRIMITIVE" }; - List queries = new List(); + List<(string, bool)> queries = new List<(string, bool)>(); foreach (string aggregateOperator in aggregateOperators) { + bool ignoreResultOrder = aggregateOperator.Equals("MAKELIST") || aggregateOperator.Equals("MAKESET"); foreach (string typeCheckFunction in typeCheckFunctions) { queries.Add( - $@" + ($@" SELECT VALUE {aggregateOperator} (c.{field}) FROM c WHERE {typeCheckFunction}(c.{field}) - "); + ", + ignoreResultOrder)); } foreach (string typeOnlyPartitionKey in typeOnlyPartitionKeys) { queries.Add( - $@" + ($@" SELECT VALUE {aggregateOperator} (c.{field}) FROM c WHERE c.{partitionKey} = ""{typeOnlyPartitionKey}"" - "); + ", + ignoreResultOrder)); } }; // mixing primitive and non primitives foreach (string minmaxop in new string[] { "MIN", "MAX" }) { + bool ignoreResultOrder = false; foreach (string key in new string[] { args.OneObjectKey, args.OneArrayKey }) { queries.Add( - $@" + ($@" SELECT VALUE {minmaxop} (c.{field}) FROM c WHERE c.{partitionKey} IN (""{key}"", ""{args.DoubleOnlyKey}"") - "); + ", + ignoreResultOrder)); } } @@ -773,7 +778,7 @@ FROM c { writer.WriteStartDocument(); writer.WriteStartElement("Results"); - foreach (string query in queries) + foreach ( (string query, bool ignoreResultOrder) in queries) { string formattedQuery = string.Join( Environment.NewLine, @@ -802,8 +807,7 @@ FROM c CosmosElement aggregateResult = items.First(); if(aggregateResult is not CosmosUndefined) { - if ((formattedQuery.Contains("MAKELIST") || formattedQuery.Contains("MAKESET")) - && (aggregateResult is CosmosArray aggregateResultArray)) + if (ignoreResultOrder && (aggregateResult is CosmosArray aggregateResultArray)) { CosmosElement[] normalizedAggregateResult = aggregateResultArray.ToArray(); Array.Sort(normalizedAggregateResult); From 1c82431f752fed7c66a03d16f56047cbcf719f7f Mon Sep 17 00:00:00 2001 From: Ezra Haleva Date: Thu, 6 Jun 2024 13:16:13 -0700 Subject: [PATCH 09/22] cleaning --- .../Query/AggregateQueryTests.cs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs index 7ab0aec408..36c481c8a1 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs @@ -221,7 +221,7 @@ async Task NonOdeImplementationAsync( } else { - if (argument.SortResult) + if (argument.IgnoreResultOrder) { // We need to sort the results for MakeList and MakeSet when comparing because these aggregates don't // provide a guarantee of the order in which elements appear, and the order can change based on the @@ -294,7 +294,7 @@ async Task OdeImplementationAsync( { Assert.AreEqual(0, items.Count, message); } - else if (argument.SortResult) + else if (argument.IgnoreResultOrder) { // We need to sort the results for MakeList and MakeSet when comparing because these aggregates don't // provide a guarantee of the order in which elements appear, and the order can change based on the @@ -394,12 +394,12 @@ private static AggregateQueryArguments[] CreateAggregateQueryArguments( aggregateOperator: "MAKELIST", expectedValue: CosmosArray.Create(makeListResult), predicate: "true", - sortResult: true), + ignoreResultOrder: true), new AggregateQueryArguments( aggregateOperator: "MAKESET", expectedValue: CosmosArray.Create(makeSetResult), predicate: "true", - sortResult: true), + ignoreResultOrder: true), new AggregateQueryArguments( aggregateOperator: "MAX", expectedValue: CosmosString.Create("xyz"), @@ -449,18 +449,18 @@ public AggregateTestArgs( private readonly struct AggregateQueryArguments { - public AggregateQueryArguments(string aggregateOperator, CosmosElement expectedValue, string predicate, bool sortResult=false) + public AggregateQueryArguments(string aggregateOperator, CosmosElement expectedValue, string predicate, bool ignoreResultOrder=false) { this.AggregateOperator = aggregateOperator; this.ExpectedValue = expectedValue; this.Predicate = predicate; - this.SortResult = sortResult; + this.IgnoreResultOrder = ignoreResultOrder; } public string AggregateOperator { get; } public CosmosElement ExpectedValue { get; } public string Predicate { get; } - public bool SortResult { get; } + public bool IgnoreResultOrder { get; } public override string ToString() { From 8353971aa71859a697381bf9e18586852f4604b3 Mon Sep 17 00:00:00 2001 From: Ezra Haleva Date: Thu, 6 Jun 2024 13:25:41 -0700 Subject: [PATCH 10/22] cleaning, update baseline --- ...ueryTests.AggregateMixedTypes_baseline.xml | 204 +++++++++--------- .../Query/AggregateQueryTests.cs | 13 +- 2 files changed, 114 insertions(+), 103 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.AggregateMixedTypes_baseline.xml b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.AggregateMixedTypes_baseline.xml index 71f90623cc..ff716c1b08 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.AggregateMixedTypes_baseline.xml +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.AggregateMixedTypes_baseline.xml @@ -98,108 +98,6 @@ WHERE c.key = "oneObjectKey"]]> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -506,6 +404,108 @@ WHERE c.key = "oneObjectKey"]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs index 36c481c8a1..640711936a 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs @@ -718,7 +718,7 @@ private async Task TestQueryCrossPartitionAggregateFunctionsWithMixedTypesHelper args.UndefinedKey }; - string[] aggregateOperators = new string[] { "AVG", "MIN", "MAKELIST", "MAKESET", "MAX", "SUM", "COUNT" }; + string[] aggregateOperators = new string[] { "AVG", "MAKELIST", "MAKESET", "MAX", "MIN", "SUM", "COUNT" }; string[] typeCheckFunctions = new string[] { "IS_ARRAY", "IS_BOOL", "IS_NULL", "IS_NUMBER", "IS_OBJECT", "IS_STRING", "IS_DEFINED", "IS_PRIMITIVE" }; List<(string, bool)> queries = new List<(string, bool)>(); foreach (string aggregateOperator in aggregateOperators) @@ -827,6 +827,17 @@ FROM c writer.WriteEndDocument(); } + //string randomStr = new Random().NextDouble().ToString(); + //string filenameOutput = $"AggregateQueryTests.AggregateMixedTypes_OUTPUT{randomStr}.xml"; + + //// Set a variable to the Documents path. + //string docPath = Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments); + + //// Write the text to a new file named "WriteFile.txt". + //File.WriteAllText(Path.Combine(docPath, filenameOutput), builder.ToString()); + + + Regex r = new Regex(">\\s+"); string normalizedBaseline = r.Replace(File.ReadAllText(baselinePath), ">"); string normalizedOutput = r.Replace(builder.ToString(), ">"); From 2758e0b01ec7af2ccc26c908d6590df2d28e885c Mon Sep 17 00:00:00 2001 From: Ezra Haleva Date: Thu, 6 Jun 2024 13:41:48 -0700 Subject: [PATCH 11/22] cleaning --- .../Aggregate/Aggregators/MakeListAggregator.cs | 14 ++++++++------ .../Aggregate/Aggregators/MakeSetAggregator.cs | 9 +++------ 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeListAggregator.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeListAggregator.cs index 993ae3bd02..43e0d515d2 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeListAggregator.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeListAggregator.cs @@ -8,6 +8,7 @@ namespace Microsoft.Azure.Cosmos.Query.Core.Pipeline.Aggregate.Aggregators using System.Collections.Generic; using System.Diagnostics; using System.Globalization; + using System.Linq; using System.Text; using Microsoft.Azure.Cosmos.CosmosElements; using Microsoft.Azure.Cosmos.CosmosElements.Numbers; @@ -34,10 +35,12 @@ public void Aggregate(CosmosElement localList) throw new ArgumentException($"{nameof(localList)} must be an array."); } - foreach (CosmosElement listItem in cosmosArray) - { - this.globalList.Add(listItem); - } + this.globalList.AddRange(cosmosArray.ToList()); + + //foreach (CosmosElement listItem in cosmosArray) + //{ + // this.globalList.Add(listItem); + //} } public CosmosElement GetResult() @@ -65,8 +68,7 @@ public static TryCatch TryCreate(CosmosElement continuationToken) partialList = CosmosArray.Empty; } - return TryCatch.FromResult( - new MakeListAggregator(initialList: partialList)); + return TryCatch.FromResult(new MakeListAggregator(initialList: partialList)); } public CosmosElement GetCosmosElementContinuationToken() diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeSetAggregator.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeSetAggregator.cs index 56aa9d6d17..f9550a47b6 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeSetAggregator.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeSetAggregator.cs @@ -8,6 +8,7 @@ namespace Microsoft.Azure.Cosmos.Query.Core.Pipeline.Aggregate.Aggregators using System.Collections.Generic; using System.Diagnostics; using System.Globalization; + using System.Linq; using System.Text; using Microsoft.Azure.Cosmos.CosmosElements; using Microsoft.Azure.Cosmos.CosmosElements.Numbers; @@ -34,10 +35,7 @@ public void Aggregate(CosmosElement localSet) throw new ArgumentException($"{nameof(localSet)} must be an array."); } - foreach (CosmosElement setItem in cosmosArray) - { - this.globalSet.Add(setItem); - } + this.globalSet.UnionWith(cosmosArray.ToList()); } public CosmosElement GetResult() @@ -70,8 +68,7 @@ public static TryCatch TryCreate(CosmosElement continuationToken) partialSet = CosmosArray.Empty; } - return TryCatch.FromResult( - new MakeSetAggregator(initialSet: partialSet)); + return TryCatch.FromResult(new MakeSetAggregator(initialSet: partialSet)); } public CosmosElement GetCosmosElementContinuationToken() From 61059ac7d02e0f2c9c2f3cb6fb1c8738698a4b75 Mon Sep 17 00:00:00 2001 From: Ezra Haleva Date: Thu, 6 Jun 2024 13:46:13 -0700 Subject: [PATCH 12/22] removed old comment --- .../Pipeline/Aggregate/Aggregators/MakeListAggregator.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeListAggregator.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeListAggregator.cs index 43e0d515d2..ad719600c3 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeListAggregator.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeListAggregator.cs @@ -36,11 +36,6 @@ public void Aggregate(CosmosElement localList) } this.globalList.AddRange(cosmosArray.ToList()); - - //foreach (CosmosElement listItem in cosmosArray) - //{ - // this.globalList.Add(listItem); - //} } public CosmosElement GetResult() From 57f63111d492af24adf1ff1abb32e4c7213b0fad Mon Sep 17 00:00:00 2001 From: Ezra Haleva Date: Mon, 10 Jun 2024 14:45:46 -0700 Subject: [PATCH 13/22] cleaning/refactoring --- .../Pipeline/Aggregate/AggregateOperator.cs | 2 +- .../Aggregators/SingleGroupAggregator.cs | 1 + .../Query/AggregateQueryTests.cs | 202 +++++++++--------- .../Query/GroupByQueryTests.cs | 68 +++--- 4 files changed, 129 insertions(+), 144 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/AggregateOperator.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/AggregateOperator.cs index 5d2020af47..86fccaef9d 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/AggregateOperator.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/AggregateOperator.cs @@ -8,9 +8,9 @@ internal enum AggregateOperator { Average, Count, - Max, MakeList, MakeSet, + Max, Min, Sum, } diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/SingleGroupAggregator.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/SingleGroupAggregator.cs index bd10c436a3..a9c47dacfb 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/SingleGroupAggregator.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/SingleGroupAggregator.cs @@ -389,6 +389,7 @@ public static TryCatch TryCreate( case AggregateOperator.Sum: tryCreateAggregator = SumAggregator.TryCreate(continuationToken); break; + default: throw new ArgumentException($"Unknown {nameof(AggregateOperator)}: {aggregateOperator}."); } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs index 640711936a..c99ed2d3f3 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs @@ -21,98 +21,6 @@ [TestCategory("Query")] public sealed class AggregateCrossPartitionQueryTests : QueryTestsBase { - - [TestMethod] - public async Task TestArrayAggregatesContinuationToken() - { - int seed = (int)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds; - uint numberOfDocuments = 100; - - Random rand = new Random(seed); - List people = new List(); - - for (int i = 0; i < numberOfDocuments; i++) - { - // Generate random people - Person person = PersonGenerator.GetRandomPerson(rand); - for (int j = 0; j < rand.Next(0, 4); j++) - { - // Force an exact duplicate - people.Add(person); - } - } - - List documents = new List(); - // Shuffle them so they end up in different pages - people = people.OrderBy((person) => Guid.NewGuid()).ToList(); - foreach (Person person in people) - { - documents.Add(JsonConvert.SerializeObject(person)); - } - - await this.CreateIngestQueryDeleteAsync( - ConnectionModes.Direct | ConnectionModes.Gateway, - CollectionTypes.MultiPartition, - documents, - ImplementationAsync, - "/id"); - - async static Task ImplementationAsync(Container container, IReadOnlyList documents) - { - foreach (string[] queriesToCompare in new string[][] - { - //new string[]{ "SELECT VALUE c.age FROM c ORDER BY c.age", "SELECT VALUE MakeList(c.age) FROM c GROUP BY c.name" }, - new string[]{ "SELECT VALUE c.age FROM c ORDER BY c.age", "SELECT VALUE MakeList(c.age) FROM c" }, - new string[]{ "SELECT DISTINCT VALUE c.age FROM c ORDER BY c.age", "SELECT VALUE MakeSet(c.age) FROM c" }, - }) - { - string queryWithoutAggregate = queriesToCompare[0]; - List expectedDocuments = await QueryTestsBase.RunQueryCombinationsAsync( - container, - queryWithoutAggregate, - new QueryRequestOptions() - { - MaxConcurrency = 10, - MaxItemCount = 100, - }, - QueryDrainingMode.ContinuationToken | QueryDrainingMode.HoldState); - - CosmosElement[] normalizedExpectedResult = expectedDocuments.ToArray(); - Array.Sort(normalizedExpectedResult); - - CosmosArray normalizedExpectedCosmosArray = CosmosArray.Create(expectedDocuments); - - foreach (int pageSize in new int[] { 1, 10, 100 }) - { - string queryWithAggregate = queriesToCompare[1]; - List actualDocuments = await QueryTestsBase.RunQueryCombinationsAsync( - container, - queryWithAggregate, - new QueryRequestOptions() - { - MaxConcurrency = 10, - MaxItemCount = pageSize - }, - QueryDrainingMode.ContinuationToken | QueryDrainingMode.HoldState | QueryDrainingMode.CosmosElementContinuationToken); - - CosmosElement aggregateResult = actualDocuments.First(); - CosmosArray normalizedActualCosmosArray = null; - if(aggregateResult is CosmosArray actualCosmosArray) - { - CosmosElement[] normalizedActualArray = actualCosmosArray.ToArray(); - Array.Sort(normalizedActualArray); - normalizedActualCosmosArray = CosmosArray.Create(normalizedActualArray); - } - - Assert.AreEqual( - expected: normalizedExpectedCosmosArray, - actual: normalizedActualCosmosArray, - message: $"Documents didn't match for {queryWithAggregate} on a Partitioned container"); - } - } - } - } - [TestMethod] public async Task TestAggregateFunctionsAsync() { @@ -576,7 +484,7 @@ async Task ImplementationAsync( MaxConcurrency = 10, }); - Assert.IsTrue((items.Single() is CosmosArray result) && result.Equals(CosmosArray.Create(CosmosNumber64.Create(valueOfInterest)))); + Assert.IsTrue((items.Count() == 1) && (items.Single() is CosmosArray result) && result.Equals(CosmosArray.Create(CosmosNumber64.Create(valueOfInterest)))); } catch (Exception ex) { @@ -827,17 +735,6 @@ FROM c writer.WriteEndDocument(); } - //string randomStr = new Random().NextDouble().ToString(); - //string filenameOutput = $"AggregateQueryTests.AggregateMixedTypes_OUTPUT{randomStr}.xml"; - - //// Set a variable to the Documents path. - //string docPath = Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments); - - //// Write the text to a new file named "WriteFile.txt". - //File.WriteAllText(Path.Combine(docPath, filenameOutput), builder.ToString()); - - - Regex r = new Regex(">\\s+"); string normalizedBaseline = r.Replace(File.ReadAllText(baselinePath), ">"); string normalizedOutput = r.Replace(builder.ToString(), ">"); @@ -1211,5 +1108,102 @@ private async Task TestNonValueAggregates( } } } + + [TestMethod] + public async Task TestArrayAggregatesWithContinuationTokenAsync() + { + //await this.TestArrayAggregatesWithContinuationToken(100, new int[] { 1, 10, 100 }); + + await this.TestArrayAggregatesWithContinuationToken(100); + } + private async Task TestArrayAggregatesWithContinuationToken(int numDocuments) + { + int seed = (int)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds; + + Random rand = new Random(seed); + List people = new List(); + + for (int i = 0; i < numDocuments; i++) + { + // Generate random people + Person person = PersonGenerator.GetRandomPerson(rand); + for (int j = 0; j < rand.Next(0, 4); j++) + { + // Force an exact duplicate + people.Add(person); + } + } + + List documents = new List(); + // Shuffle them so they end up in different pages + people = people.OrderBy((person) => Guid.NewGuid()).ToList(); + foreach (Person person in people) + { + documents.Add(JsonConvert.SerializeObject(person)); + } + + await this.CreateIngestQueryDeleteAsync( + ConnectionModes.Direct | ConnectionModes.Gateway, + CollectionTypes.MultiPartition, + documents, + ImplementationAsync, + "/id"); + + async static Task ImplementationAsync(Container container, IReadOnlyList documents) + { + foreach (string[] queriesToCompare in new string[][] + { + //new string[]{ "SELECT VALUE c.age FROM c ORDER BY c.age", "SELECT VALUE MakeList(c.age) FROM c GROUP BY c.name" }, + new string[]{ "SELECT VALUE c.age FROM c", "SELECT VALUE MakeList(c.age) FROM c" }, + new string[]{ "SELECT DISTINCT VALUE c.age FROM c ORDER BY c.age", "SELECT VALUE MakeSet(c.age) FROM c" }, + }) + { + string queryWithoutAggregate = queriesToCompare[0]; + List expectedDocuments = await QueryTestsBase.RunQueryCombinationsAsync( + container, + queryWithoutAggregate, + new QueryRequestOptions() + { + MaxConcurrency = 10, + MaxItemCount = 10000, + }, + QueryDrainingMode.ContinuationToken | QueryDrainingMode.HoldState); + + CosmosElement[] normalizedExpectedResult = expectedDocuments.ToArray(); + Array.Sort(normalizedExpectedResult); + + CosmosArray normalizedExpectedCosmosArray = CosmosArray.Create(normalizedExpectedResult); + int[] pageSizes = new int[] { 1, 10, 100 }; + //int[] pageSizes = (documents.Count() < 1000) ? new int[] { 1, 10, 100 } : new int[] { 1000 }; + foreach (int pageSize in pageSizes) + { + string queryWithAggregate = queriesToCompare[1]; + List actualDocuments = await QueryTestsBase.RunQueryCombinationsAsync( + container, + queryWithAggregate, + new QueryRequestOptions() + { + MaxConcurrency = 10, + MaxItemCount = pageSize + }, + QueryDrainingMode.ContinuationToken | QueryDrainingMode.HoldState | QueryDrainingMode.CosmosElementContinuationToken); + + CosmosElement aggregateResult = actualDocuments.First(); + CosmosArray normalizedActualCosmosArray = null; + if (aggregateResult is CosmosArray actualCosmosArray) + { + CosmosElement[] normalizedActualArray = actualCosmosArray.ToArray(); + Array.Sort(normalizedActualArray); + normalizedActualCosmosArray = CosmosArray.Create(normalizedActualArray); + } + + Assert.AreEqual( + expected: normalizedExpectedCosmosArray, + actual: normalizedActualCosmosArray, + message: $"Documents didn't match for {queryWithAggregate} on a Partitioned container"); + } + } + } + } } } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/GroupByQueryTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/GroupByQueryTests.cs index 52dbbafbca..e66508f726 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/GroupByQueryTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/GroupByQueryTests.cs @@ -500,26 +500,23 @@ FROM c { foreach (int maxItemCount in new int[] { 1, 5, 10 }) { + QueryRequestOptions queryRequestOptions = new QueryRequestOptions() + { + MaxConcurrency = 2, + MaxItemCount = maxItemCount, + MaxBufferedItemCount = 100, + }; + List actualWithoutContinuationTokens = await QueryTestsBase.QueryWithoutContinuationTokensAsync( container, query, - new QueryRequestOptions() - { - MaxConcurrency = 2, - MaxItemCount = maxItemCount, - MaxBufferedItemCount = 100, - }); + queryRequestOptions); HashSet actualWithoutContinuationTokensSet = new HashSet(actualWithoutContinuationTokens); List actualWithTryGetContinuationTokens = await QueryTestsBase.QueryWithCosmosElementContinuationTokenAsync( container, query, - new QueryRequestOptions() - { - MaxConcurrency = 2, - MaxItemCount = maxItemCount, - MaxBufferedItemCount = 100, - }); + queryRequestOptions); HashSet actualWithTryGetContinuationTokensSet = new HashSet(actualWithTryGetContinuationTokens); Assert.IsTrue( @@ -541,30 +538,26 @@ FROM c List<(string, List)> queryAndExpectedResultsListArrayAggregates = new List<(string, List)>() { ( - "SELECT c.name AS NameGroup, MAKELIST(c.age) AS list_age FROM c GROUP BY c.name", + "SELECT c.name AS Name, MAKELIST(c.age) AS AgeList FROM c GROUP BY c.name", documents .GroupBy(document => document["name"]) .Select(grouping => CosmosObject.Create( new Dictionary() { - { "NameGroup", grouping.Key }, - { "list_age", CosmosArray.Create( - grouping.Select(document => document["age"]) - )} + { "Name", grouping.Key }, + { "AgeList", CosmosArray.Create(grouping.Select(document => document["age"]))} })) .ToList() ), ( - "SELECT c.name AS NameGroup, MAKESET(c.age) AS list_age FROM c GROUP BY c.name", + "SELECT c.name AS Name, MAKESET(c.age) AS AgeSet FROM c GROUP BY c.name", documents .GroupBy(document => document["name"]) .Select(grouping => CosmosObject.Create( new Dictionary() { - { "NameGroup", grouping.Key }, - { "list_age", CosmosArray.Create( - grouping.Select(document => document["age"]).Distinct() - )} + { "Name", grouping.Key }, + { "AgeSet", CosmosArray.Create(grouping.Select(document => document["age"]).Distinct())} })) .ToList() ) @@ -574,28 +567,25 @@ FROM c { foreach (int maxItemCount in new int[] { 1, 5, 10 }) { + QueryRequestOptions queryRequestOptions = new QueryRequestOptions() + { + MaxConcurrency = 2, + MaxItemCount = maxItemCount, + MaxBufferedItemCount = 100, + }; + List actualWithoutContinuationTokens = await QueryTestsBase.QueryWithoutContinuationTokensAsync( container, query, - new QueryRequestOptions() - { - MaxConcurrency = 2, - MaxItemCount = maxItemCount, - MaxBufferedItemCount = 100, - }); + queryRequestOptions); this.NormalizeGroupByArrayAggregateResults(actualWithoutContinuationTokens); HashSet actualWithoutContinuationTokensSet = new HashSet(actualWithoutContinuationTokens); List actualWithTryGetContinuationTokens = await QueryTestsBase.QueryWithCosmosElementContinuationTokenAsync( container, - query, - new QueryRequestOptions() - { - MaxConcurrency = 2, - MaxItemCount = maxItemCount, - MaxBufferedItemCount = 100, - }); + query, + queryRequestOptions); this.NormalizeGroupByArrayAggregateResults(actualWithTryGetContinuationTokens); HashSet actualWithTryGetContinuationTokensSet = new HashSet(actualWithTryGetContinuationTokens); @@ -643,7 +633,7 @@ private void NormalizeGroupByArrayAggregateResults( { if (results[i] is CosmosObject cosmosObject) { - IDictionary myDict = new Dictionary(); + IDictionary normalizedResult = new Dictionary(); foreach (KeyValuePair kvp in cosmosObject) { @@ -651,15 +641,15 @@ private void NormalizeGroupByArrayAggregateResults( { CosmosElement[] normalizedArray = cosmosArray.ToArray(); Array.Sort(normalizedArray); - myDict.Add(kvp.Key, CosmosArray.Create(normalizedArray)); + normalizedResult.Add(kvp.Key, CosmosArray.Create(normalizedArray)); } else { - myDict.Add(kvp.Key, kvp.Value); + normalizedResult.Add(kvp.Key, kvp.Value); } } - IReadOnlyDictionary newDict = new ReadOnlyDictionary(myDict); + ReadOnlyDictionary newDict = new ReadOnlyDictionary(normalizedResult); results[i] = CosmosObject.Create(newDict); } } From 51f5d128a1bdd01217a67deb5fcdeccf93d6c237 Mon Sep 17 00:00:00 2001 From: Ezra Haleva Date: Tue, 11 Jun 2024 15:14:51 -0700 Subject: [PATCH 14/22] cleaning --- .../Aggregate/Aggregators/MakeListAggregator.cs | 12 ++++-------- .../Aggregate/Aggregators/MakeSetAggregator.cs | 8 ++------ 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeListAggregator.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeListAggregator.cs index ad719600c3..3ca13e3ec4 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeListAggregator.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeListAggregator.cs @@ -22,9 +22,9 @@ internal sealed class MakeListAggregator : IAggregator private MakeListAggregator(CosmosArray initialList) { this.globalList = new List(); - foreach (CosmosElement setItem in initialList) + foreach (CosmosElement listItem in initialList) { - this.globalList.Add(setItem); + this.globalList.Add(listItem); } } @@ -40,9 +40,7 @@ public void Aggregate(CosmosElement localList) public CosmosElement GetResult() { - CosmosElement[] cosmosElementArray = new CosmosElement[this.globalList.Count]; - this.globalList.CopyTo(cosmosElementArray); - return CosmosArray.Create(cosmosElementArray); + return CosmosArray.Create(this.globalList); } public static TryCatch TryCreate(CosmosElement continuationToken) @@ -68,9 +66,7 @@ public static TryCatch TryCreate(CosmosElement continuationToken) public CosmosElement GetCosmosElementContinuationToken() { - CosmosElement[] cosmosElementArray = new CosmosElement[this.globalList.Count]; - this.globalList.CopyTo(cosmosElementArray); - return CosmosArray.Create(cosmosElementArray); + return this.GetResult(); } } } diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeSetAggregator.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeSetAggregator.cs index f9550a47b6..7133663664 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeSetAggregator.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeSetAggregator.cs @@ -40,9 +40,7 @@ public void Aggregate(CosmosElement localSet) public CosmosElement GetResult() { - CosmosElement[] cosmosElementArray = new CosmosElement[this.globalSet.Count]; - this.globalSet.CopyTo(cosmosElementArray); - return CosmosArray.Create(cosmosElementArray); + return CosmosArray.Create(this.globalSet); } public string GetContinuationToken() @@ -73,9 +71,7 @@ public static TryCatch TryCreate(CosmosElement continuationToken) public CosmosElement GetCosmosElementContinuationToken() { - CosmosElement[] cosmosElementArray = new CosmosElement[this.globalSet.Count]; - this.globalSet.CopyTo(cosmosElementArray); - return CosmosArray.Create(cosmosElementArray); + return this.GetResult(); } } } From 5fcb08999eca182444b8132c170477712aea754c Mon Sep 17 00:00:00 2001 From: Ezra Haleva Date: Fri, 14 Jun 2024 15:10:51 -0700 Subject: [PATCH 15/22] Add explicit cases to hit continuation token limit. --- .../Query/AggregateQueryTests.cs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs index c99ed2d3f3..43b5dce7ec 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs @@ -1112,9 +1112,11 @@ private async Task TestNonValueAggregates( [TestMethod] public async Task TestArrayAggregatesWithContinuationTokenAsync() { - //await this.TestArrayAggregatesWithContinuationToken(100, new int[] { 1, 10, 100 }); - await this.TestArrayAggregatesWithContinuationToken(100); + + // using 2048 + 1 documents here to ensure list size hits continuation token limit of 16KB + // We aggregates c.age (integers) which has 8 bytes, 16KB / 8B = 2048 + await this.TestArrayAggregatesWithContinuationToken(2049); } private async Task TestArrayAggregatesWithContinuationToken(int numDocuments) { @@ -1153,7 +1155,6 @@ async static Task ImplementationAsync(Container container, IReadOnlyList Date: Fri, 14 Jun 2024 15:40:28 -0700 Subject: [PATCH 16/22] Added additional case to GroupBy tests --- .../Query/GroupByQueryTests.cs | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/GroupByQueryTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/GroupByQueryTests.cs index e66508f726..6bf19454be 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/GroupByQueryTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/GroupByQueryTests.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; using System.Collections.ObjectModel; + using System.Drawing.Printing; using System.Linq; using System.Threading.Tasks; using Microsoft.Azure.Cosmos.CosmosElements; @@ -563,6 +564,7 @@ FROM c ) }; + // Test query correctness. foreach ((string query, List expectedResults) in queryAndExpectedResultsListArrayAggregates) { foreach (int maxItemCount in new int[] { 1, 5, 10 }) @@ -578,7 +580,6 @@ FROM c container, query, queryRequestOptions); - this.NormalizeGroupByArrayAggregateResults(actualWithoutContinuationTokens); HashSet actualWithoutContinuationTokensSet = new HashSet(actualWithoutContinuationTokens); @@ -586,16 +587,29 @@ FROM c container, query, queryRequestOptions); - this.NormalizeGroupByArrayAggregateResults(actualWithTryGetContinuationTokens); HashSet actualWithTryGetContinuationTokensSet = new HashSet(actualWithTryGetContinuationTokens); + List actualWithCombinations = await QueryTestsBase.RunQueryCombinationsAsync( + container, + query, + queryRequestOptions, + QueryDrainingMode.HoldState | QueryDrainingMode.CosmosElementContinuationToken); + this.NormalizeGroupByArrayAggregateResults(actualWithCombinations); + HashSet actualWithCombinationsSet = new HashSet(actualWithCombinations); + Assert.IsTrue( actualWithoutContinuationTokensSet.SetEquals(actualWithTryGetContinuationTokensSet), $"Results did not match for query: {query} with maxItemCount: {maxItemCount}" + $"ActualWithoutContinuationTokens: {JsonConvert.SerializeObject(actualWithoutContinuationTokensSet)}" + $"ActualWithTryGetContinuationTokens: {JsonConvert.SerializeObject(actualWithTryGetContinuationTokensSet)}"); + Assert.IsTrue( + actualWithoutContinuationTokensSet.SetEquals(actualWithCombinationsSet), + $"Results did not match for query: {query} with maxItemCount: {maxItemCount}" + + $"ActualWithoutContinuationTokens: {JsonConvert.SerializeObject(actualWithoutContinuationTokensSet)}" + + $"ActualWithCombinations: {JsonConvert.SerializeObject(actualWithCombinationsSet)}"); + this.NormalizeGroupByArrayAggregateResults(expectedResults); HashSet expectedSet = new HashSet(expectedResults); @@ -633,7 +647,7 @@ private void NormalizeGroupByArrayAggregateResults( { if (results[i] is CosmosObject cosmosObject) { - IDictionary normalizedResult = new Dictionary(); + Dictionary normalizedResult = new Dictionary(); foreach (KeyValuePair kvp in cosmosObject) { @@ -649,8 +663,7 @@ private void NormalizeGroupByArrayAggregateResults( } } - ReadOnlyDictionary newDict = new ReadOnlyDictionary(normalizedResult); - results[i] = CosmosObject.Create(newDict); + results[i] = CosmosObject.Create(normalizedResult); } } } From 0defd1a2c40041f4e2d61f7b329e9d23f70597c2 Mon Sep 17 00:00:00 2001 From: Ezra Haleva Date: Fri, 14 Jun 2024 16:11:42 -0700 Subject: [PATCH 17/22] cleaning --- .../Query/GroupByQueryTests.cs | 33 ++++++++++--------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/GroupByQueryTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/GroupByQueryTests.cs index 6bf19454be..0287b2d0d4 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/GroupByQueryTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/GroupByQueryTests.cs @@ -645,26 +645,29 @@ private void NormalizeGroupByArrayAggregateResults( { for (int i = 0; i < results.Count; i++) { - if (results[i] is CosmosObject cosmosObject) + if (results[i] is not CosmosObject) { - Dictionary normalizedResult = new Dictionary(); + Assert.Fail("This function assumes results are CosmosObjects (ie, not from a SELECT VALUE query)."); + } + + Dictionary normalizedResult = new Dictionary(); - foreach (KeyValuePair kvp in cosmosObject) + CosmosObject resultObject = (CosmosObject)results[i]; + foreach (KeyValuePair kvp in resultObject) + { + if (kvp.Value is CosmosArray cosmosArray) { - if (kvp.Value is CosmosArray cosmosArray) - { - CosmosElement[] normalizedArray = cosmosArray.ToArray(); - Array.Sort(normalizedArray); - normalizedResult.Add(kvp.Key, CosmosArray.Create(normalizedArray)); - } - else - { - normalizedResult.Add(kvp.Key, kvp.Value); - } + CosmosElement[] normalizedArray = cosmosArray.ToArray(); + Array.Sort(normalizedArray); + normalizedResult.Add(kvp.Key, CosmosArray.Create(normalizedArray)); + } + else + { + normalizedResult.Add(kvp.Key, kvp.Value); } - - results[i] = CosmosObject.Create(normalizedResult); } + + results[i] = CosmosObject.Create(normalizedResult); } } } From eec3632bf0cad51a5809e34b47641316872f7ad3 Mon Sep 17 00:00:00 2001 From: Ezra Haleva Date: Mon, 17 Jun 2024 16:04:19 -0700 Subject: [PATCH 18/22] cleaning --- .../Pipeline/Aggregate/Aggregators/MakeListAggregator.cs | 8 ++++++-- .../Pipeline/Aggregate/Aggregators/MakeSetAggregator.cs | 8 ++++++-- .../Query/AggregateQueryTests.cs | 5 +++-- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeListAggregator.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeListAggregator.cs index 3ca13e3ec4..02cd7d1c48 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeListAggregator.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeListAggregator.cs @@ -22,10 +22,14 @@ internal sealed class MakeListAggregator : IAggregator private MakeListAggregator(CosmosArray initialList) { this.globalList = new List(); - foreach (CosmosElement listItem in initialList) + + // InitialList should never be null, but if it is we just keep global list as an empty list. + if (initialList == null) { - this.globalList.Add(listItem); + return; } + + this.globalList.AddRange(initialList.ToList()); } public void Aggregate(CosmosElement localList) diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeSetAggregator.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeSetAggregator.cs index 7133663664..2c8954190e 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeSetAggregator.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeSetAggregator.cs @@ -22,10 +22,14 @@ internal sealed class MakeSetAggregator : IAggregator private MakeSetAggregator(CosmosArray initialSet) { this.globalSet = new HashSet(); - foreach (CosmosElement setItem in initialSet) + + // InitialSet should never be null, but if it is we just keep globalSet as an empty set. + if (initialSet == null) { - this.globalSet.Add(setItem); + return; } + + this.globalSet.UnionWith(initialSet.ToList()); } public void Aggregate(CosmosElement localSet) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs index 43b5dce7ec..bc98897b86 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs @@ -1115,12 +1115,13 @@ public async Task TestArrayAggregatesWithContinuationTokenAsync() await this.TestArrayAggregatesWithContinuationToken(100); // using 2048 + 1 documents here to ensure list size hits continuation token limit of 16KB - // We aggregates c.age (integers) which has 8 bytes, 16KB / 8B = 2048 + // We aggregate c.age (integers) which has 8 bytes, 16KB / 8B = 2048 await this.TestArrayAggregatesWithContinuationToken(2049); } + private async Task TestArrayAggregatesWithContinuationToken(int numDocuments) { - int seed = (int)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds; + int seed = 135749376; Random rand = new Random(seed); List people = new List(); From 918c35ec7b687d7cf522169c43db2187d18c411c Mon Sep 17 00:00:00 2001 From: Ezra Haleva Date: Tue, 18 Jun 2024 09:43:32 -0700 Subject: [PATCH 19/22] cleaning, updated baseline test --- ...ueryTests.AggregateMixedTypes_baseline.xml | 394 +++++++++--------- .../Query/AggregateQueryTests.cs | 2 +- 2 files changed, 198 insertions(+), 198 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.AggregateMixedTypes_baseline.xml b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.AggregateMixedTypes_baseline.xml index ff716c1b08..7945b7db79 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.AggregateMixedTypes_baseline.xml +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.AggregateMixedTypes_baseline.xml @@ -102,616 +102,616 @@ WHERE c.key = "undefinedKey"]]> - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - - + - - + - - + - - - + - - - + - - + - - + - - + - - + - - - - - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + queries = new List<(string, bool)>(); foreach (string aggregateOperator in aggregateOperators) From 4dc9f9da587c80fa26b48d403070ee2327d7bf90 Mon Sep 17 00:00:00 2001 From: Ezra Haleva Date: Tue, 18 Jun 2024 09:47:19 -0700 Subject: [PATCH 20/22] cleaning, updated baseline test --- ...ueryTests.AggregateMixedTypes_baseline.xml | 96 +++++++++---------- .../Query/AggregateQueryTests.cs | 2 +- 2 files changed, 49 insertions(+), 49 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.AggregateMixedTypes_baseline.xml b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.AggregateMixedTypes_baseline.xml index 7945b7db79..fadf094505 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.AggregateMixedTypes_baseline.xml +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.AggregateMixedTypes_baseline.xml @@ -102,205 +102,205 @@ WHERE c.key = "undefinedKey"]]> - - - + - - - + - - - + - - - + - - + - - + - - + - - - - - - - - - + - - - + - - - + - - - + - - + - - + - - + - - - - - - diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs index 36bcf1ba51..ce31558a84 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs @@ -626,7 +626,7 @@ private async Task TestQueryCrossPartitionAggregateFunctionsWithMixedTypesHelper args.UndefinedKey }; - string[] aggregateOperators = new string[] { "AVG", "MAX", "MIN", "SUM", "COUNT", "MAKELIST", "MAKESET" }; + string[] aggregateOperators = new string[] { "AVG", "MIN", "MAX", "SUM", "COUNT", "MAKELIST", "MAKESET" }; string[] typeCheckFunctions = new string[] { "IS_ARRAY", "IS_BOOL", "IS_NULL", "IS_NUMBER", "IS_OBJECT", "IS_STRING", "IS_DEFINED", "IS_PRIMITIVE" }; List<(string, bool)> queries = new List<(string, bool)>(); foreach (string aggregateOperator in aggregateOperators) From f05d33f67e0f88d928c5aa9f9f413837a9601593 Mon Sep 17 00:00:00 2001 From: Ezra Haleva Date: Wed, 19 Jun 2024 13:09:34 -0700 Subject: [PATCH 21/22] Added coverage to QueryPlanBaselineTests.cs --- .../QueryPlanBaselineTests.Aggregates.xml | 72 ++ ...ryPlanBaselineTests.NonValueAggregates.xml | 672 +++++++++++++++++- .../Query/QueryPlanBaselineTests.cs | 14 +- 3 files changed, 745 insertions(+), 13 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BaselineTest/TestBaseline/QueryPlanBaselineTests.Aggregates.xml b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BaselineTest/TestBaseline/QueryPlanBaselineTests.Aggregates.xml index 7648693ffa..a333b20cb4 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BaselineTest/TestBaseline/QueryPlanBaselineTests.Aggregates.xml +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BaselineTest/TestBaseline/QueryPlanBaselineTests.Aggregates.xml @@ -248,6 +248,78 @@ FROM c]]> + + + + + + MAKELIST + SELECT VALUE MAKELIST(c.blah) FROM c + + /key + + Hash + + + + + None + + + + + + + + MakeList + + + + True + + + + [[],"Infinity") + + + + + + + + + MAKESET + SELECT VALUE MAKESET(c.blah) FROM c + + /key + + Hash + + + + + None + + + + + + + + MakeSet + + + + True + + + + [[],"Infinity") + + + diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BaselineTest/TestBaseline/QueryPlanBaselineTests.NonValueAggregates.xml b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BaselineTest/TestBaseline/QueryPlanBaselineTests.NonValueAggregates.xml index 29991e9575..dfbfbcf15c 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BaselineTest/TestBaseline/QueryPlanBaselineTests.NonValueAggregates.xml +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BaselineTest/TestBaseline/QueryPlanBaselineTests.NonValueAggregates.xml @@ -190,6 +190,84 @@ FROM c]]> + + + + + + Single Aggregate (MAKELIST) Without 'VALUE' and Without alias. + SELECT MAKELIST(c.blah) FROM c + + Hash + + + + + None + + + + + + + + + + $1 + MakeList + + + + $1 + + False + + + + [[],"Infinity") + + + + + + + + + Single Aggregate (MAKESET) Without 'VALUE' and Without alias. + SELECT MAKESET(c.blah) FROM c + + Hash + + + + + None + + + + + + + + + + $1 + MakeSet + + + + $1 + + False + + + + [[],"Infinity") + + + @@ -385,6 +463,84 @@ FROM c]]> + + + + + + Single Aggregate (MAKELIST) Without 'VALUE' and With alias. + SELECT MAKELIST(c.blah) as makelist_blah FROM c + + Hash + + + + + None + + + + + + + + + + makelist_blah + MakeList + + + + makelist_blah + + False + + + + [[],"Infinity") + + + + + + + + + Single Aggregate (MAKESET) Without 'VALUE' and With alias. + SELECT MAKESET(c.blah) as makeset_blah FROM c + + Hash + + + + + None + + + + + + + + + + makeset_blah + MakeSet + + + + makeset_blah + + False + + + + [[],"Infinity") + + + @@ -625,6 +781,102 @@ FROM c]]> + + + + + + Multiple Aggregates (MAKELIST) With alias. + + SELECT + MAKELIST(c.blah) as makelist_blah, + MAKELIST(c.blah) as makelist_blah2 + FROM c + + Hash + + + + + None + + + + + + + + + + makelist_blah + MakeList + + + makelist_blah2 + MakeList + + + + makelist_blah + makelist_blah2 + + False + + + + [[],"Infinity") + + + + + + + + + Multiple Aggregates (MAKESET) With alias. + + SELECT + MAKESET(c.blah) as makeset_blah, + MAKESET(c.blah) as makeset_blah2 + FROM c + + Hash + + + + + None + + + + + + + + + + makeset_blah + MakeSet + + + makeset_blah2 + MakeSet + + + + makeset_blah + makeset_blah2 + + False + + + + [[],"Infinity") + + + @@ -775,11 +1027,107 @@ FROM c]]> - Multiple Aggregates (MAX) Without alias. + Multiple Aggregates (MAX) Without alias. + + SELECT + MAX(c.blah), + MAX(c.blah) + FROM c + + Hash + + + + + None + + + + + + + + + + $1 + Max + + + $2 + Max + + + + $1 + $2 + + False + + + + [[],"Infinity") + + + + + + + + + Multiple Aggregates (AVG) Without alias. + + SELECT + AVG(c.blah), + AVG(c.blah) + FROM c + + Hash + + + + + None + + + + + + + + + + $1 + Average + + + $2 + Average + + + + $1 + $2 + + False + + + + [[],"Infinity") + + + + + + + + + Multiple Aggregates (MAKELIST) Without alias. SELECT - MAX(c.blah), - MAX(c.blah) + MAKELIST(c.blah), + MAKELIST(c.blah) FROM c Hash @@ -798,11 +1146,11 @@ FROM c]]> $1 - Max + MakeList $2 - Max + MakeList @@ -816,18 +1164,18 @@ FROM c]]> [[],"Infinity") - - Multiple Aggregates (AVG) Without alias. + Multiple Aggregates (MAKESET) Without alias. SELECT - AVG(c.blah), - AVG(c.blah) + MAKESET(c.blah), + MAKESET(c.blah) FROM c Hash @@ -846,11 +1194,11 @@ FROM c]]> $1 - Average + MakeSet $2 - Average + MakeSet @@ -864,7 +1212,7 @@ FROM c]]> [[],"Infinity") - @@ -1345,6 +1693,198 @@ FROM c]]> + + + + + + Multiple Aggregates (MAKELIST) mixed alias. + + SELECT + MAKELIST(c.blah) as makelist_blah, + MAKELIST(c.blah) + FROM c + + Hash + + + + + None + + + + + + + + + + makelist_blah + MakeList + + + $1 + MakeList + + + + makelist_blah + $1 + + False + + + + [[],"Infinity") + + + + + + + + + Multiple Aggregates (MAKELIST) mixed alias. + + SELECT + MAKELIST(c.blah), + MAKELIST(c.blah) as makelist_blah + FROM c + + Hash + + + + + None + + + + + + + + + + makelist_blah + MakeList + + + $1 + MakeList + + + + $1 + makelist_blah + + False + + + + [[],"Infinity") + + + + + + + + + Multiple Aggregates (MAKESET) mixed alias. + + SELECT + MAKESET(c.blah) as makeset_blah, + MAKESET(c.blah) + FROM c + + Hash + + + + + None + + + + + + + + + + makeset_blah + MakeSet + + + $1 + MakeSet + + + + makeset_blah + $1 + + False + + + + [[],"Infinity") + + + + + + + + + Multiple Aggregates (MAKESET) mixed alias. + + SELECT + MAKESET(c.blah), + MAKESET(c.blah) as makeset_blah + FROM c + + Hash + + + + + None + + + + + + + + + + makeset_blah + MakeSet + + + $1 + MakeSet + + + + $1 + makeset_blah + + False + + + + [[],"Infinity") + + + @@ -1615,6 +2155,114 @@ FROM c]]> + + + + + + Multiple Aggregates (MAKELIST) interleaved aliases. + + SELECT + MAKELIST(c.blah) as makelist_blah, + MAKELIST(c.blah), + MAKELIST(c.blah) as makelist_blah2 + FROM c + + Hash + + + + + None + + + + + + + + + + makelist_blah + MakeList + + + makelist_blah2 + MakeList + + + $1 + MakeList + + + + makelist_blah + $1 + makelist_blah2 + + False + + + + [[],"Infinity") + + + + + + + + + Multiple Aggregates (MAKESET) interleaved aliases. + + SELECT + MAKESET(c.blah) as makeset_blah, + MAKESET(c.blah), + MAKESET(c.blah) as makeset_blah2 + FROM c + + Hash + + + + + None + + + + + + + + + + makeset_blah + MakeSet + + + $1 + MakeSet + + + makeset_blah2 + MakeSet + + + + makeset_blah + $1 + makeset_blah2 + + False + + + + [[],"Infinity") + + + diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/QueryPlanBaselineTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/QueryPlanBaselineTests.cs index 0717a80e33..149a4010d4 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/QueryPlanBaselineTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/QueryPlanBaselineTests.cs @@ -62,6 +62,16 @@ public void Aggregates() @"SELECT VALUE COUNT(c.blah) FROM c", @"/key"), + Hash( + @"MAKELIST", + @"SELECT VALUE MAKELIST(c.blah) FROM c", + @"/key"), + + Hash( + @"MAKESET", + @"SELECT VALUE MAKESET(c.blah) FROM c", + @"/key"), + Hash( @"COUNT 1", @"SELECT VALUE COUNT(1) FROM c", @@ -114,7 +124,9 @@ public void NonValueAggregates() "COUNT", "MIN", "MAX", - "AVG" + "AVG", + "MAKELIST", + "MAKESET" }; List testVariations = new List(); From 710fc766b774e7f544711bc8b9c42c3d742ffa9f Mon Sep 17 00:00:00 2001 From: Ezra Haleva Date: Thu, 20 Jun 2024 11:38:21 -0700 Subject: [PATCH 22/22] refactored --- .../Pipeline/Aggregate/Aggregators/MakeListAggregator.cs | 8 +------- .../Pipeline/Aggregate/Aggregators/MakeSetAggregator.cs | 8 +------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeListAggregator.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeListAggregator.cs index 02cd7d1c48..7ab664bdcb 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeListAggregator.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeListAggregator.cs @@ -23,13 +23,7 @@ private MakeListAggregator(CosmosArray initialList) { this.globalList = new List(); - // InitialList should never be null, but if it is we just keep global list as an empty list. - if (initialList == null) - { - return; - } - - this.globalList.AddRange(initialList.ToList()); + this.Aggregate(initialList); } public void Aggregate(CosmosElement localList) diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeSetAggregator.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeSetAggregator.cs index 2c8954190e..79d6a9485e 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeSetAggregator.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/MakeSetAggregator.cs @@ -23,13 +23,7 @@ private MakeSetAggregator(CosmosArray initialSet) { this.globalSet = new HashSet(); - // InitialSet should never be null, but if it is we just keep globalSet as an empty set. - if (initialSet == null) - { - return; - } - - this.globalSet.UnionWith(initialSet.ToList()); + this.Aggregate(initialSet); } public void Aggregate(CosmosElement localSet)