From 0ac5be35a616251a664dcec5b9d2e577af3e4b5e Mon Sep 17 00:00:00 2001 From: Neil Deshpande Date: Fri, 18 Nov 2022 10:00:17 -0800 Subject: [PATCH 1/5] Do not maintain an independent count on QueryResponse that can go out of sync --- Microsoft.Azure.Cosmos/src/Query/v3Query/QueryResponse.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Query/v3Query/QueryResponse.cs b/Microsoft.Azure.Cosmos/src/Query/v3Query/QueryResponse.cs index e9d0b5856b..6c7777ebff 100644 --- a/Microsoft.Azure.Cosmos/src/Query/v3Query/QueryResponse.cs +++ b/Microsoft.Azure.Cosmos/src/Query/v3Query/QueryResponse.cs @@ -163,6 +163,7 @@ class QueryResponse : FeedResponse { private readonly CosmosSerializerCore serializerCore; private readonly CosmosSerializationFormatOptions serializationOptions; + private readonly IReadOnlyList resource; private QueryResponse( HttpStatusCode httpStatusCode, @@ -178,8 +179,7 @@ private QueryResponse( this.serializerCore = serializerCore; this.serializationOptions = serializationOptions; this.StatusCode = httpStatusCode; - this.Count = cosmosElements.Count; - this.Resource = CosmosElementSerializer.GetResources( + this.resource = CosmosElementSerializer.GetResources( cosmosArray: cosmosElements, serializerCore: serializerCore); @@ -197,7 +197,7 @@ private QueryResponse( public override CosmosDiagnostics Diagnostics { get; } - public override int Count { get; } + public override int Count => this.resource.Count; internal CosmosQueryResponseMessageHeaders QueryHeaders { get; } @@ -210,7 +210,7 @@ public override IEnumerator GetEnumerator() return this.Resource.GetEnumerator(); } - public override IEnumerable Resource { get; } + public override IEnumerable Resource => this.resource; internal override RequestMessage RequestMessage { get; } From 985b733a19540d41782d13c5b60d970a9aacb2da Mon Sep 17 00:00:00 2001 From: Neil Deshpande Date: Fri, 18 Nov 2022 11:11:45 -0800 Subject: [PATCH 2/5] Add more test coverage for QueryResponse.Count --- .../Query/QueryTestsBase.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/QueryTestsBase.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/QueryTestsBase.cs index 269e8448bd..e8beee81cb 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/QueryTestsBase.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/QueryTestsBase.cs @@ -645,6 +645,7 @@ internal static async Task> QueryWithContinuationTokensAsync( } List resultsFromContinuationToken = new List(); + int resultCount = 0; string continuationToken = null; do { @@ -668,6 +669,7 @@ internal static async Task> QueryWithContinuationTokensAsync( } resultsFromContinuationToken.AddRange(cosmosQueryResponse); + resultCount += cosmosQueryResponse.Count; continuationToken = cosmosQueryResponse.ContinuationToken; break; } @@ -687,6 +689,7 @@ internal static async Task> QueryWithContinuationTokensAsync( } } while (continuationToken != null); + Assert.AreEqual(resultsFromContinuationToken.Count, resultCount); return resultsFromContinuationToken; } @@ -700,6 +703,7 @@ internal static async Task> QueryWithoutContinuationTokensAsync( queryRequestOptions = new QueryRequestOptions(); } + int resultCount = 0; List results = new List(); FeedIterator itemQuery = container.GetItemQueryIterator( queryText: query, @@ -713,6 +717,7 @@ internal static async Task> QueryWithoutContinuationTokensAsync( { FeedResponse page = await itemQuery.ReadNextAsync(); results.AddRange(page); + resultCount += page.Count; if (queryRequestOptions.MaxItemCount.HasValue) { @@ -746,6 +751,7 @@ internal static async Task> QueryWithoutContinuationTokensAsync( { // The query failed and we don't have a save point, so just restart the whole thing. results = new List(); + resultCount = 0; } } } @@ -755,6 +761,7 @@ internal static async Task> QueryWithoutContinuationTokensAsync( itemQuery.Dispose(); } + Assert.AreEqual(results.Count, resultCount); return results; } From 301fd92097af74af7b7373b9b8b7bce0d0fc8dd1 Mon Sep 17 00:00:00 2001 From: Neil Deshpande Date: Fri, 18 Nov 2022 12:57:15 -0800 Subject: [PATCH 3/5] Output the correct count from CosmosElementSerializer when the input contains CosmosUndefined --- .../src/Serializer/CosmosElementSerializer.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Serializer/CosmosElementSerializer.cs b/Microsoft.Azure.Cosmos/src/Serializer/CosmosElementSerializer.cs index 1c4cb96a24..0564727b0c 100644 --- a/Microsoft.Azure.Cosmos/src/Serializer/CosmosElementSerializer.cs +++ b/Microsoft.Azure.Cosmos/src/Serializer/CosmosElementSerializer.cs @@ -74,8 +74,11 @@ internal static MemoryStream ToStream( jsonWriter.WriteArrayStart(); foreach (CosmosElement element in cosmosElements) { - count++; - element.WriteTo(jsonWriter); + if (element is not CosmosUndefined) + { + count++; + element.WriteTo(jsonWriter); + } } jsonWriter.WriteArrayEnd(); From 3233876424159f861dd42692d4ad3267e77e4181 Mon Sep 17 00:00:00 2001 From: Neil Deshpande Date: Mon, 21 Nov 2022 19:41:32 -0800 Subject: [PATCH 4/5] Add untyped tests for CosmosUndefined --- .../Query/CosmosUndefinedQueryTests.cs | 80 ++++++++++++++++++- .../Query/QueryTestsBase.cs | 17 ++++ 2 files changed, 95 insertions(+), 2 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/CosmosUndefinedQueryTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/CosmosUndefinedQueryTests.cs index 42cb70fe16..571897c4f2 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/CosmosUndefinedQueryTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/CosmosUndefinedQueryTests.cs @@ -5,6 +5,7 @@ namespace Microsoft.Azure.Cosmos.EmulatorTests.Query using System.Collections.ObjectModel; using System.Linq; using System.Threading.Tasks; + using Azure; using Microsoft.Azure.Cosmos.CosmosElements; using Microsoft.Azure.Cosmos.CosmosElements.Numbers; using Microsoft.Azure.Cosmos.Json; @@ -48,8 +49,83 @@ await this.CreateIngestQueryDeleteAsync( private static async Task RunTests(Container container, IReadOnlyList _) { - await OrderByTests(container); - await GroupByTests(container); + // await OrderByTests(container); + // await GroupByTests(container); + await UntypedTests(container); + } + + private static async Task UntypedTests(Container container) + { + UndefinedProjectionTestCase[] undefinedProjectionTestCases = new[] + { + MakeUndefinedProjectionTest( + query: "SELECT VALUE c.AlwaysUndefinedField FROM c", + expectedCount: 0), + MakeUndefinedProjectionTest( + query: "SELECT VALUE c.AlwaysUndefinedField FROM c ORDER BY c.AlwaysUndefinedField", + expectedCount: 0), + MakeUndefinedProjectionTest( + query: "SELECT c.AlwaysUndefinedField FROM c ORDER BY c.AlwaysUndefinedField", + expectedCount: DocumentCount), + MakeUndefinedProjectionTest( + query: "SELECT VALUE c.AlwaysUndefinedField FROM c GROUP BY c.AlwaysUndefinedField", + expectedCount: 0), + + //MakeUndefinedProjectionTest( + // query: "SELECT c.AlwaysUndefinedField FROM c GROUP BY c.AlwaysUndefinedField", + // expectedCount: DocumentCount), + MakeUndefinedProjectionTest( + query: $"SELECT VALUE SUM(c.{nameof(MixedTypeDocument.MixedTypeField)}) FROM c", + expectedCount: 0), + MakeUndefinedProjectionTest( + query: $"SELECT VALUE AVG(c.{nameof(MixedTypeDocument.MixedTypeField)}) FROM c", + expectedCount: 0), + + //MakeUndefinedProjectionTest( + // query: $"SELECT VALUE MIN(c.{nameof(MixedTypeDocument.MixedTypeField)}) FROM c", + // expectedCount: 0), + //MakeUndefinedProjectionTest( + // query: $"SELECT VALUE MAX(c.{nameof(MixedTypeDocument.MixedTypeField)}) FROM c", + // expectedCount: 0), + }; + + foreach (UndefinedProjectionTestCase testCase in undefinedProjectionTestCases) + { + foreach (int pageSize in PageSizes) + { + IAsyncEnumerable results = RunSimpleQueryAsync( + container, + testCase.Query, + new QueryRequestOptions { MaxItemCount = pageSize }); + + long actualCount = 0; + await foreach (ResponseMessage responseMessage in results) + { + Assert.IsTrue(responseMessage.IsSuccessStatusCode); + + string content = responseMessage.Content.ReadAsString(); + IJsonNavigator navigator = JsonNavigator.Create(System.Text.Encoding.UTF8.GetBytes(content)); + IJsonNavigatorNode rootNode = navigator.GetRootNode(); + Assert.IsTrue(navigator.TryGetObjectProperty(rootNode, "_count", out ObjectProperty countProperty)); + + long count = Number64.ToLong(navigator.GetNumber64Value(countProperty.ValueNode)); + actualCount += count; + + Assert.IsTrue(navigator.TryGetObjectProperty(rootNode, "Documents", out ObjectProperty documentsProperty)); + int documentCount = navigator.GetArrayItemCount(documentsProperty.ValueNode); + Assert.AreEqual(count, documentCount); + + for (int index= 0; index < documentCount; ++index) + { + IJsonNavigatorNode documentNode = navigator.GetArrayItemAt(documentsProperty.ValueNode, index); + int propertyCount = navigator.GetObjectPropertyCount(documentNode); + Assert.AreEqual(0, propertyCount); + } + } + + Assert.AreEqual(testCase.ExpectedResultCount, actualCount); + } + } } private static async Task OrderByTests(Container container) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/QueryTestsBase.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/QueryTestsBase.cs index e8beee81cb..4775716efa 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/QueryTestsBase.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/QueryTestsBase.cs @@ -914,6 +914,23 @@ internal static async IAsyncEnumerable> RunSimpleQueryWithNewIte } } + internal static async IAsyncEnumerable RunSimpleQueryAsync( + Container container, + string query, + QueryRequestOptions requestOptions = null) + { + using FeedIterator resultSetIterator = container.GetItemQueryStreamIterator( + query, + null, + requestOptions: requestOptions); + + while (resultSetIterator.HasMoreResults) + { + ResponseMessage response = await resultSetIterator.ReadNextAsync(); + yield return response; + } + } + internal async Task> RunSinglePartitionQuery( Container container, string query, From 25ac294eb9bacd72ac32c36223d39ce0bd691654 Mon Sep 17 00:00:00 2001 From: Neil Deshpande Date: Tue, 22 Nov 2022 09:38:16 -0800 Subject: [PATCH 5/5] Remove commented code --- .../Query/CosmosUndefinedQueryTests.cs | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/CosmosUndefinedQueryTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/CosmosUndefinedQueryTests.cs index 571897c4f2..3945169330 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/CosmosUndefinedQueryTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/CosmosUndefinedQueryTests.cs @@ -49,8 +49,8 @@ await this.CreateIngestQueryDeleteAsync( private static async Task RunTests(Container container, IReadOnlyList _) { - // await OrderByTests(container); - // await GroupByTests(container); + await OrderByTests(container); + await GroupByTests(container); await UntypedTests(container); } @@ -70,23 +70,12 @@ private static async Task UntypedTests(Container container) MakeUndefinedProjectionTest( query: "SELECT VALUE c.AlwaysUndefinedField FROM c GROUP BY c.AlwaysUndefinedField", expectedCount: 0), - - //MakeUndefinedProjectionTest( - // query: "SELECT c.AlwaysUndefinedField FROM c GROUP BY c.AlwaysUndefinedField", - // expectedCount: DocumentCount), MakeUndefinedProjectionTest( query: $"SELECT VALUE SUM(c.{nameof(MixedTypeDocument.MixedTypeField)}) FROM c", expectedCount: 0), MakeUndefinedProjectionTest( query: $"SELECT VALUE AVG(c.{nameof(MixedTypeDocument.MixedTypeField)}) FROM c", expectedCount: 0), - - //MakeUndefinedProjectionTest( - // query: $"SELECT VALUE MIN(c.{nameof(MixedTypeDocument.MixedTypeField)}) FROM c", - // expectedCount: 0), - //MakeUndefinedProjectionTest( - // query: $"SELECT VALUE MAX(c.{nameof(MixedTypeDocument.MixedTypeField)}) FROM c", - // expectedCount: 0), }; foreach (UndefinedProjectionTestCase testCase in undefinedProjectionTestCases)