From b5b6dc344aae1d652f68df5b8a51eb49b6a8921d Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Mon, 26 Aug 2019 10:27:01 -0700 Subject: [PATCH 01/54] creating executor --- .../src/Batch/BatchAsyncContainerExecutor.cs | 4 ++-- .../src/Resource/Container/ContainerCore.Items.cs | 2 ++ .../src/Resource/Container/ContainerCore.cs | 14 ++++++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncContainerExecutor.cs b/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncContainerExecutor.cs index fe36315a14..942e3fbe22 100644 --- a/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncContainerExecutor.cs +++ b/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncContainerExecutor.cs @@ -25,7 +25,7 @@ namespace Microsoft.Azure.Cosmos /// internal class BatchAsyncContainerExecutor : IDisposable { - private const int DefaultDispatchTimer = 10; + private const int DefaultDispatchTimerInSeconds = 1; private const int MinimumDispatchTimerInSeconds = 1; private readonly ContainerCore cosmosContainer; @@ -42,7 +42,7 @@ public BatchAsyncContainerExecutor( CosmosClientContext cosmosClientContext, int maxServerRequestOperationCount, int maxServerRequestBodyLength, - int dispatchTimerInSeconds = BatchAsyncContainerExecutor.DefaultDispatchTimer) + int dispatchTimerInSeconds = BatchAsyncContainerExecutor.DefaultDispatchTimerInSeconds) { if (cosmosContainer == null) { diff --git a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs index f5337ea044..2f5db40682 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs @@ -35,6 +35,8 @@ internal partial class ContainerCore : Container private readonly CosmosQueryClient queryClient; + private readonly BatchAsyncContainerExecutor batchAsyncContainerExecutor; + public override Task CreateItemStreamAsync( Stream streamPayload, PartitionKey partitionKey, diff --git a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.cs b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.cs index d21eebc43d..448106a029 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.cs @@ -44,6 +44,7 @@ internal ContainerCore( this.Scripts = new ScriptsCore(this, this.ClientContext); this.cachedUriSegmentWithoutId = this.GetResourceSegmentUriWithoutId(); this.queryClient = queryClient ?? new CosmosQueryClientCore(this.ClientContext, this); + this.batchAsyncContainerExecutor = this.InitializeBatchExecutorForContainer(); } public override string Id { get; } @@ -260,6 +261,19 @@ internal virtual Task GetRoutingMapAsync(CancellationToken .Unwrap(); } + internal virtual BatchAsyncContainerExecutor InitializeBatchExecutorForContainer() + { + int maxOperations = Constants.MaxOperationsInDirectModeBatchRequest; + int maxBodySize = Constants.MaxDirectModeBatchRequestBodySizeInBytes; + if (this.ClientContext.ClientOptions.ConnectionMode == ConnectionMode.Gateway) + { + maxOperations = Constants.MaxOperationsInGatewayModeBatchRequest; + maxBodySize = Constants.MaxGatewayModeBatchRequestBodySizeInBytes; + } + + return new BatchAsyncContainerExecutor(this, this.ClientContext, maxOperations, maxBodySize); + } + private Task ReplaceStreamInternalAsync( Stream streamPayload, ContainerRequestOptions requestOptions = null, From 47b84976eb75c10e35318ac92fcfd839d94c60bf Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Mon, 26 Aug 2019 11:22:24 -0700 Subject: [PATCH 02/54] Adding public API --- Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs | 9 +++++++++ .../src/Fluent/CosmosClientBuilder.cs | 11 +++++++++++ .../CosmosClientOptionsUnitTests.cs | 3 +++ 3 files changed, 23 insertions(+) diff --git a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs index d2230919fe..70a880f9c3 100644 --- a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs +++ b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs @@ -332,6 +332,15 @@ public CosmosSerializer Serializer } } + /// + /// Enables a high throughput mode in which ingest operations will be optimized to execute in batch requests. + /// + /// + /// There is no order of execution enforced when this mode is active. + /// This mode affects only Stream based ingest operations. + /// + public bool HighThroughputModeEnabled { get; set; } + /// /// A JSON serializer used by the CosmosClient to serialize or de-serialize cosmos request/responses. /// The default serializer is always used for all system owned types like DatabaseProperties. diff --git a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs index d7abdd84bc..c996ad3266 100644 --- a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs +++ b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs @@ -329,6 +329,17 @@ public CosmosClientBuilder WithCustomSerializer(CosmosSerializer cosmosJsonSeria return this; } + /// + /// Enables a high throughput mode in which ingest operations will be optimized to execute in batch requests. + /// + /// Whether is enabled. + /// The object + public CosmosClientBuilder WithHighThroughputMode(bool enabled) + { + this.clientOptions.HighThroughputModeEnabled = enabled; + return this; + } + /// /// The event handler to be invoked before the request is sent. /// diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs index 204000d6e4..b69302b37c 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs @@ -60,6 +60,7 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated() Assert.AreNotEqual(requestTimeout, clientOptions.RequestTimeout); Assert.AreNotEqual(userAgentSuffix, clientOptions.ApplicationName); Assert.AreNotEqual(apiType, clientOptions.ApiType); + Assert.IsFalse(clientOptions.HighThroughputModeEnabled); Assert.AreEqual(0, clientOptions.CustomHandlers.Count); Assert.IsNull(clientOptions.SerializerOptions); Assert.IsNull(clientOptions.Serializer); @@ -82,6 +83,7 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated() .AddCustomHandlers(preProcessHandler) .WithApiType(apiType) .WithThrottlingRetryOptions(maxRetryWaitTime, maxRetryAttemptsOnThrottledRequests) + .WithHighThroughputMode(true) .WithSerializerOptions(cosmosSerializerOptions); cosmosClient = cosmosClientBuilder.Build(new MockDocumentClient()); @@ -100,6 +102,7 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated() Assert.AreEqual(cosmosSerializerOptions.IgnoreNullValues, clientOptions.SerializerOptions.IgnoreNullValues); Assert.AreEqual(cosmosSerializerOptions.PropertyNamingPolicy, clientOptions.SerializerOptions.PropertyNamingPolicy); Assert.AreEqual(cosmosSerializerOptions.Indented, clientOptions.SerializerOptions.Indented); + Assert.IsTrue(clientOptions.HighThroughputModeEnabled); //Verify GetConnectionPolicy returns the correct values policy = clientOptions.GetConnectionPolicy(); From 0bd197d943b946331e9594f1d75a5dfe580b28b6 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Mon, 26 Aug 2019 11:26:48 -0700 Subject: [PATCH 03/54] Updating contract --- .../DotNetSDKAPI.json | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/DotNetSDKAPI.json b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/DotNetSDKAPI.json index 1e73ce260d..390619734b 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/DotNetSDKAPI.json +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/DotNetSDKAPI.json @@ -1225,6 +1225,18 @@ "CosmosClientOptions": { "Subclasses": {}, "Members": { + "Boolean get_HighThroughputModeEnabled()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { + "Type": "Method", + "Attributes": [ + "CompilerGeneratedAttribute" + ], + "MethodInfo": "Boolean get_HighThroughputModeEnabled()" + }, + "Boolean HighThroughputModeEnabled": { + "Type": "Property", + "Attributes": [], + "MethodInfo": null + }, "Int32 GatewayModeMaxConnectionLimit": { "Type": "Property", "Attributes": [], @@ -1425,6 +1437,13 @@ "Attributes": [], "MethodInfo": "Void set_GatewayModeMaxConnectionLimit(Int32)" }, + "Void set_HighThroughputModeEnabled(Boolean)[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { + "Type": "Method", + "Attributes": [ + "CompilerGeneratedAttribute" + ], + "MethodInfo": "Void set_HighThroughputModeEnabled(Boolean)" + }, "Void set_IdleTcpConnectionTimeout(System.Nullable`1[System.TimeSpan])": { "Type": "Method", "Attributes": [], @@ -2303,6 +2322,11 @@ "Attributes": [], "MethodInfo": "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithCustomSerializer(Microsoft.Azure.Cosmos.CosmosSerializer)" }, + "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithHighThroughputMode(Boolean)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithHighThroughputMode(Boolean)" + }, "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithRequestTimeout(System.TimeSpan)": { "Type": "Method", "Attributes": [], From f3561f34513dd6545453286920238e6d8fdd3d41 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Mon, 26 Aug 2019 12:10:58 -0700 Subject: [PATCH 04/54] Adding ToResponseMessage --- .../src/Batch/BatchOperationResult.cs | 10 ++++++ .../Batch/BatchOperationResultTests.cs | 35 +++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchOperationResultTests.cs diff --git a/Microsoft.Azure.Cosmos/src/Batch/BatchOperationResult.cs b/Microsoft.Azure.Cosmos/src/Batch/BatchOperationResult.cs index 26a1719881..0d1343fe1a 100644 --- a/Microsoft.Azure.Cosmos/src/Batch/BatchOperationResult.cs +++ b/Microsoft.Azure.Cosmos/src/Batch/BatchOperationResult.cs @@ -176,6 +176,16 @@ private static Result ReadOperationResult(ref RowReader reader, out BatchOperati return Result.Success; } + + internal ResponseMessage ToResponseMessage() + { + ResponseMessage responseMessage = new ResponseMessage(this.StatusCode); + responseMessage.Headers.SubStatusCode = this.SubStatusCode; + responseMessage.Headers.ETag = this.ETag; + responseMessage.Headers.RetryAfter = this.RetryAfter; + responseMessage.Content = this.ResourceStream; + return responseMessage; + } } /// diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchOperationResultTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchOperationResultTests.cs new file mode 100644 index 0000000000..7d5d305f53 --- /dev/null +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchOperationResultTests.cs @@ -0,0 +1,35 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Tests +{ + using System; + using System.IO; + using System.Net; + using Microsoft.Azure.Documents; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class BatchOperationResultTests + { + [TestMethod] + public void ToResponseMessage_MapsProperties() + { + BatchOperationResult result = new BatchOperationResult(HttpStatusCode.OK) + { + ResourceStream = new MemoryStream(new byte[] { 0x41, 0x42 }, index: 0, count: 2, writable: false, publiclyVisible: true), + ETag = "1234", + SubStatusCode = SubStatusCodes.CompletingSplit, + RetryAfter = TimeSpan.FromSeconds(10) + }; + + ResponseMessage response = result.ToResponseMessage(); + + Assert.AreEqual(result.ResourceStream, response.Content); + Assert.AreEqual(result.SubStatusCode, response.Headers.SubStatusCode); + Assert.AreEqual(result.RetryAfter, response.Headers.RetryAfter); + Assert.AreEqual(result.StatusCode, response.StatusCode); + } + } +} From bdc732758d9973450911951718ade74b05c6a0e5 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Mon, 26 Aug 2019 12:11:09 -0700 Subject: [PATCH 05/54] Wiring to executor --- .../Resource/Container/ContainerCore.Items.cs | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs index 2f5db40682..2ffaded148 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs @@ -43,6 +43,15 @@ public override Task CreateItemStreamAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default(CancellationToken)) { + if (this.ClientContext.ClientOptions.HighThroughputModeEnabled) + { + return this.ProcessItemStreamWithExecutorAsync( + partitionKey, + null, + streamPayload, + OperationType.Create); + } + return this.ProcessItemStreamAsync( partitionKey, null, @@ -507,6 +516,18 @@ internal async Task ProcessItemStreamAsync( cancellationToken); } + internal async Task ProcessItemStreamWithExecutorAsync( + PartitionKey partitionKey, + string id, + Stream streamPayload, + OperationType operationType, + BatchItemRequestOptions requestOptions = null) + { + ItemBatchOperation itemBatchOperation = new ItemBatchOperation(operationType, 0, partitionKey, id, streamPayload, requestOptions); + BatchOperationResult operationResult = await this.batchAsyncContainerExecutor.AddAsync(itemBatchOperation); + return operationResult.ToResponseMessage(); + } + internal async Task GetPartitionKeyValueFromStreamAsync( Stream stream, CancellationToken cancellation = default(CancellationToken)) From 5ecdd99257ae7ce4b1947698ecc216b40f2ceb35 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Mon, 26 Aug 2019 15:29:01 -0700 Subject: [PATCH 06/54] Mockable executor --- .../src/Batch/BatchAsyncContainerExecutor.cs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncContainerExecutor.cs b/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncContainerExecutor.cs index 942e3fbe22..353d0bde41 100644 --- a/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncContainerExecutor.cs +++ b/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncContainerExecutor.cs @@ -37,6 +37,13 @@ internal class BatchAsyncContainerExecutor : IDisposable private readonly ConcurrentDictionary limitersByPartitionkeyRange = new ConcurrentDictionary(); private readonly TimerPool timerPool; + /// + /// For unit testing. + /// + internal BatchAsyncContainerExecutor() + { + } + public BatchAsyncContainerExecutor( ContainerCore cosmosContainer, CosmosClientContext cosmosClientContext, @@ -72,7 +79,7 @@ public BatchAsyncContainerExecutor( this.timerPool = new TimerPool(BatchAsyncContainerExecutor.MinimumDispatchTimerInSeconds); } - public async Task AddAsync( + public virtual async Task AddAsync( ItemBatchOperation operation, ItemRequestOptions itemRequestOptions = null, CancellationToken cancellationToken = default(CancellationToken)) @@ -107,7 +114,7 @@ public void Dispose() this.timerPool.Dispose(); } - internal async Task ValidateOperationAsync( + internal virtual async Task ValidateOperationAsync( ItemBatchOperation operation, ItemRequestOptions itemRequestOptions = null, CancellationToken cancellationToken = default(CancellationToken)) From a56e84c115264891eb6da4f245177c4ba89e927c Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Mon, 26 Aug 2019 15:29:27 -0700 Subject: [PATCH 07/54] Itemrequestoptions to batchitemrequestoptions --- .../src/Batch/BatchItemRequestOptions.cs | 17 +++++++++++++++++ ...titionKeyRangeBatchExecutionResultTests.cs | 19 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/Microsoft.Azure.Cosmos/src/Batch/BatchItemRequestOptions.cs b/Microsoft.Azure.Cosmos/src/Batch/BatchItemRequestOptions.cs index 3611886715..079f8ead7b 100644 --- a/Microsoft.Azure.Cosmos/src/Batch/BatchItemRequestOptions.cs +++ b/Microsoft.Azure.Cosmos/src/Batch/BatchItemRequestOptions.cs @@ -23,5 +23,22 @@ class BatchItemRequestOptions : RequestOptions /// /// public IndexingDirective? IndexingDirective { get; set; } + + internal static BatchItemRequestOptions FromItemRequestOptions(ItemRequestOptions itemRequestOptions) + { + if (itemRequestOptions == null) + { + return null; + } + + RequestOptions requestOptions = itemRequestOptions as RequestOptions; + BatchItemRequestOptions batchItemRequestOptions = new BatchItemRequestOptions(); + batchItemRequestOptions.IndexingDirective = itemRequestOptions.IndexingDirective; + batchItemRequestOptions.IfMatchEtag = requestOptions.IfMatchEtag; + batchItemRequestOptions.IfNoneMatchEtag = requestOptions.IfNoneMatchEtag; + batchItemRequestOptions.Properties = requestOptions.Properties; + batchItemRequestOptions.IsEffectivePartitionKeyRouting = requestOptions.IsEffectivePartitionKeyRouting; + return batchItemRequestOptions; + } } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/PartitionKeyRangeBatchExecutionResultTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/PartitionKeyRangeBatchExecutionResultTests.cs index ca67af38d2..da594dde21 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/PartitionKeyRangeBatchExecutionResultTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/PartitionKeyRangeBatchExecutionResultTests.cs @@ -67,6 +67,25 @@ public async Task StatusCodesAreSetThroughResponseAsync() Assert.AreEqual(HttpStatusCode.OK, response.StatusCode); } + [TestMethod] + public void ToResponseMessage_MapsProperties() + { + BatchOperationResult result = new BatchOperationResult(HttpStatusCode.OK) + { + ResourceStream = new MemoryStream(new byte[] { 0x41, 0x42 }, index: 0, count: 2, writable: false, publiclyVisible: true), + ETag = "1234", + SubStatusCode = SubStatusCodes.CompletingSplit, + RetryAfter = TimeSpan.FromSeconds(10) + }; + + ResponseMessage response = result.ToResponseMessage(); + + Assert.AreEqual(result.ResourceStream, response.Content); + Assert.AreEqual(result.SubStatusCode, response.Headers.SubStatusCode); + Assert.AreEqual(result.RetryAfter, response.Headers.RetryAfter); + Assert.AreEqual(result.StatusCode, response.StatusCode); + } + private async Task ConstainsSplitIsTrueInternal(HttpStatusCode statusCode, SubStatusCodes subStatusCode) { List results = new List(); From dbf1b67ae13244199a7f272a4ed41e39ba83d8b4 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Mon, 26 Aug 2019 15:29:45 -0700 Subject: [PATCH 08/54] cleanup --- .../Batch/BatchAsyncContainerExecutorTests.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/BatchAsyncContainerExecutorTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/BatchAsyncContainerExecutorTests.cs index 3fbee77ab8..5de0f820e5 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/BatchAsyncContainerExecutorTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/BatchAsyncContainerExecutorTests.cs @@ -13,9 +13,7 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests using Microsoft.VisualStudio.TestTools.UnitTesting; [TestClass] -#pragma warning disable CA1001 // Types that own disposable fields should be disposable public class BatchAsyncContainerExecutorTests -#pragma warning restore CA1001 // Types that own disposable fields should be disposable { private static CosmosSerializer cosmosDefaultJsonSerializer = new CosmosJsonDotNetSerializer(); private CosmosClient cosmosClient; @@ -99,7 +97,6 @@ private static ItemBatchOperation CreateItem(string id) private class MyDocument { - //[JsonProperty("id")] public string id { get; set; } public string Status { get; set; } From a930e598b2f6f4ba2eb94f1f6b8ce2a0c98834b0 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Mon, 26 Aug 2019 15:30:10 -0700 Subject: [PATCH 09/54] Tests --- .../Batch/BatchUnitTests.cs | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchUnitTests.cs index 0af1547224..ef53c8dbe6 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchUnitTests.cs @@ -397,6 +397,39 @@ public void BatchIsWriteOperation() Assert.IsTrue(OperationType.Batch.IsWriteOperation()); } + [TestMethod] + public void FromItemRequestOptions_FromNull() + { + Assert.IsNull(BatchItemRequestOptions.FromItemRequestOptions(null)); + } + + [TestMethod] + public void FromItemRequestOptions_WithCustomValues() + { + ItemRequestOptions itemRequestOptions = new ItemRequestOptions(); + itemRequestOptions.IfMatchEtag = Guid.NewGuid().ToString(); + itemRequestOptions.IfNoneMatchEtag = Guid.NewGuid().ToString(); + itemRequestOptions.IndexingDirective = Cosmos.IndexingDirective.Exclude; + itemRequestOptions.Properties = new Dictionary() { { "test", "test" } }; + + BatchItemRequestOptions batchItemRequestOptions = BatchItemRequestOptions.FromItemRequestOptions(itemRequestOptions); + Assert.AreEqual(itemRequestOptions.IfMatchEtag, batchItemRequestOptions.IfMatchEtag); + Assert.AreEqual(itemRequestOptions.IfNoneMatchEtag, batchItemRequestOptions.IfNoneMatchEtag); + Assert.AreEqual(itemRequestOptions.IndexingDirective, batchItemRequestOptions.IndexingDirective); + Assert.AreEqual(itemRequestOptions.Properties, batchItemRequestOptions.Properties); + } + + [TestMethod] + public void FromItemRequestOptions_WithDefaultValues() + { + ItemRequestOptions itemRequestOptions = new ItemRequestOptions(); + BatchItemRequestOptions batchItemRequestOptions = BatchItemRequestOptions.FromItemRequestOptions(itemRequestOptions); + Assert.AreEqual(itemRequestOptions.IfMatchEtag, batchItemRequestOptions.IfMatchEtag); + Assert.AreEqual(itemRequestOptions.IfNoneMatchEtag, batchItemRequestOptions.IfNoneMatchEtag); + Assert.AreEqual(itemRequestOptions.IndexingDirective, batchItemRequestOptions.IndexingDirective); + Assert.AreEqual(itemRequestOptions.Properties, batchItemRequestOptions.Properties); + } + private static async Task GetBatchResponseMessageAsync(List operations, int rateLimitedOperationCount = 0) { BatchOperationResult okOperationResult = new BatchOperationResult(HttpStatusCode.OK); From ced7a82bd08074a7ea90120daa818cd9018a0026 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Mon, 26 Aug 2019 15:30:43 -0700 Subject: [PATCH 10/54] Wiring to executor --- .../src/Resource/Container/ContainerCore.Items.cs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs index 2ffaded148..5cc450bfc5 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs @@ -49,7 +49,9 @@ public override Task CreateItemStreamAsync( partitionKey, null, streamPayload, - OperationType.Create); + OperationType.Create, + requestOptions, + cancellationToken); } return this.ProcessItemStreamAsync( @@ -516,15 +518,18 @@ internal async Task ProcessItemStreamAsync( cancellationToken); } - internal async Task ProcessItemStreamWithExecutorAsync( + internal virtual async Task ProcessItemStreamWithExecutorAsync( PartitionKey partitionKey, string id, Stream streamPayload, OperationType operationType, - BatchItemRequestOptions requestOptions = null) + ItemRequestOptions itemRequestOptions = null, + CancellationToken cancellationToken = default(CancellationToken)) { - ItemBatchOperation itemBatchOperation = new ItemBatchOperation(operationType, 0, partitionKey, id, streamPayload, requestOptions); - BatchOperationResult operationResult = await this.batchAsyncContainerExecutor.AddAsync(itemBatchOperation); + BatchItemRequestOptions batchItemRequestOptions = BatchItemRequestOptions.FromItemRequestOptions(itemRequestOptions); + ItemBatchOperation itemBatchOperation = new ItemBatchOperation(operationType, /* index */ 0, partitionKey, id, streamPayload, batchItemRequestOptions); + await this.batchAsyncContainerExecutor.ValidateOperationAsync(itemBatchOperation, itemRequestOptions, cancellationToken); + BatchOperationResult operationResult = await this.batchAsyncContainerExecutor.AddAsync(itemBatchOperation, itemRequestOptions); return operationResult.ToResponseMessage(); } From ec1e6d8a8f446115c3aef347f0033deda072a17f Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Mon, 26 Aug 2019 15:30:54 -0700 Subject: [PATCH 11/54] Unit tests on executor --- .../CosmosItemUnitTests.cs | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs index ce7e5da712..fd32052292 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs @@ -11,6 +11,7 @@ namespace Microsoft.Azure.Cosmos.Tests using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.Cosmos.Client.Core.Tests; + using Microsoft.Azure.Cosmos.Query; using Microsoft.Azure.Documents; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; @@ -164,6 +165,37 @@ await Assert.ThrowsExceptionAsync(async () => { Assert.AreEqual(Cosmos.PartitionKey.Null, pkValue); } + [TestMethod] + public async Task HighThroughputSendsToExecutor() + { + Mock mockContext = new Mock(); + mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { HighThroughputModeEnabled = true }); + mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); + mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); + + DatabaseCore db = new DatabaseCore(mockContext.Object, "test"); + ExecutorContainerCore container = new ExecutorContainerCore(mockContext.Object, db, "test"); + + dynamic testItem = new + { + id = Guid.NewGuid().ToString(), + pk = "FF627B77-568E-4541-A47E-041EAC10E46F", + }; + + using (Stream itemStream = MockCosmosUtil.Serializer.ToStream(testItem)) + { + ItemRequestOptions itemRequestOptions = new ItemRequestOptions(); + Cosmos.PartitionKey partitionKey = new Cosmos.PartitionKey(testItem.pk); + using (ResponseMessage streamResponse = await container.CreateItemStreamAsync( + partitionKey: partitionKey, + streamPayload: itemStream)) + { + container.MockedExecutor.Verify(c => c.ValidateOperationAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); + container.MockedExecutor.Verify(c => c.AddAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); + } + } + } + [TestMethod] public async Task TestNestedPartitionKeyValueFromStreamAsync() { @@ -400,5 +432,21 @@ private async Task VerifyItemOperations( Assert.AreEqual(10, testHandlerHitCount, "A stream operation did not make it to the handler"); } + + private class ExecutorContainerCore : ContainerCore + { + public readonly Mock MockedExecutor = new Mock(); + public ExecutorContainerCore( + CosmosClientContext clientContext, + DatabaseCore database, + string containerId) : base (clientContext, database, containerId) + { + this.MockedExecutor + .Setup(e => e.AddAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(new BatchOperationResult(HttpStatusCode.OK)); + } + + internal override BatchAsyncContainerExecutor InitializeBatchExecutorForContainer() => this.MockedExecutor.Object; + } } } From 6cbb3b4956c45a3a414f86848b95b2d62790ccef Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Mon, 26 Aug 2019 15:31:04 -0700 Subject: [PATCH 12/54] Emulator tests on container --- .../Batch/CosmosItemHighThroughputTests.cs | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemHighThroughputTests.cs diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemHighThroughputTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemHighThroughputTests.cs new file mode 100644 index 0000000000..781a88a49c --- /dev/null +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemHighThroughputTests.cs @@ -0,0 +1,83 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Net; + using System.Threading.Tasks; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class CosmosItemHighThroughputTests + { + private static CosmosSerializer cosmosDefaultJsonSerializer = new CosmosJsonDotNetSerializer(); + + private Container container; + private Database database; + + [TestInitialize] + public async Task TestInitialize() + { + CosmosClientOptions clientOptions = new CosmosClientOptions(); + clientOptions.HighThroughputModeEnabled = true; + CosmosClient client = TestCommon.CreateCosmosClient(clientOptions); + + DatabaseResponse response = await client.CreateDatabaseIfNotExistsAsync(Guid.NewGuid().ToString()); + this.database = response.Database; + + ContainerResponse containerResponse = await this.database.CreateContainerAsync(Guid.NewGuid().ToString(), "/Status", 10000); + this.container = containerResponse; + } + + [TestCleanup] + public async Task Cleanup() + { + await this.database.DeleteAsync(); + } + + [TestMethod] + public async Task CreateItemStream_WithHighThroughput() + { + List> tasks = new List>(); + for (int i = 0; i < 100; i++) + { + tasks.Add(ExecuteAsync(this.container, CreateItem(i.ToString()))); + } + + await Task.WhenAll(tasks); + + for (int i = 0; i < 100; i++) + { + Task task = tasks[i]; + ResponseMessage result = await task; + Assert.AreEqual(HttpStatusCode.Created, result.StatusCode); + + MyDocument document = cosmosDefaultJsonSerializer.FromStream(result.Content); + Assert.AreEqual(i.ToString(), document.id); + + ItemResponse storedDoc = await this.container.ReadItemAsync(i.ToString(), new Cosmos.PartitionKey(i.ToString())); + Assert.IsNotNull(storedDoc.Resource); + } + } + + private static Task ExecuteAsync(Container container, MyDocument item) + { + return container.CreateItemStreamAsync(cosmosDefaultJsonSerializer.ToStream(item), new PartitionKey(item.Status)); + } + + private static MyDocument CreateItem(string id) => new MyDocument() { id = id, Status = id }; + + private class MyDocument + { + public string id { get; set; } + + public string Status { get; set; } + + public bool Updated { get; set; } + } + } +} From feb1f08ed2b0af96e58942e7a60d0be7da4da7ac Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Mon, 26 Aug 2019 16:37:54 -0700 Subject: [PATCH 13/54] executor retry handler --- .../src/Handler/AbstractRetryHandler.cs | 2 +- .../src/Handler/BatchExecutorRetryHandler.cs | 45 +++++++++++++++++++ .../Resource/Container/ContainerCore.Items.cs | 7 ++- .../src/Resource/Container/ContainerCore.cs | 3 +- 4 files changed, 51 insertions(+), 6 deletions(-) create mode 100644 Microsoft.Azure.Cosmos/src/Handler/BatchExecutorRetryHandler.cs diff --git a/Microsoft.Azure.Cosmos/src/Handler/AbstractRetryHandler.cs b/Microsoft.Azure.Cosmos/src/Handler/AbstractRetryHandler.cs index 5094b31bcf..e5fe55656d 100644 --- a/Microsoft.Azure.Cosmos/src/Handler/AbstractRetryHandler.cs +++ b/Microsoft.Azure.Cosmos/src/Handler/AbstractRetryHandler.cs @@ -67,7 +67,7 @@ public override async Task SendAsync( } } - private static async Task ExecuteHttpRequestAsync( + internal static async Task ExecuteHttpRequestAsync( Func> callbackMethod, Func> callShouldRetry, Func> callShouldRetryException, diff --git a/Microsoft.Azure.Cosmos/src/Handler/BatchExecutorRetryHandler.cs b/Microsoft.Azure.Cosmos/src/Handler/BatchExecutorRetryHandler.cs new file mode 100644 index 0000000000..65baedf1b4 --- /dev/null +++ b/Microsoft.Azure.Cosmos/src/Handler/BatchExecutorRetryHandler.cs @@ -0,0 +1,45 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Handlers +{ + using System.Threading; + using System.Threading.Tasks; + + internal class BatchExecutorRetryHandler + { + private readonly IDocumentClientRetryPolicy retryPolicyInstance; + private readonly BatchAsyncContainerExecutor batchAsyncContainerExecutor; + + public BatchExecutorRetryHandler( + CosmosClientContext clientContext, + BatchAsyncContainerExecutor batchAsyncContainerExecutor) + { + this.batchAsyncContainerExecutor = batchAsyncContainerExecutor; + this.retryPolicyInstance = clientContext.DocumentClient.ResetSessionTokenRetryPolicy.GetRequestPolicy(); + } + + public async Task SendAsync( + ItemBatchOperation itemBatchOperation, + ItemRequestOptions itemRequestOptions = null, + CancellationToken cancellationToken = default(CancellationToken)) + { + return await AbstractRetryHandler.ExecuteHttpRequestAsync( + callbackMethod: async () => + { + BatchOperationResult operationResult = await this.batchAsyncContainerExecutor.AddAsync(itemBatchOperation, itemRequestOptions); + return operationResult.ToResponseMessage(); + }, + callShouldRetry: (cosmosResponseMessage, token) => + { + return this.retryPolicyInstance.ShouldRetryAsync(cosmosResponseMessage, cancellationToken); + }, + callShouldRetryException: (exception, token) => + { + return this.retryPolicyInstance.ShouldRetryAsync(exception, cancellationToken); + }, + cancellationToken: cancellationToken); + } + } +} diff --git a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs index 5cc450bfc5..887755576b 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs @@ -15,6 +15,7 @@ namespace Microsoft.Azure.Cosmos using Microsoft.Azure.Cosmos.ChangeFeed; using Microsoft.Azure.Cosmos.ChangeFeed.FeedProcessing; using Microsoft.Azure.Cosmos.CosmosElements; + using Microsoft.Azure.Cosmos.Handlers; using Microsoft.Azure.Cosmos.Json; using Microsoft.Azure.Cosmos.Linq; using Microsoft.Azure.Cosmos.Query; @@ -35,7 +36,7 @@ internal partial class ContainerCore : Container private readonly CosmosQueryClient queryClient; - private readonly BatchAsyncContainerExecutor batchAsyncContainerExecutor; + private readonly BatchExecutorRetryHandler batchExecutorRetryHandler; public override Task CreateItemStreamAsync( Stream streamPayload, @@ -528,9 +529,7 @@ internal virtual async Task ProcessItemStreamWithExecutorAsync( { BatchItemRequestOptions batchItemRequestOptions = BatchItemRequestOptions.FromItemRequestOptions(itemRequestOptions); ItemBatchOperation itemBatchOperation = new ItemBatchOperation(operationType, /* index */ 0, partitionKey, id, streamPayload, batchItemRequestOptions); - await this.batchAsyncContainerExecutor.ValidateOperationAsync(itemBatchOperation, itemRequestOptions, cancellationToken); - BatchOperationResult operationResult = await this.batchAsyncContainerExecutor.AddAsync(itemBatchOperation, itemRequestOptions); - return operationResult.ToResponseMessage(); + return await this.batchExecutorRetryHandler.SendAsync(itemBatchOperation, itemRequestOptions, cancellationToken); } internal async Task GetPartitionKeyValueFromStreamAsync( diff --git a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.cs b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.cs index 448106a029..148a00d2cf 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.cs @@ -8,6 +8,7 @@ namespace Microsoft.Azure.Cosmos using System.IO; using System.Threading; using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Handlers; using Microsoft.Azure.Cosmos.Routing; using Microsoft.Azure.Cosmos.Scripts; using Microsoft.Azure.Documents; @@ -44,7 +45,7 @@ internal ContainerCore( this.Scripts = new ScriptsCore(this, this.ClientContext); this.cachedUriSegmentWithoutId = this.GetResourceSegmentUriWithoutId(); this.queryClient = queryClient ?? new CosmosQueryClientCore(this.ClientContext, this); - this.batchAsyncContainerExecutor = this.InitializeBatchExecutorForContainer(); + this.batchExecutorRetryHandler = new BatchExecutorRetryHandler(this.ClientContext, this.InitializeBatchExecutorForContainer()); } public override string Id { get; } From 883983361ba756f5e12ee54212d6b0a84c639687 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Mon, 26 Aug 2019 16:45:23 -0700 Subject: [PATCH 14/54] Retry tests --- .../CosmosItemUnitTests.cs | 50 ++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs index fd32052292..ef812c2041 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs @@ -170,6 +170,7 @@ public async Task HighThroughputSendsToExecutor() { Mock mockContext = new Mock(); mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { HighThroughputModeEnabled = true }); + mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); @@ -190,12 +191,42 @@ public async Task HighThroughputSendsToExecutor() partitionKey: partitionKey, streamPayload: itemStream)) { - container.MockedExecutor.Verify(c => c.ValidateOperationAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); container.MockedExecutor.Verify(c => c.AddAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); } } } + [TestMethod] + public async Task HighThroughputSendsToExecutor_RetriesOn429() + { + Mock mockContext = new Mock(); + mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { HighThroughputModeEnabled = true }); + mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); + mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); + mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); + + DatabaseCore db = new DatabaseCore(mockContext.Object, "test"); + ExecutorWithThrottlingContainerCore container = new ExecutorWithThrottlingContainerCore(mockContext.Object, db, "test"); + + dynamic testItem = new + { + id = Guid.NewGuid().ToString(), + pk = "FF627B77-568E-4541-A47E-041EAC10E46F", + }; + + using (Stream itemStream = MockCosmosUtil.Serializer.ToStream(testItem)) + { + ItemRequestOptions itemRequestOptions = new ItemRequestOptions(); + Cosmos.PartitionKey partitionKey = new Cosmos.PartitionKey(testItem.pk); + using (ResponseMessage streamResponse = await container.CreateItemStreamAsync( + partitionKey: partitionKey, + streamPayload: itemStream)) + { + container.MockedExecutor.Verify(c => c.AddAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(2)); + } + } + } + [TestMethod] public async Task TestNestedPartitionKeyValueFromStreamAsync() { @@ -448,5 +479,22 @@ public ExecutorContainerCore( internal override BatchAsyncContainerExecutor InitializeBatchExecutorForContainer() => this.MockedExecutor.Object; } + + private class ExecutorWithThrottlingContainerCore : ContainerCore + { + public readonly Mock MockedExecutor = new Mock(); + public ExecutorWithThrottlingContainerCore( + CosmosClientContext clientContext, + DatabaseCore database, + string containerId) : base(clientContext, database, containerId) + { + this.MockedExecutor + .SetupSequence(e => e.AddAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(new BatchOperationResult((HttpStatusCode) StatusCodes.TooManyRequests)) + .ReturnsAsync(new BatchOperationResult(HttpStatusCode.OK)); + } + + internal override BatchAsyncContainerExecutor InitializeBatchExecutorForContainer() => this.MockedExecutor.Object; + } } } From 4af418372d72ce067fe4e476239c6af5eeab769d Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Mon, 26 Aug 2019 16:55:53 -0700 Subject: [PATCH 15/54] Handler tests --- .../BatchExecutorRetryHandlerTests.cs | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BatchExecutorRetryHandlerTests.cs diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BatchExecutorRetryHandlerTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BatchExecutorRetryHandlerTests.cs new file mode 100644 index 0000000000..65e8ef88c5 --- /dev/null +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BatchExecutorRetryHandlerTests.cs @@ -0,0 +1,71 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Tests +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Net; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Client.Core.Tests; + using Microsoft.Azure.Cosmos.Handlers; + using Microsoft.Azure.Cosmos.Query; + using Microsoft.Azure.Documents; + using Microsoft.VisualStudio.TestTools.UnitTesting; + using Moq; + using Newtonsoft.Json; + using Newtonsoft.Json.Linq; + + [TestClass] + public class BatchExecutorRetryHandlerTests + { + private static CosmosSerializer cosmosDefaultJsonSerializer = new CosmosJsonDotNetSerializer(); + + [TestMethod] + public async Task NotRetryOnSuccess() + { + Mock mockContext = new Mock(); + mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); + Mock mockedExecutor = new Mock(); + mockedExecutor + .Setup(e => e.AddAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(new BatchOperationResult(HttpStatusCode.OK)); + + BatchExecutorRetryHandler batchExecutorRetryHandler = new BatchExecutorRetryHandler(mockContext.Object, mockedExecutor.Object); + ResponseMessage result = await batchExecutorRetryHandler.SendAsync(CreateItemBatchOperation()); + Assert.AreEqual(HttpStatusCode.OK, result.StatusCode); + mockedExecutor.Verify(c => c.AddAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); + } + + [TestMethod] + public async Task RetriesOn429() + { + Mock mockContext = new Mock(); + mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); + Mock mockedExecutor = new Mock(); + mockedExecutor + .SetupSequence(e => e.AddAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(new BatchOperationResult((HttpStatusCode)StatusCodes.TooManyRequests)) + .ReturnsAsync(new BatchOperationResult(HttpStatusCode.OK)); + + BatchExecutorRetryHandler batchExecutorRetryHandler = new BatchExecutorRetryHandler(mockContext.Object, mockedExecutor.Object); + ResponseMessage result = await batchExecutorRetryHandler.SendAsync(CreateItemBatchOperation()); + Assert.AreEqual(HttpStatusCode.OK, result.StatusCode); + mockedExecutor.Verify(c => c.AddAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(2)); + } + + private static ItemBatchOperation CreateItemBatchOperation() + { + dynamic testItem = new + { + id = Guid.NewGuid().ToString(), + pk = "FF627B77-568E-4541-A47E-041EAC10E46F", + }; + + return new ItemBatchOperation(OperationType.Create, /* index */ 0, new Cosmos.PartitionKey(testItem.pk), testItem.id, cosmosDefaultJsonSerializer.ToStream(testItem)); + } + } +} From 39bc6d2541a47bc77e5207a73a0ca2edb6640d69 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Tue, 27 Aug 2019 10:17:53 -0700 Subject: [PATCH 16/54] under preview --- .../src/CosmosClientOptions.cs | 8 ++++++- .../src/Fluent/CosmosClientBuilder.cs | 8 ++++++- .../DotNetSDKAPI.json | 24 ------------------- 3 files changed, 14 insertions(+), 26 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs index 70a880f9c3..c93c52d6b7 100644 --- a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs +++ b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs @@ -339,7 +339,13 @@ public CosmosSerializer Serializer /// There is no order of execution enforced when this mode is active. /// This mode affects only Stream based ingest operations. /// - public bool HighThroughputModeEnabled { get; set; } +#if PREVIEW + public +#endif +#if !PREVIEW + internal +#endif + bool HighThroughputModeEnabled { get; set; } /// /// A JSON serializer used by the CosmosClient to serialize or de-serialize cosmos request/responses. diff --git a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs index c996ad3266..eb0fbded37 100644 --- a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs +++ b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs @@ -334,7 +334,13 @@ public CosmosClientBuilder WithCustomSerializer(CosmosSerializer cosmosJsonSeria /// /// Whether is enabled. /// The object - public CosmosClientBuilder WithHighThroughputMode(bool enabled) +#if PREVIEW + public +#endif +#if !PREVIEW + internal +#endif + CosmosClientBuilder WithHighThroughputMode(bool enabled) { this.clientOptions.HighThroughputModeEnabled = enabled; return this; diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/DotNetSDKAPI.json b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/DotNetSDKAPI.json index 390619734b..1e73ce260d 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/DotNetSDKAPI.json +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/DotNetSDKAPI.json @@ -1225,18 +1225,6 @@ "CosmosClientOptions": { "Subclasses": {}, "Members": { - "Boolean get_HighThroughputModeEnabled()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { - "Type": "Method", - "Attributes": [ - "CompilerGeneratedAttribute" - ], - "MethodInfo": "Boolean get_HighThroughputModeEnabled()" - }, - "Boolean HighThroughputModeEnabled": { - "Type": "Property", - "Attributes": [], - "MethodInfo": null - }, "Int32 GatewayModeMaxConnectionLimit": { "Type": "Property", "Attributes": [], @@ -1437,13 +1425,6 @@ "Attributes": [], "MethodInfo": "Void set_GatewayModeMaxConnectionLimit(Int32)" }, - "Void set_HighThroughputModeEnabled(Boolean)[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { - "Type": "Method", - "Attributes": [ - "CompilerGeneratedAttribute" - ], - "MethodInfo": "Void set_HighThroughputModeEnabled(Boolean)" - }, "Void set_IdleTcpConnectionTimeout(System.Nullable`1[System.TimeSpan])": { "Type": "Method", "Attributes": [], @@ -2322,11 +2303,6 @@ "Attributes": [], "MethodInfo": "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithCustomSerializer(Microsoft.Azure.Cosmos.CosmosSerializer)" }, - "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithHighThroughputMode(Boolean)": { - "Type": "Method", - "Attributes": [], - "MethodInfo": "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithHighThroughputMode(Boolean)" - }, "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithRequestTimeout(System.TimeSpan)": { "Type": "Method", "Attributes": [], From 9d2af019c8b0f0e590cf8e43e454420e4376e36c Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Tue, 27 Aug 2019 10:19:09 -0700 Subject: [PATCH 17/54] else --- Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs | 3 +-- Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs index c93c52d6b7..8b5a3f3a68 100644 --- a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs +++ b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs @@ -341,8 +341,7 @@ public CosmosSerializer Serializer /// #if PREVIEW public -#endif -#if !PREVIEW +#else internal #endif bool HighThroughputModeEnabled { get; set; } diff --git a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs index eb0fbded37..90ffac49ec 100644 --- a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs +++ b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs @@ -336,8 +336,7 @@ public CosmosClientBuilder WithCustomSerializer(CosmosSerializer cosmosJsonSeria /// The object #if PREVIEW public -#endif -#if !PREVIEW +#else internal #endif CosmosClientBuilder WithHighThroughputMode(bool enabled) From bf6f78063f6a527f3e48b585e7c30325a8a00098 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Tue, 27 Aug 2019 10:52:52 -0700 Subject: [PATCH 18/54] Upsert support --- .../Resource/Container/ContainerCore.Items.cs | 11 ++++++ .../Batch/CosmosItemHighThroughputTests.cs | 34 ++++++++++++++++-- .../Batch/BatchOperationResultTests.cs | 35 ------------------- .../CosmosItemUnitTests.cs | 33 ++++++++++++++++- 4 files changed, 75 insertions(+), 38 deletions(-) delete mode 100644 Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchOperationResultTests.cs diff --git a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs index 887755576b..a8604ac733 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs @@ -124,6 +124,17 @@ public override Task UpsertItemStreamAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default(CancellationToken)) { + if (this.ClientContext.ClientOptions.HighThroughputModeEnabled) + { + return this.ProcessItemStreamWithExecutorAsync( + partitionKey, + null, + streamPayload, + OperationType.Upsert, + requestOptions, + cancellationToken); + } + return this.ProcessItemStreamAsync( partitionKey, null, diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemHighThroughputTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemHighThroughputTests.cs index 781a88a49c..451d9e3a31 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemHighThroughputTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemHighThroughputTests.cs @@ -45,7 +45,7 @@ public async Task CreateItemStream_WithHighThroughput() List> tasks = new List>(); for (int i = 0; i < 100; i++) { - tasks.Add(ExecuteAsync(this.container, CreateItem(i.ToString()))); + tasks.Add(ExecuteCreateAsync(this.container, CreateItem(i.ToString()))); } await Task.WhenAll(tasks); @@ -64,11 +64,41 @@ public async Task CreateItemStream_WithHighThroughput() } } - private static Task ExecuteAsync(Container container, MyDocument item) + [TestMethod] + public async Task UpsertItemStream_WithHighThroughput() + { + List> tasks = new List>(); + for (int i = 0; i < 100; i++) + { + tasks.Add(ExecuteUpsertAsync(this.container, CreateItem(i.ToString()))); + } + + await Task.WhenAll(tasks); + + for (int i = 0; i < 100; i++) + { + Task task = tasks[i]; + ResponseMessage result = await task; + Assert.AreEqual(HttpStatusCode.Created, result.StatusCode); + + MyDocument document = cosmosDefaultJsonSerializer.FromStream(result.Content); + Assert.AreEqual(i.ToString(), document.id); + + ItemResponse storedDoc = await this.container.ReadItemAsync(i.ToString(), new Cosmos.PartitionKey(i.ToString())); + Assert.IsNotNull(storedDoc.Resource); + } + } + + private static Task ExecuteCreateAsync(Container container, MyDocument item) { return container.CreateItemStreamAsync(cosmosDefaultJsonSerializer.ToStream(item), new PartitionKey(item.Status)); } + private static Task ExecuteUpsertAsync(Container container, MyDocument item) + { + return container.UpsertItemStreamAsync(cosmosDefaultJsonSerializer.ToStream(item), new PartitionKey(item.Status)); + } + private static MyDocument CreateItem(string id) => new MyDocument() { id = id, Status = id }; private class MyDocument diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchOperationResultTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchOperationResultTests.cs deleted file mode 100644 index 7d5d305f53..0000000000 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchOperationResultTests.cs +++ /dev/null @@ -1,35 +0,0 @@ -//------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -//------------------------------------------------------------ - -namespace Microsoft.Azure.Cosmos.Tests -{ - using System; - using System.IO; - using System.Net; - using Microsoft.Azure.Documents; - using Microsoft.VisualStudio.TestTools.UnitTesting; - - [TestClass] - public class BatchOperationResultTests - { - [TestMethod] - public void ToResponseMessage_MapsProperties() - { - BatchOperationResult result = new BatchOperationResult(HttpStatusCode.OK) - { - ResourceStream = new MemoryStream(new byte[] { 0x41, 0x42 }, index: 0, count: 2, writable: false, publiclyVisible: true), - ETag = "1234", - SubStatusCode = SubStatusCodes.CompletingSplit, - RetryAfter = TimeSpan.FromSeconds(10) - }; - - ResponseMessage response = result.ToResponseMessage(); - - Assert.AreEqual(result.ResourceStream, response.Content); - Assert.AreEqual(result.SubStatusCode, response.Headers.SubStatusCode); - Assert.AreEqual(result.RetryAfter, response.Headers.RetryAfter); - Assert.AreEqual(result.StatusCode, response.StatusCode); - } - } -} diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs index ef812c2041..271a358d17 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs @@ -166,7 +166,7 @@ await Assert.ThrowsExceptionAsync(async () => { } [TestMethod] - public async Task HighThroughputSendsToExecutor() + public async Task HighThroughputSendsToExecutor_Create() { Mock mockContext = new Mock(); mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { HighThroughputModeEnabled = true }); @@ -196,6 +196,37 @@ public async Task HighThroughputSendsToExecutor() } } + [TestMethod] + public async Task HighThroughputSendsToExecutor_Upsert() + { + Mock mockContext = new Mock(); + mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { HighThroughputModeEnabled = true }); + mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); + mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); + mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); + + DatabaseCore db = new DatabaseCore(mockContext.Object, "test"); + ExecutorContainerCore container = new ExecutorContainerCore(mockContext.Object, db, "test"); + + dynamic testItem = new + { + id = Guid.NewGuid().ToString(), + pk = "FF627B77-568E-4541-A47E-041EAC10E46F", + }; + + using (Stream itemStream = MockCosmosUtil.Serializer.ToStream(testItem)) + { + ItemRequestOptions itemRequestOptions = new ItemRequestOptions(); + Cosmos.PartitionKey partitionKey = new Cosmos.PartitionKey(testItem.pk); + using (ResponseMessage streamResponse = await container.UpsertItemStreamAsync( + partitionKey: partitionKey, + streamPayload: itemStream)) + { + container.MockedExecutor.Verify(c => c.AddAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); + } + } + } + [TestMethod] public async Task HighThroughputSendsToExecutor_RetriesOn429() { From 2ba3ed1c73cbd00c0f039b0bab1676bf92ddb4ff Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Tue, 27 Aug 2019 11:22:30 -0700 Subject: [PATCH 19/54] Using ResourceThrottleRetryPolicy --- .../src/Batch/BatchAsyncContainerExecutor.cs | 11 +++++++++-- .../src/Handler/BatchExecutorRetryHandler.cs | 5 ++++- .../BatchExecutorRetryHandlerTests.cs | 2 ++ 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncContainerExecutor.cs b/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncContainerExecutor.cs index 353d0bde41..5f731ca05d 100644 --- a/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncContainerExecutor.cs +++ b/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncContainerExecutor.cs @@ -89,6 +89,13 @@ public virtual async Task AddAsync( throw new ArgumentNullException(nameof(operation)); } + if (operation.Context != null) + { + // Retry handler would send another AddAsync on throttles + await this.RetryAsync(operation, cancellationToken); + return await operation.Context.Task; + } + await this.ValidateOperationAsync(operation, itemRequestOptions, cancellationToken); string resolvedPartitionKeyRangeId = await this.ResolvePartitionKeyRangeIdAsync(operation, cancellationToken).ConfigureAwait(false); @@ -176,7 +183,7 @@ private static void AddHeadersToRequestMessage(RequestMessage requestMessage, st requestMessage.Headers.Add(HttpConstants.HttpHeaders.IsBatchRequest, bool.TrueString); } - private async Task ReBatchAsync( + private async Task RetryAsync( ItemBatchOperation operation, CancellationToken cancellationToken) { @@ -248,7 +255,7 @@ private BatchAsyncStreamer GetOrAddStreamerForPartitionKeyRange(string partition return streamer; } - BatchAsyncStreamer newStreamer = new BatchAsyncStreamer(this.maxServerRequestOperationCount, this.maxServerRequestBodyLength, this.dispatchTimerInSeconds, this.timerPool, this.cosmosClientContext.CosmosSerializer, this.ExecuteAsync, this.ReBatchAsync); + BatchAsyncStreamer newStreamer = new BatchAsyncStreamer(this.maxServerRequestOperationCount, this.maxServerRequestBodyLength, this.dispatchTimerInSeconds, this.timerPool, this.cosmosClientContext.CosmosSerializer, this.ExecuteAsync, this.RetryAsync); if (!this.streamersByPartitionKeyRange.TryAdd(partitionKeyRangeId, newStreamer)) { newStreamer.Dispose(); diff --git a/Microsoft.Azure.Cosmos/src/Handler/BatchExecutorRetryHandler.cs b/Microsoft.Azure.Cosmos/src/Handler/BatchExecutorRetryHandler.cs index 65baedf1b4..8bb9963840 100644 --- a/Microsoft.Azure.Cosmos/src/Handler/BatchExecutorRetryHandler.cs +++ b/Microsoft.Azure.Cosmos/src/Handler/BatchExecutorRetryHandler.cs @@ -16,8 +16,11 @@ public BatchExecutorRetryHandler( CosmosClientContext clientContext, BatchAsyncContainerExecutor batchAsyncContainerExecutor) { + RetryOptions retryOptions = clientContext.ClientOptions.GetConnectionPolicy().RetryOptions; this.batchAsyncContainerExecutor = batchAsyncContainerExecutor; - this.retryPolicyInstance = clientContext.DocumentClient.ResetSessionTokenRetryPolicy.GetRequestPolicy(); + this.retryPolicyInstance = new ResourceThrottleRetryPolicy( + retryOptions.MaxRetryAttemptsOnThrottledRequests, + retryOptions.MaxRetryWaitTimeInSeconds); } public async Task SendAsync( diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BatchExecutorRetryHandlerTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BatchExecutorRetryHandlerTests.cs index 65e8ef88c5..b5444dcd54 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BatchExecutorRetryHandlerTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BatchExecutorRetryHandlerTests.cs @@ -29,6 +29,7 @@ public async Task NotRetryOnSuccess() { Mock mockContext = new Mock(); mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); + mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions()); Mock mockedExecutor = new Mock(); mockedExecutor .Setup(e => e.AddAsync(It.IsAny(), It.IsAny(), It.IsAny())) @@ -45,6 +46,7 @@ public async Task RetriesOn429() { Mock mockContext = new Mock(); mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); + mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions()); Mock mockedExecutor = new Mock(); mockedExecutor .SetupSequence(e => e.AddAsync(It.IsAny(), It.IsAny(), It.IsAny())) From 771c094f5024fcb78e287ec9cf7c7c0e8d6f3385 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Tue, 27 Aug 2019 11:37:11 -0700 Subject: [PATCH 20/54] Generating new context on retry --- .../src/Batch/BatchAsyncContainerExecutor.cs | 11 ++--------- .../src/Batch/ItemBatchOperation.cs | 7 +++++-- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncContainerExecutor.cs b/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncContainerExecutor.cs index 5f731ca05d..353d0bde41 100644 --- a/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncContainerExecutor.cs +++ b/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncContainerExecutor.cs @@ -89,13 +89,6 @@ public virtual async Task AddAsync( throw new ArgumentNullException(nameof(operation)); } - if (operation.Context != null) - { - // Retry handler would send another AddAsync on throttles - await this.RetryAsync(operation, cancellationToken); - return await operation.Context.Task; - } - await this.ValidateOperationAsync(operation, itemRequestOptions, cancellationToken); string resolvedPartitionKeyRangeId = await this.ResolvePartitionKeyRangeIdAsync(operation, cancellationToken).ConfigureAwait(false); @@ -183,7 +176,7 @@ private static void AddHeadersToRequestMessage(RequestMessage requestMessage, st requestMessage.Headers.Add(HttpConstants.HttpHeaders.IsBatchRequest, bool.TrueString); } - private async Task RetryAsync( + private async Task ReBatchAsync( ItemBatchOperation operation, CancellationToken cancellationToken) { @@ -255,7 +248,7 @@ private BatchAsyncStreamer GetOrAddStreamerForPartitionKeyRange(string partition return streamer; } - BatchAsyncStreamer newStreamer = new BatchAsyncStreamer(this.maxServerRequestOperationCount, this.maxServerRequestBodyLength, this.dispatchTimerInSeconds, this.timerPool, this.cosmosClientContext.CosmosSerializer, this.ExecuteAsync, this.RetryAsync); + BatchAsyncStreamer newStreamer = new BatchAsyncStreamer(this.maxServerRequestOperationCount, this.maxServerRequestBodyLength, this.dispatchTimerInSeconds, this.timerPool, this.cosmosClientContext.CosmosSerializer, this.ExecuteAsync, this.ReBatchAsync); if (!this.streamersByPartitionKeyRange.TryAdd(partitionKeyRangeId, newStreamer)) { newStreamer.Dispose(); diff --git a/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperation.cs b/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperation.cs index fa0967a490..84f53acfab 100644 --- a/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperation.cs +++ b/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperation.cs @@ -299,11 +299,14 @@ internal virtual async Task MaterializeResourceAsync(CosmosSerializer serializer /// /// Attached a context to the current operation to track resolution. + /// If the current context is completed, a new context can be attached, for the case of retries. /// - /// If the operation already had an attached context. + /// If the operation already had an attached context which is in progress. internal void AttachContext(ItemBatchOperationContext context) { - if (this.Context != null) + if (this.Context != null + && !this.Context.Task.IsCompleted + && !this.Context.Task.IsFaulted) { throw new InvalidOperationException("Cannot modify the current context of an operation."); } From 93800216647b942cad786c74b03bb00f296338e2 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Tue, 27 Aug 2019 11:46:35 -0700 Subject: [PATCH 21/54] Correctly obtaining retrypolicy --- .../src/Handler/BatchExecutorRetryHandler.cs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Handler/BatchExecutorRetryHandler.cs b/Microsoft.Azure.Cosmos/src/Handler/BatchExecutorRetryHandler.cs index 8bb9963840..d0c3ec21f6 100644 --- a/Microsoft.Azure.Cosmos/src/Handler/BatchExecutorRetryHandler.cs +++ b/Microsoft.Azure.Cosmos/src/Handler/BatchExecutorRetryHandler.cs @@ -9,18 +9,15 @@ namespace Microsoft.Azure.Cosmos.Handlers internal class BatchExecutorRetryHandler { - private readonly IDocumentClientRetryPolicy retryPolicyInstance; + private readonly RetryOptions retryOptions; private readonly BatchAsyncContainerExecutor batchAsyncContainerExecutor; public BatchExecutorRetryHandler( CosmosClientContext clientContext, BatchAsyncContainerExecutor batchAsyncContainerExecutor) { - RetryOptions retryOptions = clientContext.ClientOptions.GetConnectionPolicy().RetryOptions; + this.retryOptions = clientContext.ClientOptions.GetConnectionPolicy().RetryOptions; this.batchAsyncContainerExecutor = batchAsyncContainerExecutor; - this.retryPolicyInstance = new ResourceThrottleRetryPolicy( - retryOptions.MaxRetryAttemptsOnThrottledRequests, - retryOptions.MaxRetryWaitTimeInSeconds); } public async Task SendAsync( @@ -28,6 +25,10 @@ public async Task SendAsync( ItemRequestOptions itemRequestOptions = null, CancellationToken cancellationToken = default(CancellationToken)) { + IDocumentClientRetryPolicy retryPolicyInstance = new ResourceThrottleRetryPolicy( + this.retryOptions.MaxRetryAttemptsOnThrottledRequests, + this.retryOptions.MaxRetryWaitTimeInSeconds); + return await AbstractRetryHandler.ExecuteHttpRequestAsync( callbackMethod: async () => { @@ -36,11 +37,11 @@ public async Task SendAsync( }, callShouldRetry: (cosmosResponseMessage, token) => { - return this.retryPolicyInstance.ShouldRetryAsync(cosmosResponseMessage, cancellationToken); + return retryPolicyInstance.ShouldRetryAsync(cosmosResponseMessage, cancellationToken); }, callShouldRetryException: (exception, token) => { - return this.retryPolicyInstance.ShouldRetryAsync(exception, cancellationToken); + return retryPolicyInstance.ShouldRetryAsync(exception, cancellationToken); }, cancellationToken: cancellationToken); } From c40af5e3f0857114bb0d866005728dbe9331cc89 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Tue, 27 Aug 2019 11:48:01 -0700 Subject: [PATCH 22/54] Fixing unrelated test --- .../CosmosClientResourceUnitTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientResourceUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientResourceUnitTests.cs index a8555d7674..f7fd924571 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientResourceUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientResourceUnitTests.cs @@ -24,7 +24,7 @@ public void ValidateUriGenerationForResources() CosmosClientContext context = new ClientContextCore( client: null, - clientOptions: null, + clientOptions: new CosmosClientOptions(), userJsonSerializer: null, defaultJsonSerializer: null, sqlQuerySpecSerializer: null, From 19a20aba220142da67faa0d22cb3347fdd927381 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Tue, 27 Aug 2019 11:50:10 -0700 Subject: [PATCH 23/54] Removing unneeded constants --- .../src/Resource/Container/ContainerCore.cs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.cs b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.cs index 148a00d2cf..e89fb68168 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.cs @@ -264,15 +264,11 @@ internal virtual Task GetRoutingMapAsync(CancellationToken internal virtual BatchAsyncContainerExecutor InitializeBatchExecutorForContainer() { - int maxOperations = Constants.MaxOperationsInDirectModeBatchRequest; - int maxBodySize = Constants.MaxDirectModeBatchRequestBodySizeInBytes; - if (this.ClientContext.ClientOptions.ConnectionMode == ConnectionMode.Gateway) - { - maxOperations = Constants.MaxOperationsInGatewayModeBatchRequest; - maxBodySize = Constants.MaxGatewayModeBatchRequestBodySizeInBytes; - } - - return new BatchAsyncContainerExecutor(this, this.ClientContext, maxOperations, maxBodySize); + return new BatchAsyncContainerExecutor( + this, + this.ClientContext, + Constants.MaxOperationsInDirectModeBatchRequest, + Constants.MaxDirectModeBatchRequestBodySizeInBytes); } private Task ReplaceStreamInternalAsync( From 5c0fb63676b6b9d40a65d2071dc47f77d6b90416 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Tue, 27 Aug 2019 12:00:13 -0700 Subject: [PATCH 24/54] Fixing conflict tests --- .../tests/Microsoft.Azure.Cosmos.Tests/CosmosConflictTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosConflictTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosConflictTests.cs index 9608614a2e..180c51b177 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosConflictTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosConflictTests.cs @@ -122,7 +122,7 @@ private static CosmosClientContext GetMockedClientContext( return new ClientContextCore( client: client, - clientOptions: null, + clientOptions: new CosmosClientOptions(), userJsonSerializer: MockCosmosUtil.Serializer, defaultJsonSerializer: MockCosmosUtil.Serializer, sqlQuerySpecSerializer: MockCosmosUtil.Serializer, From c9ae84fd579264723ec6932aa0268095e956e1b6 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Tue, 27 Aug 2019 12:30:03 -0700 Subject: [PATCH 25/54] Replace and Delete support --- .../Resource/Container/ContainerCore.Items.cs | 22 ++++++ .../Batch/CosmosItemHighThroughputTests.cs | 77 +++++++++++++++++++ .../CosmosItemUnitTests.cs | 60 +++++++++++++++ 3 files changed, 159 insertions(+) diff --git a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs index a8604ac733..0d9777f792 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs @@ -174,6 +174,17 @@ public override Task ReplaceItemStreamAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default(CancellationToken)) { + if (this.ClientContext.ClientOptions.HighThroughputModeEnabled) + { + return this.ProcessItemStreamWithExecutorAsync( + partitionKey, + id, + streamPayload, + OperationType.Replace, + requestOptions, + cancellationToken); + } + return this.ProcessItemStreamAsync( partitionKey, id, @@ -218,6 +229,17 @@ public override Task DeleteItemStreamAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default(CancellationToken)) { + if (this.ClientContext.ClientOptions.HighThroughputModeEnabled) + { + return this.ProcessItemStreamWithExecutorAsync( + partitionKey, + id, + null, + OperationType.Delete, + requestOptions, + cancellationToken); + } + return this.ProcessItemStreamAsync( partitionKey, id, diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemHighThroughputTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemHighThroughputTests.cs index 451d9e3a31..3cdee71437 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemHighThroughputTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemHighThroughputTests.cs @@ -89,6 +89,73 @@ public async Task UpsertItemStream_WithHighThroughput() } } + [TestMethod] + public async Task DeleteItemStream_WithHighThroughput() + { + List createdDocuments = new List(); + // Create the items + List> tasks = new List>(); + for (int i = 0; i < 100; i++) + { + MyDocument createdDocument = CreateItem(i.ToString()); + createdDocuments.Add(createdDocument); + tasks.Add(ExecuteCreateAsync(this.container, createdDocument)); + } + + await Task.WhenAll(tasks); + + List> deleteTasks = new List>(); + // Delete the items + foreach (MyDocument createdDocument in createdDocuments) + { + deleteTasks.Add(ExecuteDeleteAsync(this.container, createdDocument)); + } + + await Task.WhenAll(deleteTasks); + for (int i = 0; i < 100; i++) + { + Task task = deleteTasks[i]; + ResponseMessage result = await task; + Assert.AreEqual(HttpStatusCode.NoContent, result.StatusCode); + + await Assert.ThrowsExceptionAsync(() => this.container.ReadItemAsync(i.ToString(), new Cosmos.PartitionKey(i.ToString()))); + } + } + + [TestMethod] + public async Task ReplaceItemStream_WithHighThroughput() + { + List createdDocuments = new List(); + // Create the items + List> tasks = new List>(); + for (int i = 0; i < 100; i++) + { + MyDocument createdDocument = CreateItem(i.ToString()); + createdDocuments.Add(createdDocument); + tasks.Add(ExecuteCreateAsync(this.container, createdDocument)); + } + + await Task.WhenAll(tasks); + + List> replaceTasks = new List>(); + // Replace the items + foreach (MyDocument createdDocument in createdDocuments) + { + replaceTasks.Add(ExecuteReplaceAsync(this.container, createdDocument)); + } + + await Task.WhenAll(replaceTasks); + for (int i = 0; i < 100; i++) + { + Task task = replaceTasks[i]; + ResponseMessage result = await task; + Assert.AreEqual(HttpStatusCode.OK, result.StatusCode); + + ItemResponse storedDoc = await this.container.ReadItemAsync(i.ToString(), new Cosmos.PartitionKey(i.ToString())); + Assert.IsNotNull(storedDoc.Resource); + } + } + private static Task ExecuteCreateAsync(Container container, MyDocument item) { return container.CreateItemStreamAsync(cosmosDefaultJsonSerializer.ToStream(item), new PartitionKey(item.Status)); @@ -99,6 +166,16 @@ private static Task ExecuteUpsertAsync(Container container, MyD return container.UpsertItemStreamAsync(cosmosDefaultJsonSerializer.ToStream(item), new PartitionKey(item.Status)); } + private static Task ExecuteReplaceAsync(Container container, MyDocument item) + { + return container.ReplaceItemStreamAsync(cosmosDefaultJsonSerializer.ToStream(item), item.id, new PartitionKey(item.Status)); + } + + private static Task ExecuteDeleteAsync(Container container, MyDocument item) + { + return container.DeleteItemStreamAsync(item.id, new PartitionKey(item.Status)); + } + private static MyDocument CreateItem(string id) => new MyDocument() { id = id, Status = id }; private class MyDocument diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs index 271a358d17..01764e7f3e 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs @@ -227,6 +227,66 @@ public async Task HighThroughputSendsToExecutor_Upsert() } } + [TestMethod] + public async Task HighThroughputSendsToExecutor_Replace() + { + Mock mockContext = new Mock(); + mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { HighThroughputModeEnabled = true }); + mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); + mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); + mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); + + DatabaseCore db = new DatabaseCore(mockContext.Object, "test"); + ExecutorContainerCore container = new ExecutorContainerCore(mockContext.Object, db, "test"); + + dynamic testItem = new + { + id = Guid.NewGuid().ToString(), + pk = "FF627B77-568E-4541-A47E-041EAC10E46F", + }; + + using (Stream itemStream = MockCosmosUtil.Serializer.ToStream(testItem)) + { + ItemRequestOptions itemRequestOptions = new ItemRequestOptions(); + Cosmos.PartitionKey partitionKey = new Cosmos.PartitionKey(testItem.pk); + using (ResponseMessage streamResponse = await container.ReplaceItemStreamAsync( + partitionKey: partitionKey, + id: testItem.id, + streamPayload: itemStream)) + { + container.MockedExecutor.Verify(c => c.AddAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); + } + } + } + + [TestMethod] + public async Task HighThroughputSendsToExecutor_Delete() + { + Mock mockContext = new Mock(); + mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { HighThroughputModeEnabled = true }); + mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); + mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); + mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); + + DatabaseCore db = new DatabaseCore(mockContext.Object, "test"); + ExecutorContainerCore container = new ExecutorContainerCore(mockContext.Object, db, "test"); + + dynamic testItem = new + { + id = Guid.NewGuid().ToString(), + pk = "FF627B77-568E-4541-A47E-041EAC10E46F", + }; + + ItemRequestOptions itemRequestOptions = new ItemRequestOptions(); + Cosmos.PartitionKey partitionKey = new Cosmos.PartitionKey(testItem.pk); + using (ResponseMessage streamResponse = await container.DeleteItemStreamAsync( + partitionKey: partitionKey, + id: testItem.id)) + { + container.MockedExecutor.Verify(c => c.AddAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); + } + } + [TestMethod] public async Task HighThroughputSendsToExecutor_RetriesOn429() { From 9daee99e84ab1dbc5be7ab6d84470f23cc4ce308 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Wed, 28 Aug 2019 09:27:02 -0700 Subject: [PATCH 26/54] Adding Read support --- .../Resource/Container/ContainerCore.Items.cs | 11 ++++++ .../Batch/CosmosItemHighThroughputTests.cs | 36 +++++++++++++++++++ .../CosmosItemUnitTests.cs | 31 ++++++++++++++++ 3 files changed, 78 insertions(+) diff --git a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs index 0d9777f792..863a9d091d 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs @@ -93,6 +93,17 @@ public override Task ReadItemStreamAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default(CancellationToken)) { + if (this.ClientContext.ClientOptions.HighThroughputModeEnabled) + { + return this.ProcessItemStreamWithExecutorAsync( + partitionKey, + id, + null, + OperationType.Read, + requestOptions, + cancellationToken); + } + return this.ProcessItemStreamAsync( partitionKey, id, diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemHighThroughputTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemHighThroughputTests.cs index 3cdee71437..6bce942f25 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemHighThroughputTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemHighThroughputTests.cs @@ -122,6 +122,37 @@ public async Task DeleteItemStream_WithHighThroughput() } } + [TestMethod] + public async Task ReadItemStream_WithHighThroughput() + { + List createdDocuments = new List(); + // Create the items + List> tasks = new List>(); + for (int i = 0; i < 100; i++) + { + MyDocument createdDocument = CreateItem(i.ToString()); + createdDocuments.Add(createdDocument); + tasks.Add(ExecuteCreateAsync(this.container, createdDocument)); + } + + await Task.WhenAll(tasks); + + List> readTasks = new List>(); + // Delete the items + foreach (MyDocument createdDocument in createdDocuments) + { + readTasks.Add(ExecuteReadAsync(this.container, createdDocument)); + } + + await Task.WhenAll(readTasks); + for (int i = 0; i < 100; i++) + { + Task task = readTasks[i]; + ResponseMessage result = await task; + Assert.AreEqual(HttpStatusCode.OK, result.StatusCode); + } + } + [TestMethod] public async Task ReplaceItemStream_WithHighThroughput() { @@ -176,6 +207,11 @@ private static Task ExecuteDeleteAsync(Container container, MyD return container.DeleteItemStreamAsync(item.id, new PartitionKey(item.Status)); } + private static Task ExecuteReadAsync(Container container, MyDocument item) + { + return container.ReadItemStreamAsync(item.id, new PartitionKey(item.Status)); + } + private static MyDocument CreateItem(string id) => new MyDocument() { id = id, Status = id }; private class MyDocument diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs index 01764e7f3e..66b831cac6 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs @@ -259,6 +259,37 @@ public async Task HighThroughputSendsToExecutor_Replace() } } + [TestMethod] + public async Task HighThroughputSendsToExecutor_Read() + { + Mock mockContext = new Mock(); + mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { HighThroughputModeEnabled = true }); + mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); + mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); + mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); + + DatabaseCore db = new DatabaseCore(mockContext.Object, "test"); + ExecutorContainerCore container = new ExecutorContainerCore(mockContext.Object, db, "test"); + + dynamic testItem = new + { + id = Guid.NewGuid().ToString(), + pk = "FF627B77-568E-4541-A47E-041EAC10E46F", + }; + + using (Stream itemStream = MockCosmosUtil.Serializer.ToStream(testItem)) + { + ItemRequestOptions itemRequestOptions = new ItemRequestOptions(); + Cosmos.PartitionKey partitionKey = new Cosmos.PartitionKey(testItem.pk); + using (ResponseMessage streamResponse = await container.ReadItemStreamAsync( + partitionKey: partitionKey, + id: testItem.id)) + { + container.MockedExecutor.Verify(c => c.AddAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); + } + } + } + [TestMethod] public async Task HighThroughputSendsToExecutor_Delete() { From af0a340bc3e15bc5daf01c31e79812b503d39ce4 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Wed, 28 Aug 2019 10:51:39 -0700 Subject: [PATCH 27/54] private instead of internal --- .../src/Resource/Container/ContainerCore.Items.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs index 863a9d091d..24d6fbc176 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs @@ -46,7 +46,7 @@ public override Task CreateItemStreamAsync( { if (this.ClientContext.ClientOptions.HighThroughputModeEnabled) { - return this.ProcessItemStreamWithExecutorAsync( + return this.ProcessItemStreamWithBatchExecutorAsync( partitionKey, null, streamPayload, @@ -95,7 +95,7 @@ public override Task ReadItemStreamAsync( { if (this.ClientContext.ClientOptions.HighThroughputModeEnabled) { - return this.ProcessItemStreamWithExecutorAsync( + return this.ProcessItemStreamWithBatchExecutorAsync( partitionKey, id, null, @@ -137,7 +137,7 @@ public override Task UpsertItemStreamAsync( { if (this.ClientContext.ClientOptions.HighThroughputModeEnabled) { - return this.ProcessItemStreamWithExecutorAsync( + return this.ProcessItemStreamWithBatchExecutorAsync( partitionKey, null, streamPayload, @@ -187,7 +187,7 @@ public override Task ReplaceItemStreamAsync( { if (this.ClientContext.ClientOptions.HighThroughputModeEnabled) { - return this.ProcessItemStreamWithExecutorAsync( + return this.ProcessItemStreamWithBatchExecutorAsync( partitionKey, id, streamPayload, @@ -242,7 +242,7 @@ public override Task DeleteItemStreamAsync( { if (this.ClientContext.ClientOptions.HighThroughputModeEnabled) { - return this.ProcessItemStreamWithExecutorAsync( + return this.ProcessItemStreamWithBatchExecutorAsync( partitionKey, id, null, @@ -563,7 +563,7 @@ internal async Task ProcessItemStreamAsync( cancellationToken); } - internal virtual async Task ProcessItemStreamWithExecutorAsync( + private async Task ProcessItemStreamWithBatchExecutorAsync( PartitionKey partitionKey, string id, Stream streamPayload, From 3e0cbdc01ef271660a336c53b74943e869520f7d Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Wed, 28 Aug 2019 11:18:40 -0700 Subject: [PATCH 28/54] Fix for Read and DeleteItemAsync --- .../Resource/Container/ContainerCore.Items.cs | 52 ++++++++++++++----- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs index 24d6fbc176..b8906306e8 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs @@ -104,14 +104,11 @@ public override Task ReadItemStreamAsync( cancellationToken); } - return this.ProcessItemStreamAsync( - partitionKey, + return this.ReadItemStreamInternalAsync( id, - null, - OperationType.Read, + partitionKey, requestOptions, - extractPartitionKeyIfNeeded: false, - cancellationToken: cancellationToken); + cancellationToken); } public override Task> ReadItemAsync( @@ -120,7 +117,7 @@ public override Task> ReadItemAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default(CancellationToken)) { - Task response = this.ReadItemStreamAsync( + Task response = this.ReadItemStreamInternalAsync( partitionKey: partitionKey, id: id, requestOptions: requestOptions, @@ -251,13 +248,10 @@ public override Task DeleteItemStreamAsync( cancellationToken); } - return this.ProcessItemStreamAsync( - partitionKey, + return this.DeleteItemStreamInternalAsync( id, - null, - OperationType.Delete, + partitionKey, requestOptions, - extractPartitionKeyIfNeeded: false, cancellationToken: cancellationToken); } @@ -267,7 +261,7 @@ public override Task> DeleteItemAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default(CancellationToken)) { - Task response = this.DeleteItemStreamAsync( + Task response = this.DeleteItemStreamInternalAsync( partitionKey: partitionKey, id: id, requestOptions: requestOptions, @@ -724,5 +718,37 @@ private Uri ContcatCachedUriWithId(string resourceId) { return new Uri(this.cachedUriSegmentWithoutId + Uri.EscapeUriString(resourceId), UriKind.Relative); } + + private Task ReadItemStreamInternalAsync( + string id, + PartitionKey partitionKey, + ItemRequestOptions requestOptions = null, + CancellationToken cancellationToken = default(CancellationToken)) + { + return this.ProcessItemStreamAsync( + partitionKey, + id, + null, + OperationType.Read, + requestOptions, + extractPartitionKeyIfNeeded: false, + cancellationToken: cancellationToken); + } + + private Task DeleteItemStreamInternalAsync( + string id, + PartitionKey partitionKey, + ItemRequestOptions requestOptions = null, + CancellationToken cancellationToken = default(CancellationToken)) + { + return this.ProcessItemStreamAsync( + partitionKey, + id, + null, + OperationType.Delete, + requestOptions, + extractPartitionKeyIfNeeded: false, + cancellationToken: cancellationToken); + } } } From ad827459671b0bf84110bbfc995f3c55c4d4a51d Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Thu, 29 Aug 2019 15:30:22 -0700 Subject: [PATCH 29/54] Typed API support --- .../Resource/Container/ContainerCore.Items.cs | 116 +++++------ .../Batch/CosmosItemHighThroughputTests.cs | 187 ++++++++++++++++-- .../CosmosItemUnitTests.cs | 162 ++++++++++++++- 3 files changed, 383 insertions(+), 82 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs index b8906306e8..1a0ffd89c5 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs @@ -52,7 +52,8 @@ public override Task CreateItemStreamAsync( streamPayload, OperationType.Create, requestOptions, - cancellationToken); + extractPartitionKeyIfNeeded: false, + cancellationToken: cancellationToken); } return this.ProcessItemStreamAsync( @@ -101,14 +102,18 @@ public override Task ReadItemStreamAsync( null, OperationType.Read, requestOptions, - cancellationToken); + extractPartitionKeyIfNeeded: false, + cancellationToken: cancellationToken); } - return this.ReadItemStreamInternalAsync( - id, + return this.ProcessItemStreamAsync( partitionKey, + id, + null, + OperationType.Read, requestOptions, - cancellationToken); + extractPartitionKeyIfNeeded: false, + cancellationToken: cancellationToken); } public override Task> ReadItemAsync( @@ -117,7 +122,7 @@ public override Task> ReadItemAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default(CancellationToken)) { - Task response = this.ReadItemStreamInternalAsync( + Task response = this.ReadItemStreamAsync( partitionKey: partitionKey, id: id, requestOptions: requestOptions, @@ -140,7 +145,8 @@ public override Task UpsertItemStreamAsync( streamPayload, OperationType.Upsert, requestOptions, - cancellationToken); + extractPartitionKeyIfNeeded: false, + cancellationToken: cancellationToken); } return this.ProcessItemStreamAsync( @@ -190,7 +196,8 @@ public override Task ReplaceItemStreamAsync( streamPayload, OperationType.Replace, requestOptions, - cancellationToken); + extractPartitionKeyIfNeeded: false, + cancellationToken: cancellationToken); } return this.ProcessItemStreamAsync( @@ -245,13 +252,17 @@ public override Task DeleteItemStreamAsync( null, OperationType.Delete, requestOptions, - cancellationToken); + extractPartitionKeyIfNeeded: false, + cancellationToken: cancellationToken); } - return this.DeleteItemStreamInternalAsync( - id, + return this.ProcessItemStreamAsync( partitionKey, + id, + null, + OperationType.Delete, requestOptions, + extractPartitionKeyIfNeeded: false, cancellationToken: cancellationToken); } @@ -261,7 +272,7 @@ public override Task> DeleteItemAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default(CancellationToken)) { - Task response = this.DeleteItemStreamInternalAsync( + Task response = this.DeleteItemStreamAsync( partitionKey: partitionKey, id: id, requestOptions: requestOptions, @@ -490,20 +501,36 @@ internal async Task ExtractPartitionKeyAndProcessItemStreamAsyn string itemId, Stream streamPayload, OperationType operationType, - RequestOptions requestOptions, + ItemRequestOptions requestOptions, CancellationToken cancellationToken) { PartitionKeyMismatchRetryPolicy requestRetryPolicy = null; while (true) { - ResponseMessage responseMessage = await this.ProcessItemStreamAsync( - partitionKey, - itemId, - streamPayload, - operationType, - requestOptions, - extractPartitionKeyIfNeeded: true, - cancellationToken: cancellationToken); + ResponseMessage responseMessage; + if (this.ClientContext.ClientOptions.HighThroughputModeEnabled) + { + responseMessage = await this.ProcessItemStreamWithBatchExecutorAsync( + partitionKey, + itemId, + streamPayload, + operationType, + requestOptions, + extractPartitionKeyIfNeeded: true, + cancellationToken: cancellationToken); + } + else + { + responseMessage = await this.ProcessItemStreamAsync( + partitionKey, + itemId, + streamPayload, + operationType, + requestOptions, + extractPartitionKeyIfNeeded: true, + cancellationToken: cancellationToken); + + } if (responseMessage.IsSuccessStatusCode) { @@ -558,13 +585,24 @@ internal async Task ProcessItemStreamAsync( } private async Task ProcessItemStreamWithBatchExecutorAsync( - PartitionKey partitionKey, + PartitionKey? partitionKey, string id, Stream streamPayload, OperationType operationType, - ItemRequestOptions itemRequestOptions = null, + ItemRequestOptions itemRequestOptions, + bool extractPartitionKeyIfNeeded, CancellationToken cancellationToken = default(CancellationToken)) { + if (itemRequestOptions != null && itemRequestOptions.IsEffectivePartitionKeyRouting) + { + partitionKey = null; + } + + if (extractPartitionKeyIfNeeded && partitionKey == null) + { + partitionKey = await this.GetPartitionKeyValueFromStreamAsync(streamPayload, cancellationToken); + } + BatchItemRequestOptions batchItemRequestOptions = BatchItemRequestOptions.FromItemRequestOptions(itemRequestOptions); ItemBatchOperation itemBatchOperation = new ItemBatchOperation(operationType, /* index */ 0, partitionKey, id, streamPayload, batchItemRequestOptions); return await this.batchExecutorRetryHandler.SendAsync(itemBatchOperation, itemRequestOptions, cancellationToken); @@ -718,37 +756,5 @@ private Uri ContcatCachedUriWithId(string resourceId) { return new Uri(this.cachedUriSegmentWithoutId + Uri.EscapeUriString(resourceId), UriKind.Relative); } - - private Task ReadItemStreamInternalAsync( - string id, - PartitionKey partitionKey, - ItemRequestOptions requestOptions = null, - CancellationToken cancellationToken = default(CancellationToken)) - { - return this.ProcessItemStreamAsync( - partitionKey, - id, - null, - OperationType.Read, - requestOptions, - extractPartitionKeyIfNeeded: false, - cancellationToken: cancellationToken); - } - - private Task DeleteItemStreamInternalAsync( - string id, - PartitionKey partitionKey, - ItemRequestOptions requestOptions = null, - CancellationToken cancellationToken = default(CancellationToken)) - { - return this.ProcessItemStreamAsync( - partitionKey, - id, - null, - OperationType.Delete, - requestOptions, - extractPartitionKeyIfNeeded: false, - cancellationToken: cancellationToken); - } } } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemHighThroughputTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemHighThroughputTests.cs index 6bce942f25..0c37f1633a 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemHighThroughputTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemHighThroughputTests.cs @@ -29,7 +29,7 @@ public async Task TestInitialize() DatabaseResponse response = await client.CreateDatabaseIfNotExistsAsync(Guid.NewGuid().ToString()); this.database = response.Database; - ContainerResponse containerResponse = await this.database.CreateContainerAsync(Guid.NewGuid().ToString(), "/Status", 10000); + ContainerResponse containerResponse = await this.database.CreateContainerAsync(Guid.NewGuid().ToString(), "/Status", 50000); this.container = containerResponse; } @@ -45,7 +45,7 @@ public async Task CreateItemStream_WithHighThroughput() List> tasks = new List>(); for (int i = 0; i < 100; i++) { - tasks.Add(ExecuteCreateAsync(this.container, CreateItem(i.ToString()))); + tasks.Add(ExecuteCreateStreamAsync(this.container, CreateItem(i.ToString()))); } await Task.WhenAll(tasks); @@ -58,9 +58,25 @@ public async Task CreateItemStream_WithHighThroughput() MyDocument document = cosmosDefaultJsonSerializer.FromStream(result.Content); Assert.AreEqual(i.ToString(), document.id); + } + } + + [TestMethod] + public async Task CreateItemAsync_WithHighThroughput() + { + List>> tasks = new List>>(); + for (int i = 0; i < 100; i++) + { + tasks.Add(ExecuteCreateAsync(this.container, CreateItem(i.ToString()))); + } + + await Task.WhenAll(tasks); - ItemResponse storedDoc = await this.container.ReadItemAsync(i.ToString(), new Cosmos.PartitionKey(i.ToString())); - Assert.IsNotNull(storedDoc.Resource); + for (int i = 0; i < 100; i++) + { + Task> task = tasks[i]; + ItemResponse result = await task; + Assert.AreEqual(HttpStatusCode.Created, result.StatusCode); } } @@ -70,7 +86,7 @@ public async Task UpsertItemStream_WithHighThroughput() List> tasks = new List>(); for (int i = 0; i < 100; i++) { - tasks.Add(ExecuteUpsertAsync(this.container, CreateItem(i.ToString()))); + tasks.Add(ExecuteUpsertStreamAsync(this.container, CreateItem(i.ToString()))); } await Task.WhenAll(tasks); @@ -83,9 +99,25 @@ public async Task UpsertItemStream_WithHighThroughput() MyDocument document = cosmosDefaultJsonSerializer.FromStream(result.Content); Assert.AreEqual(i.ToString(), document.id); + } + } + + [TestMethod] + public async Task UpsertItem_WithHighThroughput() + { + List>> tasks = new List>>(); + for (int i = 0; i < 100; i++) + { + tasks.Add(ExecuteUpsertAsync(this.container, CreateItem(i.ToString()))); + } + + await Task.WhenAll(tasks); - ItemResponse storedDoc = await this.container.ReadItemAsync(i.ToString(), new Cosmos.PartitionKey(i.ToString())); - Assert.IsNotNull(storedDoc.Resource); + for (int i = 0; i < 100; i++) + { + Task> task = tasks[i]; + ItemResponse result = await task; + Assert.AreEqual(HttpStatusCode.Created, result.StatusCode); } } @@ -99,7 +131,7 @@ public async Task DeleteItemStream_WithHighThroughput() { MyDocument createdDocument = CreateItem(i.ToString()); createdDocuments.Add(createdDocument); - tasks.Add(ExecuteCreateAsync(this.container, createdDocument)); + tasks.Add(ExecuteCreateStreamAsync(this.container, createdDocument)); } await Task.WhenAll(tasks); @@ -108,7 +140,7 @@ public async Task DeleteItemStream_WithHighThroughput() // Delete the items foreach (MyDocument createdDocument in createdDocuments) { - deleteTasks.Add(ExecuteDeleteAsync(this.container, createdDocument)); + deleteTasks.Add(ExecuteDeleteStreamAsync(this.container, createdDocument)); } await Task.WhenAll(deleteTasks); @@ -117,8 +149,37 @@ public async Task DeleteItemStream_WithHighThroughput() Task task = deleteTasks[i]; ResponseMessage result = await task; Assert.AreEqual(HttpStatusCode.NoContent, result.StatusCode); + } + } + + [TestMethod] + public async Task DeleteItem_WithHighThroughput() + { + List createdDocuments = new List(); + // Create the items + List>> tasks = new List>>(); + for (int i = 0; i < 100; i++) + { + MyDocument createdDocument = CreateItem(i.ToString()); + createdDocuments.Add(createdDocument); + tasks.Add(ExecuteCreateAsync(this.container, createdDocument)); + } + + await Task.WhenAll(tasks); + + List>> deleteTasks = new List>>(); + // Delete the items + foreach (MyDocument createdDocument in createdDocuments) + { + deleteTasks.Add(ExecuteDeleteAsync(this.container, createdDocument)); + } - await Assert.ThrowsExceptionAsync(() => this.container.ReadItemAsync(i.ToString(), new Cosmos.PartitionKey(i.ToString()))); + await Task.WhenAll(deleteTasks); + for (int i = 0; i < 100; i++) + { + Task> task = deleteTasks[i]; + ItemResponse result = await task; + Assert.AreEqual(HttpStatusCode.NoContent, result.StatusCode); } } @@ -132,7 +193,7 @@ public async Task ReadItemStream_WithHighThroughput() { MyDocument createdDocument = CreateItem(i.ToString()); createdDocuments.Add(createdDocument); - tasks.Add(ExecuteCreateAsync(this.container, createdDocument)); + tasks.Add(ExecuteCreateStreamAsync(this.container, createdDocument)); } await Task.WhenAll(tasks); @@ -141,7 +202,7 @@ public async Task ReadItemStream_WithHighThroughput() // Delete the items foreach (MyDocument createdDocument in createdDocuments) { - readTasks.Add(ExecuteReadAsync(this.container, createdDocument)); + readTasks.Add(ExecuteReadStreamAsync(this.container, createdDocument)); } await Task.WhenAll(readTasks); @@ -153,6 +214,37 @@ public async Task ReadItemStream_WithHighThroughput() } } + [TestMethod] + public async Task ReadItem_WithHighThroughput() + { + List createdDocuments = new List(); + // Create the items + List>> tasks = new List>>(); + for (int i = 0; i < 100; i++) + { + MyDocument createdDocument = CreateItem(i.ToString()); + createdDocuments.Add(createdDocument); + tasks.Add(ExecuteCreateAsync(this.container, createdDocument)); + } + + await Task.WhenAll(tasks); + + List>> readTasks = new List>>(); + // Delete the items + foreach (MyDocument createdDocument in createdDocuments) + { + readTasks.Add(ExecuteReadAsync(this.container, createdDocument)); + } + + await Task.WhenAll(readTasks); + for (int i = 0; i < 100; i++) + { + Task> task = readTasks[i]; + ItemResponse result = await task; + Assert.AreEqual(HttpStatusCode.OK, result.StatusCode); + } + } + [TestMethod] public async Task ReplaceItemStream_WithHighThroughput() { @@ -163,7 +255,7 @@ public async Task ReplaceItemStream_WithHighThroughput() { MyDocument createdDocument = CreateItem(i.ToString()); createdDocuments.Add(createdDocument); - tasks.Add(ExecuteCreateAsync(this.container, createdDocument)); + tasks.Add(ExecuteCreateStreamAsync(this.container, createdDocument)); } await Task.WhenAll(tasks); @@ -172,7 +264,7 @@ public async Task ReplaceItemStream_WithHighThroughput() // Replace the items foreach (MyDocument createdDocument in createdDocuments) { - replaceTasks.Add(ExecuteReplaceAsync(this.container, createdDocument)); + replaceTasks.Add(ExecuteReplaceStreamAsync(this.container, createdDocument)); } await Task.WhenAll(replaceTasks); @@ -181,33 +273,86 @@ public async Task ReplaceItemStream_WithHighThroughput() Task task = replaceTasks[i]; ResponseMessage result = await task; Assert.AreEqual(HttpStatusCode.OK, result.StatusCode); + } + } - ItemResponse storedDoc = await this.container.ReadItemAsync(i.ToString(), new Cosmos.PartitionKey(i.ToString())); - Assert.IsNotNull(storedDoc.Resource); + [TestMethod] + public async Task ReplaceItem_WithHighThroughput() + { + List createdDocuments = new List(); + // Create the items + List>> tasks = new List>>(); + for (int i = 0; i < 100; i++) + { + MyDocument createdDocument = CreateItem(i.ToString()); + createdDocuments.Add(createdDocument); + tasks.Add(ExecuteCreateAsync(this.container, createdDocument)); + } + + await Task.WhenAll(tasks); + + List>> replaceTasks = new List>>(); + // Replace the items + foreach (MyDocument createdDocument in createdDocuments) + { + replaceTasks.Add(ExecuteReplaceAsync(this.container, createdDocument)); + } + + await Task.WhenAll(replaceTasks); + for (int i = 0; i < 100; i++) + { + Task> task = replaceTasks[i]; + ItemResponse result = await task; + Assert.AreEqual(HttpStatusCode.OK, result.StatusCode); } } - private static Task ExecuteCreateAsync(Container container, MyDocument item) + private static Task> ExecuteCreateAsync(Container container, MyDocument item) + { + return container.CreateItemAsync(item, new PartitionKey(item.Status)); + } + + private static Task> ExecuteUpsertAsync(Container container, MyDocument item) + { + return container.UpsertItemAsync(item, new PartitionKey(item.Status)); + } + + private static Task> ExecuteReplaceAsync(Container container, MyDocument item) + { + return container.ReplaceItemAsync(item, item.id, new PartitionKey(item.Status)); + } + + private static Task> ExecuteDeleteAsync(Container container, MyDocument item) + { + return container.DeleteItemAsync(item.id, new PartitionKey(item.Status)); + } + + private static Task> ExecuteReadAsync(Container container, MyDocument item) + { + return container.ReadItemAsync(item.id, new PartitionKey(item.Status)); + } + + private static Task ExecuteCreateStreamAsync(Container container, MyDocument item) { return container.CreateItemStreamAsync(cosmosDefaultJsonSerializer.ToStream(item), new PartitionKey(item.Status)); } - private static Task ExecuteUpsertAsync(Container container, MyDocument item) + private static Task ExecuteUpsertStreamAsync(Container container, MyDocument item) { return container.UpsertItemStreamAsync(cosmosDefaultJsonSerializer.ToStream(item), new PartitionKey(item.Status)); } - private static Task ExecuteReplaceAsync(Container container, MyDocument item) + private static Task ExecuteReplaceStreamAsync(Container container, MyDocument item) { return container.ReplaceItemStreamAsync(cosmosDefaultJsonSerializer.ToStream(item), item.id, new PartitionKey(item.Status)); } - private static Task ExecuteDeleteAsync(Container container, MyDocument item) + private static Task ExecuteDeleteStreamAsync(Container container, MyDocument item) { return container.DeleteItemStreamAsync(item.id, new PartitionKey(item.Status)); } - private static Task ExecuteReadAsync(Container container, MyDocument item) + private static Task ExecuteReadStreamAsync(Container container, MyDocument item) { return container.ReadItemStreamAsync(item.id, new PartitionKey(item.Status)); } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs index 66b831cac6..d5abad7970 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs @@ -166,7 +166,7 @@ await Assert.ThrowsExceptionAsync(async () => { } [TestMethod] - public async Task HighThroughputSendsToExecutor_Create() + public async Task HighThroughputSendsToExecutor_CreateStream() { Mock mockContext = new Mock(); mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { HighThroughputModeEnabled = true }); @@ -197,7 +197,7 @@ public async Task HighThroughputSendsToExecutor_Create() } [TestMethod] - public async Task HighThroughputSendsToExecutor_Upsert() + public async Task HighThroughputSendsToExecutor_UpsertStream() { Mock mockContext = new Mock(); mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { HighThroughputModeEnabled = true }); @@ -228,7 +228,7 @@ public async Task HighThroughputSendsToExecutor_Upsert() } [TestMethod] - public async Task HighThroughputSendsToExecutor_Replace() + public async Task HighThroughputSendsToExecutor_ReplaceStream() { Mock mockContext = new Mock(); mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { HighThroughputModeEnabled = true }); @@ -260,7 +260,7 @@ public async Task HighThroughputSendsToExecutor_Replace() } [TestMethod] - public async Task HighThroughputSendsToExecutor_Read() + public async Task HighThroughputSendsToExecutor_ReadStream() { Mock mockContext = new Mock(); mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { HighThroughputModeEnabled = true }); @@ -291,7 +291,7 @@ public async Task HighThroughputSendsToExecutor_Read() } [TestMethod] - public async Task HighThroughputSendsToExecutor_Delete() + public async Task HighThroughputSendsToExecutor_DeleteStream() { Mock mockContext = new Mock(); mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { HighThroughputModeEnabled = true }); @@ -318,6 +318,157 @@ public async Task HighThroughputSendsToExecutor_Delete() } } + [TestMethod] + public async Task HighThroughputSendsToExecutor_Create() + { + CosmosClientOptions clientOptions = new CosmosClientOptions() { HighThroughputModeEnabled = true }; + Mock mockContext = new Mock(); + mockContext.Setup(x => x.ClientOptions).Returns(clientOptions); + mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); + mockContext.Setup(x => x.CosmosSerializer).Returns(new CosmosJsonDotNetSerializer()); + mockContext.Setup(x => x.ResponseFactory).Returns(new CosmosResponseFactory( + defaultJsonSerializer: clientOptions.PropertiesSerializer, + userJsonSerializer: new CosmosJsonDotNetSerializer())); + mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); + mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); + + DatabaseCore db = new DatabaseCore(mockContext.Object, "test"); + ExecutorContainerCore container = new ExecutorContainerCore(mockContext.Object, db, "test"); + + dynamic testItem = new + { + id = Guid.NewGuid().ToString(), + pk = "FF627B77-568E-4541-A47E-041EAC10E46F", + }; + + Cosmos.PartitionKey partitionKey = new Cosmos.PartitionKey(testItem.pk); + ItemResponse response = await container.CreateItemAsync( + testItem, + partitionKey: partitionKey); + container.MockedExecutor.Verify(c => c.AddAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); + } + + [TestMethod] + public async Task HighThroughputSendsToExecutor_Upsert() + { + CosmosClientOptions clientOptions = new CosmosClientOptions() { HighThroughputModeEnabled = true }; + Mock mockContext = new Mock(); + mockContext.Setup(x => x.ClientOptions).Returns(clientOptions); + mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); + mockContext.Setup(x => x.CosmosSerializer).Returns(new CosmosJsonDotNetSerializer()); + mockContext.Setup(x => x.ResponseFactory).Returns(new CosmosResponseFactory( + defaultJsonSerializer: clientOptions.PropertiesSerializer, + userJsonSerializer: new CosmosJsonDotNetSerializer())); + mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); + mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); + + DatabaseCore db = new DatabaseCore(mockContext.Object, "test"); + ExecutorContainerCore container = new ExecutorContainerCore(mockContext.Object, db, "test"); + + dynamic testItem = new + { + id = Guid.NewGuid().ToString(), + pk = "FF627B77-568E-4541-A47E-041EAC10E46F", + }; + + Cosmos.PartitionKey partitionKey = new Cosmos.PartitionKey(testItem.pk); + ItemResponse response = await container.UpsertItemAsync( + testItem, + partitionKey: partitionKey); + container.MockedExecutor.Verify(c => c.AddAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); + } + + [TestMethod] + public async Task HighThroughputSendsToExecutor_Replace() + { + CosmosClientOptions clientOptions = new CosmosClientOptions() { HighThroughputModeEnabled = true }; + Mock mockContext = new Mock(); + mockContext.Setup(x => x.ClientOptions).Returns(clientOptions); + mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); + mockContext.Setup(x => x.CosmosSerializer).Returns(new CosmosJsonDotNetSerializer()); + mockContext.Setup(x => x.ResponseFactory).Returns(new CosmosResponseFactory( + defaultJsonSerializer: clientOptions.PropertiesSerializer, + userJsonSerializer: new CosmosJsonDotNetSerializer())); + mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); + mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); + + DatabaseCore db = new DatabaseCore(mockContext.Object, "test"); + ExecutorContainerCore container = new ExecutorContainerCore(mockContext.Object, db, "test"); + + dynamic testItem = new + { + id = Guid.NewGuid().ToString(), + pk = "FF627B77-568E-4541-A47E-041EAC10E46F", + }; + + Cosmos.PartitionKey partitionKey = new Cosmos.PartitionKey(testItem.pk); + ItemResponse response = await container.ReplaceItemAsync( + testItem, + testItem.id); + container.MockedExecutor.Verify(c => c.AddAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); + } + + [TestMethod] + public async Task HighThroughputSendsToExecutor_Read() + { + CosmosClientOptions clientOptions = new CosmosClientOptions() { HighThroughputModeEnabled = true }; + Mock mockContext = new Mock(); + mockContext.Setup(x => x.ClientOptions).Returns(clientOptions); + mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); + mockContext.Setup(x => x.CosmosSerializer).Returns(new CosmosJsonDotNetSerializer()); + mockContext.Setup(x => x.ResponseFactory).Returns(new CosmosResponseFactory( + defaultJsonSerializer: clientOptions.PropertiesSerializer, + userJsonSerializer: new CosmosJsonDotNetSerializer())); + mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); + mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); + + DatabaseCore db = new DatabaseCore(mockContext.Object, "test"); + ExecutorContainerCore container = new ExecutorContainerCore(mockContext.Object, db, "test"); + + dynamic testItem = new + { + id = Guid.NewGuid().ToString(), + pk = "FF627B77-568E-4541-A47E-041EAC10E46F", + }; + + Cosmos.PartitionKey partitionKey = new Cosmos.PartitionKey(testItem.pk); + ItemResponse response = await container.ReadItemAsync( + id: testItem.id, + partitionKey: partitionKey); + container.MockedExecutor.Verify(c => c.AddAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); + } + + [TestMethod] + public async Task HighThroughputSendsToExecutor_Delete() + { + CosmosClientOptions clientOptions = new CosmosClientOptions() { HighThroughputModeEnabled = true }; + Mock mockContext = new Mock(); + mockContext.Setup(x => x.ClientOptions).Returns(clientOptions); + mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); + mockContext.Setup(x => x.CosmosSerializer).Returns(new CosmosJsonDotNetSerializer()); + mockContext.Setup(x => x.ResponseFactory).Returns(new CosmosResponseFactory( + defaultJsonSerializer: clientOptions.PropertiesSerializer, + userJsonSerializer: new CosmosJsonDotNetSerializer())); + mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); + mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); + + DatabaseCore db = new DatabaseCore(mockContext.Object, "test"); + ExecutorContainerCore container = new ExecutorContainerCore(mockContext.Object, db, "test"); + + dynamic testItem = new + { + id = Guid.NewGuid().ToString(), + pk = "FF627B77-568E-4541-A47E-041EAC10E46F", + }; + + Cosmos.PartitionKey partitionKey = new Cosmos.PartitionKey(testItem.pk); + ItemResponse response = await container.DeleteItemAsync( + partitionKey: partitionKey, + id: testItem.id); + + container.MockedExecutor.Verify(c => c.AddAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); + } + [TestMethod] public async Task HighThroughputSendsToExecutor_RetriesOn429() { @@ -338,7 +489,6 @@ public async Task HighThroughputSendsToExecutor_RetriesOn429() using (Stream itemStream = MockCosmosUtil.Serializer.ToStream(testItem)) { - ItemRequestOptions itemRequestOptions = new ItemRequestOptions(); Cosmos.PartitionKey partitionKey = new Cosmos.PartitionKey(testItem.pk); using (ResponseMessage streamResponse = await container.CreateItemStreamAsync( partitionKey: partitionKey, From 0f39e052ea28b6905b83e3da174ab5d4118b33e7 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Fri, 30 Aug 2019 16:26:41 -0700 Subject: [PATCH 30/54] Rename property --- .../src/CosmosClientOptions.cs | 6 ++--- .../src/Fluent/CosmosClientBuilder.cs | 6 ++--- .../Resource/Container/ContainerCore.Items.cs | 12 +++++----- .../Batch/CosmosItemHighThroughputTests.cs | 2 +- .../CosmosClientOptionsUnitTests.cs | 6 ++--- .../CosmosItemUnitTests.cs | 22 +++++++++---------- 6 files changed, 27 insertions(+), 27 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs index 8b5a3f3a68..9d87ef02bc 100644 --- a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs +++ b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs @@ -333,18 +333,18 @@ public CosmosSerializer Serializer } /// - /// Enables a high throughput mode in which ingest operations will be optimized to execute in batch requests. + /// Enables a high throughput mode in which point operations will be optimized to execute in batch requests. /// /// /// There is no order of execution enforced when this mode is active. - /// This mode affects only Stream based ingest operations. + /// This mode affects only point operations. /// #if PREVIEW public #else internal #endif - bool HighThroughputModeEnabled { get; set; } + bool OptimizeForThroughput { get; set; } /// /// A JSON serializer used by the CosmosClient to serialize or de-serialize cosmos request/responses. diff --git a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs index 90ffac49ec..ae29db4fb7 100644 --- a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs +++ b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs @@ -332,16 +332,16 @@ public CosmosClientBuilder WithCustomSerializer(CosmosSerializer cosmosJsonSeria /// /// Enables a high throughput mode in which ingest operations will be optimized to execute in batch requests. /// - /// Whether is enabled. + /// Whether is enabled. /// The object #if PREVIEW public #else internal #endif - CosmosClientBuilder WithHighThroughputMode(bool enabled) + CosmosClientBuilder WithOptimizeForThroughput(bool enabled) { - this.clientOptions.HighThroughputModeEnabled = enabled; + this.clientOptions.OptimizeForThroughput = enabled; return this; } diff --git a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs index 1a0ffd89c5..15aa936cc6 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs @@ -44,7 +44,7 @@ public override Task CreateItemStreamAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default(CancellationToken)) { - if (this.ClientContext.ClientOptions.HighThroughputModeEnabled) + if (this.ClientContext.ClientOptions.OptimizeForThroughput) { return this.ProcessItemStreamWithBatchExecutorAsync( partitionKey, @@ -94,7 +94,7 @@ public override Task ReadItemStreamAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default(CancellationToken)) { - if (this.ClientContext.ClientOptions.HighThroughputModeEnabled) + if (this.ClientContext.ClientOptions.OptimizeForThroughput) { return this.ProcessItemStreamWithBatchExecutorAsync( partitionKey, @@ -137,7 +137,7 @@ public override Task UpsertItemStreamAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default(CancellationToken)) { - if (this.ClientContext.ClientOptions.HighThroughputModeEnabled) + if (this.ClientContext.ClientOptions.OptimizeForThroughput) { return this.ProcessItemStreamWithBatchExecutorAsync( partitionKey, @@ -188,7 +188,7 @@ public override Task ReplaceItemStreamAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default(CancellationToken)) { - if (this.ClientContext.ClientOptions.HighThroughputModeEnabled) + if (this.ClientContext.ClientOptions.OptimizeForThroughput) { return this.ProcessItemStreamWithBatchExecutorAsync( partitionKey, @@ -244,7 +244,7 @@ public override Task DeleteItemStreamAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default(CancellationToken)) { - if (this.ClientContext.ClientOptions.HighThroughputModeEnabled) + if (this.ClientContext.ClientOptions.OptimizeForThroughput) { return this.ProcessItemStreamWithBatchExecutorAsync( partitionKey, @@ -508,7 +508,7 @@ internal async Task ExtractPartitionKeyAndProcessItemStreamAsyn while (true) { ResponseMessage responseMessage; - if (this.ClientContext.ClientOptions.HighThroughputModeEnabled) + if (this.ClientContext.ClientOptions.OptimizeForThroughput) { responseMessage = await this.ProcessItemStreamWithBatchExecutorAsync( partitionKey, diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemHighThroughputTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemHighThroughputTests.cs index 0c37f1633a..cfc6a8b497 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemHighThroughputTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemHighThroughputTests.cs @@ -23,7 +23,7 @@ public class CosmosItemHighThroughputTests public async Task TestInitialize() { CosmosClientOptions clientOptions = new CosmosClientOptions(); - clientOptions.HighThroughputModeEnabled = true; + clientOptions.OptimizeForThroughput = true; CosmosClient client = TestCommon.CreateCosmosClient(clientOptions); DatabaseResponse response = await client.CreateDatabaseIfNotExistsAsync(Guid.NewGuid().ToString()); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs index b69302b37c..ba3450ba34 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs @@ -60,7 +60,7 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated() Assert.AreNotEqual(requestTimeout, clientOptions.RequestTimeout); Assert.AreNotEqual(userAgentSuffix, clientOptions.ApplicationName); Assert.AreNotEqual(apiType, clientOptions.ApiType); - Assert.IsFalse(clientOptions.HighThroughputModeEnabled); + Assert.IsFalse(clientOptions.OptimizeForThroughput); Assert.AreEqual(0, clientOptions.CustomHandlers.Count); Assert.IsNull(clientOptions.SerializerOptions); Assert.IsNull(clientOptions.Serializer); @@ -83,7 +83,7 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated() .AddCustomHandlers(preProcessHandler) .WithApiType(apiType) .WithThrottlingRetryOptions(maxRetryWaitTime, maxRetryAttemptsOnThrottledRequests) - .WithHighThroughputMode(true) + .WithOptimizeForThroughput(true) .WithSerializerOptions(cosmosSerializerOptions); cosmosClient = cosmosClientBuilder.Build(new MockDocumentClient()); @@ -102,7 +102,7 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated() Assert.AreEqual(cosmosSerializerOptions.IgnoreNullValues, clientOptions.SerializerOptions.IgnoreNullValues); Assert.AreEqual(cosmosSerializerOptions.PropertyNamingPolicy, clientOptions.SerializerOptions.PropertyNamingPolicy); Assert.AreEqual(cosmosSerializerOptions.Indented, clientOptions.SerializerOptions.Indented); - Assert.IsTrue(clientOptions.HighThroughputModeEnabled); + Assert.IsTrue(clientOptions.OptimizeForThroughput); //Verify GetConnectionPolicy returns the correct values policy = clientOptions.GetConnectionPolicy(); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs index d5abad7970..30a7f4b772 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs @@ -169,7 +169,7 @@ await Assert.ThrowsExceptionAsync(async () => { public async Task HighThroughputSendsToExecutor_CreateStream() { Mock mockContext = new Mock(); - mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { HighThroughputModeEnabled = true }); + mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { OptimizeForThroughput = true }); mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); @@ -200,7 +200,7 @@ public async Task HighThroughputSendsToExecutor_CreateStream() public async Task HighThroughputSendsToExecutor_UpsertStream() { Mock mockContext = new Mock(); - mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { HighThroughputModeEnabled = true }); + mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { OptimizeForThroughput = true }); mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); @@ -231,7 +231,7 @@ public async Task HighThroughputSendsToExecutor_UpsertStream() public async Task HighThroughputSendsToExecutor_ReplaceStream() { Mock mockContext = new Mock(); - mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { HighThroughputModeEnabled = true }); + mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { OptimizeForThroughput = true }); mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); @@ -263,7 +263,7 @@ public async Task HighThroughputSendsToExecutor_ReplaceStream() public async Task HighThroughputSendsToExecutor_ReadStream() { Mock mockContext = new Mock(); - mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { HighThroughputModeEnabled = true }); + mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { OptimizeForThroughput = true }); mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); @@ -294,7 +294,7 @@ public async Task HighThroughputSendsToExecutor_ReadStream() public async Task HighThroughputSendsToExecutor_DeleteStream() { Mock mockContext = new Mock(); - mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { HighThroughputModeEnabled = true }); + mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { OptimizeForThroughput = true }); mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); @@ -321,7 +321,7 @@ public async Task HighThroughputSendsToExecutor_DeleteStream() [TestMethod] public async Task HighThroughputSendsToExecutor_Create() { - CosmosClientOptions clientOptions = new CosmosClientOptions() { HighThroughputModeEnabled = true }; + CosmosClientOptions clientOptions = new CosmosClientOptions() { OptimizeForThroughput = true }; Mock mockContext = new Mock(); mockContext.Setup(x => x.ClientOptions).Returns(clientOptions); mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); @@ -351,7 +351,7 @@ public async Task HighThroughputSendsToExecutor_Create() [TestMethod] public async Task HighThroughputSendsToExecutor_Upsert() { - CosmosClientOptions clientOptions = new CosmosClientOptions() { HighThroughputModeEnabled = true }; + CosmosClientOptions clientOptions = new CosmosClientOptions() { OptimizeForThroughput = true }; Mock mockContext = new Mock(); mockContext.Setup(x => x.ClientOptions).Returns(clientOptions); mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); @@ -381,7 +381,7 @@ public async Task HighThroughputSendsToExecutor_Upsert() [TestMethod] public async Task HighThroughputSendsToExecutor_Replace() { - CosmosClientOptions clientOptions = new CosmosClientOptions() { HighThroughputModeEnabled = true }; + CosmosClientOptions clientOptions = new CosmosClientOptions() { OptimizeForThroughput = true }; Mock mockContext = new Mock(); mockContext.Setup(x => x.ClientOptions).Returns(clientOptions); mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); @@ -411,7 +411,7 @@ public async Task HighThroughputSendsToExecutor_Replace() [TestMethod] public async Task HighThroughputSendsToExecutor_Read() { - CosmosClientOptions clientOptions = new CosmosClientOptions() { HighThroughputModeEnabled = true }; + CosmosClientOptions clientOptions = new CosmosClientOptions() { OptimizeForThroughput = true }; Mock mockContext = new Mock(); mockContext.Setup(x => x.ClientOptions).Returns(clientOptions); mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); @@ -441,7 +441,7 @@ public async Task HighThroughputSendsToExecutor_Read() [TestMethod] public async Task HighThroughputSendsToExecutor_Delete() { - CosmosClientOptions clientOptions = new CosmosClientOptions() { HighThroughputModeEnabled = true }; + CosmosClientOptions clientOptions = new CosmosClientOptions() { OptimizeForThroughput = true }; Mock mockContext = new Mock(); mockContext.Setup(x => x.ClientOptions).Returns(clientOptions); mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); @@ -473,7 +473,7 @@ public async Task HighThroughputSendsToExecutor_Delete() public async Task HighThroughputSendsToExecutor_RetriesOn429() { Mock mockContext = new Mock(); - mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { HighThroughputModeEnabled = true }); + mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { OptimizeForThroughput = true }); mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); From 03a3c55480a0916c8f101277f46e9b4b3fbfe349 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Tue, 3 Sep 2019 12:04:05 -0700 Subject: [PATCH 31/54] Rename --- .../src/CosmosClientOptions.cs | 7 +++--- .../src/Fluent/CosmosClientBuilder.cs | 9 ++++---- .../Resource/Container/ContainerCore.Items.cs | 12 +++++----- .../Batch/CosmosItemHighThroughputTests.cs | 2 +- .../CosmosClientOptionsUnitTests.cs | 6 ++--- .../CosmosItemUnitTests.cs | 22 +++++++++---------- 6 files changed, 29 insertions(+), 29 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs index 9d87ef02bc..1c33dd3a0e 100644 --- a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs +++ b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs @@ -333,18 +333,17 @@ public CosmosSerializer Serializer } /// - /// Enables a high throughput mode in which point operations will be optimized to execute in batch requests. + /// Enables the optimization of point operation requests by batching them and resolving them as a single service call. /// /// - /// There is no order of execution enforced when this mode is active. - /// This mode affects only point operations. + /// When enabled, point operations will be grouped based on their concurrency to optimize throughput. /// #if PREVIEW public #else internal #endif - bool OptimizeForThroughput { get; set; } + bool AllowBatchingRequests { get; set; } /// /// A JSON serializer used by the CosmosClient to serialize or de-serialize cosmos request/responses. diff --git a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs index ae29db4fb7..eaf6888e14 100644 --- a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs +++ b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs @@ -330,18 +330,19 @@ public CosmosClientBuilder WithCustomSerializer(CosmosSerializer cosmosJsonSeria } /// - /// Enables a high throughput mode in which ingest operations will be optimized to execute in batch requests. + /// Enables the optimization of point operation requests by batching them and resolving them as a single service call. /// - /// Whether is enabled. + /// Whether is enabled. /// The object + /// #if PREVIEW public #else internal #endif - CosmosClientBuilder WithOptimizeForThroughput(bool enabled) + CosmosClientBuilder WithBatchingRequests(bool enabled) { - this.clientOptions.OptimizeForThroughput = enabled; + this.clientOptions.AllowBatchingRequests = enabled; return this; } diff --git a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs index 15aa936cc6..4741e1afa4 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs @@ -44,7 +44,7 @@ public override Task CreateItemStreamAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default(CancellationToken)) { - if (this.ClientContext.ClientOptions.OptimizeForThroughput) + if (this.ClientContext.ClientOptions.AllowBatchingRequests) { return this.ProcessItemStreamWithBatchExecutorAsync( partitionKey, @@ -94,7 +94,7 @@ public override Task ReadItemStreamAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default(CancellationToken)) { - if (this.ClientContext.ClientOptions.OptimizeForThroughput) + if (this.ClientContext.ClientOptions.AllowBatchingRequests) { return this.ProcessItemStreamWithBatchExecutorAsync( partitionKey, @@ -137,7 +137,7 @@ public override Task UpsertItemStreamAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default(CancellationToken)) { - if (this.ClientContext.ClientOptions.OptimizeForThroughput) + if (this.ClientContext.ClientOptions.AllowBatchingRequests) { return this.ProcessItemStreamWithBatchExecutorAsync( partitionKey, @@ -188,7 +188,7 @@ public override Task ReplaceItemStreamAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default(CancellationToken)) { - if (this.ClientContext.ClientOptions.OptimizeForThroughput) + if (this.ClientContext.ClientOptions.AllowBatchingRequests) { return this.ProcessItemStreamWithBatchExecutorAsync( partitionKey, @@ -244,7 +244,7 @@ public override Task DeleteItemStreamAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default(CancellationToken)) { - if (this.ClientContext.ClientOptions.OptimizeForThroughput) + if (this.ClientContext.ClientOptions.AllowBatchingRequests) { return this.ProcessItemStreamWithBatchExecutorAsync( partitionKey, @@ -508,7 +508,7 @@ internal async Task ExtractPartitionKeyAndProcessItemStreamAsyn while (true) { ResponseMessage responseMessage; - if (this.ClientContext.ClientOptions.OptimizeForThroughput) + if (this.ClientContext.ClientOptions.AllowBatchingRequests) { responseMessage = await this.ProcessItemStreamWithBatchExecutorAsync( partitionKey, diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemHighThroughputTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemHighThroughputTests.cs index cfc6a8b497..e466b2ba00 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemHighThroughputTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemHighThroughputTests.cs @@ -23,7 +23,7 @@ public class CosmosItemHighThroughputTests public async Task TestInitialize() { CosmosClientOptions clientOptions = new CosmosClientOptions(); - clientOptions.OptimizeForThroughput = true; + clientOptions.AllowBatchingRequests = true; CosmosClient client = TestCommon.CreateCosmosClient(clientOptions); DatabaseResponse response = await client.CreateDatabaseIfNotExistsAsync(Guid.NewGuid().ToString()); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs index ba3450ba34..f7cf792a8d 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs @@ -60,7 +60,7 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated() Assert.AreNotEqual(requestTimeout, clientOptions.RequestTimeout); Assert.AreNotEqual(userAgentSuffix, clientOptions.ApplicationName); Assert.AreNotEqual(apiType, clientOptions.ApiType); - Assert.IsFalse(clientOptions.OptimizeForThroughput); + Assert.IsFalse(clientOptions.AllowBatchingRequests); Assert.AreEqual(0, clientOptions.CustomHandlers.Count); Assert.IsNull(clientOptions.SerializerOptions); Assert.IsNull(clientOptions.Serializer); @@ -83,7 +83,7 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated() .AddCustomHandlers(preProcessHandler) .WithApiType(apiType) .WithThrottlingRetryOptions(maxRetryWaitTime, maxRetryAttemptsOnThrottledRequests) - .WithOptimizeForThroughput(true) + .WithBatchingRequests(true) .WithSerializerOptions(cosmosSerializerOptions); cosmosClient = cosmosClientBuilder.Build(new MockDocumentClient()); @@ -102,7 +102,7 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated() Assert.AreEqual(cosmosSerializerOptions.IgnoreNullValues, clientOptions.SerializerOptions.IgnoreNullValues); Assert.AreEqual(cosmosSerializerOptions.PropertyNamingPolicy, clientOptions.SerializerOptions.PropertyNamingPolicy); Assert.AreEqual(cosmosSerializerOptions.Indented, clientOptions.SerializerOptions.Indented); - Assert.IsTrue(clientOptions.OptimizeForThroughput); + Assert.IsTrue(clientOptions.AllowBatchingRequests); //Verify GetConnectionPolicy returns the correct values policy = clientOptions.GetConnectionPolicy(); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs index 30a7f4b772..c2ec81e636 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs @@ -169,7 +169,7 @@ await Assert.ThrowsExceptionAsync(async () => { public async Task HighThroughputSendsToExecutor_CreateStream() { Mock mockContext = new Mock(); - mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { OptimizeForThroughput = true }); + mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { AllowBatchingRequests = true }); mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); @@ -200,7 +200,7 @@ public async Task HighThroughputSendsToExecutor_CreateStream() public async Task HighThroughputSendsToExecutor_UpsertStream() { Mock mockContext = new Mock(); - mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { OptimizeForThroughput = true }); + mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { AllowBatchingRequests = true }); mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); @@ -231,7 +231,7 @@ public async Task HighThroughputSendsToExecutor_UpsertStream() public async Task HighThroughputSendsToExecutor_ReplaceStream() { Mock mockContext = new Mock(); - mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { OptimizeForThroughput = true }); + mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { AllowBatchingRequests = true }); mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); @@ -263,7 +263,7 @@ public async Task HighThroughputSendsToExecutor_ReplaceStream() public async Task HighThroughputSendsToExecutor_ReadStream() { Mock mockContext = new Mock(); - mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { OptimizeForThroughput = true }); + mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { AllowBatchingRequests = true }); mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); @@ -294,7 +294,7 @@ public async Task HighThroughputSendsToExecutor_ReadStream() public async Task HighThroughputSendsToExecutor_DeleteStream() { Mock mockContext = new Mock(); - mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { OptimizeForThroughput = true }); + mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { AllowBatchingRequests = true }); mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); @@ -321,7 +321,7 @@ public async Task HighThroughputSendsToExecutor_DeleteStream() [TestMethod] public async Task HighThroughputSendsToExecutor_Create() { - CosmosClientOptions clientOptions = new CosmosClientOptions() { OptimizeForThroughput = true }; + CosmosClientOptions clientOptions = new CosmosClientOptions() { AllowBatchingRequests = true }; Mock mockContext = new Mock(); mockContext.Setup(x => x.ClientOptions).Returns(clientOptions); mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); @@ -351,7 +351,7 @@ public async Task HighThroughputSendsToExecutor_Create() [TestMethod] public async Task HighThroughputSendsToExecutor_Upsert() { - CosmosClientOptions clientOptions = new CosmosClientOptions() { OptimizeForThroughput = true }; + CosmosClientOptions clientOptions = new CosmosClientOptions() { AllowBatchingRequests = true }; Mock mockContext = new Mock(); mockContext.Setup(x => x.ClientOptions).Returns(clientOptions); mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); @@ -381,7 +381,7 @@ public async Task HighThroughputSendsToExecutor_Upsert() [TestMethod] public async Task HighThroughputSendsToExecutor_Replace() { - CosmosClientOptions clientOptions = new CosmosClientOptions() { OptimizeForThroughput = true }; + CosmosClientOptions clientOptions = new CosmosClientOptions() { AllowBatchingRequests = true }; Mock mockContext = new Mock(); mockContext.Setup(x => x.ClientOptions).Returns(clientOptions); mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); @@ -411,7 +411,7 @@ public async Task HighThroughputSendsToExecutor_Replace() [TestMethod] public async Task HighThroughputSendsToExecutor_Read() { - CosmosClientOptions clientOptions = new CosmosClientOptions() { OptimizeForThroughput = true }; + CosmosClientOptions clientOptions = new CosmosClientOptions() { AllowBatchingRequests = true }; Mock mockContext = new Mock(); mockContext.Setup(x => x.ClientOptions).Returns(clientOptions); mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); @@ -441,7 +441,7 @@ public async Task HighThroughputSendsToExecutor_Read() [TestMethod] public async Task HighThroughputSendsToExecutor_Delete() { - CosmosClientOptions clientOptions = new CosmosClientOptions() { OptimizeForThroughput = true }; + CosmosClientOptions clientOptions = new CosmosClientOptions() { AllowBatchingRequests = true }; Mock mockContext = new Mock(); mockContext.Setup(x => x.ClientOptions).Returns(clientOptions); mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); @@ -473,7 +473,7 @@ public async Task HighThroughputSendsToExecutor_Delete() public async Task HighThroughputSendsToExecutor_RetriesOn429() { Mock mockContext = new Mock(); - mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { OptimizeForThroughput = true }); + mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { AllowBatchingRequests = true }); mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); From 8bfcf4ed576804e3cea39d9f351999bcc46315aa Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Wed, 4 Sep 2019 14:00:34 -0700 Subject: [PATCH 32/54] Refactor on ClientContext --- .../src/Resource/ClientContextCore.cs | 49 ++++ .../Resource/Container/ContainerCore.Items.cs | 105 +------ .../src/Resource/Container/ContainerCore.cs | 4 +- .../CosmosItemUnitTests.cs | 272 ++++++++++-------- 4 files changed, 212 insertions(+), 218 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Resource/ClientContextCore.cs b/Microsoft.Azure.Cosmos/src/Resource/ClientContextCore.cs index e6e505d614..5236bf3b8a 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/ClientContextCore.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/ClientContextCore.cs @@ -102,6 +102,20 @@ internal override Task ProcessResourceOperationStreamAsync( Action requestEnricher, CancellationToken cancellationToken) { + if (this.IsBatchingSupported(resourceType, operationType)) + { + return this.ProcessResourceOperationWithBatchExecutorAsync( + resourceUri: resourceUri, + resourceType: resourceType, + operationType: operationType, + requestOptions: requestOptions, + cosmosContainerCore: cosmosContainerCore, + partitionKey: partitionKey, + streamPayload: streamPayload, + requestEnricher: requestEnricher, + cancellationToken: cancellationToken); + } + return this.RequestHandler.SendAsync( resourceUri: resourceUri, resourceType: resourceType, @@ -138,5 +152,40 @@ internal override Task ProcessResourceOperationAsync( responseCreator: responseCreator, cancellationToken: cancellationToken); } + + private async Task ProcessResourceOperationWithBatchExecutorAsync( + Uri resourceUri, + ResourceType resourceType, + OperationType operationType, + RequestOptions requestOptions, + ContainerCore cosmosContainerCore, + PartitionKey? partitionKey, + Stream streamPayload, + Action requestEnricher, + CancellationToken cancellationToken) + { + ItemRequestOptions itemRequestOptions = requestOptions as ItemRequestOptions; + BatchItemRequestOptions batchItemRequestOptions = BatchItemRequestOptions.FromItemRequestOptions(itemRequestOptions); + ItemBatchOperation itemBatchOperation = new ItemBatchOperation(operationType, /* index */ 0, partitionKey, /* id */ null, streamPayload, batchItemRequestOptions); + BatchOperationResult batchOperationResult = await cosmosContainerCore.BatchExecutor.AddAsync(itemBatchOperation, itemRequestOptions, cancellationToken); + return batchOperationResult.ToResponseMessage(); + } + + private bool IsBatchingSupported( + ResourceType resourceType, + OperationType operationType) + { + if (!this.ClientOptions.AllowBatchingRequests) + { + return false; + } + + return resourceType == ResourceType.Document + && (operationType == OperationType.Create + || operationType == OperationType.Upsert + || operationType == OperationType.Read + || operationType == OperationType.Delete + || operationType == OperationType.Replace); + } } } diff --git a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs index 3b61082c01..3bbd5f8a32 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs @@ -15,7 +15,6 @@ namespace Microsoft.Azure.Cosmos using Microsoft.Azure.Cosmos.ChangeFeed; using Microsoft.Azure.Cosmos.ChangeFeed.FeedProcessing; using Microsoft.Azure.Cosmos.CosmosElements; - using Microsoft.Azure.Cosmos.Handlers; using Microsoft.Azure.Cosmos.Json; using Microsoft.Azure.Cosmos.Linq; using Microsoft.Azure.Cosmos.Query; @@ -36,26 +35,12 @@ internal partial class ContainerCore : Container private readonly CosmosQueryClient queryClient; - private readonly BatchExecutorRetryHandler batchExecutorRetryHandler; - public override Task CreateItemStreamAsync( Stream streamPayload, PartitionKey partitionKey, ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default(CancellationToken)) { - if (this.ClientContext.ClientOptions.AllowBatchingRequests) - { - return this.ProcessItemStreamWithBatchExecutorAsync( - partitionKey, - null, - streamPayload, - OperationType.Create, - requestOptions, - extractPartitionKeyIfNeeded: false, - cancellationToken: cancellationToken); - } - return this.ProcessItemStreamAsync( partitionKey, null, @@ -94,18 +79,6 @@ public override Task ReadItemStreamAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default(CancellationToken)) { - if (this.ClientContext.ClientOptions.AllowBatchingRequests) - { - return this.ProcessItemStreamWithBatchExecutorAsync( - partitionKey, - id, - null, - OperationType.Read, - requestOptions, - extractPartitionKeyIfNeeded: false, - cancellationToken: cancellationToken); - } - return this.ProcessItemStreamAsync( partitionKey, id, @@ -137,18 +110,6 @@ public override Task UpsertItemStreamAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default(CancellationToken)) { - if (this.ClientContext.ClientOptions.AllowBatchingRequests) - { - return this.ProcessItemStreamWithBatchExecutorAsync( - partitionKey, - null, - streamPayload, - OperationType.Upsert, - requestOptions, - extractPartitionKeyIfNeeded: false, - cancellationToken: cancellationToken); - } - return this.ProcessItemStreamAsync( partitionKey, null, @@ -188,18 +149,6 @@ public override Task ReplaceItemStreamAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default(CancellationToken)) { - if (this.ClientContext.ClientOptions.AllowBatchingRequests) - { - return this.ProcessItemStreamWithBatchExecutorAsync( - partitionKey, - id, - streamPayload, - OperationType.Replace, - requestOptions, - extractPartitionKeyIfNeeded: false, - cancellationToken: cancellationToken); - } - return this.ProcessItemStreamAsync( partitionKey, id, @@ -244,18 +193,6 @@ public override Task DeleteItemStreamAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default(CancellationToken)) { - if (this.ClientContext.ClientOptions.AllowBatchingRequests) - { - return this.ProcessItemStreamWithBatchExecutorAsync( - partitionKey, - id, - null, - OperationType.Delete, - requestOptions, - extractPartitionKeyIfNeeded: false, - cancellationToken: cancellationToken); - } - return this.ProcessItemStreamAsync( partitionKey, id, @@ -508,10 +445,7 @@ internal async Task ExtractPartitionKeyAndProcessItemStreamAsyn PartitionKeyMismatchRetryPolicy requestRetryPolicy = null; while (true) { - ResponseMessage responseMessage; - if (this.ClientContext.ClientOptions.AllowBatchingRequests) - { - responseMessage = await this.ProcessItemStreamWithBatchExecutorAsync( + ResponseMessage responseMessage = await this.ProcessItemStreamAsync( partitionKey, itemId, streamPayload, @@ -519,19 +453,6 @@ internal async Task ExtractPartitionKeyAndProcessItemStreamAsyn requestOptions, extractPartitionKeyIfNeeded: true, cancellationToken: cancellationToken); - } - else - { - responseMessage = await this.ProcessItemStreamAsync( - partitionKey, - itemId, - streamPayload, - operationType, - requestOptions, - extractPartitionKeyIfNeeded: true, - cancellationToken: cancellationToken); - - } if (responseMessage.IsSuccessStatusCode) { @@ -585,30 +506,6 @@ internal async Task ProcessItemStreamAsync( cancellationToken); } - private async Task ProcessItemStreamWithBatchExecutorAsync( - PartitionKey? partitionKey, - string id, - Stream streamPayload, - OperationType operationType, - ItemRequestOptions itemRequestOptions, - bool extractPartitionKeyIfNeeded, - CancellationToken cancellationToken = default(CancellationToken)) - { - if (itemRequestOptions != null && itemRequestOptions.IsEffectivePartitionKeyRouting) - { - partitionKey = null; - } - - if (extractPartitionKeyIfNeeded && partitionKey == null) - { - partitionKey = await this.GetPartitionKeyValueFromStreamAsync(streamPayload, cancellationToken); - } - - BatchItemRequestOptions batchItemRequestOptions = BatchItemRequestOptions.FromItemRequestOptions(itemRequestOptions); - ItemBatchOperation itemBatchOperation = new ItemBatchOperation(operationType, /* index */ 0, partitionKey, id, streamPayload, batchItemRequestOptions); - return await this.batchExecutorRetryHandler.SendAsync(itemBatchOperation, itemRequestOptions, cancellationToken); - } - internal async Task GetPartitionKeyValueFromStreamAsync( Stream stream, CancellationToken cancellation = default(CancellationToken)) diff --git a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.cs b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.cs index e89fb68168..2de7bbe418 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.cs @@ -45,7 +45,7 @@ internal ContainerCore( this.Scripts = new ScriptsCore(this, this.ClientContext); this.cachedUriSegmentWithoutId = this.GetResourceSegmentUriWithoutId(); this.queryClient = queryClient ?? new CosmosQueryClientCore(this.ClientContext, this); - this.batchExecutorRetryHandler = new BatchExecutorRetryHandler(this.ClientContext, this.InitializeBatchExecutorForContainer()); + this.BatchExecutor = this.InitializeBatchExecutorForContainer(); } public override string Id { get; } @@ -56,6 +56,8 @@ internal ContainerCore( internal virtual CosmosClientContext ClientContext { get; } + internal virtual BatchAsyncContainerExecutor BatchExecutor { get; } + public override Conflicts Conflicts { get; } public override Scripts.Scripts Scripts { get; } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs index c2ec81e636..09b7cf252b 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs @@ -168,14 +168,20 @@ await Assert.ThrowsExceptionAsync(async () => { [TestMethod] public async Task HighThroughputSendsToExecutor_CreateStream() { - Mock mockContext = new Mock(); - mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { AllowBatchingRequests = true }); - mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); - mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); - mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); - - DatabaseCore db = new DatabaseCore(mockContext.Object, "test"); - ExecutorContainerCore container = new ExecutorContainerCore(mockContext.Object, db, "test"); + ClientContextCore clientContextCore = new ClientContextCore( + MockCosmosUtil.CreateMockCosmosClient(), + new CosmosClientOptions() { AllowBatchingRequests = true }, + new CosmosJsonDotNetSerializer(), + new CosmosJsonDotNetSerializer(), + null, + null, + null, + new MockDocumentClient(), + Mock.Of() + ); + + DatabaseCore db = new DatabaseCore(clientContextCore, "test"); + ExecutorContainerCore container = new ExecutorContainerCore(clientContextCore, db, "test"); dynamic testItem = new { @@ -199,14 +205,20 @@ public async Task HighThroughputSendsToExecutor_CreateStream() [TestMethod] public async Task HighThroughputSendsToExecutor_UpsertStream() { - Mock mockContext = new Mock(); - mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { AllowBatchingRequests = true }); - mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); - mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); - mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); - - DatabaseCore db = new DatabaseCore(mockContext.Object, "test"); - ExecutorContainerCore container = new ExecutorContainerCore(mockContext.Object, db, "test"); + ClientContextCore clientContextCore = new ClientContextCore( + MockCosmosUtil.CreateMockCosmosClient(), + new CosmosClientOptions() { AllowBatchingRequests = true }, + new CosmosJsonDotNetSerializer(), + new CosmosJsonDotNetSerializer(), + null, + null, + null, + new MockDocumentClient(), + Mock.Of() + ); + + DatabaseCore db = new DatabaseCore(clientContextCore, "test"); + ExecutorContainerCore container = new ExecutorContainerCore(clientContextCore, db, "test"); dynamic testItem = new { @@ -230,14 +242,20 @@ public async Task HighThroughputSendsToExecutor_UpsertStream() [TestMethod] public async Task HighThroughputSendsToExecutor_ReplaceStream() { - Mock mockContext = new Mock(); - mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { AllowBatchingRequests = true }); - mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); - mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); - mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); - - DatabaseCore db = new DatabaseCore(mockContext.Object, "test"); - ExecutorContainerCore container = new ExecutorContainerCore(mockContext.Object, db, "test"); + ClientContextCore clientContextCore = new ClientContextCore( + MockCosmosUtil.CreateMockCosmosClient(), + new CosmosClientOptions() { AllowBatchingRequests = true }, + new CosmosJsonDotNetSerializer(), + new CosmosJsonDotNetSerializer(), + null, + null, + null, + new MockDocumentClient(), + Mock.Of() + ); + + DatabaseCore db = new DatabaseCore(clientContextCore, "test"); + ExecutorContainerCore container = new ExecutorContainerCore(clientContextCore, db, "test"); dynamic testItem = new { @@ -262,14 +280,20 @@ public async Task HighThroughputSendsToExecutor_ReplaceStream() [TestMethod] public async Task HighThroughputSendsToExecutor_ReadStream() { - Mock mockContext = new Mock(); - mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { AllowBatchingRequests = true }); - mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); - mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); - mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); - - DatabaseCore db = new DatabaseCore(mockContext.Object, "test"); - ExecutorContainerCore container = new ExecutorContainerCore(mockContext.Object, db, "test"); + ClientContextCore clientContextCore = new ClientContextCore( + MockCosmosUtil.CreateMockCosmosClient(), + new CosmosClientOptions() { AllowBatchingRequests = true }, + new CosmosJsonDotNetSerializer(), + new CosmosJsonDotNetSerializer(), + null, + null, + null, + new MockDocumentClient(), + Mock.Of() + ); + + DatabaseCore db = new DatabaseCore(clientContextCore, "test"); + ExecutorContainerCore container = new ExecutorContainerCore(clientContextCore, db, "test"); dynamic testItem = new { @@ -293,14 +317,20 @@ public async Task HighThroughputSendsToExecutor_ReadStream() [TestMethod] public async Task HighThroughputSendsToExecutor_DeleteStream() { - Mock mockContext = new Mock(); - mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { AllowBatchingRequests = true }); - mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); - mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); - mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); - - DatabaseCore db = new DatabaseCore(mockContext.Object, "test"); - ExecutorContainerCore container = new ExecutorContainerCore(mockContext.Object, db, "test"); + ClientContextCore clientContextCore = new ClientContextCore( + MockCosmosUtil.CreateMockCosmosClient(), + new CosmosClientOptions() { AllowBatchingRequests = true }, + new CosmosJsonDotNetSerializer(), + new CosmosJsonDotNetSerializer(), + null, + null, + null, + new MockDocumentClient(), + Mock.Of() + ); + + DatabaseCore db = new DatabaseCore(clientContextCore, "test"); + ExecutorContainerCore container = new ExecutorContainerCore(clientContextCore, db, "test"); dynamic testItem = new { @@ -321,19 +351,21 @@ public async Task HighThroughputSendsToExecutor_DeleteStream() [TestMethod] public async Task HighThroughputSendsToExecutor_Create() { - CosmosClientOptions clientOptions = new CosmosClientOptions() { AllowBatchingRequests = true }; - Mock mockContext = new Mock(); - mockContext.Setup(x => x.ClientOptions).Returns(clientOptions); - mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); - mockContext.Setup(x => x.CosmosSerializer).Returns(new CosmosJsonDotNetSerializer()); - mockContext.Setup(x => x.ResponseFactory).Returns(new CosmosResponseFactory( - defaultJsonSerializer: clientOptions.PropertiesSerializer, - userJsonSerializer: new CosmosJsonDotNetSerializer())); - mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); - mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); - - DatabaseCore db = new DatabaseCore(mockContext.Object, "test"); - ExecutorContainerCore container = new ExecutorContainerCore(mockContext.Object, db, "test"); + CosmosClient cosmosClient = MockCosmosUtil.CreateMockCosmosClient(); + ClientContextCore clientContextCore = new ClientContextCore( + cosmosClient, + new CosmosClientOptions() { AllowBatchingRequests = true }, + new CosmosJsonDotNetSerializer(), + new CosmosJsonDotNetSerializer(), + null, + cosmosClient.ResponseFactory, + null, + new MockDocumentClient(), + Mock.Of() + ); + + DatabaseCore db = new DatabaseCore(clientContextCore, "test"); + ExecutorContainerCore container = new ExecutorContainerCore(clientContextCore, db, "test"); dynamic testItem = new { @@ -351,19 +383,21 @@ public async Task HighThroughputSendsToExecutor_Create() [TestMethod] public async Task HighThroughputSendsToExecutor_Upsert() { - CosmosClientOptions clientOptions = new CosmosClientOptions() { AllowBatchingRequests = true }; - Mock mockContext = new Mock(); - mockContext.Setup(x => x.ClientOptions).Returns(clientOptions); - mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); - mockContext.Setup(x => x.CosmosSerializer).Returns(new CosmosJsonDotNetSerializer()); - mockContext.Setup(x => x.ResponseFactory).Returns(new CosmosResponseFactory( - defaultJsonSerializer: clientOptions.PropertiesSerializer, - userJsonSerializer: new CosmosJsonDotNetSerializer())); - mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); - mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); - - DatabaseCore db = new DatabaseCore(mockContext.Object, "test"); - ExecutorContainerCore container = new ExecutorContainerCore(mockContext.Object, db, "test"); + CosmosClient cosmosClient = MockCosmosUtil.CreateMockCosmosClient(); + ClientContextCore clientContextCore = new ClientContextCore( + cosmosClient, + new CosmosClientOptions() { AllowBatchingRequests = true }, + new CosmosJsonDotNetSerializer(), + new CosmosJsonDotNetSerializer(), + null, + cosmosClient.ResponseFactory, + null, + new MockDocumentClient(), + Mock.Of() + ); + + DatabaseCore db = new DatabaseCore(clientContextCore, "test"); + ExecutorContainerCore container = new ExecutorContainerCore(clientContextCore, db, "test"); dynamic testItem = new { @@ -381,19 +415,21 @@ public async Task HighThroughputSendsToExecutor_Upsert() [TestMethod] public async Task HighThroughputSendsToExecutor_Replace() { - CosmosClientOptions clientOptions = new CosmosClientOptions() { AllowBatchingRequests = true }; - Mock mockContext = new Mock(); - mockContext.Setup(x => x.ClientOptions).Returns(clientOptions); - mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); - mockContext.Setup(x => x.CosmosSerializer).Returns(new CosmosJsonDotNetSerializer()); - mockContext.Setup(x => x.ResponseFactory).Returns(new CosmosResponseFactory( - defaultJsonSerializer: clientOptions.PropertiesSerializer, - userJsonSerializer: new CosmosJsonDotNetSerializer())); - mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); - mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); - - DatabaseCore db = new DatabaseCore(mockContext.Object, "test"); - ExecutorContainerCore container = new ExecutorContainerCore(mockContext.Object, db, "test"); + CosmosClient cosmosClient = MockCosmosUtil.CreateMockCosmosClient(); + ClientContextCore clientContextCore = new ClientContextCore( + cosmosClient, + new CosmosClientOptions() { AllowBatchingRequests = true }, + new CosmosJsonDotNetSerializer(), + new CosmosJsonDotNetSerializer(), + null, + cosmosClient.ResponseFactory, + null, + new MockDocumentClient(), + Mock.Of() + ); + + DatabaseCore db = new DatabaseCore(clientContextCore, "test"); + ExecutorContainerCore container = new ExecutorContainerCore(clientContextCore, db, "test"); dynamic testItem = new { @@ -411,19 +447,21 @@ public async Task HighThroughputSendsToExecutor_Replace() [TestMethod] public async Task HighThroughputSendsToExecutor_Read() { - CosmosClientOptions clientOptions = new CosmosClientOptions() { AllowBatchingRequests = true }; - Mock mockContext = new Mock(); - mockContext.Setup(x => x.ClientOptions).Returns(clientOptions); - mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); - mockContext.Setup(x => x.CosmosSerializer).Returns(new CosmosJsonDotNetSerializer()); - mockContext.Setup(x => x.ResponseFactory).Returns(new CosmosResponseFactory( - defaultJsonSerializer: clientOptions.PropertiesSerializer, - userJsonSerializer: new CosmosJsonDotNetSerializer())); - mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); - mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); - - DatabaseCore db = new DatabaseCore(mockContext.Object, "test"); - ExecutorContainerCore container = new ExecutorContainerCore(mockContext.Object, db, "test"); + CosmosClient cosmosClient = MockCosmosUtil.CreateMockCosmosClient(); + ClientContextCore clientContextCore = new ClientContextCore( + cosmosClient, + new CosmosClientOptions() { AllowBatchingRequests = true }, + new CosmosJsonDotNetSerializer(), + new CosmosJsonDotNetSerializer(), + null, + cosmosClient.ResponseFactory, + null, + new MockDocumentClient(), + Mock.Of() + ); + + DatabaseCore db = new DatabaseCore(clientContextCore, "test"); + ExecutorContainerCore container = new ExecutorContainerCore(clientContextCore, db, "test"); dynamic testItem = new { @@ -441,19 +479,21 @@ public async Task HighThroughputSendsToExecutor_Read() [TestMethod] public async Task HighThroughputSendsToExecutor_Delete() { - CosmosClientOptions clientOptions = new CosmosClientOptions() { AllowBatchingRequests = true }; - Mock mockContext = new Mock(); - mockContext.Setup(x => x.ClientOptions).Returns(clientOptions); - mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); - mockContext.Setup(x => x.CosmosSerializer).Returns(new CosmosJsonDotNetSerializer()); - mockContext.Setup(x => x.ResponseFactory).Returns(new CosmosResponseFactory( - defaultJsonSerializer: clientOptions.PropertiesSerializer, - userJsonSerializer: new CosmosJsonDotNetSerializer())); - mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); - mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); - - DatabaseCore db = new DatabaseCore(mockContext.Object, "test"); - ExecutorContainerCore container = new ExecutorContainerCore(mockContext.Object, db, "test"); + CosmosClient cosmosClient = MockCosmosUtil.CreateMockCosmosClient(); + ClientContextCore clientContextCore = new ClientContextCore( + cosmosClient, + new CosmosClientOptions() { AllowBatchingRequests = true }, + new CosmosJsonDotNetSerializer(), + new CosmosJsonDotNetSerializer(), + null, + cosmosClient.ResponseFactory, + null, + new MockDocumentClient(), + Mock.Of() + ); + + DatabaseCore db = new DatabaseCore(clientContextCore, "test"); + ExecutorContainerCore container = new ExecutorContainerCore(clientContextCore, db, "test"); dynamic testItem = new { @@ -472,14 +512,20 @@ public async Task HighThroughputSendsToExecutor_Delete() [TestMethod] public async Task HighThroughputSendsToExecutor_RetriesOn429() { - Mock mockContext = new Mock(); - mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions() { AllowBatchingRequests = true }); - mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); - mockContext.Setup(x => x.DocumentQueryClient).Returns(Mock.Of()); - mockContext.Setup(x => x.CreateLink(It.IsAny(), It.IsAny(), It.IsAny())).Returns(UriFactory.CreateDocumentCollectionUri("test", "test")); - - DatabaseCore db = new DatabaseCore(mockContext.Object, "test"); - ExecutorWithThrottlingContainerCore container = new ExecutorWithThrottlingContainerCore(mockContext.Object, db, "test"); + ClientContextCore clientContextCore = new ClientContextCore( + MockCosmosUtil.CreateMockCosmosClient(), + new CosmosClientOptions() { AllowBatchingRequests = true }, + new CosmosJsonDotNetSerializer(), + new CosmosJsonDotNetSerializer(), + null, + null, + null, + new MockDocumentClient(), + Mock.Of() + ); + + DatabaseCore db = new DatabaseCore(clientContextCore, "test"); + ExecutorContainerCore container = new ExecutorContainerCore(clientContextCore, db, "test"); dynamic testItem = new { From 41cae1df5cd6f3c0a192d3ee81781e2a40b915e9 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Wed, 4 Sep 2019 15:02:01 -0700 Subject: [PATCH 33/54] Refactoring through policies and batcher --- .../src/Batch/BatchAsyncBatcher.cs | 21 ++-- .../src/Batch/BatchAsyncContainerExecutor.cs | 12 ++- .../src/Batch/ItemBatchOperation.cs | 4 +- .../src/Batch/ItemBatchOperationContext.cs | 26 ++++- .../BulkPartitionKeyRangeGoneRetryPolicy.cs | 96 +++++++++++++++++++ .../src/CosmosClientOptions.cs | 5 +- .../src/Fluent/CosmosClientBuilder.cs | 2 +- .../src/Handler/BatchExecutorRetryHandler.cs | 49 ---------- .../src/Resource/ClientContextCore.cs | 8 +- ...oughputTests.cs => CosmosItemBulkTests.cs} | 25 +++-- ...icrosoft.Azure.Cosmos.EmulatorTests.csproj | 3 - .../Batch/BatchAsyncBatcherTests.cs | 10 +- .../Batch/BatchAsyncContainerExecutorTests.cs | 82 ++++++++++++++++ .../Batch/BatchAsyncOperationContextTests.cs | 47 +++++++++ .../BatchExecutorRetryHandlerTests.cs | 73 -------------- ...lkPartitionKeyRangeGoneRetryPolicyTests.cs | 49 ++++++++++ .../CosmosItemUnitTests.cs | 56 ++--------- 17 files changed, 357 insertions(+), 211 deletions(-) create mode 100644 Microsoft.Azure.Cosmos/src/BulkPartitionKeyRangeGoneRetryPolicy.cs delete mode 100644 Microsoft.Azure.Cosmos/src/Handler/BatchExecutorRetryHandler.cs rename Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/{CosmosItemHighThroughputTests.cs => CosmosItemBulkTests.cs} (95%) delete mode 100644 Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BatchExecutorRetryHandlerTests.cs create mode 100644 Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BulkPartitionKeyRangeGoneRetryPolicyTests.cs diff --git a/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncBatcher.cs b/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncBatcher.cs index 21dda95a7c..499eab03d3 100644 --- a/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncBatcher.cs +++ b/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncBatcher.cs @@ -152,22 +152,21 @@ public virtual bool TryAdd(ItemBatchOperation operation) try { PartitionKeyRangeBatchExecutionResult result = await this.executor(serverRequest, cancellationToken); - - if (result.IsSplit()) - { - foreach (ItemBatchOperation operationToRetry in result.Operations) - { - await this.retrier(operationToRetry, cancellationToken); - } - - return; - } - using (PartitionKeyRangeBatchResponse batchResponse = new PartitionKeyRangeBatchResponse(serverRequest.Operations.Count, result.ServerResponse, this.cosmosSerializer)) { foreach (ItemBatchOperation itemBatchOperation in batchResponse.Operations) { BatchOperationResult response = batchResponse[itemBatchOperation.OperationIndex]; + if (!response.IsSuccessStatusCode) + { + Documents.ShouldRetryResult shouldRetry = await itemBatchOperation.Context.ShouldRetryAsync(response, cancellationToken); + if (shouldRetry.ShouldRetry) + { + await this.retrier(itemBatchOperation, cancellationToken); + continue; + } + } + itemBatchOperation.Context.Complete(this, response); } } diff --git a/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncContainerExecutor.cs b/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncContainerExecutor.cs index 8490b4c321..5f3219a1a5 100644 --- a/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncContainerExecutor.cs +++ b/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncContainerExecutor.cs @@ -36,6 +36,7 @@ internal class BatchAsyncContainerExecutor : IDisposable private readonly ConcurrentDictionary streamersByPartitionKeyRange = new ConcurrentDictionary(); private readonly ConcurrentDictionary limitersByPartitionkeyRange = new ConcurrentDictionary(); private readonly TimerPool timerPool; + private readonly RetryOptions retryOptions; /// /// For unit testing. @@ -77,6 +78,7 @@ public BatchAsyncContainerExecutor( this.maxServerRequestOperationCount = maxServerRequestOperationCount; this.dispatchTimerInSeconds = dispatchTimerInSeconds; this.timerPool = new TimerPool(BatchAsyncContainerExecutor.MinimumDispatchTimerInSeconds); + this.retryOptions = cosmosClientContext.ClientOptions.GetConnectionPolicy().RetryOptions; } public virtual async Task AddAsync( @@ -93,7 +95,7 @@ public virtual async Task AddAsync( string resolvedPartitionKeyRangeId = await this.ResolvePartitionKeyRangeIdAsync(operation, cancellationToken).ConfigureAwait(false); BatchAsyncStreamer streamer = this.GetOrAddStreamerForPartitionKeyRange(resolvedPartitionKeyRangeId); - ItemBatchOperationContext context = new ItemBatchOperationContext(resolvedPartitionKeyRangeId); + ItemBatchOperationContext context = new ItemBatchOperationContext(resolvedPartitionKeyRangeId, BatchAsyncContainerExecutor.GetRetryPolicy(this.retryOptions)); operation.AttachContext(context); streamer.Add(operation); return await context.Task; @@ -142,6 +144,14 @@ internal virtual async Task ValidateOperationAsync( } } + private static IDocumentClientRetryPolicy GetRetryPolicy(RetryOptions retryOptions) + { + return new BulkPartitionKeyRangeGoneRetryPolicy( + new ResourceThrottleRetryPolicy( + retryOptions.MaxRetryAttemptsOnThrottledRequests, + retryOptions.MaxRetryWaitTimeInSeconds)); + } + private static bool ValidateOperationEPK( ItemBatchOperation operation, ItemRequestOptions itemRequestOptions) diff --git a/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperation.cs b/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperation.cs index b6c3b07f67..b2e7e7dfd9 100644 --- a/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperation.cs +++ b/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperation.cs @@ -304,9 +304,7 @@ internal virtual async Task MaterializeResourceAsync(CosmosSerializer serializer /// If the operation already had an attached context which is in progress. internal void AttachContext(ItemBatchOperationContext context) { - if (this.Context != null - && !this.Context.Task.IsCompleted - && !this.Context.Task.IsFaulted) + if (this.Context != null) { throw new InvalidOperationException("Cannot modify the current context of an operation."); } diff --git a/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperationContext.cs b/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperationContext.cs index e1d264fdbc..1483bc0a90 100644 --- a/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperationContext.cs +++ b/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperationContext.cs @@ -5,8 +5,10 @@ namespace Microsoft.Azure.Cosmos { using System; + using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.Cosmos.Core.Trace; + using Microsoft.Azure.Documents; /// /// Context for a particular Batch operation. @@ -19,11 +21,33 @@ internal class ItemBatchOperationContext : IDisposable public Task Task => this.taskCompletionSource.Task; + private readonly IDocumentClientRetryPolicy retryPolicy; + private TaskCompletionSource taskCompletionSource = new TaskCompletionSource(); - public ItemBatchOperationContext(string partitionKeyRangeId) + public ItemBatchOperationContext( + string partitionKeyRangeId, + IDocumentClientRetryPolicy retryPolicy = null) { this.PartitionKeyRangeId = partitionKeyRangeId; + this.retryPolicy = retryPolicy; + } + + /// + /// Based on the Retry Policy, if a failed response should retry. + /// + public Task ShouldRetryAsync( + BatchOperationResult batchOperationResult, + CancellationToken cancellationToken) + { + if (this.retryPolicy == null + || batchOperationResult.IsSuccessStatusCode) + { + return System.Threading.Tasks.Task.FromResult(ShouldRetryResult.NoRetry()); + } + + ResponseMessage responseMessage = batchOperationResult.ToResponseMessage(); + return this.retryPolicy.ShouldRetryAsync(responseMessage, cancellationToken); } public void Complete( diff --git a/Microsoft.Azure.Cosmos/src/BulkPartitionKeyRangeGoneRetryPolicy.cs b/Microsoft.Azure.Cosmos/src/BulkPartitionKeyRangeGoneRetryPolicy.cs new file mode 100644 index 0000000000..f0cbd6dd10 --- /dev/null +++ b/Microsoft.Azure.Cosmos/src/BulkPartitionKeyRangeGoneRetryPolicy.cs @@ -0,0 +1,96 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos +{ + using System; + using System.Net; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Azure.Documents; + + /// + /// Used only in the context of Bulk Stream operations. + /// + /// + /// + internal sealed class BulkPartitionKeyRangeGoneRetryPolicy : IDocumentClientRetryPolicy + { + private const int MaxRetries = 1; + + private readonly IDocumentClientRetryPolicy nextRetryPolicy; + + private int retriesAttempted; + + public BulkPartitionKeyRangeGoneRetryPolicy(IDocumentClientRetryPolicy nextRetryPolicy) + { + this.nextRetryPolicy = nextRetryPolicy; + } + + public Task ShouldRetryAsync( + Exception exception, + CancellationToken cancellationToken) + { + DocumentClientException clientException = exception as DocumentClientException; + + ShouldRetryResult shouldRetryResult = this.ShouldRetryInternal( + clientException?.StatusCode, + clientException?.GetSubStatus(), + clientException?.ResourceAddress); + + if (shouldRetryResult != null) + { + return Task.FromResult(shouldRetryResult); + } + + if (this.nextRetryPolicy == null) + { + return Task.FromResult(ShouldRetryResult.NoRetry()); + } + + return this.nextRetryPolicy.ShouldRetryAsync(exception, cancellationToken); + } + + public Task ShouldRetryAsync( + ResponseMessage cosmosResponseMessage, + CancellationToken cancellationToken) + { + ShouldRetryResult shouldRetryResult = this.ShouldRetryInternal(cosmosResponseMessage?.StatusCode, + cosmosResponseMessage?.Headers.SubStatusCode, + cosmosResponseMessage?.GetResourceAddress()); + if (shouldRetryResult != null) + { + return Task.FromResult(shouldRetryResult); + } + + if (this.nextRetryPolicy == null) + { + return Task.FromResult(ShouldRetryResult.NoRetry()); + } + + return this.nextRetryPolicy.ShouldRetryAsync(cosmosResponseMessage, cancellationToken); + } + + public void OnBeforeSendRequest(DocumentServiceRequest request) + { + this.nextRetryPolicy.OnBeforeSendRequest(request); + } + + private ShouldRetryResult ShouldRetryInternal( + HttpStatusCode? statusCode, + SubStatusCodes? subStatusCode, + string resourceIdOrFullName) + { + if (statusCode == HttpStatusCode.Gone + && subStatusCode == SubStatusCodes.PartitionKeyRangeGone + && this.retriesAttempted < MaxRetries) + { + this.retriesAttempted++; + return ShouldRetryResult.RetryAfter(TimeSpan.Zero); + } + + return null; + } + } +} diff --git a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs index 1eee184eab..3a9b1a35e9 100644 --- a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs +++ b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs @@ -323,11 +323,8 @@ public CosmosSerializer Serializer } /// - /// Enables the optimization of point operation requests by batching them and resolving them as a single service call. + /// Allows optimistic batching of requests to service. Setting this option might impact the latency of the operations. Hence this option is recommended for throughput scenarios only. /// - /// - /// When enabled, point operations will be grouped based on their concurrency to optimize throughput. - /// #if PREVIEW public #else diff --git a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs index 92ff3eac6f..f29ddb740e 100644 --- a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs +++ b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs @@ -329,7 +329,7 @@ public CosmosClientBuilder WithCustomSerializer(CosmosSerializer cosmosJsonSeria } /// - /// Enables the optimization of point operation requests by batching them and resolving them as a single service call. + /// Allows optimistic batching of requests to service. Setting this option might impact the latency of the operations. Hence this option is recommended for throughput scenarios only. /// /// Whether is enabled. /// The object diff --git a/Microsoft.Azure.Cosmos/src/Handler/BatchExecutorRetryHandler.cs b/Microsoft.Azure.Cosmos/src/Handler/BatchExecutorRetryHandler.cs deleted file mode 100644 index d0c3ec21f6..0000000000 --- a/Microsoft.Azure.Cosmos/src/Handler/BatchExecutorRetryHandler.cs +++ /dev/null @@ -1,49 +0,0 @@ -//------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -//------------------------------------------------------------ - -namespace Microsoft.Azure.Cosmos.Handlers -{ - using System.Threading; - using System.Threading.Tasks; - - internal class BatchExecutorRetryHandler - { - private readonly RetryOptions retryOptions; - private readonly BatchAsyncContainerExecutor batchAsyncContainerExecutor; - - public BatchExecutorRetryHandler( - CosmosClientContext clientContext, - BatchAsyncContainerExecutor batchAsyncContainerExecutor) - { - this.retryOptions = clientContext.ClientOptions.GetConnectionPolicy().RetryOptions; - this.batchAsyncContainerExecutor = batchAsyncContainerExecutor; - } - - public async Task SendAsync( - ItemBatchOperation itemBatchOperation, - ItemRequestOptions itemRequestOptions = null, - CancellationToken cancellationToken = default(CancellationToken)) - { - IDocumentClientRetryPolicy retryPolicyInstance = new ResourceThrottleRetryPolicy( - this.retryOptions.MaxRetryAttemptsOnThrottledRequests, - this.retryOptions.MaxRetryWaitTimeInSeconds); - - return await AbstractRetryHandler.ExecuteHttpRequestAsync( - callbackMethod: async () => - { - BatchOperationResult operationResult = await this.batchAsyncContainerExecutor.AddAsync(itemBatchOperation, itemRequestOptions); - return operationResult.ToResponseMessage(); - }, - callShouldRetry: (cosmosResponseMessage, token) => - { - return retryPolicyInstance.ShouldRetryAsync(cosmosResponseMessage, cancellationToken); - }, - callShouldRetryException: (exception, token) => - { - return retryPolicyInstance.ShouldRetryAsync(exception, cancellationToken); - }, - cancellationToken: cancellationToken); - } - } -} diff --git a/Microsoft.Azure.Cosmos/src/Resource/ClientContextCore.cs b/Microsoft.Azure.Cosmos/src/Resource/ClientContextCore.cs index 5236bf3b8a..f7c04360b4 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/ClientContextCore.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/ClientContextCore.cs @@ -102,9 +102,9 @@ internal override Task ProcessResourceOperationStreamAsync( Action requestEnricher, CancellationToken cancellationToken) { - if (this.IsBatchingSupported(resourceType, operationType)) + if (this.IsBulkOperationSupported(resourceType, operationType)) { - return this.ProcessResourceOperationWithBatchExecutorAsync( + return this.ProcessResourceOperationAsBulkStreamAsync( resourceUri: resourceUri, resourceType: resourceType, operationType: operationType, @@ -153,7 +153,7 @@ internal override Task ProcessResourceOperationAsync( cancellationToken: cancellationToken); } - private async Task ProcessResourceOperationWithBatchExecutorAsync( + private async Task ProcessResourceOperationAsBulkStreamAsync( Uri resourceUri, ResourceType resourceType, OperationType operationType, @@ -171,7 +171,7 @@ private async Task ProcessResourceOperationWithBatchExecutorAsy return batchOperationResult.ToResponseMessage(); } - private bool IsBatchingSupported( + private bool IsBulkOperationSupported( ResourceType resourceType, OperationType operationType) { diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemHighThroughputTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemBulkTests.cs similarity index 95% rename from Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemHighThroughputTests.cs rename to Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemBulkTests.cs index e466b2ba00..26b531f1d2 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemHighThroughputTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemBulkTests.cs @@ -6,13 +6,12 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests { using System; using System.Collections.Generic; - using System.IO; using System.Net; using System.Threading.Tasks; using Microsoft.VisualStudio.TestTools.UnitTesting; [TestClass] - public class CosmosItemHighThroughputTests + public class CosmosItemBulkTests { private static CosmosSerializer cosmosDefaultJsonSerializer = new CosmosJsonDotNetSerializer(); @@ -29,7 +28,7 @@ public async Task TestInitialize() DatabaseResponse response = await client.CreateDatabaseIfNotExistsAsync(Guid.NewGuid().ToString()); this.database = response.Database; - ContainerResponse containerResponse = await this.database.CreateContainerAsync(Guid.NewGuid().ToString(), "/Status", 50000); + ContainerResponse containerResponse = await this.database.CreateContainerAsync(Guid.NewGuid().ToString(), "/Status", 10000); this.container = containerResponse; } @@ -40,7 +39,7 @@ public async Task Cleanup() } [TestMethod] - public async Task CreateItemStream_WithHighThroughput() + public async Task CreateItemStream_WithBulk() { List> tasks = new List>(); for (int i = 0; i < 100; i++) @@ -62,7 +61,7 @@ public async Task CreateItemStream_WithHighThroughput() } [TestMethod] - public async Task CreateItemAsync_WithHighThroughput() + public async Task CreateItemAsync_WithBulk() { List>> tasks = new List>>(); for (int i = 0; i < 100; i++) @@ -81,7 +80,7 @@ public async Task CreateItemAsync_WithHighThroughput() } [TestMethod] - public async Task UpsertItemStream_WithHighThroughput() + public async Task UpsertItemStream_WithBulk() { List> tasks = new List>(); for (int i = 0; i < 100; i++) @@ -103,7 +102,7 @@ public async Task UpsertItemStream_WithHighThroughput() } [TestMethod] - public async Task UpsertItem_WithHighThroughput() + public async Task UpsertItem_WithBulk() { List>> tasks = new List>>(); for (int i = 0; i < 100; i++) @@ -122,7 +121,7 @@ public async Task UpsertItem_WithHighThroughput() } [TestMethod] - public async Task DeleteItemStream_WithHighThroughput() + public async Task DeleteItemStream_WithBulk() { List createdDocuments = new List(); // Create the items @@ -153,7 +152,7 @@ public async Task DeleteItemStream_WithHighThroughput() } [TestMethod] - public async Task DeleteItem_WithHighThroughput() + public async Task DeleteItem_WithBulk() { List createdDocuments = new List(); // Create the items @@ -184,7 +183,7 @@ public async Task DeleteItem_WithHighThroughput() } [TestMethod] - public async Task ReadItemStream_WithHighThroughput() + public async Task ReadItemStream_WithBulk() { List createdDocuments = new List(); // Create the items @@ -215,7 +214,7 @@ public async Task ReadItemStream_WithHighThroughput() } [TestMethod] - public async Task ReadItem_WithHighThroughput() + public async Task ReadItem_WithBulk() { List createdDocuments = new List(); // Create the items @@ -246,7 +245,7 @@ public async Task ReadItem_WithHighThroughput() } [TestMethod] - public async Task ReplaceItemStream_WithHighThroughput() + public async Task ReplaceItemStream_WithBulk() { List createdDocuments = new List(); // Create the items @@ -277,7 +276,7 @@ public async Task ReplaceItemStream_WithHighThroughput() } [TestMethod] - public async Task ReplaceItem_WithHighThroughput() + public async Task ReplaceItem_WithBulk() { List createdDocuments = new List(); // Create the items diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Microsoft.Azure.Cosmos.EmulatorTests.csproj b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Microsoft.Azure.Cosmos.EmulatorTests.csproj index 9c947bc5cd..8411b546f4 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Microsoft.Azure.Cosmos.EmulatorTests.csproj +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Microsoft.Azure.Cosmos.EmulatorTests.csproj @@ -242,9 +242,6 @@ PreserveNewest - - - C:\Users\abpai\.nuget\packages\microsoft.azure.cosmos.direct.myget\3.0.0.33-preview\runtimes\any\lib\netstandard2.0\Microsoft.Azure.Cosmos.Core.dll diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncBatcherTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncBatcherTests.cs index 4244c84f61..fed3fcc2b3 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncBatcherTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncBatcherTests.cs @@ -341,10 +341,16 @@ public async Task CannotAddToDispatchedBatch() [TestMethod] public async Task RetrierGetsCalledOnSplit() { + IDocumentClientRetryPolicy retryPolicy1 = new BulkPartitionKeyRangeGoneRetryPolicy( + new ResourceThrottleRetryPolicy(1)); + + IDocumentClientRetryPolicy retryPolicy2 = new BulkPartitionKeyRangeGoneRetryPolicy( + new ResourceThrottleRetryPolicy(1)); + ItemBatchOperation operation1 = this.CreateItemBatchOperation(); ItemBatchOperation operation2 = this.CreateItemBatchOperation(); - operation1.AttachContext(new ItemBatchOperationContext(string.Empty)); - operation2.AttachContext(new ItemBatchOperationContext(string.Empty)); + operation1.AttachContext(new ItemBatchOperationContext(string.Empty, retryPolicy1)); + operation2.AttachContext(new ItemBatchOperationContext(string.Empty, retryPolicy2)); Mock retryDelegate = new Mock(); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncContainerExecutorTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncContainerExecutorTests.cs index ab6b342f88..8bb5ae30c0 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncContainerExecutorTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncContainerExecutorTests.cs @@ -27,6 +27,7 @@ public async Task RetryOnSplit() ItemBatchOperation itemBatchOperation = CreateItem("test"); Mock mockedContext = new Mock(); + mockedContext.Setup(c => c.ClientOptions).Returns(new CosmosClientOptions()); mockedContext .SetupSequence(c => c.ProcessResourceOperationStreamAsync( It.IsAny(), @@ -74,12 +75,67 @@ public async Task RetryOnSplit() Assert.AreEqual(HttpStatusCode.OK, result.StatusCode); } + [TestMethod] + public async Task RetryOn429() + { + ItemBatchOperation itemBatchOperation = CreateItem("test"); + + Mock mockedContext = new Mock(); + mockedContext.Setup(c => c.ClientOptions).Returns(new CosmosClientOptions()); + mockedContext + .SetupSequence(c => c.ProcessResourceOperationStreamAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny>(), + It.IsAny())) + .Returns(this.Generate429ResponseAsync(itemBatchOperation)) + .Returns(this.GenerateOkResponseAsync(itemBatchOperation)); + + mockedContext.Setup(c => c.CosmosSerializer).Returns(new CosmosJsonDotNetSerializer()); + + Uri link = new Uri($"/dbs/db/colls/colls", UriKind.Relative); + Mock mockContainer = new Mock(); + mockContainer.Setup(x => x.LinkUri).Returns(link); + mockContainer.Setup(x => x.GetPartitionKeyDefinitionAsync(It.IsAny())).Returns(Task.FromResult(new PartitionKeyDefinition() { Paths = new Collection() { "/id" } })); + + CollectionRoutingMap routingMap = CollectionRoutingMap.TryCreateCompleteRoutingMap( + new[] + { + Tuple.Create(new PartitionKeyRange{ Id = "0", MinInclusive = "", MaxExclusive = "FF"}, (ServiceIdentity)null) + }, + string.Empty); + mockContainer.Setup(x => x.GetRoutingMapAsync(It.IsAny())).Returns(Task.FromResult(routingMap)); + BatchAsyncContainerExecutor executor = new BatchAsyncContainerExecutor(mockContainer.Object, mockedContext.Object, 20, Constants.MaxDirectModeBatchRequestBodySizeInBytes, 1); + BatchOperationResult result = await executor.AddAsync(itemBatchOperation); + + Mock.Get(mockContainer.Object) + .Verify(x => x.GetPartitionKeyDefinitionAsync(It.IsAny()), Times.Exactly(2)); + Mock.Get(mockedContext.Object) + .Verify(c => c.ProcessResourceOperationStreamAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny>(), + It.IsAny()), Times.Exactly(2)); + Assert.AreEqual(HttpStatusCode.OK, result.StatusCode); + } + [TestMethod] public async Task DoesNotRecalculatePartitionKeyRangeOnNoSplits() { ItemBatchOperation itemBatchOperation = CreateItem("test"); Mock mockedContext = new Mock(); + mockedContext.Setup(c => c.ClientOptions).Returns(new CosmosClientOptions()); mockedContext .Setup(c => c.ProcessResourceOperationStreamAsync( It.IsAny(), @@ -154,6 +210,32 @@ private async Task GenerateSplitResponseAsync(ItemBatchOperatio return responseMessage; } + private async Task Generate429ResponseAsync(ItemBatchOperation itemBatchOperation) + { + List results = new List(); + ItemBatchOperation[] arrayOperations = new ItemBatchOperation[1]; + results.Add( + new BatchOperationResult((HttpStatusCode) StatusCodes.TooManyRequests) + { + ETag = itemBatchOperation.Id + }); + + arrayOperations[0] = itemBatchOperation; + + MemoryStream responseContent = await new BatchResponsePayloadWriter(results).GeneratePayloadAsync(); + + SinglePartitionKeyServerBatchRequest batchRequest = await SinglePartitionKeyServerBatchRequest.CreateAsync( + partitionKey: null, + operations: new ArraySegment(arrayOperations), + maxBodyLength: 100, + maxOperationCount: 1, + serializer: new CosmosJsonDotNetSerializer(), + cancellationToken: CancellationToken.None); + + ResponseMessage responseMessage = new ResponseMessage((HttpStatusCode)StatusCodes.TooManyRequests) { Content = responseContent }; + return responseMessage; + } + private async Task GenerateOkResponseAsync(ItemBatchOperation itemBatchOperation) { List results = new List(); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncOperationContextTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncOperationContextTests.cs index d060ee7fe8..1aed5b29f5 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncOperationContextTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncOperationContextTests.cs @@ -6,6 +6,7 @@ namespace Microsoft.Azure.Cosmos.Tests { using System; using System.Net; + using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.Documents; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -76,5 +77,51 @@ public void CannotAttachMoreThanOnce() operation.AttachContext(new ItemBatchOperationContext(string.Empty)); Assert.ThrowsException(() => operation.AttachContext(new ItemBatchOperationContext(string.Empty))); } + + [TestMethod] + public async Task ShouldRetry_NoPolicy() + { + BatchOperationResult result = new BatchOperationResult(HttpStatusCode.OK); + ItemBatchOperation operation = new ItemBatchOperation(OperationType.Create, 0); + operation.AttachContext(new ItemBatchOperationContext(string.Empty)); + ShouldRetryResult shouldRetryResult = await operation.Context.ShouldRetryAsync(result, default(CancellationToken)); + Assert.IsFalse(shouldRetryResult.ShouldRetry); + } + + [TestMethod] + public async Task ShouldRetry_WithPolicy_OnSuccess() + { + IDocumentClientRetryPolicy retryPolicy = new BulkPartitionKeyRangeGoneRetryPolicy( + new ResourceThrottleRetryPolicy(1)); + BatchOperationResult result = new BatchOperationResult(HttpStatusCode.OK); + ItemBatchOperation operation = new ItemBatchOperation(OperationType.Create, 0); + operation.AttachContext(new ItemBatchOperationContext(string.Empty, retryPolicy)); + ShouldRetryResult shouldRetryResult = await operation.Context.ShouldRetryAsync(result, default(CancellationToken)); + Assert.IsFalse(shouldRetryResult.ShouldRetry); + } + + [TestMethod] + public async Task ShouldRetry_WithPolicy_On429() + { + IDocumentClientRetryPolicy retryPolicy = new BulkPartitionKeyRangeGoneRetryPolicy( + new ResourceThrottleRetryPolicy(1)); + BatchOperationResult result = new BatchOperationResult((HttpStatusCode)StatusCodes.TooManyRequests); + ItemBatchOperation operation = new ItemBatchOperation(OperationType.Create, 0); + operation.AttachContext(new ItemBatchOperationContext(string.Empty, retryPolicy)); + ShouldRetryResult shouldRetryResult = await operation.Context.ShouldRetryAsync(result, default(CancellationToken)); + Assert.IsTrue(shouldRetryResult.ShouldRetry); + } + + [TestMethod] + public async Task ShouldRetry_WithPolicy_OnSplit() + { + IDocumentClientRetryPolicy retryPolicy = new BulkPartitionKeyRangeGoneRetryPolicy( + new ResourceThrottleRetryPolicy(1)); + BatchOperationResult result = new BatchOperationResult(HttpStatusCode.Gone) { SubStatusCode = SubStatusCodes.PartitionKeyRangeGone }; + ItemBatchOperation operation = new ItemBatchOperation(OperationType.Create, 0); + operation.AttachContext(new ItemBatchOperationContext(string.Empty, retryPolicy)); + ShouldRetryResult shouldRetryResult = await operation.Context.ShouldRetryAsync(result, default(CancellationToken)); + Assert.IsTrue(shouldRetryResult.ShouldRetry); + } } } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BatchExecutorRetryHandlerTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BatchExecutorRetryHandlerTests.cs deleted file mode 100644 index b5444dcd54..0000000000 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BatchExecutorRetryHandlerTests.cs +++ /dev/null @@ -1,73 +0,0 @@ -//------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -//------------------------------------------------------------ - -namespace Microsoft.Azure.Cosmos.Tests -{ - using System; - using System.Collections.Generic; - using System.IO; - using System.Net; - using System.Threading; - using System.Threading.Tasks; - using Microsoft.Azure.Cosmos.Client.Core.Tests; - using Microsoft.Azure.Cosmos.Handlers; - using Microsoft.Azure.Cosmos.Query; - using Microsoft.Azure.Documents; - using Microsoft.VisualStudio.TestTools.UnitTesting; - using Moq; - using Newtonsoft.Json; - using Newtonsoft.Json.Linq; - - [TestClass] - public class BatchExecutorRetryHandlerTests - { - private static CosmosSerializer cosmosDefaultJsonSerializer = new CosmosJsonDotNetSerializer(); - - [TestMethod] - public async Task NotRetryOnSuccess() - { - Mock mockContext = new Mock(); - mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); - mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions()); - Mock mockedExecutor = new Mock(); - mockedExecutor - .Setup(e => e.AddAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync(new BatchOperationResult(HttpStatusCode.OK)); - - BatchExecutorRetryHandler batchExecutorRetryHandler = new BatchExecutorRetryHandler(mockContext.Object, mockedExecutor.Object); - ResponseMessage result = await batchExecutorRetryHandler.SendAsync(CreateItemBatchOperation()); - Assert.AreEqual(HttpStatusCode.OK, result.StatusCode); - mockedExecutor.Verify(c => c.AddAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); - } - - [TestMethod] - public async Task RetriesOn429() - { - Mock mockContext = new Mock(); - mockContext.Setup(x => x.DocumentClient).Returns(new MockDocumentClient()); - mockContext.Setup(x => x.ClientOptions).Returns(new CosmosClientOptions()); - Mock mockedExecutor = new Mock(); - mockedExecutor - .SetupSequence(e => e.AddAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync(new BatchOperationResult((HttpStatusCode)StatusCodes.TooManyRequests)) - .ReturnsAsync(new BatchOperationResult(HttpStatusCode.OK)); - - BatchExecutorRetryHandler batchExecutorRetryHandler = new BatchExecutorRetryHandler(mockContext.Object, mockedExecutor.Object); - ResponseMessage result = await batchExecutorRetryHandler.SendAsync(CreateItemBatchOperation()); - Assert.AreEqual(HttpStatusCode.OK, result.StatusCode); - mockedExecutor.Verify(c => c.AddAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(2)); - } - - private static ItemBatchOperation CreateItemBatchOperation() - { - dynamic testItem = new - { - id = Guid.NewGuid().ToString(), - pk = "FF627B77-568E-4541-A47E-041EAC10E46F", - }; - - return new ItemBatchOperation(OperationType.Create, /* index */ 0, new Cosmos.PartitionKey(testItem.pk), testItem.id, cosmosDefaultJsonSerializer.ToStream(testItem)); - } - } -} diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BulkPartitionKeyRangeGoneRetryPolicyTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BulkPartitionKeyRangeGoneRetryPolicyTests.cs new file mode 100644 index 0000000000..2e220f1ad0 --- /dev/null +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BulkPartitionKeyRangeGoneRetryPolicyTests.cs @@ -0,0 +1,49 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Tests +{ + using System.Net; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Azure.Documents; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class BulkPartitionKeyRangeGoneRetryPolicyTests + { + [TestMethod] + public async Task NotRetryOnSuccess() + { + IDocumentClientRetryPolicy retryPolicy = new BulkPartitionKeyRangeGoneRetryPolicy( + new ResourceThrottleRetryPolicy(1)); + + BatchOperationResult result = new BatchOperationResult(HttpStatusCode.OK); + ShouldRetryResult shouldRetryResult = await retryPolicy.ShouldRetryAsync(result.ToResponseMessage(), default(CancellationToken)); + Assert.IsFalse(shouldRetryResult.ShouldRetry); + } + + [TestMethod] + public async Task RetriesOn429() + { + IDocumentClientRetryPolicy retryPolicy = new BulkPartitionKeyRangeGoneRetryPolicy( + new ResourceThrottleRetryPolicy(1)); + + BatchOperationResult result = new BatchOperationResult((HttpStatusCode)StatusCodes.TooManyRequests); + ShouldRetryResult shouldRetryResult = await retryPolicy.ShouldRetryAsync(result.ToResponseMessage(), default(CancellationToken)); + Assert.IsTrue(shouldRetryResult.ShouldRetry); + } + + [TestMethod] + public async Task RetriesOnSplits() + { + IDocumentClientRetryPolicy retryPolicy = new BulkPartitionKeyRangeGoneRetryPolicy( + new ResourceThrottleRetryPolicy(1)); + + BatchOperationResult result = new BatchOperationResult(HttpStatusCode.Gone) { SubStatusCode = SubStatusCodes.PartitionKeyRangeGone }; + ShouldRetryResult shouldRetryResult = await retryPolicy.ShouldRetryAsync(result.ToResponseMessage(), default(CancellationToken)); + Assert.IsTrue(shouldRetryResult.ShouldRetry); + } + } +} diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs index 09b7cf252b..2dd36b445a 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs @@ -166,7 +166,7 @@ await Assert.ThrowsExceptionAsync(async () => { } [TestMethod] - public async Task HighThroughputSendsToExecutor_CreateStream() + public async Task AllowBatchingRequestsSendsToExecutor_CreateStream() { ClientContextCore clientContextCore = new ClientContextCore( MockCosmosUtil.CreateMockCosmosClient(), @@ -203,7 +203,7 @@ public async Task HighThroughputSendsToExecutor_CreateStream() } [TestMethod] - public async Task HighThroughputSendsToExecutor_UpsertStream() + public async Task AllowBatchingRequestsSendsToExecutor_UpsertStream() { ClientContextCore clientContextCore = new ClientContextCore( MockCosmosUtil.CreateMockCosmosClient(), @@ -240,7 +240,7 @@ public async Task HighThroughputSendsToExecutor_UpsertStream() } [TestMethod] - public async Task HighThroughputSendsToExecutor_ReplaceStream() + public async Task AllowBatchingRequestsSendsToExecutor_ReplaceStream() { ClientContextCore clientContextCore = new ClientContextCore( MockCosmosUtil.CreateMockCosmosClient(), @@ -278,7 +278,7 @@ public async Task HighThroughputSendsToExecutor_ReplaceStream() } [TestMethod] - public async Task HighThroughputSendsToExecutor_ReadStream() + public async Task AllowBatchingRequestsSendsToExecutor_ReadStream() { ClientContextCore clientContextCore = new ClientContextCore( MockCosmosUtil.CreateMockCosmosClient(), @@ -315,7 +315,7 @@ public async Task HighThroughputSendsToExecutor_ReadStream() } [TestMethod] - public async Task HighThroughputSendsToExecutor_DeleteStream() + public async Task AllowBatchingRequestsSendsToExecutor_DeleteStream() { ClientContextCore clientContextCore = new ClientContextCore( MockCosmosUtil.CreateMockCosmosClient(), @@ -349,7 +349,7 @@ public async Task HighThroughputSendsToExecutor_DeleteStream() } [TestMethod] - public async Task HighThroughputSendsToExecutor_Create() + public async Task AllowBatchingRequestsSendsToExecutor_Create() { CosmosClient cosmosClient = MockCosmosUtil.CreateMockCosmosClient(); ClientContextCore clientContextCore = new ClientContextCore( @@ -381,7 +381,7 @@ public async Task HighThroughputSendsToExecutor_Create() } [TestMethod] - public async Task HighThroughputSendsToExecutor_Upsert() + public async Task AllowBatchingRequestsSendsToExecutor_Upsert() { CosmosClient cosmosClient = MockCosmosUtil.CreateMockCosmosClient(); ClientContextCore clientContextCore = new ClientContextCore( @@ -413,7 +413,7 @@ public async Task HighThroughputSendsToExecutor_Upsert() } [TestMethod] - public async Task HighThroughputSendsToExecutor_Replace() + public async Task AllowBatchingRequestsSendsToExecutor_Replace() { CosmosClient cosmosClient = MockCosmosUtil.CreateMockCosmosClient(); ClientContextCore clientContextCore = new ClientContextCore( @@ -445,7 +445,7 @@ public async Task HighThroughputSendsToExecutor_Replace() } [TestMethod] - public async Task HighThroughputSendsToExecutor_Read() + public async Task AllowBatchingRequestsSendsToExecutor_Read() { CosmosClient cosmosClient = MockCosmosUtil.CreateMockCosmosClient(); ClientContextCore clientContextCore = new ClientContextCore( @@ -477,7 +477,7 @@ public async Task HighThroughputSendsToExecutor_Read() } [TestMethod] - public async Task HighThroughputSendsToExecutor_Delete() + public async Task AllowBatchingRequestsSendsToExecutor_Delete() { CosmosClient cosmosClient = MockCosmosUtil.CreateMockCosmosClient(); ClientContextCore clientContextCore = new ClientContextCore( @@ -509,42 +509,6 @@ public async Task HighThroughputSendsToExecutor_Delete() container.MockedExecutor.Verify(c => c.AddAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); } - [TestMethod] - public async Task HighThroughputSendsToExecutor_RetriesOn429() - { - ClientContextCore clientContextCore = new ClientContextCore( - MockCosmosUtil.CreateMockCosmosClient(), - new CosmosClientOptions() { AllowBatchingRequests = true }, - new CosmosJsonDotNetSerializer(), - new CosmosJsonDotNetSerializer(), - null, - null, - null, - new MockDocumentClient(), - Mock.Of() - ); - - DatabaseCore db = new DatabaseCore(clientContextCore, "test"); - ExecutorContainerCore container = new ExecutorContainerCore(clientContextCore, db, "test"); - - dynamic testItem = new - { - id = Guid.NewGuid().ToString(), - pk = "FF627B77-568E-4541-A47E-041EAC10E46F", - }; - - using (Stream itemStream = MockCosmosUtil.Serializer.ToStream(testItem)) - { - Cosmos.PartitionKey partitionKey = new Cosmos.PartitionKey(testItem.pk); - using (ResponseMessage streamResponse = await container.CreateItemStreamAsync( - partitionKey: partitionKey, - streamPayload: itemStream)) - { - container.MockedExecutor.Verify(c => c.AddAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(2)); - } - } - } - [TestMethod] public async Task TestNestedPartitionKeyValueFromStreamAsync() { From fda4292fb3268a9da91c1385161343d80ada2838 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Wed, 4 Sep 2019 15:33:45 -0700 Subject: [PATCH 34/54] ItemId is required --- .../src/Resource/ClientContextCore.cs | 28 ++++++++++++++++++- .../Resource/Container/ContainerCore.Items.cs | 1 + .../src/Resource/CosmosClientContext.cs | 16 +++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos/src/Resource/ClientContextCore.cs b/Microsoft.Azure.Cosmos/src/Resource/ClientContextCore.cs index f7c04360b4..dfac65a40b 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/ClientContextCore.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/ClientContextCore.cs @@ -98,6 +98,7 @@ internal override Task ProcessResourceOperationStreamAsync( RequestOptions requestOptions, ContainerCore cosmosContainerCore, PartitionKey? partitionKey, + string itemId, Stream streamPayload, Action requestEnricher, CancellationToken cancellationToken) @@ -111,11 +112,35 @@ internal override Task ProcessResourceOperationStreamAsync( requestOptions: requestOptions, cosmosContainerCore: cosmosContainerCore, partitionKey: partitionKey, + itemId: itemId, streamPayload: streamPayload, requestEnricher: requestEnricher, cancellationToken: cancellationToken); } + return this.ProcessResourceOperationStreamAsync( + resourceUri: resourceUri, + resourceType: resourceType, + operationType: operationType, + requestOptions: requestOptions, + cosmosContainerCore: cosmosContainerCore, + partitionKey: partitionKey, + streamPayload: streamPayload, + requestEnricher: requestEnricher, + cancellationToken: cancellationToken); + } + + internal override Task ProcessResourceOperationStreamAsync( + Uri resourceUri, + ResourceType resourceType, + OperationType operationType, + RequestOptions requestOptions, + ContainerCore cosmosContainerCore, + PartitionKey? partitionKey, + Stream streamPayload, + Action requestEnricher, + CancellationToken cancellationToken) + { return this.RequestHandler.SendAsync( resourceUri: resourceUri, resourceType: resourceType, @@ -160,13 +185,14 @@ private async Task ProcessResourceOperationAsBulkStreamAsync( RequestOptions requestOptions, ContainerCore cosmosContainerCore, PartitionKey? partitionKey, + string itemId, Stream streamPayload, Action requestEnricher, CancellationToken cancellationToken) { ItemRequestOptions itemRequestOptions = requestOptions as ItemRequestOptions; BatchItemRequestOptions batchItemRequestOptions = BatchItemRequestOptions.FromItemRequestOptions(itemRequestOptions); - ItemBatchOperation itemBatchOperation = new ItemBatchOperation(operationType, /* index */ 0, partitionKey, /* id */ null, streamPayload, batchItemRequestOptions); + ItemBatchOperation itemBatchOperation = new ItemBatchOperation(operationType, /* index */ 0, partitionKey, itemId, streamPayload, batchItemRequestOptions); BatchOperationResult batchOperationResult = await cosmosContainerCore.BatchExecutor.AddAsync(itemBatchOperation, itemRequestOptions, cancellationToken); return batchOperationResult.ToResponseMessage(); } diff --git a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs index 3bbd5f8a32..c71991c8de 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs @@ -501,6 +501,7 @@ internal async Task ProcessItemStreamAsync( requestOptions, this, partitionKey, + itemId, streamPayload, null, cancellationToken); diff --git a/Microsoft.Azure.Cosmos/src/Resource/CosmosClientContext.cs b/Microsoft.Azure.Cosmos/src/Resource/CosmosClientContext.cs index 81498860bb..7dbe957ca7 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/CosmosClientContext.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/CosmosClientContext.cs @@ -53,6 +53,22 @@ internal abstract Uri CreateLink( internal abstract void ValidateResource(string id); + /// + /// This is a wrapper around ExecUtil method. This allows the calls to be mocked so logic done + /// in a resource can be unit tested. + /// + internal abstract Task ProcessResourceOperationStreamAsync( + Uri resourceUri, + ResourceType resourceType, + OperationType operationType, + RequestOptions requestOptions, + ContainerCore cosmosContainerCore, + PartitionKey? partitionKey, + string itemId, + Stream streamPayload, + Action requestEnricher, + CancellationToken cancellationToken); + /// /// This is a wrapper around ExecUtil method. This allows the calls to be mocked so logic done /// in a resource can be unit tested. From 8fd1d1f8b47bd75d34ebdc5daf8a5af2b353abe5 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Wed, 4 Sep 2019 15:45:10 -0700 Subject: [PATCH 35/54] Undo unnecessary changes --- .../src/Handler/AbstractRetryHandler.cs | 2 +- .../Resource/Container/ContainerCore.Items.cs | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Handler/AbstractRetryHandler.cs b/Microsoft.Azure.Cosmos/src/Handler/AbstractRetryHandler.cs index e5fe55656d..5094b31bcf 100644 --- a/Microsoft.Azure.Cosmos/src/Handler/AbstractRetryHandler.cs +++ b/Microsoft.Azure.Cosmos/src/Handler/AbstractRetryHandler.cs @@ -67,7 +67,7 @@ public override async Task SendAsync( } } - internal static async Task ExecuteHttpRequestAsync( + private static async Task ExecuteHttpRequestAsync( Func> callbackMethod, Func> callShouldRetry, Func> callShouldRetryException, diff --git a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs index c71991c8de..680428ad4f 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs @@ -439,20 +439,20 @@ internal async Task ExtractPartitionKeyAndProcessItemStreamAsyn string itemId, Stream streamPayload, OperationType operationType, - ItemRequestOptions requestOptions, + RequestOptions requestOptions, CancellationToken cancellationToken) { PartitionKeyMismatchRetryPolicy requestRetryPolicy = null; while (true) { ResponseMessage responseMessage = await this.ProcessItemStreamAsync( - partitionKey, - itemId, - streamPayload, - operationType, - requestOptions, - extractPartitionKeyIfNeeded: true, - cancellationToken: cancellationToken); + partitionKey, + itemId, + streamPayload, + operationType, + requestOptions, + extractPartitionKeyIfNeeded: true, + cancellationToken: cancellationToken); if (responseMessage.IsSuccessStatusCode) { From aebff8f30f7cf9aa28f1df1f16daf9b3c9c88f6e Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Wed, 4 Sep 2019 09:01:24 -0700 Subject: [PATCH 36/54] Diagnostics from response --- .../src/Batch/BatchResponse.cs | 15 +++++++- .../Batch/PartitionKeyRangeBatchResponse.cs | 15 ++++---- .../Batch/PartitionKeyBatchResponseTests.cs | 38 +++++++++++++++++++ 3 files changed, 59 insertions(+), 9 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Batch/BatchResponse.cs b/Microsoft.Azure.Cosmos/src/Batch/BatchResponse.cs index a630436535..c210979837 100644 --- a/Microsoft.Azure.Cosmos/src/Batch/BatchResponse.cs +++ b/Microsoft.Azure.Cosmos/src/Batch/BatchResponse.cs @@ -36,9 +36,10 @@ internal BatchResponse( double requestCharge, TimeSpan? retryAfter, string activityId, + CosmosDiagnostics cosmosDiagnostics, ServerBatchRequest serverRequest, CosmosSerializer serializer) - : this(statusCode, subStatusCode, errorMessage, requestCharge, retryAfter, activityId, serverRequest.Operations, serializer) + : this(statusCode, subStatusCode, errorMessage, requestCharge, retryAfter, activityId, cosmosDiagnostics, serverRequest.Operations, serializer) { } @@ -61,6 +62,7 @@ internal BatchResponse( requestCharge: 0, retryAfter: null, activityId: Guid.Empty.ToString(), + cosmosDiagnostics: null, operations: operations, serializer: null) { @@ -80,6 +82,7 @@ private BatchResponse( double requestCharge, TimeSpan? retryAfter, string activityId, + CosmosDiagnostics cosmosDiagnostics, IReadOnlyList operations, CosmosSerializer serializer) { @@ -91,6 +94,7 @@ private BatchResponse( this.RequestCharge = requestCharge; this.RetryAfter = retryAfter; this.ActivityId = activityId; + this.Diagnostics = cosmosDiagnostics; } /// @@ -140,6 +144,11 @@ public virtual bool IsSuccessStatusCode /// public virtual int Count => this.results?.Count ?? 0; + /// + /// Gets the cosmos diagnostic information for the current request to Azure Cosmos DB service + /// + public virtual CosmosDiagnostics Diagnostics { get; set; } + internal virtual SubStatusCodes SubStatusCode { get; } internal virtual CosmosSerializer Serializer { get; } @@ -232,6 +241,7 @@ internal static async Task FromResponseMessageAsync( responseMessage.Headers.RequestCharge, responseMessage.Headers.RetryAfter, responseMessage.Headers.ActivityId, + responseMessage.Diagnostics, serverRequest, serializer); } @@ -245,6 +255,7 @@ internal static async Task FromResponseMessageAsync( responseMessage.Headers.RequestCharge, responseMessage.Headers.RetryAfter, responseMessage.Headers.ActivityId, + responseMessage.Diagnostics, serverRequest, serializer); } @@ -262,6 +273,7 @@ internal static async Task FromResponseMessageAsync( responseMessage.Headers.RequestCharge, responseMessage.Headers.RetryAfter, responseMessage.Headers.ActivityId, + responseMessage.Diagnostics, serverRequest, serializer); } @@ -334,6 +346,7 @@ record => responseMessage.Headers.RequestCharge, responseMessage.Headers.RetryAfter, responseMessage.Headers.ActivityId, + responseMessage.Diagnostics, serverRequest, serializer); diff --git a/Microsoft.Azure.Cosmos/src/Batch/PartitionKeyRangeBatchResponse.cs b/Microsoft.Azure.Cosmos/src/Batch/PartitionKeyRangeBatchResponse.cs index c2ff0fc1ab..5feb05c498 100644 --- a/Microsoft.Azure.Cosmos/src/Batch/PartitionKeyRangeBatchResponse.cs +++ b/Microsoft.Azure.Cosmos/src/Batch/PartitionKeyRangeBatchResponse.cs @@ -17,6 +17,7 @@ internal class PartitionKeyRangeBatchResponse : BatchResponse { // Results sorted in the order operations had been added. private readonly BatchOperationResult[] resultsByOperationIndex; + private readonly BatchResponse serverResponse; private bool isDisposed; /// @@ -49,11 +50,10 @@ internal PartitionKeyRangeBatchResponse( { this.StatusCode = serverResponse.StatusCode; - this.ServerResponse = serverResponse; + this.serverResponse = serverResponse; this.resultsByOperationIndex = new BatchOperationResult[originalOperationsCount]; StringBuilder errorMessageBuilder = new StringBuilder(); - List activityIds = new List(); List itemBatchOperations = new List(); // We expect number of results == number of operations here for (int index = 0; index < serverResponse.Operations.Count; index++) @@ -74,7 +74,6 @@ internal PartitionKeyRangeBatchResponse( errorMessageBuilder.AppendFormat("{0}; ", serverResponse.ErrorMessage); } - this.ActivityId = serverResponse.ActivityId; this.ErrorMessage = errorMessageBuilder.Length > 2 ? errorMessageBuilder.ToString(0, errorMessageBuilder.Length - 2) : null; this.Operations = itemBatchOperations; this.Serializer = serializer; @@ -83,12 +82,12 @@ internal PartitionKeyRangeBatchResponse( /// /// Gets the ActivityId that identifies the server request made to execute the batch request. /// - public override string ActivityId { get; } + public override string ActivityId => this.serverResponse.ActivityId; - internal override CosmosSerializer Serializer { get; } + /// + public override CosmosDiagnostics Diagnostics => this.serverResponse.Diagnostics; - // for unit testing only - internal BatchResponse ServerResponse { get; private set; } + internal override CosmosSerializer Serializer { get; } /// /// Gets the number of operation results. @@ -148,7 +147,7 @@ protected override void Dispose(bool disposing) if (disposing && !this.isDisposed) { this.isDisposed = true; - this.ServerResponse?.Dispose(); + this.serverResponse?.Dispose(); } base.Dispose(disposing); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/PartitionKeyBatchResponseTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/PartitionKeyBatchResponseTests.cs index 3deb8b54e4..f544256fe9 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/PartitionKeyBatchResponseTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/PartitionKeyBatchResponseTests.cs @@ -61,5 +61,43 @@ public async Task StatusCodesAreSetThroughResponseAsync() PartitionKeyRangeBatchResponse response = new PartitionKeyRangeBatchResponse(arrayOperations.Length, batchresponse, new CosmosJsonDotNetSerializer()); Assert.AreEqual(HttpStatusCode.OK, response.StatusCode); } + + [TestMethod] + public async Task DiagnosticsAreSetThroughResponseAsync() + { + List results = new List(); + ItemBatchOperation[] arrayOperations = new ItemBatchOperation[1]; + + ItemBatchOperation operation = new ItemBatchOperation(OperationType.AddComputeGatewayRequestCharges, 0, "0"); + + results.Add( + new BatchOperationResult(HttpStatusCode.OK) + { + ResourceStream = new MemoryStream(new byte[] { 0x41, 0x42 }, index: 0, count: 2, writable: false, publiclyVisible: true), + ETag = operation.Id + }); + + arrayOperations[0] = operation; + + MemoryStream responseContent = await new BatchResponsePayloadWriter(results).GeneratePayloadAsync(); + + SinglePartitionKeyServerBatchRequest batchRequest = await SinglePartitionKeyServerBatchRequest.CreateAsync( + partitionKey: null, + operations: new ArraySegment(arrayOperations), + maxBodyLength: 100, + maxOperationCount: 1, + serializer: new CosmosJsonDotNetSerializer(), + cancellationToken: default(CancellationToken)); + + CosmosDiagnostics diagnostics = new PointOperationStatistics(new CosmosClientSideRequestStatistics()); + + BatchResponse batchresponse = await BatchResponse.PopulateFromContentAsync( + new ResponseMessage(HttpStatusCode.OK) { Content = responseContent, Diagnostics = diagnostics }, + batchRequest, + new CosmosJsonDotNetSerializer()); + + PartitionKeyRangeBatchResponse response = new PartitionKeyRangeBatchResponse(arrayOperations.Length, batchresponse, new CosmosJsonDotNetSerializer()); + Assert.AreEqual(diagnostics, response.Diagnostics); + } } } From 36ea84ad6e06fef907a0ee9f14af946df787d02b Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Wed, 4 Sep 2019 10:03:10 -0700 Subject: [PATCH 37/54] Introducing itembatchoperationstatistics --- .../Settings/ItemBatchOperationStatistics.cs | 35 ++++++++++++++++ .../ItemBatchOperationStatisticsTests.cs | 42 +++++++++++++++++++ .../Microsoft.Azure.Cosmos.Tests.csproj | 3 -- 3 files changed, 77 insertions(+), 3 deletions(-) create mode 100644 Microsoft.Azure.Cosmos/src/Resource/Settings/ItemBatchOperationStatistics.cs create mode 100644 Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/ItemBatchOperationStatisticsTests.cs diff --git a/Microsoft.Azure.Cosmos/src/Resource/Settings/ItemBatchOperationStatistics.cs b/Microsoft.Azure.Cosmos/src/Resource/Settings/ItemBatchOperationStatistics.cs new file mode 100644 index 0000000000..4e684f425e --- /dev/null +++ b/Microsoft.Azure.Cosmos/src/Resource/Settings/ItemBatchOperationStatistics.cs @@ -0,0 +1,35 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos +{ + using System.Collections.Generic; + using System.Text; + + internal class ItemBatchOperationStatistics : CosmosDiagnostics + { + private readonly List pointOperationStatistics = new List(); + + public void AppendPointOperation(PointOperationStatistics pointOperationStatistic) + { + this.pointOperationStatistics.Add(pointOperationStatistic); + } + + public override string ToString() + { + if (this.pointOperationStatistics.Count == 0) + { + return string.Empty; + } + + StringBuilder statistics = new StringBuilder(); + foreach (PointOperationStatistics pointOperationStatistic in this.pointOperationStatistics) + { + statistics.AppendLine(pointOperationStatistic.ToString()); + } + + return statistics.ToString(); + } + } +} diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/ItemBatchOperationStatisticsTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/ItemBatchOperationStatisticsTests.cs new file mode 100644 index 0000000000..3562f3cf58 --- /dev/null +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/ItemBatchOperationStatisticsTests.cs @@ -0,0 +1,42 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Tests +{ + using System; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class ItemBatchOperationStatisticsTests + { + [TestMethod] + public void ToString_WhenEmptyReturnEmptyString() + { + ItemBatchOperationStatistics itemBatchOperationStatistics = new ItemBatchOperationStatistics(); + Assert.IsTrue(string.IsNullOrEmpty(itemBatchOperationStatistics.ToString())); + } + + [TestMethod] + public void ToString_ReturnItemsToString() + { + ItemBatchOperationStatistics itemBatchOperationStatistics = new ItemBatchOperationStatistics(); + + CosmosClientSideRequestStatistics cosmosClientSideRequestStatistics1 = new CosmosClientSideRequestStatistics(); + cosmosClientSideRequestStatistics1.ContactedReplicas.Add(new Uri("https://one.com")); + PointOperationStatistics pointOperation1 = new PointOperationStatistics(cosmosClientSideRequestStatistics1); + + CosmosClientSideRequestStatistics cosmosClientSideRequestStatistics2 = new CosmosClientSideRequestStatistics(); + cosmosClientSideRequestStatistics2.ContactedReplicas.Add(new Uri("https://two.com")); + PointOperationStatistics pointOperation2 = new PointOperationStatistics(cosmosClientSideRequestStatistics2); + + itemBatchOperationStatistics.AppendPointOperation(pointOperation1); + itemBatchOperationStatistics.AppendPointOperation(pointOperation2); + + string toString = itemBatchOperationStatistics.ToString(); + + Assert.IsTrue(toString.Contains(pointOperation1.ToString())); + Assert.IsTrue(toString.Contains(pointOperation2.ToString())); + } + } +} diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Microsoft.Azure.Cosmos.Tests.csproj b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Microsoft.Azure.Cosmos.Tests.csproj index 5e2ceebd69..e17f7729dd 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Microsoft.Azure.Cosmos.Tests.csproj +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Microsoft.Azure.Cosmos.Tests.csproj @@ -225,9 +225,6 @@ PreserveNewest - - - C:\Users\abpai\.nuget\packages\microsoft.azure.cosmos.direct.myget\3.0.0.33-preview\runtimes\any\lib\netstandard2.0\Microsoft.Azure.Cosmos.Core.dll From 0304ba4323e207aabc8d788371fd6ce9ee5bf3e3 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Wed, 4 Sep 2019 10:03:58 -0700 Subject: [PATCH 38/54] Description --- .../src/Resource/Settings/ItemBatchOperationStatistics.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Microsoft.Azure.Cosmos/src/Resource/Settings/ItemBatchOperationStatistics.cs b/Microsoft.Azure.Cosmos/src/Resource/Settings/ItemBatchOperationStatistics.cs index 4e684f425e..77696972c9 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Settings/ItemBatchOperationStatistics.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Settings/ItemBatchOperationStatistics.cs @@ -7,6 +7,9 @@ namespace Microsoft.Azure.Cosmos using System.Collections.Generic; using System.Text; + /// + /// A batch operation might extend multiple requests due to retries. + /// internal class ItemBatchOperationStatistics : CosmosDiagnostics { private readonly List pointOperationStatistics = new List(); From 7d77faf6db0619c230b0e5415b485c3809832555 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Wed, 4 Sep 2019 10:30:47 -0700 Subject: [PATCH 39/54] More generic --- .../src/Resource/Settings/ItemBatchOperationStatistics.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Resource/Settings/ItemBatchOperationStatistics.cs b/Microsoft.Azure.Cosmos/src/Resource/Settings/ItemBatchOperationStatistics.cs index 77696972c9..980b34e011 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Settings/ItemBatchOperationStatistics.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Settings/ItemBatchOperationStatistics.cs @@ -12,9 +12,9 @@ namespace Microsoft.Azure.Cosmos /// internal class ItemBatchOperationStatistics : CosmosDiagnostics { - private readonly List pointOperationStatistics = new List(); + private readonly List pointOperationStatistics = new List(); - public void AppendPointOperation(PointOperationStatistics pointOperationStatistic) + public void AppendPointOperation(CosmosDiagnostics pointOperationStatistic) { this.pointOperationStatistics.Add(pointOperationStatistic); } @@ -27,7 +27,7 @@ public override string ToString() } StringBuilder statistics = new StringBuilder(); - foreach (PointOperationStatistics pointOperationStatistic in this.pointOperationStatistics) + foreach (CosmosDiagnostics pointOperationStatistic in this.pointOperationStatistics) { statistics.AppendLine(pointOperationStatistic.ToString()); } From 94460dc014f49d8908c3badc195c2ed593b05657 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Wed, 4 Sep 2019 10:38:15 -0700 Subject: [PATCH 40/54] Batcher appends diagnostics --- .../src/Batch/BatchAsyncBatcher.cs | 1 + .../src/Batch/ItemBatchOperation.cs | 2 ++ .../Batch/BatchAsyncBatcherTests.cs | 21 ++++++++++++------- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncBatcher.cs b/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncBatcher.cs index 499eab03d3..5dcf8580e6 100644 --- a/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncBatcher.cs +++ b/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncBatcher.cs @@ -157,6 +157,7 @@ public virtual bool TryAdd(ItemBatchOperation operation) foreach (ItemBatchOperation itemBatchOperation in batchResponse.Operations) { BatchOperationResult response = batchResponse[itemBatchOperation.OperationIndex]; + itemBatchOperation.Diagnostics.AppendPointOperation(batchResponse.Diagnostics); if (!response.IsSuccessStatusCode) { Documents.ShouldRetryResult shouldRetry = await itemBatchOperation.Context.ShouldRetryAsync(response, cancellationToken); diff --git a/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperation.cs b/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperation.cs index b2e7e7dfd9..fec5b6cf93 100644 --- a/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperation.cs +++ b/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperation.cs @@ -67,6 +67,8 @@ public ItemBatchOperation( public int OperationIndex { get; internal set; } + internal ItemBatchOperationStatistics Diagnostics { get; } = new ItemBatchOperationStatistics(); + internal string PartitionKeyJson { get; set; } internal Documents.PartitionKey ParsedPartitionKey { get; set; } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncBatcherTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncBatcherTests.cs index fed3fcc2b3..f69a6d5b9b 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncBatcherTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncBatcherTests.cs @@ -59,7 +59,7 @@ private BatchAsyncBatcherExecuteDelegate Executor cancellationToken: cancellationToken); BatchResponse batchresponse = await BatchResponse.PopulateFromContentAsync( - new ResponseMessage(HttpStatusCode.OK) { Content = responseContent }, + new ResponseMessage(HttpStatusCode.OK) { Content = responseContent, Diagnostics = new PointOperationStatistics(new CosmosClientSideRequestStatistics()) }, batchRequest, new CosmosJsonDotNetSerializer()); @@ -94,7 +94,7 @@ private BatchAsyncBatcherExecuteDelegate ExecutorWithSplit serializer: new CosmosJsonDotNetSerializer(), cancellationToken: cancellationToken); - ResponseMessage responseMessage = new ResponseMessage(HttpStatusCode.Gone) { Content = responseContent }; + ResponseMessage responseMessage = new ResponseMessage(HttpStatusCode.Gone) { Content = responseContent, Diagnostics = new PointOperationStatistics(new CosmosClientSideRequestStatistics()) }; responseMessage.Headers.SubStatusCode = SubStatusCodes.PartitionKeyRangeGone; BatchResponse batchresponse = await BatchResponse.PopulateFromContentAsync( @@ -238,13 +238,13 @@ public async Task ExceptionsFailOperationsAsync() public async Task DispatchProcessInOrderAsync() { BatchAsyncBatcher batchAsyncBatcher = new BatchAsyncBatcher(10, 1000, new CosmosJsonDotNetSerializer(), this.Executor, this.Retrier); - List contexts = new List(10); + List operations = new List(10); for (int i = 0; i < 10; i++) { ItemBatchOperation operation = new ItemBatchOperation(OperationType.Create, i, i.ToString()); ItemBatchOperationContext context = new ItemBatchOperationContext(string.Empty); operation.AttachContext(context); - contexts.Add(context); + operations.Add(operation); Assert.IsTrue(batchAsyncBatcher.TryAdd(operation)); } @@ -252,10 +252,13 @@ public async Task DispatchProcessInOrderAsync() for (int i = 0; i < 10; i++) { - ItemBatchOperationContext context = contexts[i]; - Assert.AreEqual(TaskStatus.RanToCompletion, context.Task.Status); - BatchOperationResult result = await context.Task; + ItemBatchOperation operation = operations[i]; + Assert.AreEqual(TaskStatus.RanToCompletion, operation.Context.Task.Status); + BatchOperationResult result = await operation.Context.Task; Assert.AreEqual(i.ToString(), result.ETag); + + Assert.IsNotNull(operation.Diagnostics); + Assert.IsTrue(!string.IsNullOrEmpty(operation.Diagnostics.ToString())); } } @@ -361,6 +364,10 @@ public async Task RetrierGetsCalledOnSplit() retryDelegate.Verify(a => a(It.Is(o => o == operation1), It.IsAny()), Times.Once); retryDelegate.Verify(a => a(It.Is(o => o == operation2), It.IsAny()), Times.Once); retryDelegate.Verify(a => a(It.IsAny(), It.IsAny()), Times.Exactly(2)); + Assert.IsNotNull(operation1.Diagnostics); + Assert.IsNotNull(operation2.Diagnostics); + Assert.IsTrue(!string.IsNullOrEmpty(operation1.Diagnostics.ToString())); + Assert.IsTrue(!string.IsNullOrEmpty(operation2.Diagnostics.ToString())); } [TestMethod] From 7dbc381b1ca0ebde2daf9ae416de565cd5c1c6e4 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Wed, 4 Sep 2019 12:00:25 -0700 Subject: [PATCH 41/54] Test to verify multiple operations --- .../src/Resource/Settings/ItemBatchOperationStatistics.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Resource/Settings/ItemBatchOperationStatistics.cs b/Microsoft.Azure.Cosmos/src/Resource/Settings/ItemBatchOperationStatistics.cs index 980b34e011..e07536c124 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Settings/ItemBatchOperationStatistics.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Settings/ItemBatchOperationStatistics.cs @@ -12,22 +12,22 @@ namespace Microsoft.Azure.Cosmos /// internal class ItemBatchOperationStatistics : CosmosDiagnostics { - private readonly List pointOperationStatistics = new List(); + private readonly List cosmosDiagnostics = new List(); public void AppendPointOperation(CosmosDiagnostics pointOperationStatistic) { - this.pointOperationStatistics.Add(pointOperationStatistic); + this.cosmosDiagnostics.Add(pointOperationStatistic); } public override string ToString() { - if (this.pointOperationStatistics.Count == 0) + if (this.cosmosDiagnostics.Count == 0) { return string.Empty; } StringBuilder statistics = new StringBuilder(); - foreach (CosmosDiagnostics pointOperationStatistic in this.pointOperationStatistics) + foreach (CosmosDiagnostics pointOperationStatistic in this.cosmosDiagnostics) { statistics.AppendLine(pointOperationStatistic.ToString()); } From 27085ecdc9d88d8d54fc487a1c2a6f4b2b2a629a Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Wed, 4 Sep 2019 12:07:26 -0700 Subject: [PATCH 42/54] Emulator tests --- .../Batch/CosmosItemBulkTests.cs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemBulkTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemBulkTests.cs index 26b531f1d2..e43ace478d 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemBulkTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemBulkTests.cs @@ -54,7 +54,7 @@ public async Task CreateItemStream_WithBulk() Task task = tasks[i]; ResponseMessage result = await task; Assert.AreEqual(HttpStatusCode.Created, result.StatusCode); - + Assert.IsFalse(string.IsNullOrEmpty(result.Diagnostics.ToString())); MyDocument document = cosmosDefaultJsonSerializer.FromStream(result.Content); Assert.AreEqual(i.ToString(), document.id); } @@ -75,6 +75,7 @@ public async Task CreateItemAsync_WithBulk() { Task> task = tasks[i]; ItemResponse result = await task; + Assert.IsFalse(string.IsNullOrEmpty(result.Diagnostics.ToString())); Assert.AreEqual(HttpStatusCode.Created, result.StatusCode); } } @@ -95,7 +96,7 @@ public async Task UpsertItemStream_WithBulk() Task task = tasks[i]; ResponseMessage result = await task; Assert.AreEqual(HttpStatusCode.Created, result.StatusCode); - + Assert.IsFalse(string.IsNullOrEmpty(result.Diagnostics.ToString())); MyDocument document = cosmosDefaultJsonSerializer.FromStream(result.Content); Assert.AreEqual(i.ToString(), document.id); } @@ -116,6 +117,7 @@ public async Task UpsertItem_WithBulk() { Task> task = tasks[i]; ItemResponse result = await task; + Assert.IsFalse(string.IsNullOrEmpty(result.Diagnostics.ToString())); Assert.AreEqual(HttpStatusCode.Created, result.StatusCode); } } @@ -147,6 +149,7 @@ public async Task DeleteItemStream_WithBulk() { Task task = deleteTasks[i]; ResponseMessage result = await task; + Assert.IsFalse(string.IsNullOrEmpty(result.Diagnostics.ToString())); Assert.AreEqual(HttpStatusCode.NoContent, result.StatusCode); } } @@ -178,6 +181,7 @@ public async Task DeleteItem_WithBulk() { Task> task = deleteTasks[i]; ItemResponse result = await task; + Assert.IsFalse(string.IsNullOrEmpty(result.Diagnostics.ToString())); Assert.AreEqual(HttpStatusCode.NoContent, result.StatusCode); } } @@ -198,7 +202,7 @@ public async Task ReadItemStream_WithBulk() await Task.WhenAll(tasks); List> readTasks = new List>(); - // Delete the items + // Read the items foreach (MyDocument createdDocument in createdDocuments) { readTasks.Add(ExecuteReadStreamAsync(this.container, createdDocument)); @@ -209,6 +213,7 @@ public async Task ReadItemStream_WithBulk() { Task task = readTasks[i]; ResponseMessage result = await task; + Assert.IsFalse(string.IsNullOrEmpty(result.Diagnostics.ToString())); Assert.AreEqual(HttpStatusCode.OK, result.StatusCode); } } @@ -229,7 +234,7 @@ public async Task ReadItem_WithBulk() await Task.WhenAll(tasks); List>> readTasks = new List>>(); - // Delete the items + // Read the items foreach (MyDocument createdDocument in createdDocuments) { readTasks.Add(ExecuteReadAsync(this.container, createdDocument)); @@ -240,6 +245,7 @@ public async Task ReadItem_WithBulk() { Task> task = readTasks[i]; ItemResponse result = await task; + Assert.IsFalse(string.IsNullOrEmpty(result.Diagnostics.ToString())); Assert.AreEqual(HttpStatusCode.OK, result.StatusCode); } } @@ -271,6 +277,7 @@ public async Task ReplaceItemStream_WithBulk() { Task task = replaceTasks[i]; ResponseMessage result = await task; + Assert.IsFalse(string.IsNullOrEmpty(result.Diagnostics.ToString())); Assert.AreEqual(HttpStatusCode.OK, result.StatusCode); } } @@ -302,6 +309,7 @@ public async Task ReplaceItem_WithBulk() { Task> task = replaceTasks[i]; ItemResponse result = await task; + Assert.IsFalse(string.IsNullOrEmpty(result.Diagnostics.ToString())); Assert.AreEqual(HttpStatusCode.OK, result.StatusCode); } } From c54d8a27ec744badbdea37aa104d6089200b07f8 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Wed, 4 Sep 2019 16:05:04 -0700 Subject: [PATCH 43/54] On Context --- .../src/Batch/BatchAsyncBatcher.cs | 2 +- .../src/Batch/BatchOperationResult.cs | 6 ++++ .../src/Batch/ItemBatchOperation.cs | 2 -- .../src/Batch/ItemBatchOperationContext.cs | 3 ++ .../Settings/ItemBatchOperationStatistics.cs | 4 +-- .../Batch/BatchAsyncBatcherTests.cs | 13 ++++---- .../Batch/BatchAsyncContainerExecutorTests.cs | 30 +++++++++++++++++-- .../ItemBatchOperationStatisticsTests.cs | 4 +-- ...titionKeyRangeBatchExecutionResultTests.cs | 4 ++- 9 files changed, 51 insertions(+), 17 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncBatcher.cs b/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncBatcher.cs index 5dcf8580e6..c0472328ad 100644 --- a/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncBatcher.cs +++ b/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncBatcher.cs @@ -157,7 +157,7 @@ public virtual bool TryAdd(ItemBatchOperation operation) foreach (ItemBatchOperation itemBatchOperation in batchResponse.Operations) { BatchOperationResult response = batchResponse[itemBatchOperation.OperationIndex]; - itemBatchOperation.Diagnostics.AppendPointOperation(batchResponse.Diagnostics); + itemBatchOperation.Context.Diagnostics.AppendDiagnostics(batchResponse.Diagnostics); if (!response.IsSuccessStatusCode) { Documents.ShouldRetryResult shouldRetry = await itemBatchOperation.Context.ShouldRetryAsync(response, cancellationToken); diff --git a/Microsoft.Azure.Cosmos/src/Batch/BatchOperationResult.cs b/Microsoft.Azure.Cosmos/src/Batch/BatchOperationResult.cs index 0d1343fe1a..e7866ba274 100644 --- a/Microsoft.Azure.Cosmos/src/Batch/BatchOperationResult.cs +++ b/Microsoft.Azure.Cosmos/src/Batch/BatchOperationResult.cs @@ -85,6 +85,11 @@ public virtual bool IsSuccessStatusCode /// internal virtual SubStatusCodes SubStatusCode { get; set; } + /// + /// Gets the cosmos diagnostic information for the current request to Azure Cosmos DB service + /// + internal virtual CosmosDiagnostics Diagnostics { get; set; } + internal static Result ReadOperationResult(Memory input, out BatchOperationResult batchOperationResult) { RowBuffer row = new RowBuffer(input.Length); @@ -184,6 +189,7 @@ internal ResponseMessage ToResponseMessage() responseMessage.Headers.ETag = this.ETag; responseMessage.Headers.RetryAfter = this.RetryAfter; responseMessage.Content = this.ResourceStream; + responseMessage.Diagnostics = this.Diagnostics; return responseMessage; } } diff --git a/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperation.cs b/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperation.cs index fec5b6cf93..b2e7e7dfd9 100644 --- a/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperation.cs +++ b/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperation.cs @@ -67,8 +67,6 @@ public ItemBatchOperation( public int OperationIndex { get; internal set; } - internal ItemBatchOperationStatistics Diagnostics { get; } = new ItemBatchOperationStatistics(); - internal string PartitionKeyJson { get; set; } internal Documents.PartitionKey ParsedPartitionKey { get; set; } diff --git a/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperationContext.cs b/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperationContext.cs index 1483bc0a90..cb52450ff4 100644 --- a/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperationContext.cs +++ b/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperationContext.cs @@ -21,6 +21,8 @@ internal class ItemBatchOperationContext : IDisposable public Task Task => this.taskCompletionSource.Task; + public ItemBatchOperationStatistics Diagnostics { get; } = new ItemBatchOperationStatistics(); + private readonly IDocumentClientRetryPolicy retryPolicy; private TaskCompletionSource taskCompletionSource = new TaskCompletionSource(); @@ -56,6 +58,7 @@ public void Complete( { if (this.AssertBatcher(completer)) { + result.Diagnostics = this.Diagnostics; this.taskCompletionSource.SetResult(result); } diff --git a/Microsoft.Azure.Cosmos/src/Resource/Settings/ItemBatchOperationStatistics.cs b/Microsoft.Azure.Cosmos/src/Resource/Settings/ItemBatchOperationStatistics.cs index e07536c124..976750d2ae 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Settings/ItemBatchOperationStatistics.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Settings/ItemBatchOperationStatistics.cs @@ -14,9 +14,9 @@ internal class ItemBatchOperationStatistics : CosmosDiagnostics { private readonly List cosmosDiagnostics = new List(); - public void AppendPointOperation(CosmosDiagnostics pointOperationStatistic) + public void AppendDiagnostics(CosmosDiagnostics diagnostics) { - this.cosmosDiagnostics.Add(pointOperationStatistic); + this.cosmosDiagnostics.Add(diagnostics); } public override string ToString() diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncBatcherTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncBatcherTests.cs index f69a6d5b9b..038a0a5b7c 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncBatcherTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncBatcherTests.cs @@ -257,8 +257,9 @@ public async Task DispatchProcessInOrderAsync() BatchOperationResult result = await operation.Context.Task; Assert.AreEqual(i.ToString(), result.ETag); - Assert.IsNotNull(operation.Diagnostics); - Assert.IsTrue(!string.IsNullOrEmpty(operation.Diagnostics.ToString())); + Assert.IsNotNull(operation.Context.Diagnostics); + Assert.AreEqual(operation.Context.Diagnostics.ToString(), result.Diagnostics.ToString()); + Assert.IsFalse(string.IsNullOrEmpty(operation.Context.Diagnostics.ToString())); } } @@ -364,10 +365,10 @@ public async Task RetrierGetsCalledOnSplit() retryDelegate.Verify(a => a(It.Is(o => o == operation1), It.IsAny()), Times.Once); retryDelegate.Verify(a => a(It.Is(o => o == operation2), It.IsAny()), Times.Once); retryDelegate.Verify(a => a(It.IsAny(), It.IsAny()), Times.Exactly(2)); - Assert.IsNotNull(operation1.Diagnostics); - Assert.IsNotNull(operation2.Diagnostics); - Assert.IsTrue(!string.IsNullOrEmpty(operation1.Diagnostics.ToString())); - Assert.IsTrue(!string.IsNullOrEmpty(operation2.Diagnostics.ToString())); + Assert.IsNotNull(operation1.Context.Diagnostics); + Assert.IsNotNull(operation2.Context.Diagnostics); + Assert.IsTrue(!string.IsNullOrEmpty(operation1.Context.Diagnostics.ToString())); + Assert.IsTrue(!string.IsNullOrEmpty(operation2.Context.Diagnostics.ToString())); } [TestMethod] diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncContainerExecutorTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncContainerExecutorTests.cs index 8bb5ae30c0..7c72f09185 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncContainerExecutorTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncContainerExecutorTests.cs @@ -73,6 +73,18 @@ public async Task RetryOnSplit() It.IsAny>(), It.IsAny()), Times.Exactly(2)); Assert.AreEqual(HttpStatusCode.OK, result.StatusCode); + Assert.IsNotNull(result.Diagnostics); + + int diagnosticsLines = 0; + string diagnosticsString = result.Diagnostics.ToString(); + int index = diagnosticsString.IndexOf(Environment.NewLine); + while(index > -1) + { + diagnosticsLines++; + index = diagnosticsString.IndexOf(Environment.NewLine, index + 1); + } + + Assert.AreEqual(2, diagnosticsLines); } [TestMethod] @@ -127,6 +139,18 @@ public async Task RetryOn429() It.IsAny>(), It.IsAny()), Times.Exactly(2)); Assert.AreEqual(HttpStatusCode.OK, result.StatusCode); + Assert.IsNotNull(result.Diagnostics); + + int diagnosticsLines = 0; + string diagnosticsString = result.Diagnostics.ToString(); + int index = diagnosticsString.IndexOf(Environment.NewLine); + while (index > -1) + { + diagnosticsLines++; + index = diagnosticsString.IndexOf(Environment.NewLine, index + 1); + } + + Assert.AreEqual(2, diagnosticsLines); } [TestMethod] @@ -205,7 +229,7 @@ private async Task GenerateSplitResponseAsync(ItemBatchOperatio serializer: new CosmosJsonDotNetSerializer(), cancellationToken: CancellationToken.None); - ResponseMessage responseMessage = new ResponseMessage(HttpStatusCode.Gone) { Content = responseContent }; + ResponseMessage responseMessage = new ResponseMessage(HttpStatusCode.Gone) { Content = responseContent, Diagnostics = new PointOperationStatistics(new CosmosClientSideRequestStatistics()) }; responseMessage.Headers.SubStatusCode = SubStatusCodes.PartitionKeyRangeGone; return responseMessage; } @@ -232,7 +256,7 @@ private async Task Generate429ResponseAsync(ItemBatchOperation serializer: new CosmosJsonDotNetSerializer(), cancellationToken: CancellationToken.None); - ResponseMessage responseMessage = new ResponseMessage((HttpStatusCode)StatusCodes.TooManyRequests) { Content = responseContent }; + ResponseMessage responseMessage = new ResponseMessage((HttpStatusCode)StatusCodes.TooManyRequests) { Content = responseContent, Diagnostics = new PointOperationStatistics(new CosmosClientSideRequestStatistics()) }; return responseMessage; } @@ -258,7 +282,7 @@ private async Task GenerateOkResponseAsync(ItemBatchOperation i serializer: new CosmosJsonDotNetSerializer(), cancellationToken: CancellationToken.None); - ResponseMessage responseMessage = new ResponseMessage(HttpStatusCode.OK) { Content = responseContent }; + ResponseMessage responseMessage = new ResponseMessage(HttpStatusCode.OK) { Content = responseContent, Diagnostics = new PointOperationStatistics(new CosmosClientSideRequestStatistics()) }; return responseMessage; } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/ItemBatchOperationStatisticsTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/ItemBatchOperationStatisticsTests.cs index 3562f3cf58..707a7b0d44 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/ItemBatchOperationStatisticsTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/ItemBatchOperationStatisticsTests.cs @@ -30,8 +30,8 @@ public void ToString_ReturnItemsToString() cosmosClientSideRequestStatistics2.ContactedReplicas.Add(new Uri("https://two.com")); PointOperationStatistics pointOperation2 = new PointOperationStatistics(cosmosClientSideRequestStatistics2); - itemBatchOperationStatistics.AppendPointOperation(pointOperation1); - itemBatchOperationStatistics.AppendPointOperation(pointOperation2); + itemBatchOperationStatistics.AppendDiagnostics(pointOperation1); + itemBatchOperationStatistics.AppendDiagnostics(pointOperation2); string toString = itemBatchOperationStatistics.ToString(); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/PartitionKeyRangeBatchExecutionResultTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/PartitionKeyRangeBatchExecutionResultTests.cs index da594dde21..5c933f51a9 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/PartitionKeyRangeBatchExecutionResultTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/PartitionKeyRangeBatchExecutionResultTests.cs @@ -75,7 +75,8 @@ public void ToResponseMessage_MapsProperties() ResourceStream = new MemoryStream(new byte[] { 0x41, 0x42 }, index: 0, count: 2, writable: false, publiclyVisible: true), ETag = "1234", SubStatusCode = SubStatusCodes.CompletingSplit, - RetryAfter = TimeSpan.FromSeconds(10) + RetryAfter = TimeSpan.FromSeconds(10), + Diagnostics = new PointOperationStatistics(new CosmosClientSideRequestStatistics()) }; ResponseMessage response = result.ToResponseMessage(); @@ -84,6 +85,7 @@ public void ToResponseMessage_MapsProperties() Assert.AreEqual(result.SubStatusCode, response.Headers.SubStatusCode); Assert.AreEqual(result.RetryAfter, response.Headers.RetryAfter); Assert.AreEqual(result.StatusCode, response.StatusCode); + Assert.AreEqual(result.Diagnostics, response.Diagnostics); } private async Task ConstainsSplitIsTrueInternal(HttpStatusCode statusCode, SubStatusCodes subStatusCode) From 2252f6f713f8c6f7f8677bf410372e1af6036698 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Wed, 4 Sep 2019 16:07:33 -0700 Subject: [PATCH 44/54] Rename --- .../src/CosmosClientOptions.cs | 2 +- .../src/Fluent/CosmosClientBuilder.cs | 8 ++++---- .../src/Resource/ClientContextCore.cs | 2 +- .../Batch/CosmosItemBulkTests.cs | 2 +- .../CosmosClientOptionsUnitTests.cs | 6 +++--- .../CosmosItemUnitTests.cs | 20 +++++++++---------- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs index d44efea0f6..d0a56b2746 100644 --- a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs +++ b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs @@ -348,7 +348,7 @@ public CosmosSerializer Serializer #else internal #endif - bool AllowBatchingRequests { get; set; } + bool AllowBatchRequests { get; set; } /// /// A JSON serializer used by the CosmosClient to serialize or de-serialize cosmos request/responses. diff --git a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs index 3be823aa84..d2ceba7bce 100644 --- a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs +++ b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs @@ -337,17 +337,17 @@ public CosmosClientBuilder WithCustomSerializer(CosmosSerializer cosmosJsonSeria /// /// Allows optimistic batching of requests to service. Setting this option might impact the latency of the operations. Hence this option is recommended for throughput scenarios only. /// - /// Whether is enabled. + /// Whether is enabled. /// The object - /// + /// #if PREVIEW public #else internal #endif - CosmosClientBuilder WithBatchingRequests(bool enabled) + CosmosClientBuilder WithBatchRequests(bool enabled) { - this.clientOptions.AllowBatchingRequests = enabled; + this.clientOptions.AllowBatchRequests = enabled; return this; } diff --git a/Microsoft.Azure.Cosmos/src/Resource/ClientContextCore.cs b/Microsoft.Azure.Cosmos/src/Resource/ClientContextCore.cs index dfac65a40b..292ca66186 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/ClientContextCore.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/ClientContextCore.cs @@ -201,7 +201,7 @@ private bool IsBulkOperationSupported( ResourceType resourceType, OperationType operationType) { - if (!this.ClientOptions.AllowBatchingRequests) + if (!this.ClientOptions.AllowBatchRequests) { return false; } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemBulkTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemBulkTests.cs index 26b531f1d2..46644d196e 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemBulkTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemBulkTests.cs @@ -22,7 +22,7 @@ public class CosmosItemBulkTests public async Task TestInitialize() { CosmosClientOptions clientOptions = new CosmosClientOptions(); - clientOptions.AllowBatchingRequests = true; + clientOptions.AllowBatchRequests = true; CosmosClient client = TestCommon.CreateCosmosClient(clientOptions); DatabaseResponse response = await client.CreateDatabaseIfNotExistsAsync(Guid.NewGuid().ToString()); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs index 98793f4723..ca841edf54 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs @@ -63,7 +63,7 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated() Assert.AreNotEqual(requestTimeout, clientOptions.RequestTimeout); Assert.AreNotEqual(userAgentSuffix, clientOptions.ApplicationName); Assert.AreNotEqual(apiType, clientOptions.ApiType); - Assert.IsFalse(clientOptions.AllowBatchingRequests); + Assert.IsFalse(clientOptions.AllowBatchRequests); Assert.AreEqual(0, clientOptions.CustomHandlers.Count); Assert.IsNull(clientOptions.SerializerOptions); Assert.IsNull(clientOptions.Serializer); @@ -87,7 +87,7 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated() .AddCustomHandlers(preProcessHandler) .WithApiType(apiType) .WithThrottlingRetryOptions(maxRetryWaitTime, maxRetryAttemptsOnThrottledRequests) - .WithBatchingRequests(true) + .WithBatchRequests(true) .WithSerializerOptions(cosmosSerializerOptions); cosmosClient = cosmosClientBuilder.Build(new MockDocumentClient()); @@ -107,7 +107,7 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated() Assert.AreEqual(cosmosSerializerOptions.PropertyNamingPolicy, clientOptions.SerializerOptions.PropertyNamingPolicy); Assert.AreEqual(cosmosSerializerOptions.Indented, clientOptions.SerializerOptions.Indented); Assert.IsTrue(object.ReferenceEquals(webProxy, clientOptions.WebProxy)); - Assert.IsTrue(clientOptions.AllowBatchingRequests); + Assert.IsTrue(clientOptions.AllowBatchRequests); //Verify GetConnectionPolicy returns the correct values policy = clientOptions.GetConnectionPolicy(); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs index 2dd36b445a..03be7fbc8b 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs @@ -170,7 +170,7 @@ public async Task AllowBatchingRequestsSendsToExecutor_CreateStream() { ClientContextCore clientContextCore = new ClientContextCore( MockCosmosUtil.CreateMockCosmosClient(), - new CosmosClientOptions() { AllowBatchingRequests = true }, + new CosmosClientOptions() { AllowBatchRequests = true }, new CosmosJsonDotNetSerializer(), new CosmosJsonDotNetSerializer(), null, @@ -207,7 +207,7 @@ public async Task AllowBatchingRequestsSendsToExecutor_UpsertStream() { ClientContextCore clientContextCore = new ClientContextCore( MockCosmosUtil.CreateMockCosmosClient(), - new CosmosClientOptions() { AllowBatchingRequests = true }, + new CosmosClientOptions() { AllowBatchRequests = true }, new CosmosJsonDotNetSerializer(), new CosmosJsonDotNetSerializer(), null, @@ -244,7 +244,7 @@ public async Task AllowBatchingRequestsSendsToExecutor_ReplaceStream() { ClientContextCore clientContextCore = new ClientContextCore( MockCosmosUtil.CreateMockCosmosClient(), - new CosmosClientOptions() { AllowBatchingRequests = true }, + new CosmosClientOptions() { AllowBatchRequests = true }, new CosmosJsonDotNetSerializer(), new CosmosJsonDotNetSerializer(), null, @@ -282,7 +282,7 @@ public async Task AllowBatchingRequestsSendsToExecutor_ReadStream() { ClientContextCore clientContextCore = new ClientContextCore( MockCosmosUtil.CreateMockCosmosClient(), - new CosmosClientOptions() { AllowBatchingRequests = true }, + new CosmosClientOptions() { AllowBatchRequests = true }, new CosmosJsonDotNetSerializer(), new CosmosJsonDotNetSerializer(), null, @@ -319,7 +319,7 @@ public async Task AllowBatchingRequestsSendsToExecutor_DeleteStream() { ClientContextCore clientContextCore = new ClientContextCore( MockCosmosUtil.CreateMockCosmosClient(), - new CosmosClientOptions() { AllowBatchingRequests = true }, + new CosmosClientOptions() { AllowBatchRequests = true }, new CosmosJsonDotNetSerializer(), new CosmosJsonDotNetSerializer(), null, @@ -354,7 +354,7 @@ public async Task AllowBatchingRequestsSendsToExecutor_Create() CosmosClient cosmosClient = MockCosmosUtil.CreateMockCosmosClient(); ClientContextCore clientContextCore = new ClientContextCore( cosmosClient, - new CosmosClientOptions() { AllowBatchingRequests = true }, + new CosmosClientOptions() { AllowBatchRequests = true }, new CosmosJsonDotNetSerializer(), new CosmosJsonDotNetSerializer(), null, @@ -386,7 +386,7 @@ public async Task AllowBatchingRequestsSendsToExecutor_Upsert() CosmosClient cosmosClient = MockCosmosUtil.CreateMockCosmosClient(); ClientContextCore clientContextCore = new ClientContextCore( cosmosClient, - new CosmosClientOptions() { AllowBatchingRequests = true }, + new CosmosClientOptions() { AllowBatchRequests = true }, new CosmosJsonDotNetSerializer(), new CosmosJsonDotNetSerializer(), null, @@ -418,7 +418,7 @@ public async Task AllowBatchingRequestsSendsToExecutor_Replace() CosmosClient cosmosClient = MockCosmosUtil.CreateMockCosmosClient(); ClientContextCore clientContextCore = new ClientContextCore( cosmosClient, - new CosmosClientOptions() { AllowBatchingRequests = true }, + new CosmosClientOptions() { AllowBatchRequests = true }, new CosmosJsonDotNetSerializer(), new CosmosJsonDotNetSerializer(), null, @@ -450,7 +450,7 @@ public async Task AllowBatchingRequestsSendsToExecutor_Read() CosmosClient cosmosClient = MockCosmosUtil.CreateMockCosmosClient(); ClientContextCore clientContextCore = new ClientContextCore( cosmosClient, - new CosmosClientOptions() { AllowBatchingRequests = true }, + new CosmosClientOptions() { AllowBatchRequests = true }, new CosmosJsonDotNetSerializer(), new CosmosJsonDotNetSerializer(), null, @@ -482,7 +482,7 @@ public async Task AllowBatchingRequestsSendsToExecutor_Delete() CosmosClient cosmosClient = MockCosmosUtil.CreateMockCosmosClient(); ClientContextCore clientContextCore = new ClientContextCore( cosmosClient, - new CosmosClientOptions() { AllowBatchingRequests = true }, + new CosmosClientOptions() { AllowBatchRequests = true }, new CosmosJsonDotNetSerializer(), new CosmosJsonDotNetSerializer(), null, From 6870378d0b8841e1c409233f4d71c1d244c33096 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Wed, 4 Sep 2019 17:04:00 -0700 Subject: [PATCH 45/54] Undoing comment --- Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperation.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperation.cs b/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperation.cs index b2e7e7dfd9..9a491ea637 100644 --- a/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperation.cs +++ b/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperation.cs @@ -299,9 +299,8 @@ internal virtual async Task MaterializeResourceAsync(CosmosSerializer serializer /// /// Attached a context to the current operation to track resolution. - /// If the current context is completed, a new context can be attached, for the case of retries. /// - /// If the operation already had an attached context which is in progress. + /// If the operation already had an attached context. internal void AttachContext(ItemBatchOperationContext context) { if (this.Context != null) From a3cf72c81cbe11343ac70f1ca9ae8ab34926c296 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Thu, 5 Sep 2019 06:42:29 -0700 Subject: [PATCH 46/54] Description --- Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs | 2 +- Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs index d0a56b2746..65a48c74c5 100644 --- a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs +++ b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs @@ -341,7 +341,7 @@ public CosmosSerializer Serializer } /// - /// Allows optimistic batching of requests to service. Setting this option might impact the latency of the operations. Hence this option is recommended for throughput scenarios only. + /// Allows optimistic batching of requests to service. Setting this option might impact the latency of the operations. Hence this option is recommended for non-latency sensitive scenarios only. /// #if PREVIEW public diff --git a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs index d2ceba7bce..6299de1d7e 100644 --- a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs +++ b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs @@ -335,7 +335,7 @@ public CosmosClientBuilder WithCustomSerializer(CosmosSerializer cosmosJsonSeria } /// - /// Allows optimistic batching of requests to service. Setting this option might impact the latency of the operations. Hence this option is recommended for throughput scenarios only. + /// Allows optimistic batching of requests to service. Setting this option might impact the latency of the operations. Hence this option is recommended for non-latency sensitive scenarios only. /// /// Whether is enabled. /// The object From 3ec10c6f8c348f2db006c33d6b8b5681eaef4976 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Thu, 5 Sep 2019 07:19:43 -0700 Subject: [PATCH 47/54] Renaming variable --- .../src/Batch/BatchAsyncContainerExecutor.cs | 2 +- .../src/Batch/ItemBatchOperationContext.cs | 4 ++-- .../Batch/BatchAsyncBatcherTests.cs | 24 +++++++++---------- .../Batch/BatchAsyncOperationContextTests.cs | 16 ++++++------- .../Batch/BatchAsyncStreamerTests.cs | 6 ++--- 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncContainerExecutor.cs b/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncContainerExecutor.cs index 5f3219a1a5..499cbcf49a 100644 --- a/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncContainerExecutor.cs +++ b/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncContainerExecutor.cs @@ -98,7 +98,7 @@ public virtual async Task AddAsync( ItemBatchOperationContext context = new ItemBatchOperationContext(resolvedPartitionKeyRangeId, BatchAsyncContainerExecutor.GetRetryPolicy(this.retryOptions)); operation.AttachContext(context); streamer.Add(operation); - return await context.Task; + return await context.OperationTask; } public void Dispose() diff --git a/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperationContext.cs b/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperationContext.cs index 1483bc0a90..833729d2a6 100644 --- a/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperationContext.cs +++ b/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperationContext.cs @@ -19,7 +19,7 @@ internal class ItemBatchOperationContext : IDisposable public BatchAsyncBatcher CurrentBatcher { get; set; } - public Task Task => this.taskCompletionSource.Task; + public Task OperationTask => this.taskCompletionSource.Task; private readonly IDocumentClientRetryPolicy retryPolicy; @@ -43,7 +43,7 @@ public Task ShouldRetryAsync( if (this.retryPolicy == null || batchOperationResult.IsSuccessStatusCode) { - return System.Threading.Tasks.Task.FromResult(ShouldRetryResult.NoRetry()); + return Task.FromResult(ShouldRetryResult.NoRetry()); } ResponseMessage responseMessage = batchOperationResult.ToResponseMessage(); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncBatcherTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncBatcherTests.cs index fed3fcc2b3..aea5279738 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncBatcherTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncBatcherTests.cs @@ -228,10 +228,10 @@ public async Task ExceptionsFailOperationsAsync() batchAsyncBatcher.TryAdd(operation2); await batchAsyncBatcher.DispatchAsync(); - Assert.AreEqual(TaskStatus.Faulted, context1.Task.Status); - Assert.AreEqual(TaskStatus.Faulted, context2.Task.Status); - Assert.AreEqual(expectedException, context1.Task.Exception.InnerException); - Assert.AreEqual(expectedException, context2.Task.Exception.InnerException); + Assert.AreEqual(TaskStatus.Faulted, context1.OperationTask.Status); + Assert.AreEqual(TaskStatus.Faulted, context2.OperationTask.Status); + Assert.AreEqual(expectedException, context1.OperationTask.Exception.InnerException); + Assert.AreEqual(expectedException, context2.OperationTask.Exception.InnerException); } [TestMethod] @@ -253,8 +253,8 @@ public async Task DispatchProcessInOrderAsync() for (int i = 0; i < 10; i++) { ItemBatchOperationContext context = contexts[i]; - Assert.AreEqual(TaskStatus.RanToCompletion, context.Task.Status); - BatchOperationResult result = await context.Task; + Assert.AreEqual(TaskStatus.RanToCompletion, context.OperationTask.Status); + BatchOperationResult result = await context.OperationTask; Assert.AreEqual(i.ToString(), result.ETag); } } @@ -283,15 +283,15 @@ public async Task DispatchWithLessResponses() // Some tasks should not be resolved if(i == 0 || i == 9) { - Assert.IsTrue(operation.Context.Task.Status == TaskStatus.WaitingForActivation); + Assert.IsTrue(operation.Context.OperationTask.Status == TaskStatus.WaitingForActivation); } else { - Assert.IsTrue(operation.Context.Task.Status == TaskStatus.RanToCompletion); + Assert.IsTrue(operation.Context.OperationTask.Status == TaskStatus.RanToCompletion); } - if (operation.Context.Task.Status == TaskStatus.RanToCompletion) + if (operation.Context.OperationTask.Status == TaskStatus.RanToCompletion) { - BatchOperationResult result = await operation.Context.Task; + BatchOperationResult result = await operation.Context.OperationTask; Assert.AreEqual(i.ToString(), result.ETag); } else @@ -306,8 +306,8 @@ public async Task DispatchWithLessResponses() for (int i = 0; i < 10; i++) { ItemBatchOperation operation = operations[i]; - Assert.AreEqual(TaskStatus.RanToCompletion, operation.Context.Task.Status); - BatchOperationResult result = await operation.Context.Task; + Assert.AreEqual(TaskStatus.RanToCompletion, operation.Context.OperationTask.Status); + BatchOperationResult result = await operation.Context.OperationTask; Assert.AreEqual(i.ToString(), result.ETag); } } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncOperationContextTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncOperationContextTests.cs index 1aed5b29f5..453cb159fa 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncOperationContextTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncOperationContextTests.cs @@ -22,10 +22,10 @@ public void PartitionKeyRangeIdIsSetOnInitialization() ItemBatchOperationContext batchAsyncOperationContext = new ItemBatchOperationContext(expectedPkRangeId); operation.AttachContext(batchAsyncOperationContext); - Assert.IsNotNull(batchAsyncOperationContext.Task); + Assert.IsNotNull(batchAsyncOperationContext.OperationTask); Assert.AreEqual(batchAsyncOperationContext, operation.Context); Assert.AreEqual(expectedPkRangeId, batchAsyncOperationContext.PartitionKeyRangeId); - Assert.AreEqual(TaskStatus.WaitingForActivation, batchAsyncOperationContext.Task.Status); + Assert.AreEqual(TaskStatus.WaitingForActivation, batchAsyncOperationContext.OperationTask.Status); } [TestMethod] @@ -35,9 +35,9 @@ public void TaskIsCreatedOnInitialization() ItemBatchOperationContext batchAsyncOperationContext = new ItemBatchOperationContext(string.Empty); operation.AttachContext(batchAsyncOperationContext); - Assert.IsNotNull(batchAsyncOperationContext.Task); + Assert.IsNotNull(batchAsyncOperationContext.OperationTask); Assert.AreEqual(batchAsyncOperationContext, operation.Context); - Assert.AreEqual(TaskStatus.WaitingForActivation, batchAsyncOperationContext.Task.Status); + Assert.AreEqual(TaskStatus.WaitingForActivation, batchAsyncOperationContext.OperationTask.Status); } [TestMethod] @@ -51,8 +51,8 @@ public async Task TaskResultIsSetOnCompleteAsync() batchAsyncOperationContext.Complete(null, expected); - Assert.AreEqual(expected, await batchAsyncOperationContext.Task); - Assert.AreEqual(TaskStatus.RanToCompletion, batchAsyncOperationContext.Task.Status); + Assert.AreEqual(expected, await batchAsyncOperationContext.OperationTask); + Assert.AreEqual(TaskStatus.RanToCompletion, batchAsyncOperationContext.OperationTask.Status); } [TestMethod] @@ -65,9 +65,9 @@ public async Task ExceptionIsSetOnFailAsync() batchAsyncOperationContext.Fail(null, failure); - Exception capturedException = await Assert.ThrowsExceptionAsync(() => batchAsyncOperationContext.Task); + Exception capturedException = await Assert.ThrowsExceptionAsync(() => batchAsyncOperationContext.OperationTask); Assert.AreEqual(failure, capturedException); - Assert.AreEqual(TaskStatus.Faulted, batchAsyncOperationContext.Task.Status); + Assert.AreEqual(TaskStatus.Faulted, batchAsyncOperationContext.OperationTask.Status); } [TestMethod] diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncStreamerTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncStreamerTests.cs index f0a4b47b29..780d350f27 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncStreamerTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncStreamerTests.cs @@ -114,7 +114,7 @@ public async Task ExceptionsOnBatchBubbleUpAsync() BatchAsyncStreamer batchAsyncStreamer = new BatchAsyncStreamer(2, MaxBatchByteSize, DispatchTimerInSeconds, this.TimerPool, new CosmosJsonDotNetSerializer(), this.ExecutorWithFailure, this.Retrier); ItemBatchOperationContext context = AttachContext(this.ItemBatchOperation); batchAsyncStreamer.Add(this.ItemBatchOperation); - Exception capturedException = await Assert.ThrowsExceptionAsync(() => context.Task); + Exception capturedException = await Assert.ThrowsExceptionAsync(() => context.OperationTask); Assert.AreEqual(expectedException, capturedException); } @@ -125,7 +125,7 @@ public async Task TimerDispatchesAsync() BatchAsyncStreamer batchAsyncStreamer = new BatchAsyncStreamer(2, MaxBatchByteSize, DispatchTimerInSeconds, this.TimerPool, new CosmosJsonDotNetSerializer(), this.Executor, this.Retrier); ItemBatchOperationContext context = AttachContext(this.ItemBatchOperation); batchAsyncStreamer.Add(this.ItemBatchOperation); - BatchOperationResult result = await context.Task; + BatchOperationResult result = await context.OperationTask; Assert.AreEqual(this.ItemBatchOperation.Id, result.ETag); } @@ -141,7 +141,7 @@ public async Task DispatchesAsync() ItemBatchOperation operation = new ItemBatchOperation(OperationType.Create, i, i.ToString()); ItemBatchOperationContext context = AttachContext(operation); batchAsyncStreamer.Add(operation); - contexts.Add(context.Task); + contexts.Add(context.OperationTask); } await Task.WhenAll(contexts); From afec0cacea0739a4b2385aab3ab8ac9b2719361c Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Thu, 5 Sep 2019 08:57:06 -0700 Subject: [PATCH 48/54] Fixing test --- .../CosmosItemUnitTests.cs | 40 +++++++------------ 1 file changed, 15 insertions(+), 25 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs index 03be7fbc8b..35e4b364ba 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs @@ -173,11 +173,10 @@ public async Task AllowBatchingRequestsSendsToExecutor_CreateStream() new CosmosClientOptions() { AllowBatchRequests = true }, new CosmosJsonDotNetSerializer(), new CosmosJsonDotNetSerializer(), + new CosmosJsonDotNetSerializer(), null, null, - null, - new MockDocumentClient(), - Mock.Of() + new MockDocumentClient() ); DatabaseCore db = new DatabaseCore(clientContextCore, "test"); @@ -210,11 +209,10 @@ public async Task AllowBatchingRequestsSendsToExecutor_UpsertStream() new CosmosClientOptions() { AllowBatchRequests = true }, new CosmosJsonDotNetSerializer(), new CosmosJsonDotNetSerializer(), + new CosmosJsonDotNetSerializer(), null, null, - null, - new MockDocumentClient(), - Mock.Of() + new MockDocumentClient() ); DatabaseCore db = new DatabaseCore(clientContextCore, "test"); @@ -247,11 +245,10 @@ public async Task AllowBatchingRequestsSendsToExecutor_ReplaceStream() new CosmosClientOptions() { AllowBatchRequests = true }, new CosmosJsonDotNetSerializer(), new CosmosJsonDotNetSerializer(), + new CosmosJsonDotNetSerializer(), null, null, - null, - new MockDocumentClient(), - Mock.Of() + new MockDocumentClient() ); DatabaseCore db = new DatabaseCore(clientContextCore, "test"); @@ -285,11 +282,10 @@ public async Task AllowBatchingRequestsSendsToExecutor_ReadStream() new CosmosClientOptions() { AllowBatchRequests = true }, new CosmosJsonDotNetSerializer(), new CosmosJsonDotNetSerializer(), + new CosmosJsonDotNetSerializer(), null, null, - null, - new MockDocumentClient(), - Mock.Of() + new MockDocumentClient() ); DatabaseCore db = new DatabaseCore(clientContextCore, "test"); @@ -322,11 +318,10 @@ public async Task AllowBatchingRequestsSendsToExecutor_DeleteStream() new CosmosClientOptions() { AllowBatchRequests = true }, new CosmosJsonDotNetSerializer(), new CosmosJsonDotNetSerializer(), + new CosmosJsonDotNetSerializer(), null, null, - null, - new MockDocumentClient(), - Mock.Of() + new MockDocumentClient() ); DatabaseCore db = new DatabaseCore(clientContextCore, "test"); @@ -360,8 +355,7 @@ public async Task AllowBatchingRequestsSendsToExecutor_Create() null, cosmosClient.ResponseFactory, null, - new MockDocumentClient(), - Mock.Of() + new MockDocumentClient() ); DatabaseCore db = new DatabaseCore(clientContextCore, "test"); @@ -392,8 +386,7 @@ public async Task AllowBatchingRequestsSendsToExecutor_Upsert() null, cosmosClient.ResponseFactory, null, - new MockDocumentClient(), - Mock.Of() + new MockDocumentClient() ); DatabaseCore db = new DatabaseCore(clientContextCore, "test"); @@ -424,8 +417,7 @@ public async Task AllowBatchingRequestsSendsToExecutor_Replace() null, cosmosClient.ResponseFactory, null, - new MockDocumentClient(), - Mock.Of() + new MockDocumentClient() ); DatabaseCore db = new DatabaseCore(clientContextCore, "test"); @@ -456,8 +448,7 @@ public async Task AllowBatchingRequestsSendsToExecutor_Read() null, cosmosClient.ResponseFactory, null, - new MockDocumentClient(), - Mock.Of() + new MockDocumentClient() ); DatabaseCore db = new DatabaseCore(clientContextCore, "test"); @@ -488,8 +479,7 @@ public async Task AllowBatchingRequestsSendsToExecutor_Delete() null, cosmosClient.ResponseFactory, null, - new MockDocumentClient(), - Mock.Of() + new MockDocumentClient() ); DatabaseCore db = new DatabaseCore(clientContextCore, "test"); From 62827129825d1d21ccad6ff1f361d93d39b7933c Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Thu, 5 Sep 2019 11:31:23 -0700 Subject: [PATCH 49/54] Rename --- .../src/CosmosClientOptions.cs | 2 +- .../src/Fluent/CosmosClientBuilder.cs | 8 ++++---- .../src/Resource/ClientContextCore.cs | 2 +- .../Batch/CosmosItemBulkTests.cs | 2 +- .../CosmosClientOptionsUnitTests.cs | 6 +++--- .../CosmosItemUnitTests.cs | 20 +++++++++---------- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs index 65a48c74c5..559044d54b 100644 --- a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs +++ b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs @@ -348,7 +348,7 @@ public CosmosSerializer Serializer #else internal #endif - bool AllowBatchRequests { get; set; } + bool AllowBulkExecution { get; set; } /// /// A JSON serializer used by the CosmosClient to serialize or de-serialize cosmos request/responses. diff --git a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs index 6299de1d7e..7c8a00e852 100644 --- a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs +++ b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs @@ -337,17 +337,17 @@ public CosmosClientBuilder WithCustomSerializer(CosmosSerializer cosmosJsonSeria /// /// Allows optimistic batching of requests to service. Setting this option might impact the latency of the operations. Hence this option is recommended for non-latency sensitive scenarios only. /// - /// Whether is enabled. + /// Whether is enabled. /// The object - /// + /// #if PREVIEW public #else internal #endif - CosmosClientBuilder WithBatchRequests(bool enabled) + CosmosClientBuilder WithBulkexecution(bool enabled) { - this.clientOptions.AllowBatchRequests = enabled; + this.clientOptions.AllowBulkExecution = enabled; return this; } diff --git a/Microsoft.Azure.Cosmos/src/Resource/ClientContextCore.cs b/Microsoft.Azure.Cosmos/src/Resource/ClientContextCore.cs index b07745ee47..c21dda602f 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/ClientContextCore.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/ClientContextCore.cs @@ -216,7 +216,7 @@ private bool IsBulkOperationSupported( ResourceType resourceType, OperationType operationType) { - if (!this.ClientOptions.AllowBatchRequests) + if (!this.ClientOptions.AllowBulkExecution) { return false; } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemBulkTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemBulkTests.cs index 46644d196e..e6b4f46d70 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemBulkTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Batch/CosmosItemBulkTests.cs @@ -22,7 +22,7 @@ public class CosmosItemBulkTests public async Task TestInitialize() { CosmosClientOptions clientOptions = new CosmosClientOptions(); - clientOptions.AllowBatchRequests = true; + clientOptions.AllowBulkExecution = true; CosmosClient client = TestCommon.CreateCosmosClient(clientOptions); DatabaseResponse response = await client.CreateDatabaseIfNotExistsAsync(Guid.NewGuid().ToString()); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs index ca841edf54..cde7ba84a3 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs @@ -63,7 +63,7 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated() Assert.AreNotEqual(requestTimeout, clientOptions.RequestTimeout); Assert.AreNotEqual(userAgentSuffix, clientOptions.ApplicationName); Assert.AreNotEqual(apiType, clientOptions.ApiType); - Assert.IsFalse(clientOptions.AllowBatchRequests); + Assert.IsFalse(clientOptions.AllowBulkExecution); Assert.AreEqual(0, clientOptions.CustomHandlers.Count); Assert.IsNull(clientOptions.SerializerOptions); Assert.IsNull(clientOptions.Serializer); @@ -87,7 +87,7 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated() .AddCustomHandlers(preProcessHandler) .WithApiType(apiType) .WithThrottlingRetryOptions(maxRetryWaitTime, maxRetryAttemptsOnThrottledRequests) - .WithBatchRequests(true) + .WithBulkexecution(true) .WithSerializerOptions(cosmosSerializerOptions); cosmosClient = cosmosClientBuilder.Build(new MockDocumentClient()); @@ -107,7 +107,7 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated() Assert.AreEqual(cosmosSerializerOptions.PropertyNamingPolicy, clientOptions.SerializerOptions.PropertyNamingPolicy); Assert.AreEqual(cosmosSerializerOptions.Indented, clientOptions.SerializerOptions.Indented); Assert.IsTrue(object.ReferenceEquals(webProxy, clientOptions.WebProxy)); - Assert.IsTrue(clientOptions.AllowBatchRequests); + Assert.IsTrue(clientOptions.AllowBulkExecution); //Verify GetConnectionPolicy returns the correct values policy = clientOptions.GetConnectionPolicy(); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs index 35e4b364ba..a50063df5c 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs @@ -170,7 +170,7 @@ public async Task AllowBatchingRequestsSendsToExecutor_CreateStream() { ClientContextCore clientContextCore = new ClientContextCore( MockCosmosUtil.CreateMockCosmosClient(), - new CosmosClientOptions() { AllowBatchRequests = true }, + new CosmosClientOptions() { AllowBulkExecution = true }, new CosmosJsonDotNetSerializer(), new CosmosJsonDotNetSerializer(), new CosmosJsonDotNetSerializer(), @@ -206,7 +206,7 @@ public async Task AllowBatchingRequestsSendsToExecutor_UpsertStream() { ClientContextCore clientContextCore = new ClientContextCore( MockCosmosUtil.CreateMockCosmosClient(), - new CosmosClientOptions() { AllowBatchRequests = true }, + new CosmosClientOptions() { AllowBulkExecution = true }, new CosmosJsonDotNetSerializer(), new CosmosJsonDotNetSerializer(), new CosmosJsonDotNetSerializer(), @@ -242,7 +242,7 @@ public async Task AllowBatchingRequestsSendsToExecutor_ReplaceStream() { ClientContextCore clientContextCore = new ClientContextCore( MockCosmosUtil.CreateMockCosmosClient(), - new CosmosClientOptions() { AllowBatchRequests = true }, + new CosmosClientOptions() { AllowBulkExecution = true }, new CosmosJsonDotNetSerializer(), new CosmosJsonDotNetSerializer(), new CosmosJsonDotNetSerializer(), @@ -279,7 +279,7 @@ public async Task AllowBatchingRequestsSendsToExecutor_ReadStream() { ClientContextCore clientContextCore = new ClientContextCore( MockCosmosUtil.CreateMockCosmosClient(), - new CosmosClientOptions() { AllowBatchRequests = true }, + new CosmosClientOptions() { AllowBulkExecution = true }, new CosmosJsonDotNetSerializer(), new CosmosJsonDotNetSerializer(), new CosmosJsonDotNetSerializer(), @@ -315,7 +315,7 @@ public async Task AllowBatchingRequestsSendsToExecutor_DeleteStream() { ClientContextCore clientContextCore = new ClientContextCore( MockCosmosUtil.CreateMockCosmosClient(), - new CosmosClientOptions() { AllowBatchRequests = true }, + new CosmosClientOptions() { AllowBulkExecution = true }, new CosmosJsonDotNetSerializer(), new CosmosJsonDotNetSerializer(), new CosmosJsonDotNetSerializer(), @@ -349,7 +349,7 @@ public async Task AllowBatchingRequestsSendsToExecutor_Create() CosmosClient cosmosClient = MockCosmosUtil.CreateMockCosmosClient(); ClientContextCore clientContextCore = new ClientContextCore( cosmosClient, - new CosmosClientOptions() { AllowBatchRequests = true }, + new CosmosClientOptions() { AllowBulkExecution = true }, new CosmosJsonDotNetSerializer(), new CosmosJsonDotNetSerializer(), null, @@ -380,7 +380,7 @@ public async Task AllowBatchingRequestsSendsToExecutor_Upsert() CosmosClient cosmosClient = MockCosmosUtil.CreateMockCosmosClient(); ClientContextCore clientContextCore = new ClientContextCore( cosmosClient, - new CosmosClientOptions() { AllowBatchRequests = true }, + new CosmosClientOptions() { AllowBulkExecution = true }, new CosmosJsonDotNetSerializer(), new CosmosJsonDotNetSerializer(), null, @@ -411,7 +411,7 @@ public async Task AllowBatchingRequestsSendsToExecutor_Replace() CosmosClient cosmosClient = MockCosmosUtil.CreateMockCosmosClient(); ClientContextCore clientContextCore = new ClientContextCore( cosmosClient, - new CosmosClientOptions() { AllowBatchRequests = true }, + new CosmosClientOptions() { AllowBulkExecution = true }, new CosmosJsonDotNetSerializer(), new CosmosJsonDotNetSerializer(), null, @@ -442,7 +442,7 @@ public async Task AllowBatchingRequestsSendsToExecutor_Read() CosmosClient cosmosClient = MockCosmosUtil.CreateMockCosmosClient(); ClientContextCore clientContextCore = new ClientContextCore( cosmosClient, - new CosmosClientOptions() { AllowBatchRequests = true }, + new CosmosClientOptions() { AllowBulkExecution = true }, new CosmosJsonDotNetSerializer(), new CosmosJsonDotNetSerializer(), null, @@ -473,7 +473,7 @@ public async Task AllowBatchingRequestsSendsToExecutor_Delete() CosmosClient cosmosClient = MockCosmosUtil.CreateMockCosmosClient(); ClientContextCore clientContextCore = new ClientContextCore( cosmosClient, - new CosmosClientOptions() { AllowBatchRequests = true }, + new CosmosClientOptions() { AllowBulkExecution = true }, new CosmosJsonDotNetSerializer(), new CosmosJsonDotNetSerializer(), null, From 0420d83eb9ec0260e5ab2dfdeabd08bfa223b9fb Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Thu, 5 Sep 2019 13:57:52 -0700 Subject: [PATCH 50/54] Duplicat test --- .../Batch/BatchAsyncContainerExecutorTests.cs | 54 ------------------- 1 file changed, 54 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncContainerExecutorTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncContainerExecutorTests.cs index f162ed2b18..7c72f09185 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncContainerExecutorTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncContainerExecutorTests.cs @@ -153,60 +153,6 @@ public async Task RetryOn429() Assert.AreEqual(2, diagnosticsLines); } - [TestMethod] - public async Task RetryOn429() - { - ItemBatchOperation itemBatchOperation = CreateItem("test"); - - Mock mockedContext = new Mock(); - mockedContext.Setup(c => c.ClientOptions).Returns(new CosmosClientOptions()); - mockedContext - .SetupSequence(c => c.ProcessResourceOperationStreamAsync( - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny>(), - It.IsAny())) - .Returns(this.Generate429ResponseAsync(itemBatchOperation)) - .Returns(this.GenerateOkResponseAsync(itemBatchOperation)); - - mockedContext.Setup(c => c.CosmosSerializer).Returns(new CosmosJsonDotNetSerializer()); - - Uri link = new Uri($"/dbs/db/colls/colls", UriKind.Relative); - Mock mockContainer = new Mock(); - mockContainer.Setup(x => x.LinkUri).Returns(link); - mockContainer.Setup(x => x.GetPartitionKeyDefinitionAsync(It.IsAny())).Returns(Task.FromResult(new PartitionKeyDefinition() { Paths = new Collection() { "/id" } })); - - CollectionRoutingMap routingMap = CollectionRoutingMap.TryCreateCompleteRoutingMap( - new[] - { - Tuple.Create(new PartitionKeyRange{ Id = "0", MinInclusive = "", MaxExclusive = "FF"}, (ServiceIdentity)null) - }, - string.Empty); - mockContainer.Setup(x => x.GetRoutingMapAsync(It.IsAny())).Returns(Task.FromResult(routingMap)); - BatchAsyncContainerExecutor executor = new BatchAsyncContainerExecutor(mockContainer.Object, mockedContext.Object, 20, Constants.MaxDirectModeBatchRequestBodySizeInBytes, 1); - BatchOperationResult result = await executor.AddAsync(itemBatchOperation); - - Mock.Get(mockContainer.Object) - .Verify(x => x.GetPartitionKeyDefinitionAsync(It.IsAny()), Times.Exactly(2)); - Mock.Get(mockedContext.Object) - .Verify(c => c.ProcessResourceOperationStreamAsync( - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny>(), - It.IsAny()), Times.Exactly(2)); - Assert.AreEqual(HttpStatusCode.OK, result.StatusCode); - } - [TestMethod] public async Task DoesNotRecalculatePartitionKeyRangeOnNoSplits() { From 2364c9f87c656e75882cad6d0b51624b9920b894 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Thu, 5 Sep 2019 16:14:12 -0700 Subject: [PATCH 51/54] Adding created and completed --- .../src/Batch/ItemBatchOperationContext.cs | 2 +- .../Settings/ItemBatchOperationStatistics.cs | 16 ++++++++++++- .../Batch/BatchAsyncContainerExecutorTests.cs | 4 ++-- .../ItemBatchOperationStatisticsTests.cs | 23 +++++++++++++++++++ 4 files changed, 41 insertions(+), 4 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperationContext.cs b/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperationContext.cs index bc8b65607f..4c0bb7bcd2 100644 --- a/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperationContext.cs +++ b/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperationContext.cs @@ -58,7 +58,7 @@ public void Complete( { if (this.AssertBatcher(completer)) { - result.Diagnostics = this.Diagnostics; + this.Diagnostics.Complete(result); this.taskCompletionSource.SetResult(result); } diff --git a/Microsoft.Azure.Cosmos/src/Resource/Settings/ItemBatchOperationStatistics.cs b/Microsoft.Azure.Cosmos/src/Resource/Settings/ItemBatchOperationStatistics.cs index 976750d2ae..e8744ae205 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Settings/ItemBatchOperationStatistics.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Settings/ItemBatchOperationStatistics.cs @@ -4,6 +4,7 @@ namespace Microsoft.Azure.Cosmos { + using System; using System.Collections.Generic; using System.Text; @@ -12,13 +13,21 @@ namespace Microsoft.Azure.Cosmos /// internal class ItemBatchOperationStatistics : CosmosDiagnostics { + private readonly DateTime created = DateTime.UtcNow; private readonly List cosmosDiagnostics = new List(); + private DateTime completed; public void AppendDiagnostics(CosmosDiagnostics diagnostics) { this.cosmosDiagnostics.Add(diagnostics); } + public void Complete(BatchOperationResult result) + { + this.completed = DateTime.UtcNow; + result.Diagnostics = this; + } + public override string ToString() { if (this.cosmosDiagnostics.Count == 0) @@ -26,7 +35,12 @@ public override string ToString() return string.Empty; } - StringBuilder statistics = new StringBuilder(); + StringBuilder statistics = new StringBuilder($"Bulk operation started at {this.created}. "); + if (this.completed != null) + { + statistics.Append($"Completed at {this.completed}. "); + } + foreach (CosmosDiagnostics pointOperationStatistic in this.cosmosDiagnostics) { statistics.AppendLine(pointOperationStatistic.ToString()); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncContainerExecutorTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncContainerExecutorTests.cs index 7c72f09185..cfb658af6e 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncContainerExecutorTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncContainerExecutorTests.cs @@ -84,7 +84,7 @@ public async Task RetryOnSplit() index = diagnosticsString.IndexOf(Environment.NewLine, index + 1); } - Assert.AreEqual(2, diagnosticsLines); + Assert.IsTrue(diagnosticsLines > 2); } [TestMethod] @@ -150,7 +150,7 @@ public async Task RetryOn429() index = diagnosticsString.IndexOf(Environment.NewLine, index + 1); } - Assert.AreEqual(2, diagnosticsLines); + Assert.IsTrue(diagnosticsLines > 2); } [TestMethod] diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/ItemBatchOperationStatisticsTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/ItemBatchOperationStatisticsTests.cs index 707a7b0d44..4c32274f88 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/ItemBatchOperationStatisticsTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/ItemBatchOperationStatisticsTests.cs @@ -5,6 +5,7 @@ namespace Microsoft.Azure.Cosmos.Tests { using System; + using System.Net; using Microsoft.VisualStudio.TestTools.UnitTesting; [TestClass] @@ -38,5 +39,27 @@ public void ToString_ReturnItemsToString() Assert.IsTrue(toString.Contains(pointOperation1.ToString())); Assert.IsTrue(toString.Contains(pointOperation2.ToString())); } + + [TestMethod] + public void Complete_AttachesDiagnostics() + { + ItemBatchOperationStatistics itemBatchOperationStatistics = new ItemBatchOperationStatistics(); + + CosmosClientSideRequestStatistics cosmosClientSideRequestStatistics1 = new CosmosClientSideRequestStatistics(); + cosmosClientSideRequestStatistics1.ContactedReplicas.Add(new Uri("https://one.com")); + PointOperationStatistics pointOperation1 = new PointOperationStatistics(cosmosClientSideRequestStatistics1); + + CosmosClientSideRequestStatistics cosmosClientSideRequestStatistics2 = new CosmosClientSideRequestStatistics(); + cosmosClientSideRequestStatistics2.ContactedReplicas.Add(new Uri("https://two.com")); + PointOperationStatistics pointOperation2 = new PointOperationStatistics(cosmosClientSideRequestStatistics2); + + itemBatchOperationStatistics.AppendDiagnostics(pointOperation1); + itemBatchOperationStatistics.AppendDiagnostics(pointOperation2); + + BatchOperationResult result = new BatchOperationResult(HttpStatusCode.OK); + itemBatchOperationStatistics.Complete(result); + + Assert.AreEqual(itemBatchOperationStatistics, result.Diagnostics); + } } } From b6c7c30346ab2365f5f60658eccc774549f0e7a5 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Fri, 6 Sep 2019 14:40:11 -0700 Subject: [PATCH 52/54] set is private --- Microsoft.Azure.Cosmos/src/Batch/BatchResponse.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos/src/Batch/BatchResponse.cs b/Microsoft.Azure.Cosmos/src/Batch/BatchResponse.cs index c210979837..fc4184b19a 100644 --- a/Microsoft.Azure.Cosmos/src/Batch/BatchResponse.cs +++ b/Microsoft.Azure.Cosmos/src/Batch/BatchResponse.cs @@ -147,7 +147,7 @@ public virtual bool IsSuccessStatusCode /// /// Gets the cosmos diagnostic information for the current request to Azure Cosmos DB service /// - public virtual CosmosDiagnostics Diagnostics { get; set; } + public virtual CosmosDiagnostics Diagnostics { get; } internal virtual SubStatusCodes SubStatusCode { get; } From a234ffd04f4dfb5ad24185692ed705b90053fce9 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Fri, 6 Sep 2019 15:38:45 -0700 Subject: [PATCH 53/54] Avoid creating executor instances --- .../src/Resource/Container/ContainerCore.cs | 6 ++- .../CosmosClientResourceUnitTests.cs | 42 +++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.cs b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.cs index 2de7bbe418..b31f25dd19 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.cs @@ -8,7 +8,6 @@ namespace Microsoft.Azure.Cosmos using System.IO; using System.Threading; using System.Threading.Tasks; - using Microsoft.Azure.Cosmos.Handlers; using Microsoft.Azure.Cosmos.Routing; using Microsoft.Azure.Cosmos.Scripts; using Microsoft.Azure.Documents; @@ -266,6 +265,11 @@ internal virtual Task GetRoutingMapAsync(CancellationToken internal virtual BatchAsyncContainerExecutor InitializeBatchExecutorForContainer() { + if (!this.ClientContext.ClientOptions.AllowBulkExecution) + { + return null; + } + return new BatchAsyncContainerExecutor( this, this.ClientContext, diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientResourceUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientResourceUnitTests.cs index 97ba44a1dd..f2198540f8 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientResourceUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientResourceUnitTests.cs @@ -114,5 +114,47 @@ public void ValidateSetItemRequestOptions() Assert.IsTrue(httpRequest.Headers.TryGetValue(HttpConstants.HttpHeaders.PostTriggerInclude, out string postTriggerHeader)); } + [TestMethod] + public void InitializeBatchExecutorForContainer_Null_WhenAllowBulk_False() + { + string databaseId = "db1234"; + string crId = "cr42"; + + CosmosClientContext context = new ClientContextCore( + client: null, + clientOptions: new CosmosClientOptions(), + userJsonSerializer: null, + defaultJsonSerializer: null, + sqlQuerySpecSerializer: null, + cosmosResponseFactory: null, + requestHandler: null, + documentClient: null); + + DatabaseCore db = new DatabaseCore(context, databaseId); + ContainerCore container = new ContainerCore(context, db, crId); + Assert.IsNull(container.BatchExecutor); + } + + [TestMethod] + public void InitializeBatchExecutorForContainer_NotNull_WhenAllowBulk_True() + { + string databaseId = "db1234"; + string crId = "cr42"; + + CosmosClientContext context = new ClientContextCore( + client: null, + clientOptions: new CosmosClientOptions() { AllowBulkExecution = true }, + userJsonSerializer: null, + defaultJsonSerializer: null, + sqlQuerySpecSerializer: null, + cosmosResponseFactory: null, + requestHandler: null, + documentClient: null); + + DatabaseCore db = new DatabaseCore(context, databaseId); + ContainerCore container = new ContainerCore(context, db, crId); + Assert.IsNotNull(container.BatchExecutor); + } + } } From bcba0dd39e9b4ce0e5f281abe1fc70658a3c9c90 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Mon, 9 Sep 2019 06:44:12 -0700 Subject: [PATCH 54/54] Set diagnostics independently --- .../src/Batch/ItemBatchOperationContext.cs | 3 ++- .../src/Resource/Settings/ItemBatchOperationStatistics.cs | 3 +-- .../Batch/ItemBatchOperationStatisticsTests.cs | 8 +++----- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperationContext.cs b/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperationContext.cs index 4c0bb7bcd2..18ca705c71 100644 --- a/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperationContext.cs +++ b/Microsoft.Azure.Cosmos/src/Batch/ItemBatchOperationContext.cs @@ -58,7 +58,8 @@ public void Complete( { if (this.AssertBatcher(completer)) { - this.Diagnostics.Complete(result); + this.Diagnostics.Complete(); + result.Diagnostics = this.Diagnostics; this.taskCompletionSource.SetResult(result); } diff --git a/Microsoft.Azure.Cosmos/src/Resource/Settings/ItemBatchOperationStatistics.cs b/Microsoft.Azure.Cosmos/src/Resource/Settings/ItemBatchOperationStatistics.cs index e8744ae205..f747ccad70 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Settings/ItemBatchOperationStatistics.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Settings/ItemBatchOperationStatistics.cs @@ -22,10 +22,9 @@ public void AppendDiagnostics(CosmosDiagnostics diagnostics) this.cosmosDiagnostics.Add(diagnostics); } - public void Complete(BatchOperationResult result) + public void Complete() { this.completed = DateTime.UtcNow; - result.Diagnostics = this; } public override string ToString() diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/ItemBatchOperationStatisticsTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/ItemBatchOperationStatisticsTests.cs index 4c32274f88..1291218c4f 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/ItemBatchOperationStatisticsTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/ItemBatchOperationStatisticsTests.cs @@ -41,7 +41,7 @@ public void ToString_ReturnItemsToString() } [TestMethod] - public void Complete_AttachesDiagnostics() + public void Complete_AddsCompleteTime() { ItemBatchOperationStatistics itemBatchOperationStatistics = new ItemBatchOperationStatistics(); @@ -55,11 +55,9 @@ public void Complete_AttachesDiagnostics() itemBatchOperationStatistics.AppendDiagnostics(pointOperation1); itemBatchOperationStatistics.AppendDiagnostics(pointOperation2); + itemBatchOperationStatistics.Complete(); - BatchOperationResult result = new BatchOperationResult(HttpStatusCode.OK); - itemBatchOperationStatistics.Complete(result); - - Assert.AreEqual(itemBatchOperationStatistics, result.Diagnostics); + Assert.IsTrue(itemBatchOperationStatistics.ToString().Contains("Completed at")); } } }