diff --git a/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncContainerExecutor.cs b/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncContainerExecutor.cs index d8e6b54008..7e90e4fd99 100644 --- a/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncContainerExecutor.cs +++ b/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncContainerExecutor.cs @@ -84,7 +84,7 @@ public virtual async Task AddAsync( string resolvedPartitionKeyRangeId = await this.ResolvePartitionKeyRangeIdAsync(operation, cancellationToken).ConfigureAwait(false); BatchAsyncStreamer streamer = this.GetOrAddStreamerForPartitionKeyRange(resolvedPartitionKeyRangeId); - ItemBatchOperationContext context = new ItemBatchOperationContext(resolvedPartitionKeyRangeId, BatchAsyncContainerExecutor.GetRetryPolicy(this.cosmosContainer, this.retryOptions)); + ItemBatchOperationContext context = new ItemBatchOperationContext(resolvedPartitionKeyRangeId, BatchAsyncContainerExecutor.GetRetryPolicy(this.cosmosContainer, operation.OperationType, this.retryOptions)); operation.AttachContext(context); streamer.Add(operation); return await context.OperationTask; @@ -133,10 +133,12 @@ internal virtual async Task ValidateOperationAsync( private static IDocumentClientRetryPolicy GetRetryPolicy( ContainerInternal containerInternal, + OperationType operationType, RetryOptions retryOptions) { - return new BulkPartitionKeyRangeGoneRetryPolicy( + return new BulkExecutionRetryPolicy( containerInternal, + operationType, new ResourceThrottleRetryPolicy( retryOptions.MaxRetryAttemptsOnThrottledRequests, retryOptions.MaxRetryWaitTimeInSeconds)); diff --git a/Microsoft.Azure.Cosmos/src/BulkPartitionKeyRangeGoneRetryPolicy.cs b/Microsoft.Azure.Cosmos/src/BulkExecutionRetryPolicy.cs similarity index 79% rename from Microsoft.Azure.Cosmos/src/BulkPartitionKeyRangeGoneRetryPolicy.cs rename to Microsoft.Azure.Cosmos/src/BulkExecutionRetryPolicy.cs index 0191c13e52..9195415be5 100644 --- a/Microsoft.Azure.Cosmos/src/BulkPartitionKeyRangeGoneRetryPolicy.cs +++ b/Microsoft.Azure.Cosmos/src/BulkExecutionRetryPolicy.cs @@ -16,16 +16,21 @@ namespace Microsoft.Azure.Cosmos /// /// /// - internal sealed class BulkPartitionKeyRangeGoneRetryPolicy : IDocumentClientRetryPolicy + internal sealed class BulkExecutionRetryPolicy : IDocumentClientRetryPolicy { + private const int MaxRetryOn410 = 10; private readonly IDocumentClientRetryPolicy nextRetryPolicy; + private readonly OperationType operationType; private readonly ContainerInternal container; + private int retriesOn410 = 0; - public BulkPartitionKeyRangeGoneRetryPolicy( + public BulkExecutionRetryPolicy( ContainerInternal container, + OperationType operationType, IDocumentClientRetryPolicy nextRetryPolicy) { this.container = container ?? throw new ArgumentNullException(nameof(container)); + this.operationType = operationType; this.nextRetryPolicy = nextRetryPolicy; } @@ -80,6 +85,8 @@ public void OnBeforeSendRequest(DocumentServiceRequest request) this.nextRetryPolicy.OnBeforeSendRequest(request); } + private bool IsReadRequest => this.operationType == OperationType.Read; + private async Task ShouldRetryInternalAsync( HttpStatusCode? statusCode, SubStatusCodes? subStatusCode, @@ -87,6 +94,13 @@ private async Task ShouldRetryInternalAsync( { if (statusCode == HttpStatusCode.Gone) { + this.retriesOn410++; + + if (this.retriesOn410 > MaxRetryOn410) + { + return ShouldRetryResult.NoRetry(); + } + if (subStatusCode == SubStatusCodes.PartitionKeyRangeGone || subStatusCode == SubStatusCodes.CompletingSplit || subStatusCode == SubStatusCodes.CompletingPartitionMigration) @@ -103,6 +117,14 @@ private async Task ShouldRetryInternalAsync( } } + // Batch API can return 413 which means the response is bigger than 4Mb. + // Operations that exceed the 4Mb limit are returned as 413, while the operations within the 4Mb limit will be 200 + if (this.IsReadRequest + && statusCode == HttpStatusCode.RequestEntityTooLarge) + { + return ShouldRetryResult.RetryAfter(TimeSpan.Zero); + } + return null; } } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncBatcherTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncBatcherTests.cs index e0e85b85a2..5576c6f42a 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncBatcherTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncBatcherTests.cs @@ -203,6 +203,59 @@ private BatchAsyncBatcherExecuteDelegate ExecutorWithCompletingPartitionMigratio return new PartitionKeyRangeBatchExecutionResult(request.PartitionKeyRangeId, request.Operations, batchresponse); }; + private readonly BatchAsyncBatcherExecuteDelegate ExecutorWith413 + = async (PartitionKeyRangeServerBatchRequest request, CancellationToken cancellationToken) => + { + List results = new List(); + ItemBatchOperation[] arrayOperations = new ItemBatchOperation[request.Operations.Count]; + int index = 0; + foreach (ItemBatchOperation operation in request.Operations) + { + if (index == 0) + { + // First operation is fine + results.Add( + new TransactionalBatchOperationResult(HttpStatusCode.OK) + { + ETag = operation.Id + }); + } + else + { + // second operation is too big + results.Add( + new TransactionalBatchOperationResult(HttpStatusCode.RequestEntityTooLarge) + { + ETag = operation.Id + }); + } + + arrayOperations[index++] = operation; + } + + MemoryStream responseContent = await new BatchResponsePayloadWriter(results).GeneratePayloadAsync(); + + SinglePartitionKeyServerBatchRequest batchRequest = await SinglePartitionKeyServerBatchRequest.CreateAsync( + partitionKey: null, + operations: new ArraySegment(arrayOperations), + serializerCore: MockCosmosUtil.Serializer, + cancellationToken: cancellationToken); + + ResponseMessage responseMessage = new ResponseMessage((HttpStatusCode)207) + { + Content = responseContent + }; + + TransactionalBatchResponse batchresponse = await TransactionalBatchResponse.FromResponseMessageAsync( + responseMessage, + batchRequest, + MockCosmosUtil.Serializer, + true, + CancellationToken.None); + + return new PartitionKeyRangeBatchExecutionResult(request.PartitionKeyRangeId, request.Operations, batchresponse); + }; + // The response will include all but 2 operation responses private BatchAsyncBatcherExecuteDelegate ExecutorWithLessResponses = async (PartitionKeyRangeServerBatchRequest request, CancellationToken cancellationToken) => @@ -448,12 +501,14 @@ public async Task CannotAddToDispatchedBatch() [TestMethod] public async Task RetrierGetsCalledOnSplit() { - IDocumentClientRetryPolicy retryPolicy1 = new BulkPartitionKeyRangeGoneRetryPolicy( + IDocumentClientRetryPolicy retryPolicy1 = new BulkExecutionRetryPolicy( GetSplitEnabledContainer(), + OperationType.Read, new ResourceThrottleRetryPolicy(1)); - IDocumentClientRetryPolicy retryPolicy2 = new BulkPartitionKeyRangeGoneRetryPolicy( + IDocumentClientRetryPolicy retryPolicy2 = new BulkExecutionRetryPolicy( GetSplitEnabledContainer(), + OperationType.Read, new ResourceThrottleRetryPolicy(1)); ItemBatchOperation operation1 = this.CreateItemBatchOperation(); @@ -477,12 +532,14 @@ public async Task RetrierGetsCalledOnSplit() [TestMethod] public async Task RetrierGetsCalledOnCompletingSplit() { - IDocumentClientRetryPolicy retryPolicy1 = new BulkPartitionKeyRangeGoneRetryPolicy( + IDocumentClientRetryPolicy retryPolicy1 = new BulkExecutionRetryPolicy( GetSplitEnabledContainer(), + OperationType.Read, new ResourceThrottleRetryPolicy(1)); - IDocumentClientRetryPolicy retryPolicy2 = new BulkPartitionKeyRangeGoneRetryPolicy( + IDocumentClientRetryPolicy retryPolicy2 = new BulkExecutionRetryPolicy( GetSplitEnabledContainer(), + OperationType.Read, new ResourceThrottleRetryPolicy(1)); ItemBatchOperation operation1 = this.CreateItemBatchOperation(); @@ -506,12 +563,14 @@ public async Task RetrierGetsCalledOnCompletingSplit() [TestMethod] public async Task RetrierGetsCalledOnCompletingPartitionMigration() { - IDocumentClientRetryPolicy retryPolicy1 = new BulkPartitionKeyRangeGoneRetryPolicy( + IDocumentClientRetryPolicy retryPolicy1 = new BulkExecutionRetryPolicy( GetSplitEnabledContainer(), + OperationType.Read, new ResourceThrottleRetryPolicy(1)); - IDocumentClientRetryPolicy retryPolicy2 = new BulkPartitionKeyRangeGoneRetryPolicy( + IDocumentClientRetryPolicy retryPolicy2 = new BulkExecutionRetryPolicy( GetSplitEnabledContainer(), + OperationType.Read, new ResourceThrottleRetryPolicy(1)); ItemBatchOperation operation1 = this.CreateItemBatchOperation(); @@ -552,6 +611,64 @@ public async Task RetrierGetsCalledOnOverFlow() retryDelegate.Verify(a => a(It.IsAny(), It.IsAny()), Times.Once); } + [TestMethod] + public async Task RetrierGetsCalledOn413_OnRead() + { + IDocumentClientRetryPolicy retryPolicy1 = new BulkExecutionRetryPolicy( + GetSplitEnabledContainer(), + OperationType.Read, + new ResourceThrottleRetryPolicy(1)); + + IDocumentClientRetryPolicy retryPolicy2 = new BulkExecutionRetryPolicy( + GetSplitEnabledContainer(), + OperationType.Read, + new ResourceThrottleRetryPolicy(1)); + + ItemBatchOperation operation1 = this.CreateItemBatchOperation(); + ItemBatchOperation operation2 = this.CreateItemBatchOperation(); + operation1.AttachContext(new ItemBatchOperationContext(string.Empty, retryPolicy1)); + operation2.AttachContext(new ItemBatchOperationContext(string.Empty, retryPolicy2)); + + Mock retryDelegate = new Mock(); + + BatchAsyncBatcher batchAsyncBatcher = new BatchAsyncBatcher(2, 1000, MockCosmosUtil.Serializer, this.ExecutorWith413, retryDelegate.Object); + Assert.IsTrue(batchAsyncBatcher.TryAdd(operation1)); + Assert.IsTrue(batchAsyncBatcher.TryAdd(operation2)); + await batchAsyncBatcher.DispatchAsync(metric); + retryDelegate.Verify(a => a(It.Is(o => o == operation1), It.IsAny()), Times.Never); + retryDelegate.Verify(a => a(It.Is(o => o == operation2), It.IsAny()), Times.Once); + retryDelegate.Verify(a => a(It.IsAny(), It.IsAny()), Times.Once); + } + + [TestMethod] + public async Task RetrierGetsCalledOn413_OnWrite() + { + IDocumentClientRetryPolicy retryPolicy1 = new BulkExecutionRetryPolicy( + GetSplitEnabledContainer(), + OperationType.Create, + new ResourceThrottleRetryPolicy(1)); + + IDocumentClientRetryPolicy retryPolicy2 = new BulkExecutionRetryPolicy( + GetSplitEnabledContainer(), + OperationType.Create, + new ResourceThrottleRetryPolicy(1)); + + ItemBatchOperation operation1 = this.CreateItemBatchOperation(); + ItemBatchOperation operation2 = this.CreateItemBatchOperation(); + operation1.AttachContext(new ItemBatchOperationContext(string.Empty, retryPolicy1)); + operation2.AttachContext(new ItemBatchOperationContext(string.Empty, retryPolicy2)); + + Mock retryDelegate = new Mock(); + + BatchAsyncBatcher batchAsyncBatcher = new BatchAsyncBatcher(2, 1000, MockCosmosUtil.Serializer, this.ExecutorWith413, retryDelegate.Object); + Assert.IsTrue(batchAsyncBatcher.TryAdd(operation1)); + Assert.IsTrue(batchAsyncBatcher.TryAdd(operation2)); + await batchAsyncBatcher.DispatchAsync(metric); + retryDelegate.Verify(a => a(It.Is(o => o == operation1), It.IsAny()), Times.Never); + retryDelegate.Verify(a => a(It.Is(o => o == operation2), It.IsAny()), Times.Never); + retryDelegate.Verify(a => a(It.IsAny(), It.IsAny()), Times.Never); + } + private static ContainerInternal GetSplitEnabledContainer() { Mock container = new Mock(); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncOperationContextTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncOperationContextTests.cs index 989506ff6d..dc218d6f61 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncOperationContextTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncOperationContextTests.cs @@ -94,8 +94,9 @@ public async Task ShouldRetry_NoPolicy() [TestMethod] public async Task ShouldRetry_WithPolicy_OnSuccess() { - IDocumentClientRetryPolicy retryPolicy = new BulkPartitionKeyRangeGoneRetryPolicy( + IDocumentClientRetryPolicy retryPolicy = new BulkExecutionRetryPolicy( Mock.Of(), + OperationType.Read, new ResourceThrottleRetryPolicy(1)); TransactionalBatchOperationResult result = new TransactionalBatchOperationResult(HttpStatusCode.OK); ItemBatchOperation operation = new ItemBatchOperation(OperationType.Create, 0, Cosmos.PartitionKey.Null); @@ -107,8 +108,9 @@ public async Task ShouldRetry_WithPolicy_OnSuccess() [TestMethod] public async Task ShouldRetry_WithPolicy_On429() { - IDocumentClientRetryPolicy retryPolicy = new BulkPartitionKeyRangeGoneRetryPolicy( + IDocumentClientRetryPolicy retryPolicy = new BulkExecutionRetryPolicy( Mock.Of(), + OperationType.Read, new ResourceThrottleRetryPolicy(1)); TransactionalBatchOperationResult result = new TransactionalBatchOperationResult((HttpStatusCode)StatusCodes.TooManyRequests); ItemBatchOperation operation = new ItemBatchOperation(OperationType.Create, 0, Cosmos.PartitionKey.Null); @@ -117,11 +119,40 @@ public async Task ShouldRetry_WithPolicy_On429() Assert.IsTrue(shouldRetryResult.ShouldRetry); } + [TestMethod] + public async Task ShouldRetry_WithPolicy_On413_OnRead() + { + IDocumentClientRetryPolicy retryPolicy = new BulkExecutionRetryPolicy( + Mock.Of(), + OperationType.Read, + new ResourceThrottleRetryPolicy(1)); + TransactionalBatchOperationResult result = new TransactionalBatchOperationResult(HttpStatusCode.RequestEntityTooLarge); + ItemBatchOperation operation = new ItemBatchOperation(OperationType.Create, 0, Cosmos.PartitionKey.Null); + operation.AttachContext(new ItemBatchOperationContext(string.Empty, retryPolicy)); + ShouldRetryResult shouldRetryResult = await operation.Context.ShouldRetryAsync(result, default); + Assert.IsTrue(shouldRetryResult.ShouldRetry); + } + + [TestMethod] + public async Task ShouldRetry_WithPolicy_On413_OnWrite() + { + IDocumentClientRetryPolicy retryPolicy = new BulkExecutionRetryPolicy( + Mock.Of(), + OperationType.Create, + new ResourceThrottleRetryPolicy(1)); + TransactionalBatchOperationResult result = new TransactionalBatchOperationResult(HttpStatusCode.RequestEntityTooLarge); + ItemBatchOperation operation = new ItemBatchOperation(OperationType.Create, 0, Cosmos.PartitionKey.Null); + operation.AttachContext(new ItemBatchOperationContext(string.Empty, retryPolicy)); + ShouldRetryResult shouldRetryResult = await operation.Context.ShouldRetryAsync(result, default); + Assert.IsFalse(shouldRetryResult.ShouldRetry); + } + [TestMethod] public async Task ShouldRetry_WithPolicy_OnSplit() { - IDocumentClientRetryPolicy retryPolicy = new BulkPartitionKeyRangeGoneRetryPolicy( + IDocumentClientRetryPolicy retryPolicy = new BulkExecutionRetryPolicy( GetSplitEnabledContainer(), + OperationType.Read, new ResourceThrottleRetryPolicy(1)); TransactionalBatchOperationResult result = new TransactionalBatchOperationResult(HttpStatusCode.Gone) { SubStatusCode = SubStatusCodes.PartitionKeyRangeGone }; ItemBatchOperation operation = new ItemBatchOperation(OperationType.Create, 0, Cosmos.PartitionKey.Null); @@ -133,8 +164,9 @@ public async Task ShouldRetry_WithPolicy_OnSplit() [TestMethod] public async Task ShouldRetry_WithPolicy_OnCompletingSplit() { - IDocumentClientRetryPolicy retryPolicy = new BulkPartitionKeyRangeGoneRetryPolicy( + IDocumentClientRetryPolicy retryPolicy = new BulkExecutionRetryPolicy( GetSplitEnabledContainer(), + OperationType.Read, new ResourceThrottleRetryPolicy(1)); TransactionalBatchOperationResult result = new TransactionalBatchOperationResult(HttpStatusCode.Gone) { SubStatusCode = SubStatusCodes.CompletingSplit }; ItemBatchOperation operation = new ItemBatchOperation(OperationType.Create, 0, Cosmos.PartitionKey.Null); @@ -146,8 +178,9 @@ public async Task ShouldRetry_WithPolicy_OnCompletingSplit() [TestMethod] public async Task ShouldRetry_WithPolicy_OnCompletingPartitionMigration() { - IDocumentClientRetryPolicy retryPolicy = new BulkPartitionKeyRangeGoneRetryPolicy( + IDocumentClientRetryPolicy retryPolicy = new BulkExecutionRetryPolicy( GetSplitEnabledContainer(), + OperationType.Read, new ResourceThrottleRetryPolicy(1)); TransactionalBatchOperationResult result = new TransactionalBatchOperationResult(HttpStatusCode.Gone) { SubStatusCode = SubStatusCodes.CompletingPartitionMigration }; ItemBatchOperation operation = new ItemBatchOperation(OperationType.Create, 0, Cosmos.PartitionKey.Null); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BulkPartitionKeyRangeGoneRetryPolicyTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BulkPartitionKeyRangeGoneRetryPolicyTests.cs index 0548932c00..b86de0d195 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BulkPartitionKeyRangeGoneRetryPolicyTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BulkPartitionKeyRangeGoneRetryPolicyTests.cs @@ -20,8 +20,9 @@ public class BulkPartitionKeyRangeGoneRetryPolicyTests [TestMethod] public async Task NotRetryOnSuccess() { - IDocumentClientRetryPolicy retryPolicy = new BulkPartitionKeyRangeGoneRetryPolicy( + IDocumentClientRetryPolicy retryPolicy = new BulkExecutionRetryPolicy( Mock.Of(), + OperationType.Read, new ResourceThrottleRetryPolicy(1)); TransactionalBatchOperationResult result = new TransactionalBatchOperationResult(HttpStatusCode.OK); @@ -32,8 +33,9 @@ public async Task NotRetryOnSuccess() [TestMethod] public async Task RetriesOn429() { - IDocumentClientRetryPolicy retryPolicy = new BulkPartitionKeyRangeGoneRetryPolicy( + IDocumentClientRetryPolicy retryPolicy = new BulkExecutionRetryPolicy( Mock.Of(), + OperationType.Read, new ResourceThrottleRetryPolicy(1)); TransactionalBatchOperationResult result = new TransactionalBatchOperationResult((HttpStatusCode)StatusCodes.TooManyRequests); @@ -41,11 +43,38 @@ public async Task RetriesOn429() Assert.IsTrue(shouldRetryResult.ShouldRetry); } + [TestMethod] + public async Task RetriesOn413_OnRead() + { + IDocumentClientRetryPolicy retryPolicy = new BulkExecutionRetryPolicy( + Mock.Of(), + OperationType.Read, + new ResourceThrottleRetryPolicy(1)); + + TransactionalBatchOperationResult result = new TransactionalBatchOperationResult(HttpStatusCode.RequestEntityTooLarge); + ShouldRetryResult shouldRetryResult = await retryPolicy.ShouldRetryAsync(result.ToResponseMessage(), default); + Assert.IsTrue(shouldRetryResult.ShouldRetry); + } + + [TestMethod] + public async Task RetriesOn413_OnWrite() + { + IDocumentClientRetryPolicy retryPolicy = new BulkExecutionRetryPolicy( + Mock.Of(), + OperationType.Create, + new ResourceThrottleRetryPolicy(1)); + + TransactionalBatchOperationResult result = new TransactionalBatchOperationResult(HttpStatusCode.RequestEntityTooLarge); + ShouldRetryResult shouldRetryResult = await retryPolicy.ShouldRetryAsync(result.ToResponseMessage(), default); + Assert.IsFalse(shouldRetryResult.ShouldRetry); + } + [TestMethod] public async Task RetriesOnSplits() { - IDocumentClientRetryPolicy retryPolicy = new BulkPartitionKeyRangeGoneRetryPolicy( + IDocumentClientRetryPolicy retryPolicy = new BulkExecutionRetryPolicy( GetSplitEnabledContainer(), + OperationType.Read, new ResourceThrottleRetryPolicy(1)); TransactionalBatchOperationResult result = new TransactionalBatchOperationResult(HttpStatusCode.Gone) { SubStatusCode = SubStatusCodes.PartitionKeyRangeGone }; @@ -53,11 +82,32 @@ public async Task RetriesOnSplits() Assert.IsTrue(shouldRetryResult.ShouldRetry); } + [TestMethod] + public async Task RetriesOnSplits_UpToMax() + { + IDocumentClientRetryPolicy retryPolicy = new BulkExecutionRetryPolicy( + GetSplitEnabledContainer(), + OperationType.Read, + new ResourceThrottleRetryPolicy(1)); + + TransactionalBatchOperationResult result = new TransactionalBatchOperationResult(HttpStatusCode.Gone) { SubStatusCode = SubStatusCodes.PartitionKeyRangeGone }; + ShouldRetryResult shouldRetryResult; + for (int i = 0; i < 10; i++) + { + shouldRetryResult = await retryPolicy.ShouldRetryAsync(result.ToResponseMessage(), default); + Assert.IsTrue(shouldRetryResult.ShouldRetry); + } + + shouldRetryResult = await retryPolicy.ShouldRetryAsync(result.ToResponseMessage(), default); + Assert.IsFalse(shouldRetryResult.ShouldRetry); + } + [TestMethod] public async Task RetriesOnCompletingSplits() { - IDocumentClientRetryPolicy retryPolicy = new BulkPartitionKeyRangeGoneRetryPolicy( + IDocumentClientRetryPolicy retryPolicy = new BulkExecutionRetryPolicy( GetSplitEnabledContainer(), + OperationType.Read, new ResourceThrottleRetryPolicy(1)); TransactionalBatchOperationResult result = new TransactionalBatchOperationResult(HttpStatusCode.Gone) { SubStatusCode = SubStatusCodes.CompletingSplit }; @@ -68,8 +118,9 @@ public async Task RetriesOnCompletingSplits() [TestMethod] public async Task RetriesOnCompletingPartitionMigrationSplits() { - IDocumentClientRetryPolicy retryPolicy = new BulkPartitionKeyRangeGoneRetryPolicy( + IDocumentClientRetryPolicy retryPolicy = new BulkExecutionRetryPolicy( GetSplitEnabledContainer(), + OperationType.Read, new ResourceThrottleRetryPolicy(1)); TransactionalBatchOperationResult result = new TransactionalBatchOperationResult(HttpStatusCode.Gone) { SubStatusCode = SubStatusCodes.CompletingPartitionMigration };