From ef78559caccec2db64ab2cb01a666f137019288c Mon Sep 17 00:00:00 2001 From: Debdatta Kunda Date: Thu, 7 Sep 2023 11:55:58 -0700 Subject: [PATCH] Revert "Code changes to add retry logic for GW returned 503.9002." This reverts commit 53ef5f3c1b038d14dbb1473cafa18223b33af2ce. --- .../src/ClientRetryPolicy.cs | 873 +++++------ .../src/CosmosClientOptions.cs | 2 +- .../src/Routing/LocationCache.cs | 1336 ++++++++--------- 3 files changed, 1108 insertions(+), 1103 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/ClientRetryPolicy.cs b/Microsoft.Azure.Cosmos/src/ClientRetryPolicy.cs index 2f007c6fbf..2933baa1a9 100644 --- a/Microsoft.Azure.Cosmos/src/ClientRetryPolicy.cs +++ b/Microsoft.Azure.Cosmos/src/ClientRetryPolicy.cs @@ -1,435 +1,440 @@ -//------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -//------------------------------------------------------------ - -namespace Microsoft.Azure.Cosmos -{ - using System; - using System.Collections.Generic; - using System.Collections.ObjectModel; - using System.Net; - using System.Net.Http; - using System.Threading; - using System.Threading.Tasks; - using Microsoft.Azure.Cosmos.Core.Trace; - using Microsoft.Azure.Cosmos.Routing; - using Microsoft.Azure.Documents; - - /// - /// Client policy is combination of endpoint change retry + throttling retry. - /// - internal sealed class ClientRetryPolicy : IDocumentClientRetryPolicy - { - private const int RetryIntervalInMS = 1000; // Once we detect failover wait for 1 second before retrying request. - private const int MaxRetryCount = 120; - private const int MaxServiceUnavailableRetryCount = 1; - - private readonly IDocumentClientRetryPolicy throttlingRetry; - private readonly GlobalEndpointManager globalEndpointManager; - private readonly GlobalPartitionEndpointManager partitionKeyRangeLocationCache; - private readonly bool enableEndpointDiscovery; - private int failoverRetryCount; - - private int sessionTokenRetryCount; - private int serviceUnavailableRetryCount; - private bool isReadRequest; - private bool canUseMultipleWriteLocations; - private Uri locationEndpoint; - private RetryContext retryContext; - private DocumentServiceRequest documentServiceRequest; - - public ClientRetryPolicy( - GlobalEndpointManager globalEndpointManager, - GlobalPartitionEndpointManager partitionKeyRangeLocationCache, - bool enableEndpointDiscovery, - RetryOptions retryOptions) - { - this.throttlingRetry = new ResourceThrottleRetryPolicy( - retryOptions.MaxRetryAttemptsOnThrottledRequests, - retryOptions.MaxRetryWaitTimeInSeconds); - - this.globalEndpointManager = globalEndpointManager; - this.partitionKeyRangeLocationCache = partitionKeyRangeLocationCache; - this.failoverRetryCount = 0; - this.enableEndpointDiscovery = enableEndpointDiscovery; - this.sessionTokenRetryCount = 0; - this.serviceUnavailableRetryCount = 0; - this.canUseMultipleWriteLocations = false; - } - - /// - /// Should the caller retry the operation. - /// - /// Exception that occurred when the operation was tried - /// - /// True indicates caller should retry, False otherwise - public async Task ShouldRetryAsync( - Exception exception, - CancellationToken cancellationToken) - { - this.retryContext = null; - // Received Connection error (HttpRequestException), initiate the endpoint rediscovery - if (exception is HttpRequestException _) - { - DefaultTrace.TraceWarning("ClientRetryPolicy: Gateway HttpRequestException Endpoint not reachable. Failed Location: {0}; ResourceAddress: {1}", - this.documentServiceRequest?.RequestContext?.LocationEndpointToRoute?.ToString() ?? string.Empty, - this.documentServiceRequest?.ResourceAddress ?? string.Empty); - - // Mark both read and write requests because it gateway exception. - // This means all requests going to the region will fail. - return await this.ShouldRetryOnEndpointFailureAsync( - isReadRequest: this.isReadRequest, - markBothReadAndWriteAsUnavailable: true, - forceRefresh: false, - retryOnPreferredLocations: true); - } - - if (exception is DocumentClientException clientException) - { - ShouldRetryResult shouldRetryResult = await this.ShouldRetryInternalAsync( - clientException?.StatusCode, - clientException?.GetSubStatus()); - if (shouldRetryResult != null) - { - return shouldRetryResult; - } - } - - return await this.throttlingRetry.ShouldRetryAsync(exception, cancellationToken); - } - - /// - /// Should the caller retry the operation. - /// - /// in return of the request - /// - /// True indicates caller should retry, False otherwise - public async Task ShouldRetryAsync( - ResponseMessage cosmosResponseMessage, - CancellationToken cancellationToken) - { - this.retryContext = null; - - ShouldRetryResult shouldRetryResult = await this.ShouldRetryInternalAsync( - cosmosResponseMessage?.StatusCode, - cosmosResponseMessage?.Headers.SubStatusCode); - if (shouldRetryResult != null) - { - return shouldRetryResult; - } - - return await this.throttlingRetry.ShouldRetryAsync(cosmosResponseMessage, cancellationToken); - } - - /// - /// Method that is called before a request is sent to allow the retry policy implementation - /// to modify the state of the request. - /// - /// The request being sent to the service. - public void OnBeforeSendRequest(DocumentServiceRequest request) - { - this.isReadRequest = request.IsReadOnlyRequest; - this.canUseMultipleWriteLocations = this.globalEndpointManager.CanUseMultipleWriteLocations(request); - this.documentServiceRequest = request; - - // clear previous location-based routing directive - request.RequestContext.ClearRouteToLocation(); - - if (this.retryContext != null) - { - if (this.retryContext.RouteToHub) - { - request.RequestContext.RouteToLocation(this.globalEndpointManager.GetHubUri()); - } - else - { - // set location-based routing directive based on request retry context - request.RequestContext.RouteToLocation(this.retryContext.RetryLocationIndex, this.retryContext.RetryRequestOnPreferredLocations); - } - } - - // Resolve the endpoint for the request and pin the resolution to the resolved endpoint - // This enables marking the endpoint unavailability on endpoint failover/unreachability - this.locationEndpoint = this.globalEndpointManager.ResolveServiceEndpoint(request); - request.RequestContext.RouteToLocation(this.locationEndpoint); - } - - private async Task ShouldRetryInternalAsync( - HttpStatusCode? statusCode, - SubStatusCodes? subStatusCode) - { - if (!statusCode.HasValue - && (!subStatusCode.HasValue - || subStatusCode.Value == SubStatusCodes.Unknown)) - { - return null; - } - - // Console.WriteLine("Status Code: " + statusCode.Value + "Sub Status Code: " + subStatusCode.Value + "IsRead Request: " + this.isReadRequest); - - // Received request timeout - if (statusCode == HttpStatusCode.RequestTimeout) - { - DefaultTrace.TraceWarning("ClientRetryPolicy: RequestTimeout. Failed Location: {0}; ResourceAddress: {1}", - this.documentServiceRequest?.RequestContext?.LocationEndpointToRoute?.ToString() ?? string.Empty, - this.documentServiceRequest?.ResourceAddress ?? string.Empty); - - // Mark the partition key range as unavailable to retry future request on a new region. - this.partitionKeyRangeLocationCache.TryMarkEndpointUnavailableForPartitionKeyRange( - this.documentServiceRequest); - } - - // Received 403.3 on write region, initiate the endpoint rediscovery - if (statusCode == HttpStatusCode.Forbidden - && subStatusCode == SubStatusCodes.WriteForbidden) - { - // It's a write forbidden so it safe to retry - if (this.partitionKeyRangeLocationCache.TryMarkEndpointUnavailableForPartitionKeyRange( - this.documentServiceRequest)) - { - return ShouldRetryResult.RetryAfter(TimeSpan.Zero); - } - - DefaultTrace.TraceWarning("ClientRetryPolicy: Endpoint not writable. Refresh cache and retry. Failed Location: {0}; ResourceAddress: {1}", - this.documentServiceRequest?.RequestContext?.LocationEndpointToRoute?.ToString() ?? string.Empty, - this.documentServiceRequest?.ResourceAddress ?? string.Empty); - - if (this.globalEndpointManager.IsMultimasterMetadataWriteRequest(this.documentServiceRequest)) - { - bool forceRefresh = false; - - if (this.retryContext != null && this.retryContext.RouteToHub) - { - forceRefresh = true; - - } - - ShouldRetryResult retryResult = await this.ShouldRetryOnEndpointFailureAsync( - isReadRequest: false, - markBothReadAndWriteAsUnavailable: false, - forceRefresh: forceRefresh, - retryOnPreferredLocations: false, - overwriteEndpointDiscovery: true); - - if (retryResult.ShouldRetry) - { - this.retryContext.RouteToHub = true; - } - - return retryResult; - } - - return await this.ShouldRetryOnEndpointFailureAsync( - isReadRequest: false, - markBothReadAndWriteAsUnavailable: false, - forceRefresh: true, - retryOnPreferredLocations: false); - } - - // Regional endpoint is not available yet for reads (e.g. add/ online of region is in progress) - if (statusCode == HttpStatusCode.Forbidden - && subStatusCode == SubStatusCodes.DatabaseAccountNotFound - && (this.isReadRequest || this.canUseMultipleWriteLocations)) - { - DefaultTrace.TraceWarning("ClientRetryPolicy: Endpoint not available for reads. Refresh cache and retry. Failed Location: {0}; ResourceAddress: {1}", - this.documentServiceRequest?.RequestContext?.LocationEndpointToRoute?.ToString() ?? string.Empty, - this.documentServiceRequest?.ResourceAddress ?? string.Empty); - - return await this.ShouldRetryOnEndpointFailureAsync( - isReadRequest: this.isReadRequest, - markBothReadAndWriteAsUnavailable: false, - forceRefresh: false, - retryOnPreferredLocations: false); - } - - if (statusCode == HttpStatusCode.NotFound - && subStatusCode == SubStatusCodes.ReadSessionNotAvailable) - { - return this.ShouldRetryOnSessionNotAvailable(); - } - - // Received 503 due to client connect timeout or Gateway - if (statusCode == HttpStatusCode.ServiceUnavailable) - { - DefaultTrace.TraceWarning("ClientRetryPolicy: ServiceUnavailable. Refresh cache and retry. Failed Location: {0}; ResourceAddress: {1}", - this.documentServiceRequest?.RequestContext?.LocationEndpointToRoute?.ToString() ?? string.Empty, - this.documentServiceRequest?.ResourceAddress ?? string.Empty); - - // Mark the partition as unavailable. - // Let the ClientRetry logic decide if the request should be retried - this.partitionKeyRangeLocationCache.TryMarkEndpointUnavailableForPartitionKeyRange( - this.documentServiceRequest); - - return this.ShouldRetryOnServiceUnavailable(); - } - - return null; - } - - private async Task ShouldRetryOnEndpointFailureAsync( - bool isReadRequest, - bool markBothReadAndWriteAsUnavailable, - bool forceRefresh, - bool retryOnPreferredLocations, - bool overwriteEndpointDiscovery = false) - { - if (this.failoverRetryCount > MaxRetryCount || (!this.enableEndpointDiscovery && !overwriteEndpointDiscovery)) - { - DefaultTrace.TraceInformation("ClientRetryPolicy: ShouldRetryOnEndpointFailureAsync() Not retrying. Retry count = {0}, Endpoint = {1}", - this.failoverRetryCount, - this.locationEndpoint?.ToString() ?? string.Empty); - return ShouldRetryResult.NoRetry(); - } - - this.failoverRetryCount++; - - if (this.locationEndpoint != null && !overwriteEndpointDiscovery) - { - if (isReadRequest || markBothReadAndWriteAsUnavailable) - { - this.globalEndpointManager.MarkEndpointUnavailableForRead(this.locationEndpoint); - } - - if (!isReadRequest || markBothReadAndWriteAsUnavailable) - { - this.globalEndpointManager.MarkEndpointUnavailableForWrite(this.locationEndpoint); - } - } - - TimeSpan retryDelay = TimeSpan.Zero; - if (!isReadRequest) - { - DefaultTrace.TraceInformation("ClientRetryPolicy: Failover happening. retryCount {0}", this.failoverRetryCount); - - if (this.failoverRetryCount > 1) - { - //if retried both endpoints, follow regular retry interval. - retryDelay = TimeSpan.FromMilliseconds(ClientRetryPolicy.RetryIntervalInMS); - } - } - else - { - retryDelay = TimeSpan.FromMilliseconds(ClientRetryPolicy.RetryIntervalInMS); - } - - await this.globalEndpointManager.RefreshLocationAsync(forceRefresh); - - int retryLocationIndex = this.failoverRetryCount; // Used to generate a round-robin effect - if (retryOnPreferredLocations) - { - retryLocationIndex = 0; // When the endpoint is marked as unavailable, it is moved to the bottom of the preferrence list - } - - this.retryContext = new RetryContext - { - RetryLocationIndex = retryLocationIndex, - RetryRequestOnPreferredLocations = retryOnPreferredLocations, - }; - - return ShouldRetryResult.RetryAfter(retryDelay); - } - - private ShouldRetryResult ShouldRetryOnSessionNotAvailable() - { - this.sessionTokenRetryCount++; - - if (!this.enableEndpointDiscovery) - { - // if endpoint discovery is disabled, the request cannot be retried anywhere else - return ShouldRetryResult.NoRetry(); - } - else - { - if (this.canUseMultipleWriteLocations) - { - ReadOnlyCollection endpoints = this.isReadRequest ? this.globalEndpointManager.ReadEndpoints : this.globalEndpointManager.WriteEndpoints; - - if (this.sessionTokenRetryCount > endpoints.Count) - { - // When use multiple write locations is true and the request has been tried - // on all locations, then don't retry the request - return ShouldRetryResult.NoRetry(); - } - else - { - this.retryContext = new RetryContext() - { - RetryLocationIndex = this.sessionTokenRetryCount, - RetryRequestOnPreferredLocations = true - }; - - return ShouldRetryResult.RetryAfter(TimeSpan.Zero); - } - } - else - { - if (this.sessionTokenRetryCount > 1) - { - // When cannot use multiple write locations, then don't retry the request if - // we have already tried this request on the write location - return ShouldRetryResult.NoRetry(); - } - else - { - this.retryContext = new RetryContext - { - RetryLocationIndex = 0, - RetryRequestOnPreferredLocations = false - }; - - return ShouldRetryResult.RetryAfter(TimeSpan.Zero); - } - } - } - } - - /// - /// For a ServiceUnavailable (503.0) we could be having a timeout from Direct/TCP locally or a request to Gateway request with a similar response due to an endpoint not yet available. - /// We try and retry the request only if there are other regions available. - /// - private ShouldRetryResult ShouldRetryOnServiceUnavailable() - { - if (this.serviceUnavailableRetryCount++ >= ClientRetryPolicy.MaxServiceUnavailableRetryCount) - { - DefaultTrace.TraceInformation($"ClientRetryPolicy: ShouldRetryOnServiceUnavailable() Not retrying. Retry count = {this.serviceUnavailableRetryCount}."); - return ShouldRetryResult.NoRetry(); - } - - /*if (!this.canUseMultipleWriteLocations - && !this.isReadRequest) - { - // Write requests on single master cannot be retried, no other regions available - return ShouldRetryResult.NoRetry(); - }*/ - - int availablePreferredLocations = this.globalEndpointManager.PreferredLocationCount; - - if (availablePreferredLocations <= 1) - { - // No other regions to retry on - DefaultTrace.TraceInformation($"ClientRetryPolicy: ShouldRetryOnServiceUnavailable() Not retrying. No other regions available for the request. AvailablePreferredLocations = {availablePreferredLocations}."); - return ShouldRetryResult.NoRetry(); - } - - DefaultTrace.TraceInformation($"ClientRetryPolicy: ShouldRetryOnServiceUnavailable() Retrying. Received on endpoint {this.locationEndpoint}, IsReadRequest = {this.isReadRequest}."); - - // Retrying on second PreferredLocations - // RetryCount is used as zero-based index - this.retryContext = new RetryContext() - { - RetryLocationIndex = this.serviceUnavailableRetryCount, - RetryRequestOnPreferredLocations = true - }; - - return ShouldRetryResult.RetryAfter(TimeSpan.Zero); - } - - private sealed class RetryContext - { - public int RetryLocationIndex { get; set; } - public bool RetryRequestOnPreferredLocations { get; set; } - - public bool RouteToHub { get; set; } - } - } +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos +{ + using System; + using System.Collections.Generic; + using System.Collections.ObjectModel; + using System.Net; + using System.Net.Http; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Core.Trace; + using Microsoft.Azure.Cosmos.Routing; + using Microsoft.Azure.Documents; + + /// + /// Client policy is combination of endpoint change retry + throttling retry. + /// + internal sealed class ClientRetryPolicy : IDocumentClientRetryPolicy + { + private const int RetryIntervalInMS = 1000; // Once we detect failover wait for 1 second before retrying request. + private const int MaxRetryCount = 120; + private const int MaxServiceUnavailableRetryCount = 1; + + private readonly IDocumentClientRetryPolicy throttlingRetry; + private readonly GlobalEndpointManager globalEndpointManager; + private readonly GlobalPartitionEndpointManager partitionKeyRangeLocationCache; + private readonly bool enableEndpointDiscovery; + private int failoverRetryCount; + + private int sessionTokenRetryCount; + private int serviceUnavailableRetryCount; + private bool isReadRequest; + private bool canUseMultipleWriteLocations; + private Uri locationEndpoint; + private RetryContext retryContext; + private DocumentServiceRequest documentServiceRequest; + + public ClientRetryPolicy( + GlobalEndpointManager globalEndpointManager, + GlobalPartitionEndpointManager partitionKeyRangeLocationCache, + bool enableEndpointDiscovery, + RetryOptions retryOptions) + { + this.throttlingRetry = new ResourceThrottleRetryPolicy( + retryOptions.MaxRetryAttemptsOnThrottledRequests, + retryOptions.MaxRetryWaitTimeInSeconds); + + this.globalEndpointManager = globalEndpointManager; + this.partitionKeyRangeLocationCache = partitionKeyRangeLocationCache; + this.failoverRetryCount = 0; + this.enableEndpointDiscovery = enableEndpointDiscovery; + this.sessionTokenRetryCount = 0; + this.serviceUnavailableRetryCount = 0; + this.canUseMultipleWriteLocations = false; + } + + /// + /// Should the caller retry the operation. + /// + /// Exception that occurred when the operation was tried + /// + /// True indicates caller should retry, False otherwise + public async Task ShouldRetryAsync( + Exception exception, + CancellationToken cancellationToken) + { + this.retryContext = null; + // Received Connection error (HttpRequestException), initiate the endpoint rediscovery + if (exception is HttpRequestException _) + { + DefaultTrace.TraceWarning("ClientRetryPolicy: Gateway HttpRequestException Endpoint not reachable. Failed Location: {0}; ResourceAddress: {1}", + this.documentServiceRequest?.RequestContext?.LocationEndpointToRoute?.ToString() ?? string.Empty, + this.documentServiceRequest?.ResourceAddress ?? string.Empty); + + // Mark both read and write requests because it gateway exception. + // This means all requests going to the region will fail. + return await this.ShouldRetryOnEndpointFailureAsync( + isReadRequest: this.isReadRequest, + markBothReadAndWriteAsUnavailable: true, + forceRefresh: false, + retryOnPreferredLocations: true); + } + + if (exception is DocumentClientException clientException) + { + ShouldRetryResult shouldRetryResult = await this.ShouldRetryInternalAsync( + clientException?.StatusCode, + clientException?.GetSubStatus()); + if (shouldRetryResult != null) + { + return shouldRetryResult; + } + } + + return await this.throttlingRetry.ShouldRetryAsync(exception, cancellationToken); + } + + /// + /// Should the caller retry the operation. + /// + /// in return of the request + /// + /// True indicates caller should retry, False otherwise + public async Task ShouldRetryAsync( + ResponseMessage cosmosResponseMessage, + CancellationToken cancellationToken) + { + this.retryContext = null; + + ShouldRetryResult shouldRetryResult = await this.ShouldRetryInternalAsync( + cosmosResponseMessage?.StatusCode, + cosmosResponseMessage?.Headers.SubStatusCode); + if (shouldRetryResult != null) + { + return shouldRetryResult; + } + + return await this.throttlingRetry.ShouldRetryAsync(cosmosResponseMessage, cancellationToken); + } + + /// + /// Method that is called before a request is sent to allow the retry policy implementation + /// to modify the state of the request. + /// + /// The request being sent to the service. + public void OnBeforeSendRequest(DocumentServiceRequest request) + { + this.isReadRequest = request.IsReadOnlyRequest; + this.canUseMultipleWriteLocations = this.globalEndpointManager.CanUseMultipleWriteLocations(request); + this.documentServiceRequest = request; + + // clear previous location-based routing directive + request.RequestContext.ClearRouteToLocation(); + + if (this.retryContext != null) + { + if (this.retryContext.RouteToHub) + { + request.RequestContext.RouteToLocation(this.globalEndpointManager.GetHubUri()); + } + else + { + // set location-based routing directive based on request retry context + request.RequestContext.RouteToLocation(this.retryContext.RetryLocationIndex, this.retryContext.RetryRequestOnPreferredLocations); + } + } + + // Resolve the endpoint for the request and pin the resolution to the resolved endpoint + // This enables marking the endpoint unavailability on endpoint failover/unreachability + this.locationEndpoint = this.globalEndpointManager.ResolveServiceEndpoint(request); + request.RequestContext.RouteToLocation(this.locationEndpoint); + } + + private async Task ShouldRetryInternalAsync( + HttpStatusCode? statusCode, + SubStatusCodes? subStatusCode) + { + if (!statusCode.HasValue + && (!subStatusCode.HasValue + || subStatusCode.Value == SubStatusCodes.Unknown)) + { + return null; + } + + // Received request timeout + if (statusCode == HttpStatusCode.RequestTimeout) + { + DefaultTrace.TraceWarning("ClientRetryPolicy: RequestTimeout. Failed Location: {0}; ResourceAddress: {1}", + this.documentServiceRequest?.RequestContext?.LocationEndpointToRoute?.ToString() ?? string.Empty, + this.documentServiceRequest?.ResourceAddress ?? string.Empty); + + // Mark the partition key range as unavailable to retry future request on a new region. + this.partitionKeyRangeLocationCache.TryMarkEndpointUnavailableForPartitionKeyRange( + this.documentServiceRequest); + } + + // Received 403.3 on write region, initiate the endpoint rediscovery + if (statusCode == HttpStatusCode.Forbidden + && subStatusCode == SubStatusCodes.WriteForbidden) + { + // It's a write forbidden so it safe to retry + if (this.partitionKeyRangeLocationCache.TryMarkEndpointUnavailableForPartitionKeyRange( + this.documentServiceRequest)) + { + return ShouldRetryResult.RetryAfter(TimeSpan.Zero); + } + + DefaultTrace.TraceWarning("ClientRetryPolicy: Endpoint not writable. Refresh cache and retry. Failed Location: {0}; ResourceAddress: {1}", + this.documentServiceRequest?.RequestContext?.LocationEndpointToRoute?.ToString() ?? string.Empty, + this.documentServiceRequest?.ResourceAddress ?? string.Empty); + + if (this.globalEndpointManager.IsMultimasterMetadataWriteRequest(this.documentServiceRequest)) + { + bool forceRefresh = false; + + if (this.retryContext != null && this.retryContext.RouteToHub) + { + forceRefresh = true; + + } + + ShouldRetryResult retryResult = await this.ShouldRetryOnEndpointFailureAsync( + isReadRequest: false, + markBothReadAndWriteAsUnavailable: false, + forceRefresh: forceRefresh, + retryOnPreferredLocations: false, + overwriteEndpointDiscovery: true); + + if (retryResult.ShouldRetry) + { + this.retryContext.RouteToHub = true; + } + + return retryResult; + } + + return await this.ShouldRetryOnEndpointFailureAsync( + isReadRequest: false, + markBothReadAndWriteAsUnavailable: false, + forceRefresh: true, + retryOnPreferredLocations: false); + } + + // Regional endpoint is not available yet for reads (e.g. add/ online of region is in progress) + if (statusCode == HttpStatusCode.Forbidden + && subStatusCode == SubStatusCodes.DatabaseAccountNotFound + && (this.isReadRequest || this.canUseMultipleWriteLocations)) + { + DefaultTrace.TraceWarning("ClientRetryPolicy: Endpoint not available for reads. Refresh cache and retry. Failed Location: {0}; ResourceAddress: {1}", + this.documentServiceRequest?.RequestContext?.LocationEndpointToRoute?.ToString() ?? string.Empty, + this.documentServiceRequest?.ResourceAddress ?? string.Empty); + + return await this.ShouldRetryOnEndpointFailureAsync( + isReadRequest: this.isReadRequest, + markBothReadAndWriteAsUnavailable: false, + forceRefresh: false, + retryOnPreferredLocations: false); + } + + if (statusCode == HttpStatusCode.NotFound + && subStatusCode == SubStatusCodes.ReadSessionNotAvailable) + { + return this.ShouldRetryOnSessionNotAvailable(); + } + + // Received 503 due to client connect timeout or Gateway + if (statusCode == HttpStatusCode.ServiceUnavailable + && ClientRetryPolicy.IsRetriableServiceUnavailable(subStatusCode)) + { + DefaultTrace.TraceWarning("ClientRetryPolicy: ServiceUnavailable. Refresh cache and retry. Failed Location: {0}; ResourceAddress: {1}", + this.documentServiceRequest?.RequestContext?.LocationEndpointToRoute?.ToString() ?? string.Empty, + this.documentServiceRequest?.ResourceAddress ?? string.Empty); + + // Mark the partition as unavailable. + // Let the ClientRetry logic decide if the request should be retried + this.partitionKeyRangeLocationCache.TryMarkEndpointUnavailableForPartitionKeyRange( + this.documentServiceRequest); + + return this.ShouldRetryOnServiceUnavailable(); + } + + return null; + } + + private static bool IsRetriableServiceUnavailable(SubStatusCodes? subStatusCode) + { + return subStatusCode == SubStatusCodes.Unknown || + (subStatusCode.HasValue && subStatusCode.Value.IsSDKGeneratedSubStatus()); + } + + private async Task ShouldRetryOnEndpointFailureAsync( + bool isReadRequest, + bool markBothReadAndWriteAsUnavailable, + bool forceRefresh, + bool retryOnPreferredLocations, + bool overwriteEndpointDiscovery = false) + { + if (this.failoverRetryCount > MaxRetryCount || (!this.enableEndpointDiscovery && !overwriteEndpointDiscovery)) + { + DefaultTrace.TraceInformation("ClientRetryPolicy: ShouldRetryOnEndpointFailureAsync() Not retrying. Retry count = {0}, Endpoint = {1}", + this.failoverRetryCount, + this.locationEndpoint?.ToString() ?? string.Empty); + return ShouldRetryResult.NoRetry(); + } + + this.failoverRetryCount++; + + if (this.locationEndpoint != null && !overwriteEndpointDiscovery) + { + if (isReadRequest || markBothReadAndWriteAsUnavailable) + { + this.globalEndpointManager.MarkEndpointUnavailableForRead(this.locationEndpoint); + } + + if (!isReadRequest || markBothReadAndWriteAsUnavailable) + { + this.globalEndpointManager.MarkEndpointUnavailableForWrite(this.locationEndpoint); + } + } + + TimeSpan retryDelay = TimeSpan.Zero; + if (!isReadRequest) + { + DefaultTrace.TraceInformation("ClientRetryPolicy: Failover happening. retryCount {0}", this.failoverRetryCount); + + if (this.failoverRetryCount > 1) + { + //if retried both endpoints, follow regular retry interval. + retryDelay = TimeSpan.FromMilliseconds(ClientRetryPolicy.RetryIntervalInMS); + } + } + else + { + retryDelay = TimeSpan.FromMilliseconds(ClientRetryPolicy.RetryIntervalInMS); + } + + await this.globalEndpointManager.RefreshLocationAsync(forceRefresh); + + int retryLocationIndex = this.failoverRetryCount; // Used to generate a round-robin effect + if (retryOnPreferredLocations) + { + retryLocationIndex = 0; // When the endpoint is marked as unavailable, it is moved to the bottom of the preferrence list + } + + this.retryContext = new RetryContext + { + RetryLocationIndex = retryLocationIndex, + RetryRequestOnPreferredLocations = retryOnPreferredLocations, + }; + + return ShouldRetryResult.RetryAfter(retryDelay); + } + + private ShouldRetryResult ShouldRetryOnSessionNotAvailable() + { + this.sessionTokenRetryCount++; + + if (!this.enableEndpointDiscovery) + { + // if endpoint discovery is disabled, the request cannot be retried anywhere else + return ShouldRetryResult.NoRetry(); + } + else + { + if (this.canUseMultipleWriteLocations) + { + ReadOnlyCollection endpoints = this.isReadRequest ? this.globalEndpointManager.ReadEndpoints : this.globalEndpointManager.WriteEndpoints; + + if (this.sessionTokenRetryCount > endpoints.Count) + { + // When use multiple write locations is true and the request has been tried + // on all locations, then don't retry the request + return ShouldRetryResult.NoRetry(); + } + else + { + this.retryContext = new RetryContext() + { + RetryLocationIndex = this.sessionTokenRetryCount, + RetryRequestOnPreferredLocations = true + }; + + return ShouldRetryResult.RetryAfter(TimeSpan.Zero); + } + } + else + { + if (this.sessionTokenRetryCount > 1) + { + // When cannot use multiple write locations, then don't retry the request if + // we have already tried this request on the write location + return ShouldRetryResult.NoRetry(); + } + else + { + this.retryContext = new RetryContext + { + RetryLocationIndex = 0, + RetryRequestOnPreferredLocations = false + }; + + return ShouldRetryResult.RetryAfter(TimeSpan.Zero); + } + } + } + } + + /// + /// For a ServiceUnavailable (503.0) we could be having a timeout from Direct/TCP locally or a request to Gateway request with a similar response due to an endpoint not yet available. + /// We try and retry the request only if there are other regions available. + /// + private ShouldRetryResult ShouldRetryOnServiceUnavailable() + { + if (this.serviceUnavailableRetryCount++ >= ClientRetryPolicy.MaxServiceUnavailableRetryCount) + { + DefaultTrace.TraceInformation($"ClientRetryPolicy: ShouldRetryOnServiceUnavailable() Not retrying. Retry count = {this.serviceUnavailableRetryCount}."); + return ShouldRetryResult.NoRetry(); + } + + if (!this.canUseMultipleWriteLocations + && !this.isReadRequest) + { + // Write requests on single master cannot be retried, no other regions available + return ShouldRetryResult.NoRetry(); + } + + int availablePreferredLocations = this.globalEndpointManager.PreferredLocationCount; + + if (availablePreferredLocations <= 1) + { + // No other regions to retry on + DefaultTrace.TraceInformation($"ClientRetryPolicy: ShouldRetryOnServiceUnavailable() Not retrying. No other regions available for the request. AvailablePreferredLocations = {availablePreferredLocations}."); + return ShouldRetryResult.NoRetry(); + } + + DefaultTrace.TraceInformation($"ClientRetryPolicy: ShouldRetryOnServiceUnavailable() Retrying. Received on endpoint {this.locationEndpoint}, IsReadRequest = {this.isReadRequest}."); + + // Retrying on second PreferredLocations + // RetryCount is used as zero-based index + this.retryContext = new RetryContext() + { + RetryLocationIndex = this.serviceUnavailableRetryCount, + RetryRequestOnPreferredLocations = true + }; + + return ShouldRetryResult.RetryAfter(TimeSpan.Zero); + } + + private sealed class RetryContext + { + public int RetryLocationIndex { get; set; } + public bool RetryRequestOnPreferredLocations { get; set; } + + public bool RouteToHub { get; set; } + } + } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs index ee67522d6b..2c07f060f8 100644 --- a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs +++ b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs @@ -608,7 +608,7 @@ public Func HttpClientFactory /// /// Enable partition key level failover /// - public bool EnablePartitionLevelFailover { get; set; } = false; + internal bool EnablePartitionLevelFailover { get; set; } = false; /// /// Quorum Read allowed with eventual consistency account or consistent prefix account. diff --git a/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs b/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs index 6f07b7a52a..9c6308d8b6 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs @@ -1,668 +1,668 @@ -//------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -//------------------------------------------------------------ - -namespace Microsoft.Azure.Cosmos.Routing -{ - using System; - using System.Collections.Concurrent; - using System.Collections.Generic; - using System.Collections.ObjectModel; - using System.Globalization; - using System.Linq; - using System.Net; - using global::Azure.Core; - using Microsoft.Azure.Cosmos.Core.Trace; - using Microsoft.Azure.Documents; - - /// - /// Implements the abstraction to resolve target location for geo-replicated DatabaseAccount - /// with multiple writable and readable locations. - /// - internal sealed class LocationCache - { - private const string UnavailableLocationsExpirationTimeInSeconds = "UnavailableLocationsExpirationTimeInSeconds"; - private static int DefaultUnavailableLocationsExpirationTimeInSeconds = 5 * 60; - - private readonly bool enableEndpointDiscovery; - private readonly Uri defaultEndpoint; - private readonly bool useMultipleWriteLocations; - private readonly object lockObject; - private readonly TimeSpan unavailableLocationsExpirationTime; - private readonly int connectionLimit; - private readonly ConcurrentDictionary locationUnavailablityInfoByEndpoint; - - private DatabaseAccountLocationsInfo locationInfo; - private DateTime lastCacheUpdateTimestamp; - private bool enableMultipleWriteLocations; - - public LocationCache( - ReadOnlyCollection preferredLocations, - Uri defaultEndpoint, - bool enableEndpointDiscovery, - int connectionLimit, - bool useMultipleWriteLocations) - { - this.locationInfo = new DatabaseAccountLocationsInfo(preferredLocations, defaultEndpoint); - this.defaultEndpoint = defaultEndpoint; - this.enableEndpointDiscovery = enableEndpointDiscovery; - this.useMultipleWriteLocations = useMultipleWriteLocations; - this.connectionLimit = connectionLimit; - - this.lockObject = new object(); - this.locationUnavailablityInfoByEndpoint = new ConcurrentDictionary(); - this.lastCacheUpdateTimestamp = DateTime.MinValue; - this.enableMultipleWriteLocations = false; - this.unavailableLocationsExpirationTime = TimeSpan.FromSeconds(LocationCache.DefaultUnavailableLocationsExpirationTimeInSeconds); - -#if !(NETSTANDARD15 || NETSTANDARD16) -#if NETSTANDARD20 - // GetEntryAssembly returns null when loaded from native netstandard2.0 - if (System.Reflection.Assembly.GetEntryAssembly() != null) - { -#endif - string unavailableLocationsExpirationTimeInSecondsConfig = System.Configuration.ConfigurationManager.AppSettings[LocationCache.UnavailableLocationsExpirationTimeInSeconds]; - if (!string.IsNullOrEmpty(unavailableLocationsExpirationTimeInSecondsConfig)) - { - int unavailableLocationsExpirationTimeinSecondsConfigValue; - - if (!int.TryParse(unavailableLocationsExpirationTimeInSecondsConfig, out unavailableLocationsExpirationTimeinSecondsConfigValue)) - { - this.unavailableLocationsExpirationTime = TimeSpan.FromSeconds(LocationCache.DefaultUnavailableLocationsExpirationTimeInSeconds); - } - else - { - this.unavailableLocationsExpirationTime = TimeSpan.FromSeconds(unavailableLocationsExpirationTimeinSecondsConfigValue); - } - } -#if NETSTANDARD20 - } -#endif -#endif - } - - /// - /// Gets list of read endpoints ordered by - /// 1. Preferred location - /// 2. Endpoint availablity - /// - public ReadOnlyCollection ReadEndpoints - { - get - { - // Hot-path: avoid ConcurrentDictionary methods which acquire locks - if (DateTime.UtcNow - this.lastCacheUpdateTimestamp > this.unavailableLocationsExpirationTime - && this.locationUnavailablityInfoByEndpoint.Any()) - { - this.UpdateLocationCache(); - } - - return this.locationInfo.ReadEndpoints; - } - } - - /// - /// Gets list of write endpoints ordered by - /// 1. Preferred location - /// 2. Endpoint availablity - /// - public ReadOnlyCollection WriteEndpoints - { - get - { - // Hot-path: avoid ConcurrentDictionary methods which acquire locks - if (DateTime.UtcNow - this.lastCacheUpdateTimestamp > this.unavailableLocationsExpirationTime - && this.locationUnavailablityInfoByEndpoint.Any()) - { - this.UpdateLocationCache(); - } - - return this.locationInfo.WriteEndpoints; - } - } - - /// - /// Returns the location corresponding to the endpoint if location specific endpoint is provided. - /// For the defaultEndPoint, we will return the first available write location. - /// Returns null, in other cases. - /// - /// - /// Today we return null for defaultEndPoint if multiple write locations can be used. - /// This needs to be modifed to figure out proper location in such case. - /// - public string GetLocation(Uri endpoint) - { - string location = this.locationInfo.AvailableWriteEndpointByLocation.FirstOrDefault(uri => uri.Value == endpoint).Key ?? this.locationInfo.AvailableReadEndpointByLocation.FirstOrDefault(uri => uri.Value == endpoint).Key; - - if (location == null && endpoint == this.defaultEndpoint && !this.CanUseMultipleWriteLocations()) - { - if (this.locationInfo.AvailableWriteEndpointByLocation.Any()) - { - return this.locationInfo.AvailableWriteEndpointByLocation.First().Key; - } - } - - return location; - } - - /// - /// Set region name for a location if present in the locationcache otherwise set region name as null. - /// If endpoint's hostname is same as default endpoint hostname, set regionName as null. - /// - /// - /// - /// true if region found else false - public bool TryGetLocationForGatewayDiagnostics(Uri endpoint, out string regionName) - { - if (Uri.Compare( - endpoint, - this.defaultEndpoint, - UriComponents.Host, - UriFormat.SafeUnescaped, - StringComparison.OrdinalIgnoreCase) == 0) - { - regionName = null; - return false; - } - - regionName = this.GetLocation(endpoint); - return true; - } - - /// - /// Marks the current location unavailable for read - /// - public void MarkEndpointUnavailableForRead(Uri endpoint) - { - this.MarkEndpointUnavailable(endpoint, OperationType.Read); - } - - /// - /// Marks the current location unavailable for write - /// - public void MarkEndpointUnavailableForWrite(Uri endpoint) - { - this.MarkEndpointUnavailable(endpoint, OperationType.Write); - } - - /// - /// Invoked when is read - /// - /// Read DatabaseAccoaunt - public void OnDatabaseAccountRead(AccountProperties databaseAccount) - { - this.UpdateLocationCache( - databaseAccount.WritableRegions, - databaseAccount.ReadableRegions, - preferenceList: null, - enableMultipleWriteLocations: databaseAccount.EnableMultipleWriteLocations); - } - - /// - /// Invoked when changes - /// - /// - public void OnLocationPreferenceChanged(ReadOnlyCollection preferredLocations) - { - this.UpdateLocationCache( - preferenceList: preferredLocations); - } - - public bool IsMetaData(DocumentServiceRequest request) - { - return (request.OperationType != Documents.OperationType.ExecuteJavaScript && request.ResourceType == ResourceType.StoredProcedure) || - request.ResourceType != ResourceType.Document; - - } - public bool IsMultimasterMetadataWriteRequest(DocumentServiceRequest request) - { - return !request.IsReadOnlyRequest && this.locationInfo.AvailableWriteLocations.Count > 1 - && this.IsMetaData(request) - && this.CanUseMultipleWriteLocations(); - - } - - public Uri GetHubUri() - { - DatabaseAccountLocationsInfo currentLocationInfo = this.locationInfo; - string writeLocation = currentLocationInfo.AvailableWriteLocations[0]; - Uri locationEndpointToRoute = currentLocationInfo.AvailableWriteEndpointByLocation[writeLocation]; - return locationEndpointToRoute; - } - - /// - /// Resolves request to service endpoint. - /// 1. If this is a write request - /// (a) If UseMultipleWriteLocations = true - /// (i) For document writes, resolve to most preferred and available write endpoint. - /// Once the endpoint is marked unavailable, it is moved to the end of available write endpoint. Current request will - /// be retried on next preferred available write endpoint. - /// (ii) For all other resources, always resolve to first/second (regardless of preferred locations) - /// write endpoint in . - /// Endpoint of first write location in is the only endpoint that supports - /// write operation on all resource types (except during that region's failover). - /// Only during manual failover, client would retry write on second write location in . - /// (b) Else resolve the request to first write endpoint in OR - /// second write endpoint in in case of manual failover of that location. - /// 2. Else resolve the request to most preferred available read endpoint (automatic failover for read requests) - /// - /// Request for which endpoint is to be resolved - /// Resolved endpoint - public Uri ResolveServiceEndpoint(DocumentServiceRequest request) - { - if (request.RequestContext != null && request.RequestContext.LocationEndpointToRoute != null) - { - return request.RequestContext.LocationEndpointToRoute; - } - - int locationIndex = request.RequestContext.LocationIndexToRoute.GetValueOrDefault(0); - - Uri locationEndpointToRoute = this.defaultEndpoint; - - if (!request.RequestContext.UsePreferredLocations.GetValueOrDefault(true) // Should not use preferred location ? - || (request.OperationType.IsWriteOperation() && !this.CanUseMultipleWriteLocations(request))) - { - // For non-document resource types in case of client can use multiple write locations - // or when client cannot use multiple write locations, flip-flop between the - // first and the second writable region in DatabaseAccount (for manual failover) - DatabaseAccountLocationsInfo currentLocationInfo = this.locationInfo; - - if (this.enableEndpointDiscovery && currentLocationInfo.AvailableWriteLocations.Count > 0) - { - locationIndex = Math.Min(locationIndex % 2, currentLocationInfo.AvailableWriteLocations.Count - 1); - string writeLocation = currentLocationInfo.AvailableWriteLocations[locationIndex]; - locationEndpointToRoute = currentLocationInfo.AvailableWriteEndpointByLocation[writeLocation]; - } - } - else - { - ReadOnlyCollection endpoints = this.ReadEndpoints; - locationEndpointToRoute = endpoints[locationIndex % endpoints.Count]; - } - - request.RequestContext.RouteToLocation(locationEndpointToRoute); - return locationEndpointToRoute; - } - - public bool ShouldRefreshEndpoints(out bool canRefreshInBackground) - { - canRefreshInBackground = true; - DatabaseAccountLocationsInfo currentLocationInfo = this.locationInfo; - - string mostPreferredLocation = currentLocationInfo.PreferredLocations.FirstOrDefault(); - - // we should schedule refresh in background if we are unable to target the user's most preferredLocation. - if (this.enableEndpointDiscovery) - { - // Refresh if client opts-in to useMultipleWriteLocations but server-side setting is disabled - bool shouldRefresh = this.useMultipleWriteLocations && !this.enableMultipleWriteLocations; - - ReadOnlyCollection readLocationEndpoints = currentLocationInfo.ReadEndpoints; - - if (this.IsEndpointUnavailable(readLocationEndpoints[0], OperationType.Read)) - { - canRefreshInBackground = readLocationEndpoints.Count > 1; - DefaultTrace.TraceInformation("ShouldRefreshEndpoints = true since the first read endpoint {0} is not available for read. canRefreshInBackground = {1}", - readLocationEndpoints[0], - canRefreshInBackground); - - return true; - } - - if (!string.IsNullOrEmpty(mostPreferredLocation)) - { - Uri mostPreferredReadEndpoint; - - if (currentLocationInfo.AvailableReadEndpointByLocation.TryGetValue(mostPreferredLocation, out mostPreferredReadEndpoint)) - { - if (mostPreferredReadEndpoint != readLocationEndpoints[0]) - { - // For reads, we can always refresh in background as we can alternate to - // other available read endpoints - DefaultTrace.TraceInformation("ShouldRefreshEndpoints = true since most preferred location {0} is not available for read.", mostPreferredLocation); - return true; - } - } - else - { - DefaultTrace.TraceInformation("ShouldRefreshEndpoints = true since most preferred location {0} is not in available read locations.", mostPreferredLocation); - return true; - } - } - - Uri mostPreferredWriteEndpoint; - ReadOnlyCollection writeLocationEndpoints = currentLocationInfo.WriteEndpoints; - - if (!this.CanUseMultipleWriteLocations()) - { - if (this.IsEndpointUnavailable(writeLocationEndpoints[0], OperationType.Write)) - { - // Since most preferred write endpoint is unavailable, we can only refresh in background if - // we have an alternate write endpoint - canRefreshInBackground = writeLocationEndpoints.Count > 1; - DefaultTrace.TraceInformation("ShouldRefreshEndpoints = true since most preferred location {0} endpoint {1} is not available for write. canRefreshInBackground = {2}", - mostPreferredLocation, - writeLocationEndpoints[0], - canRefreshInBackground); - - return true; - } - else - { - return shouldRefresh; - } - } - else if (!string.IsNullOrEmpty(mostPreferredLocation)) - { - if (currentLocationInfo.AvailableWriteEndpointByLocation.TryGetValue(mostPreferredLocation, out mostPreferredWriteEndpoint)) - { - shouldRefresh |= mostPreferredWriteEndpoint != writeLocationEndpoints[0]; - DefaultTrace.TraceInformation("ShouldRefreshEndpoints = {0} since most preferred location {1} is not available for write.", shouldRefresh, mostPreferredLocation); - return shouldRefresh; - } - else - { - DefaultTrace.TraceInformation("ShouldRefreshEndpoints = true since most preferred location {0} is not in available write locations", mostPreferredLocation); - return true; - } - } - else - { - return shouldRefresh; - } - } - else - { - return false; - } - } - - public bool CanUseMultipleWriteLocations(DocumentServiceRequest request) - { - return this.CanUseMultipleWriteLocations() && - (request.ResourceType == ResourceType.Document || - (request.ResourceType == ResourceType.StoredProcedure && request.OperationType == Documents.OperationType.ExecuteJavaScript)); - } - - private void ClearStaleEndpointUnavailabilityInfo() - { - if (this.locationUnavailablityInfoByEndpoint.Any()) - { - List unavailableEndpoints = this.locationUnavailablityInfoByEndpoint.Keys.ToList(); - - foreach (Uri unavailableEndpoint in unavailableEndpoints) - { - LocationUnavailabilityInfo unavailabilityInfo; - LocationUnavailabilityInfo removed; - - if (this.locationUnavailablityInfoByEndpoint.TryGetValue(unavailableEndpoint, out unavailabilityInfo) - && DateTime.UtcNow - unavailabilityInfo.LastUnavailabilityCheckTimeStamp > this.unavailableLocationsExpirationTime - && this.locationUnavailablityInfoByEndpoint.TryRemove(unavailableEndpoint, out removed)) - { - DefaultTrace.TraceInformation( - "Removed endpoint {0} unavailable for operations {1} from unavailableEndpoints", - unavailableEndpoint, - unavailabilityInfo.UnavailableOperations); - } - } - } - } - - private bool IsEndpointUnavailable(Uri endpoint, OperationType expectedAvailableOperations) - { - LocationUnavailabilityInfo unavailabilityInfo; - - if (expectedAvailableOperations == OperationType.None - || !this.locationUnavailablityInfoByEndpoint.TryGetValue(endpoint, out unavailabilityInfo) - || !unavailabilityInfo.UnavailableOperations.HasFlag(expectedAvailableOperations)) - { - return false; - } - else - { - if (DateTime.UtcNow - unavailabilityInfo.LastUnavailabilityCheckTimeStamp > this.unavailableLocationsExpirationTime) - { - return false; - } - else - { - DefaultTrace.TraceInformation( - "Endpoint {0} unavailable for operations {1} present in unavailableEndpoints", - endpoint, - unavailabilityInfo.UnavailableOperations); - // Unexpired entry present. Endpoint is unavailable - return true; - } - } - } - - private void MarkEndpointUnavailable( - Uri unavailableEndpoint, - OperationType unavailableOperationType) - { - DateTime currentTime = DateTime.UtcNow; - LocationUnavailabilityInfo updatedInfo = this.locationUnavailablityInfoByEndpoint.AddOrUpdate( - unavailableEndpoint, - (Uri endpoint) => - { - return new LocationUnavailabilityInfo() - { - LastUnavailabilityCheckTimeStamp = currentTime, - UnavailableOperations = unavailableOperationType, - }; - }, - (Uri endpoint, LocationUnavailabilityInfo info) => - { - info.LastUnavailabilityCheckTimeStamp = currentTime; - info.UnavailableOperations |= unavailableOperationType; - return info; - }); - - this.UpdateLocationCache(); - - DefaultTrace.TraceInformation( - "Endpoint {0} unavailable for {1} added/updated to unavailableEndpoints with timestamp {2}", - unavailableEndpoint, - unavailableOperationType, - updatedInfo.LastUnavailabilityCheckTimeStamp); - } - - private void UpdateLocationCache( - IEnumerable writeLocations = null, - IEnumerable readLocations = null, - ReadOnlyCollection preferenceList = null, - bool? enableMultipleWriteLocations = null) - { - lock (this.lockObject) - { - DatabaseAccountLocationsInfo nextLocationInfo = new DatabaseAccountLocationsInfo(this.locationInfo); - - if (preferenceList != null) - { - nextLocationInfo.PreferredLocations = preferenceList; - } - - if (enableMultipleWriteLocations.HasValue) - { - this.enableMultipleWriteLocations = enableMultipleWriteLocations.Value; - } - - this.ClearStaleEndpointUnavailabilityInfo(); - - if (readLocations != null) - { - ReadOnlyCollection availableReadLocations; - nextLocationInfo.AvailableReadEndpointByLocation = this.GetEndpointByLocation(readLocations, out availableReadLocations); - nextLocationInfo.AvailableReadLocations = availableReadLocations; - } - - if (writeLocations != null) - { - ReadOnlyCollection availableWriteLocations; - nextLocationInfo.AvailableWriteEndpointByLocation = this.GetEndpointByLocation(writeLocations, out availableWriteLocations); - nextLocationInfo.AvailableWriteLocations = availableWriteLocations; - } - - nextLocationInfo.WriteEndpoints = this.GetPreferredAvailableEndpoints(nextLocationInfo.AvailableWriteEndpointByLocation, nextLocationInfo.AvailableWriteLocations, OperationType.Write, this.defaultEndpoint); - nextLocationInfo.ReadEndpoints = this.GetPreferredAvailableEndpoints(nextLocationInfo.AvailableReadEndpointByLocation, nextLocationInfo.AvailableReadLocations, OperationType.Read, nextLocationInfo.WriteEndpoints[0]); - this.lastCacheUpdateTimestamp = DateTime.UtcNow; - - DefaultTrace.TraceInformation("Current WriteEndpoints = ({0}) ReadEndpoints = ({1})", - string.Join(", ", nextLocationInfo.WriteEndpoints.Select(endpoint => endpoint.ToString())), - string.Join(", ", nextLocationInfo.ReadEndpoints.Select(endpoint => endpoint.ToString()))); - - this.locationInfo = nextLocationInfo; - } - } - - private ReadOnlyCollection GetPreferredAvailableEndpoints(ReadOnlyDictionary endpointsByLocation, ReadOnlyCollection orderedLocations, OperationType expectedAvailableOperation, Uri fallbackEndpoint) - { - List endpoints = new List(); - DatabaseAccountLocationsInfo currentLocationInfo = this.locationInfo; - - // if enableEndpointDiscovery is false, we always use the defaultEndpoint that user passed in during documentClient init - if (this.enableEndpointDiscovery) - { - if (this.CanUseMultipleWriteLocations() || expectedAvailableOperation.HasFlag(OperationType.Read)) - { - List unavailableEndpoints = new List(); - - // When client can not use multiple write locations, preferred locations list should only be used - // determining read endpoints order. - // If client can use multiple write locations, preferred locations list should be used for determining - // both read and write endpoints order. - - foreach (string location in currentLocationInfo.PreferredLocations) - { - Uri endpoint; - if (endpointsByLocation.TryGetValue(location, out endpoint)) - { - if (this.IsEndpointUnavailable(endpoint, expectedAvailableOperation)) - { - unavailableEndpoints.Add(endpoint); - } - else - { - endpoints.Add(endpoint); - } - } - } - - if (endpoints.Count == 0) - { - endpoints.Add(fallbackEndpoint); - unavailableEndpoints.Remove(fallbackEndpoint); - } - - endpoints.AddRange(unavailableEndpoints); - } - else - { - foreach (string location in orderedLocations) - { - Uri endpoint; - if (!string.IsNullOrEmpty(location) && // location is empty during manual failover - endpointsByLocation.TryGetValue(location, out endpoint)) - { - endpoints.Add(endpoint); - } - } - } - } - - if (endpoints.Count == 0) - { - endpoints.Add(fallbackEndpoint); - } - - return endpoints.AsReadOnly(); - } - - private ReadOnlyDictionary GetEndpointByLocation(IEnumerable locations, out ReadOnlyCollection orderedLocations) - { - Dictionary endpointsByLocation = new Dictionary(StringComparer.OrdinalIgnoreCase); - List parsedLocations = new List(); - - foreach (AccountRegion location in locations) - { - Uri endpoint; - if (!string.IsNullOrEmpty(location.Name) - && Uri.TryCreate(location.Endpoint, UriKind.Absolute, out endpoint)) - { - endpointsByLocation[location.Name] = endpoint; - parsedLocations.Add(location.Name); - this.SetServicePointConnectionLimit(endpoint); - } - else - { - DefaultTrace.TraceInformation("GetAvailableEndpointsByLocation() - skipping add for location = {0} as it is location name is either empty or endpoint is malformed {1}", - location.Name, - location.Endpoint); - } - } - - orderedLocations = parsedLocations.AsReadOnly(); - return new ReadOnlyDictionary(endpointsByLocation); - } - - private bool CanUseMultipleWriteLocations() - { - return this.useMultipleWriteLocations && this.enableMultipleWriteLocations; - } - - private void SetServicePointConnectionLimit(Uri endpoint) - { -#if !NETSTANDARD16 - ServicePointAccessor servicePoint = ServicePointAccessor.FindServicePoint(endpoint); - servicePoint.ConnectionLimit = this.connectionLimit; -#endif - } - - private sealed class LocationUnavailabilityInfo - { - public DateTime LastUnavailabilityCheckTimeStamp { get; set; } - public OperationType UnavailableOperations { get; set; } - } - - private sealed class DatabaseAccountLocationsInfo - { - public DatabaseAccountLocationsInfo(ReadOnlyCollection preferredLocations, Uri defaultEndpoint) - { - this.PreferredLocations = preferredLocations; - this.AvailableWriteLocations = new List().AsReadOnly(); - this.AvailableReadLocations = new List().AsReadOnly(); - this.AvailableWriteEndpointByLocation = new ReadOnlyDictionary(new Dictionary(StringComparer.OrdinalIgnoreCase)); - this.AvailableReadEndpointByLocation = new ReadOnlyDictionary(new Dictionary(StringComparer.OrdinalIgnoreCase)); - this.WriteEndpoints = new List() { defaultEndpoint }.AsReadOnly(); - this.ReadEndpoints = new List() { defaultEndpoint }.AsReadOnly(); - } - - public DatabaseAccountLocationsInfo(DatabaseAccountLocationsInfo other) - { - this.PreferredLocations = other.PreferredLocations; - this.AvailableWriteLocations = other.AvailableWriteLocations; - this.AvailableReadLocations = other.AvailableReadLocations; - this.AvailableWriteEndpointByLocation = other.AvailableWriteEndpointByLocation; - this.AvailableReadEndpointByLocation = other.AvailableReadEndpointByLocation; - this.WriteEndpoints = other.WriteEndpoints; - this.ReadEndpoints = other.ReadEndpoints; - } - - public ReadOnlyCollection PreferredLocations { get; set; } - public ReadOnlyCollection AvailableWriteLocations { get; set; } - public ReadOnlyCollection AvailableReadLocations { get; set; } - public ReadOnlyDictionary AvailableWriteEndpointByLocation { get; set; } - public ReadOnlyDictionary AvailableReadEndpointByLocation { get; set; } - public ReadOnlyCollection WriteEndpoints { get; set; } - public ReadOnlyCollection ReadEndpoints { get; set; } - } - - [Flags] - private enum OperationType - { - None = 0x0, - Read = 0x1, - Write = 0x2 - } - } -} +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Routing +{ + using System; + using System.Collections.Concurrent; + using System.Collections.Generic; + using System.Collections.ObjectModel; + using System.Globalization; + using System.Linq; + using System.Net; + using global::Azure.Core; + using Microsoft.Azure.Cosmos.Core.Trace; + using Microsoft.Azure.Documents; + + /// + /// Implements the abstraction to resolve target location for geo-replicated DatabaseAccount + /// with multiple writable and readable locations. + /// + internal sealed class LocationCache + { + private const string UnavailableLocationsExpirationTimeInSeconds = "UnavailableLocationsExpirationTimeInSeconds"; + private static int DefaultUnavailableLocationsExpirationTimeInSeconds = 5 * 60; + + private readonly bool enableEndpointDiscovery; + private readonly Uri defaultEndpoint; + private readonly bool useMultipleWriteLocations; + private readonly object lockObject; + private readonly TimeSpan unavailableLocationsExpirationTime; + private readonly int connectionLimit; + private readonly ConcurrentDictionary locationUnavailablityInfoByEndpoint; + + private DatabaseAccountLocationsInfo locationInfo; + private DateTime lastCacheUpdateTimestamp; + private bool enableMultipleWriteLocations; + + public LocationCache( + ReadOnlyCollection preferredLocations, + Uri defaultEndpoint, + bool enableEndpointDiscovery, + int connectionLimit, + bool useMultipleWriteLocations) + { + this.locationInfo = new DatabaseAccountLocationsInfo(preferredLocations, defaultEndpoint); + this.defaultEndpoint = defaultEndpoint; + this.enableEndpointDiscovery = enableEndpointDiscovery; + this.useMultipleWriteLocations = useMultipleWriteLocations; + this.connectionLimit = connectionLimit; + + this.lockObject = new object(); + this.locationUnavailablityInfoByEndpoint = new ConcurrentDictionary(); + this.lastCacheUpdateTimestamp = DateTime.MinValue; + this.enableMultipleWriteLocations = false; + this.unavailableLocationsExpirationTime = TimeSpan.FromSeconds(LocationCache.DefaultUnavailableLocationsExpirationTimeInSeconds); + +#if !(NETSTANDARD15 || NETSTANDARD16) +#if NETSTANDARD20 + // GetEntryAssembly returns null when loaded from native netstandard2.0 + if (System.Reflection.Assembly.GetEntryAssembly() != null) + { +#endif + string unavailableLocationsExpirationTimeInSecondsConfig = System.Configuration.ConfigurationManager.AppSettings[LocationCache.UnavailableLocationsExpirationTimeInSeconds]; + if (!string.IsNullOrEmpty(unavailableLocationsExpirationTimeInSecondsConfig)) + { + int unavailableLocationsExpirationTimeinSecondsConfigValue; + + if (!int.TryParse(unavailableLocationsExpirationTimeInSecondsConfig, out unavailableLocationsExpirationTimeinSecondsConfigValue)) + { + this.unavailableLocationsExpirationTime = TimeSpan.FromSeconds(LocationCache.DefaultUnavailableLocationsExpirationTimeInSeconds); + } + else + { + this.unavailableLocationsExpirationTime = TimeSpan.FromSeconds(unavailableLocationsExpirationTimeinSecondsConfigValue); + } + } +#if NETSTANDARD20 + } +#endif +#endif + } + + /// + /// Gets list of read endpoints ordered by + /// 1. Preferred location + /// 2. Endpoint availablity + /// + public ReadOnlyCollection ReadEndpoints + { + get + { + // Hot-path: avoid ConcurrentDictionary methods which acquire locks + if (DateTime.UtcNow - this.lastCacheUpdateTimestamp > this.unavailableLocationsExpirationTime + && this.locationUnavailablityInfoByEndpoint.Any()) + { + this.UpdateLocationCache(); + } + + return this.locationInfo.ReadEndpoints; + } + } + + /// + /// Gets list of write endpoints ordered by + /// 1. Preferred location + /// 2. Endpoint availablity + /// + public ReadOnlyCollection WriteEndpoints + { + get + { + // Hot-path: avoid ConcurrentDictionary methods which acquire locks + if (DateTime.UtcNow - this.lastCacheUpdateTimestamp > this.unavailableLocationsExpirationTime + && this.locationUnavailablityInfoByEndpoint.Any()) + { + this.UpdateLocationCache(); + } + + return this.locationInfo.WriteEndpoints; + } + } + + /// + /// Returns the location corresponding to the endpoint if location specific endpoint is provided. + /// For the defaultEndPoint, we will return the first available write location. + /// Returns null, in other cases. + /// + /// + /// Today we return null for defaultEndPoint if multiple write locations can be used. + /// This needs to be modifed to figure out proper location in such case. + /// + public string GetLocation(Uri endpoint) + { + string location = this.locationInfo.AvailableWriteEndpointByLocation.FirstOrDefault(uri => uri.Value == endpoint).Key ?? this.locationInfo.AvailableReadEndpointByLocation.FirstOrDefault(uri => uri.Value == endpoint).Key; + + if (location == null && endpoint == this.defaultEndpoint && !this.CanUseMultipleWriteLocations()) + { + if (this.locationInfo.AvailableWriteEndpointByLocation.Any()) + { + return this.locationInfo.AvailableWriteEndpointByLocation.First().Key; + } + } + + return location; + } + + /// + /// Set region name for a location if present in the locationcache otherwise set region name as null. + /// If endpoint's hostname is same as default endpoint hostname, set regionName as null. + /// + /// + /// + /// true if region found else false + public bool TryGetLocationForGatewayDiagnostics(Uri endpoint, out string regionName) + { + if (Uri.Compare( + endpoint, + this.defaultEndpoint, + UriComponents.Host, + UriFormat.SafeUnescaped, + StringComparison.OrdinalIgnoreCase) == 0) + { + regionName = null; + return false; + } + + regionName = this.GetLocation(endpoint); + return true; + } + + /// + /// Marks the current location unavailable for read + /// + public void MarkEndpointUnavailableForRead(Uri endpoint) + { + this.MarkEndpointUnavailable(endpoint, OperationType.Read); + } + + /// + /// Marks the current location unavailable for write + /// + public void MarkEndpointUnavailableForWrite(Uri endpoint) + { + this.MarkEndpointUnavailable(endpoint, OperationType.Write); + } + + /// + /// Invoked when is read + /// + /// Read DatabaseAccoaunt + public void OnDatabaseAccountRead(AccountProperties databaseAccount) + { + this.UpdateLocationCache( + databaseAccount.WritableRegions, + databaseAccount.ReadableRegions, + preferenceList: null, + enableMultipleWriteLocations: databaseAccount.EnableMultipleWriteLocations); + } + + /// + /// Invoked when changes + /// + /// + public void OnLocationPreferenceChanged(ReadOnlyCollection preferredLocations) + { + this.UpdateLocationCache( + preferenceList: preferredLocations); + } + + public bool IsMetaData(DocumentServiceRequest request) + { + return (request.OperationType != Documents.OperationType.ExecuteJavaScript && request.ResourceType == ResourceType.StoredProcedure) || + request.ResourceType != ResourceType.Document; + + } + public bool IsMultimasterMetadataWriteRequest(DocumentServiceRequest request) + { + return !request.IsReadOnlyRequest && this.locationInfo.AvailableWriteLocations.Count > 1 + && this.IsMetaData(request) + && this.CanUseMultipleWriteLocations(); + + } + + public Uri GetHubUri() + { + DatabaseAccountLocationsInfo currentLocationInfo = this.locationInfo; + string writeLocation = currentLocationInfo.AvailableWriteLocations[0]; + Uri locationEndpointToRoute = currentLocationInfo.AvailableWriteEndpointByLocation[writeLocation]; + return locationEndpointToRoute; + } + + /// + /// Resolves request to service endpoint. + /// 1. If this is a write request + /// (a) If UseMultipleWriteLocations = true + /// (i) For document writes, resolve to most preferred and available write endpoint. + /// Once the endpoint is marked unavailable, it is moved to the end of available write endpoint. Current request will + /// be retried on next preferred available write endpoint. + /// (ii) For all other resources, always resolve to first/second (regardless of preferred locations) + /// write endpoint in . + /// Endpoint of first write location in is the only endpoint that supports + /// write operation on all resource types (except during that region's failover). + /// Only during manual failover, client would retry write on second write location in . + /// (b) Else resolve the request to first write endpoint in OR + /// second write endpoint in in case of manual failover of that location. + /// 2. Else resolve the request to most preferred available read endpoint (automatic failover for read requests) + /// + /// Request for which endpoint is to be resolved + /// Resolved endpoint + public Uri ResolveServiceEndpoint(DocumentServiceRequest request) + { + if (request.RequestContext != null && request.RequestContext.LocationEndpointToRoute != null) + { + return request.RequestContext.LocationEndpointToRoute; + } + + int locationIndex = request.RequestContext.LocationIndexToRoute.GetValueOrDefault(0); + + Uri locationEndpointToRoute = this.defaultEndpoint; + + if (!request.RequestContext.UsePreferredLocations.GetValueOrDefault(true) // Should not use preferred location ? + || (request.OperationType.IsWriteOperation() && !this.CanUseMultipleWriteLocations(request))) + { + // For non-document resource types in case of client can use multiple write locations + // or when client cannot use multiple write locations, flip-flop between the + // first and the second writable region in DatabaseAccount (for manual failover) + DatabaseAccountLocationsInfo currentLocationInfo = this.locationInfo; + + if (this.enableEndpointDiscovery && currentLocationInfo.AvailableWriteLocations.Count > 0) + { + locationIndex = Math.Min(locationIndex % 2, currentLocationInfo.AvailableWriteLocations.Count - 1); + string writeLocation = currentLocationInfo.AvailableWriteLocations[locationIndex]; + locationEndpointToRoute = currentLocationInfo.AvailableWriteEndpointByLocation[writeLocation]; + } + } + else + { + ReadOnlyCollection endpoints = request.OperationType.IsWriteOperation() ? this.WriteEndpoints : this.ReadEndpoints; + locationEndpointToRoute = endpoints[locationIndex % endpoints.Count]; + } + + request.RequestContext.RouteToLocation(locationEndpointToRoute); + return locationEndpointToRoute; + } + + public bool ShouldRefreshEndpoints(out bool canRefreshInBackground) + { + canRefreshInBackground = true; + DatabaseAccountLocationsInfo currentLocationInfo = this.locationInfo; + + string mostPreferredLocation = currentLocationInfo.PreferredLocations.FirstOrDefault(); + + // we should schedule refresh in background if we are unable to target the user's most preferredLocation. + if (this.enableEndpointDiscovery) + { + // Refresh if client opts-in to useMultipleWriteLocations but server-side setting is disabled + bool shouldRefresh = this.useMultipleWriteLocations && !this.enableMultipleWriteLocations; + + ReadOnlyCollection readLocationEndpoints = currentLocationInfo.ReadEndpoints; + + if (this.IsEndpointUnavailable(readLocationEndpoints[0], OperationType.Read)) + { + canRefreshInBackground = readLocationEndpoints.Count > 1; + DefaultTrace.TraceInformation("ShouldRefreshEndpoints = true since the first read endpoint {0} is not available for read. canRefreshInBackground = {1}", + readLocationEndpoints[0], + canRefreshInBackground); + + return true; + } + + if (!string.IsNullOrEmpty(mostPreferredLocation)) + { + Uri mostPreferredReadEndpoint; + + if (currentLocationInfo.AvailableReadEndpointByLocation.TryGetValue(mostPreferredLocation, out mostPreferredReadEndpoint)) + { + if (mostPreferredReadEndpoint != readLocationEndpoints[0]) + { + // For reads, we can always refresh in background as we can alternate to + // other available read endpoints + DefaultTrace.TraceInformation("ShouldRefreshEndpoints = true since most preferred location {0} is not available for read.", mostPreferredLocation); + return true; + } + } + else + { + DefaultTrace.TraceInformation("ShouldRefreshEndpoints = true since most preferred location {0} is not in available read locations.", mostPreferredLocation); + return true; + } + } + + Uri mostPreferredWriteEndpoint; + ReadOnlyCollection writeLocationEndpoints = currentLocationInfo.WriteEndpoints; + + if (!this.CanUseMultipleWriteLocations()) + { + if (this.IsEndpointUnavailable(writeLocationEndpoints[0], OperationType.Write)) + { + // Since most preferred write endpoint is unavailable, we can only refresh in background if + // we have an alternate write endpoint + canRefreshInBackground = writeLocationEndpoints.Count > 1; + DefaultTrace.TraceInformation("ShouldRefreshEndpoints = true since most preferred location {0} endpoint {1} is not available for write. canRefreshInBackground = {2}", + mostPreferredLocation, + writeLocationEndpoints[0], + canRefreshInBackground); + + return true; + } + else + { + return shouldRefresh; + } + } + else if (!string.IsNullOrEmpty(mostPreferredLocation)) + { + if (currentLocationInfo.AvailableWriteEndpointByLocation.TryGetValue(mostPreferredLocation, out mostPreferredWriteEndpoint)) + { + shouldRefresh |= mostPreferredWriteEndpoint != writeLocationEndpoints[0]; + DefaultTrace.TraceInformation("ShouldRefreshEndpoints = {0} since most preferred location {1} is not available for write.", shouldRefresh, mostPreferredLocation); + return shouldRefresh; + } + else + { + DefaultTrace.TraceInformation("ShouldRefreshEndpoints = true since most preferred location {0} is not in available write locations", mostPreferredLocation); + return true; + } + } + else + { + return shouldRefresh; + } + } + else + { + return false; + } + } + + public bool CanUseMultipleWriteLocations(DocumentServiceRequest request) + { + return this.CanUseMultipleWriteLocations() && + (request.ResourceType == ResourceType.Document || + (request.ResourceType == ResourceType.StoredProcedure && request.OperationType == Documents.OperationType.ExecuteJavaScript)); + } + + private void ClearStaleEndpointUnavailabilityInfo() + { + if (this.locationUnavailablityInfoByEndpoint.Any()) + { + List unavailableEndpoints = this.locationUnavailablityInfoByEndpoint.Keys.ToList(); + + foreach (Uri unavailableEndpoint in unavailableEndpoints) + { + LocationUnavailabilityInfo unavailabilityInfo; + LocationUnavailabilityInfo removed; + + if (this.locationUnavailablityInfoByEndpoint.TryGetValue(unavailableEndpoint, out unavailabilityInfo) + && DateTime.UtcNow - unavailabilityInfo.LastUnavailabilityCheckTimeStamp > this.unavailableLocationsExpirationTime + && this.locationUnavailablityInfoByEndpoint.TryRemove(unavailableEndpoint, out removed)) + { + DefaultTrace.TraceInformation( + "Removed endpoint {0} unavailable for operations {1} from unavailableEndpoints", + unavailableEndpoint, + unavailabilityInfo.UnavailableOperations); + } + } + } + } + + private bool IsEndpointUnavailable(Uri endpoint, OperationType expectedAvailableOperations) + { + LocationUnavailabilityInfo unavailabilityInfo; + + if (expectedAvailableOperations == OperationType.None + || !this.locationUnavailablityInfoByEndpoint.TryGetValue(endpoint, out unavailabilityInfo) + || !unavailabilityInfo.UnavailableOperations.HasFlag(expectedAvailableOperations)) + { + return false; + } + else + { + if (DateTime.UtcNow - unavailabilityInfo.LastUnavailabilityCheckTimeStamp > this.unavailableLocationsExpirationTime) + { + return false; + } + else + { + DefaultTrace.TraceInformation( + "Endpoint {0} unavailable for operations {1} present in unavailableEndpoints", + endpoint, + unavailabilityInfo.UnavailableOperations); + // Unexpired entry present. Endpoint is unavailable + return true; + } + } + } + + private void MarkEndpointUnavailable( + Uri unavailableEndpoint, + OperationType unavailableOperationType) + { + DateTime currentTime = DateTime.UtcNow; + LocationUnavailabilityInfo updatedInfo = this.locationUnavailablityInfoByEndpoint.AddOrUpdate( + unavailableEndpoint, + (Uri endpoint) => + { + return new LocationUnavailabilityInfo() + { + LastUnavailabilityCheckTimeStamp = currentTime, + UnavailableOperations = unavailableOperationType, + }; + }, + (Uri endpoint, LocationUnavailabilityInfo info) => + { + info.LastUnavailabilityCheckTimeStamp = currentTime; + info.UnavailableOperations |= unavailableOperationType; + return info; + }); + + this.UpdateLocationCache(); + + DefaultTrace.TraceInformation( + "Endpoint {0} unavailable for {1} added/updated to unavailableEndpoints with timestamp {2}", + unavailableEndpoint, + unavailableOperationType, + updatedInfo.LastUnavailabilityCheckTimeStamp); + } + + private void UpdateLocationCache( + IEnumerable writeLocations = null, + IEnumerable readLocations = null, + ReadOnlyCollection preferenceList = null, + bool? enableMultipleWriteLocations = null) + { + lock (this.lockObject) + { + DatabaseAccountLocationsInfo nextLocationInfo = new DatabaseAccountLocationsInfo(this.locationInfo); + + if (preferenceList != null) + { + nextLocationInfo.PreferredLocations = preferenceList; + } + + if (enableMultipleWriteLocations.HasValue) + { + this.enableMultipleWriteLocations = enableMultipleWriteLocations.Value; + } + + this.ClearStaleEndpointUnavailabilityInfo(); + + if (readLocations != null) + { + ReadOnlyCollection availableReadLocations; + nextLocationInfo.AvailableReadEndpointByLocation = this.GetEndpointByLocation(readLocations, out availableReadLocations); + nextLocationInfo.AvailableReadLocations = availableReadLocations; + } + + if (writeLocations != null) + { + ReadOnlyCollection availableWriteLocations; + nextLocationInfo.AvailableWriteEndpointByLocation = this.GetEndpointByLocation(writeLocations, out availableWriteLocations); + nextLocationInfo.AvailableWriteLocations = availableWriteLocations; + } + + nextLocationInfo.WriteEndpoints = this.GetPreferredAvailableEndpoints(nextLocationInfo.AvailableWriteEndpointByLocation, nextLocationInfo.AvailableWriteLocations, OperationType.Write, this.defaultEndpoint); + nextLocationInfo.ReadEndpoints = this.GetPreferredAvailableEndpoints(nextLocationInfo.AvailableReadEndpointByLocation, nextLocationInfo.AvailableReadLocations, OperationType.Read, nextLocationInfo.WriteEndpoints[0]); + this.lastCacheUpdateTimestamp = DateTime.UtcNow; + + DefaultTrace.TraceInformation("Current WriteEndpoints = ({0}) ReadEndpoints = ({1})", + string.Join(", ", nextLocationInfo.WriteEndpoints.Select(endpoint => endpoint.ToString())), + string.Join(", ", nextLocationInfo.ReadEndpoints.Select(endpoint => endpoint.ToString()))); + + this.locationInfo = nextLocationInfo; + } + } + + private ReadOnlyCollection GetPreferredAvailableEndpoints(ReadOnlyDictionary endpointsByLocation, ReadOnlyCollection orderedLocations, OperationType expectedAvailableOperation, Uri fallbackEndpoint) + { + List endpoints = new List(); + DatabaseAccountLocationsInfo currentLocationInfo = this.locationInfo; + + // if enableEndpointDiscovery is false, we always use the defaultEndpoint that user passed in during documentClient init + if (this.enableEndpointDiscovery) + { + if (this.CanUseMultipleWriteLocations() || expectedAvailableOperation.HasFlag(OperationType.Read)) + { + List unavailableEndpoints = new List(); + + // When client can not use multiple write locations, preferred locations list should only be used + // determining read endpoints order. + // If client can use multiple write locations, preferred locations list should be used for determining + // both read and write endpoints order. + + foreach (string location in currentLocationInfo.PreferredLocations) + { + Uri endpoint; + if (endpointsByLocation.TryGetValue(location, out endpoint)) + { + if (this.IsEndpointUnavailable(endpoint, expectedAvailableOperation)) + { + unavailableEndpoints.Add(endpoint); + } + else + { + endpoints.Add(endpoint); + } + } + } + + if (endpoints.Count == 0) + { + endpoints.Add(fallbackEndpoint); + unavailableEndpoints.Remove(fallbackEndpoint); + } + + endpoints.AddRange(unavailableEndpoints); + } + else + { + foreach (string location in orderedLocations) + { + Uri endpoint; + if (!string.IsNullOrEmpty(location) && // location is empty during manual failover + endpointsByLocation.TryGetValue(location, out endpoint)) + { + endpoints.Add(endpoint); + } + } + } + } + + if (endpoints.Count == 0) + { + endpoints.Add(fallbackEndpoint); + } + + return endpoints.AsReadOnly(); + } + + private ReadOnlyDictionary GetEndpointByLocation(IEnumerable locations, out ReadOnlyCollection orderedLocations) + { + Dictionary endpointsByLocation = new Dictionary(StringComparer.OrdinalIgnoreCase); + List parsedLocations = new List(); + + foreach (AccountRegion location in locations) + { + Uri endpoint; + if (!string.IsNullOrEmpty(location.Name) + && Uri.TryCreate(location.Endpoint, UriKind.Absolute, out endpoint)) + { + endpointsByLocation[location.Name] = endpoint; + parsedLocations.Add(location.Name); + this.SetServicePointConnectionLimit(endpoint); + } + else + { + DefaultTrace.TraceInformation("GetAvailableEndpointsByLocation() - skipping add for location = {0} as it is location name is either empty or endpoint is malformed {1}", + location.Name, + location.Endpoint); + } + } + + orderedLocations = parsedLocations.AsReadOnly(); + return new ReadOnlyDictionary(endpointsByLocation); + } + + private bool CanUseMultipleWriteLocations() + { + return this.useMultipleWriteLocations && this.enableMultipleWriteLocations; + } + + private void SetServicePointConnectionLimit(Uri endpoint) + { +#if !NETSTANDARD16 + ServicePointAccessor servicePoint = ServicePointAccessor.FindServicePoint(endpoint); + servicePoint.ConnectionLimit = this.connectionLimit; +#endif + } + + private sealed class LocationUnavailabilityInfo + { + public DateTime LastUnavailabilityCheckTimeStamp { get; set; } + public OperationType UnavailableOperations { get; set; } + } + + private sealed class DatabaseAccountLocationsInfo + { + public DatabaseAccountLocationsInfo(ReadOnlyCollection preferredLocations, Uri defaultEndpoint) + { + this.PreferredLocations = preferredLocations; + this.AvailableWriteLocations = new List().AsReadOnly(); + this.AvailableReadLocations = new List().AsReadOnly(); + this.AvailableWriteEndpointByLocation = new ReadOnlyDictionary(new Dictionary(StringComparer.OrdinalIgnoreCase)); + this.AvailableReadEndpointByLocation = new ReadOnlyDictionary(new Dictionary(StringComparer.OrdinalIgnoreCase)); + this.WriteEndpoints = new List() { defaultEndpoint }.AsReadOnly(); + this.ReadEndpoints = new List() { defaultEndpoint }.AsReadOnly(); + } + + public DatabaseAccountLocationsInfo(DatabaseAccountLocationsInfo other) + { + this.PreferredLocations = other.PreferredLocations; + this.AvailableWriteLocations = other.AvailableWriteLocations; + this.AvailableReadLocations = other.AvailableReadLocations; + this.AvailableWriteEndpointByLocation = other.AvailableWriteEndpointByLocation; + this.AvailableReadEndpointByLocation = other.AvailableReadEndpointByLocation; + this.WriteEndpoints = other.WriteEndpoints; + this.ReadEndpoints = other.ReadEndpoints; + } + + public ReadOnlyCollection PreferredLocations { get; set; } + public ReadOnlyCollection AvailableWriteLocations { get; set; } + public ReadOnlyCollection AvailableReadLocations { get; set; } + public ReadOnlyDictionary AvailableWriteEndpointByLocation { get; set; } + public ReadOnlyDictionary AvailableReadEndpointByLocation { get; set; } + public ReadOnlyCollection WriteEndpoints { get; set; } + public ReadOnlyCollection ReadEndpoints { get; set; } + } + + [Flags] + private enum OperationType + { + None = 0x0, + Read = 0x1, + Write = 0x2 + } + } +}