Merge branch 'master' into users/wahyuen/isreplacepending

Microsoft.Azure.Cosmos/src/GatewayStoreClient.cs

@@ -324,19 +324,21 @@ private async Task<HttpResponseMessage> InvokeClientAsync(
                     DateTime sendTimeUtc = DateTime.UtcNow;
                     Guid localGuid = Guid.NewGuid();  // For correlating HttpRequest and HttpResponse Traces
 
+                    Guid requestedActivityId = Trace.CorrelationManager.ActivityId;
                     this.eventSource.Request(
-                        Guid.Empty,
+                        requestedActivityId,
                         localGuid,
                         requestMessage.RequestUri.ToString(),
                         resourceType.ToResourceTypeString(),
                         requestMessage.Headers);
 
+                    TimeSpan durationTimeSpan;
                     try
                     {
                         HttpResponseMessage responseMessage = await this.httpClient.SendAsync(requestMessage, cancellationToken);
 
                         DateTime receivedTimeUtc = DateTime.UtcNow;
-                        double durationInMilliSeconds = (receivedTimeUtc - sendTimeUtc).TotalMilliseconds;
+                        durationTimeSpan = receivedTimeUtc - sendTimeUtc;
 
                         IEnumerable<string> headerValues;
                         Guid activityId = Guid.Empty;
@@ -350,7 +352,7 @@ private async Task<HttpResponseMessage> InvokeClientAsync(
                             activityId,
                             localGuid,
                             (short)responseMessage.StatusCode,
-                            durationInMilliSeconds,
+                            durationTimeSpan.TotalMilliseconds,
                             responseMessage.Headers);
 
                         return responseMessage;
@@ -360,7 +362,23 @@ private async Task<HttpResponseMessage> InvokeClientAsync(
                         if (!cancellationToken.IsCancellationRequested)
                         {
                             // throw timeout if the cancellationToken is not canceled (i.e. httpClient timed out)
-                            throw new RequestTimeoutException(ex, requestMessage.RequestUri);
+                            durationTimeSpan = DateTime.UtcNow - sendTimeUtc;
+                            string message = $"GatewayStoreClient Request Timeout. Start Time:{sendTimeUtc}; Total Duration:{durationTimeSpan}; Http Client Timeout:{this.httpClient.Timeout}; Activity id: {requestedActivityId}; Inner Message: {ex.Message};";
+                            throw new RequestTimeoutException(message, ex, requestMessage.RequestUri);
+                        }
+                        else
+                        {
+                            throw;
+                        }
+                    }
+                    catch (OperationCanceledException ex)
+                    {
+                        if (!cancellationToken.IsCancellationRequested)
+                        {
+                            // throw timeout if the cancellationToken is not canceled (i.e. httpClient timed out)
+                            durationTimeSpan = DateTime.UtcNow - sendTimeUtc;
+                            string message = $"GatewayStoreClient Request Timeout. Start Time:{sendTimeUtc}; Total Duration:{durationTimeSpan}; Http Client Timeout:{this.httpClient.Timeout}; Activity id: {requestedActivityId}; Inner Message: {ex.Message};";
+                            throw new RequestTimeoutException(message, ex, requestMessage.RequestUri);
                         }
                         else
                         {

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosGatewayTimeoutTests.cs

@@ -0,0 +1,76 @@
+//------------------------------------------------------------
+// Copyright (c) Microsoft Corporation.  All rights reserved.
+//------------------------------------------------------------
+
+namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
+{
+    using System;
+    using System.Net.Http;
+    using System.Reflection;
+    using System.Threading;
+    using System.Threading.Tasks;
+    using Microsoft.Azure.Documents;
+    using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+    [TestClass]
+    public class CosmosGatewayTimeoutTests
+    {
+        [TestMethod]
+        public async Task GatewayStoreClientTimeout()
+        {
+            using (CosmosClient client = TestCommon.CreateCosmosClient(useGateway: true))
+            {
+                // Creates the store clients in the document client
+                await client.DocumentClient.EnsureValidClientAsync();
+
+                // Get the GatewayStoreModel
+                GatewayStoreModel gatewayStore;
+                using (DocumentServiceRequest serviceRequest = new DocumentServiceRequest(
+                                operationType: OperationType.Read,
+                                resourceIdOrFullName: null,
+                                resourceType: ResourceType.Database,
+                                body: null,
+                                headers: null,
+                                isNameBased: false,
+                                authorizationTokenType: AuthorizationTokenType.PrimaryMasterKey))
+                {
+                    serviceRequest.UseGatewayMode = true;
+                    gatewayStore = (GatewayStoreModel)client.DocumentClient.GetStoreProxy(serviceRequest);
+                }
+
+                // Get the GatewayStoreClient
+                FieldInfo gatewayStoreClientProperty = gatewayStore.GetType().GetField("gatewayStoreClient", BindingFlags.NonPublic | BindingFlags.Instance);
+                GatewayStoreClient storeClient = (GatewayStoreClient)gatewayStoreClientProperty.GetValue(gatewayStore);
+
+                // Set the http request timeout to 10 ms to cause a timeout exception
+                HttpClient httpClient = new HttpClient(new TimeOutHttpClientHandler());
+                FieldInfo httpClientProperty = storeClient.GetType().GetField("httpClient", BindingFlags.NonPublic | BindingFlags.Instance);
+                httpClientProperty.SetValue(storeClient, httpClient);
+
+                // Verify the failure has the required info
+                CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
+                try
+                {
+                    await client.CreateDatabaseAsync("TestGatewayTimeoutDb" + Guid.NewGuid().ToString());
+                    Assert.Fail("Operation should have timed out:");
+                }
+                catch (CosmosException rte)
+                {
+                    string message = rte.ToString();
+                    Assert.IsTrue(message.Contains("Start Time"), "Start Time:" + message);
+                    Assert.IsTrue(message.Contains("Total Duration"), "Total Duration:" + message);
+                    Assert.IsTrue(message.Contains("Http Client Timeout"), "Http Client Timeout:" + message);
+                    Assert.IsTrue(message.Contains("Activity id"), "Activity id:" + message);
+                }
+            }
+        }
+
+        private class TimeOutHttpClientHandler : DelegatingHandler
+        {
+            protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+            {
+                throw new TaskCanceledException();
+            }
+        }
+    }
+}

changelog.md

@@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - [#944](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/944) Change Feed Processor won't use user serializer for internal operations
 - [#988](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/988) Fixed query mutating due to retry of gone / name cache is stale.
 - [#999](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/999) Fixed grabbing extra page and updated continuation token on exception path.
+- [#1013](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/1013) Gateway OperationCanceledException are now returned as request timeouts
 - [#1023](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/1023) Fixed ThroughputResponse.IsReplacePending header mapping
 
 ## <a name="3.4.1"/> [3.4.1](https://www.nuget.org/packages/Microsoft.Azure.Cosmos/3.4.1) - 2019-11-06