Diagnostics: Adds GetStartTimeUtc and GetFailedRequestCount

Microsoft.Azure.Cosmos.Encryption/src/EncryptionCosmosDiagnostics.cs

@@ -86,5 +86,17 @@ public override string ToString()
 
             return stringWriter.ToString();
         }
+
+#if SDKPROJECTREF
+        public override DateTime? GetStartTimeUtc()
+        {
+            return this.coreDiagnostics.GetStartTimeUtc();
+        }
+        public override int GetFailedRequestCount()
+        {
+            return this.coreDiagnostics.GetFailedRequestCount();
+        }
+#endif
+
     }
-}
+}

Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnostics.cs

@@ -23,6 +23,26 @@ public virtual TimeSpan GetClientElapsedTime()
             throw new NotImplementedException($"CosmosDiagnostics.GetElapsedTime");
         }
 
+        /// <summary>
+        /// This represents the start time of the request.
+        /// </summary>
+        /// <returns>This returns the start time of the request.</returns>
+        public virtual DateTime? GetStartTimeUtc()
+        {
+            // Default implementation avoids breaking change for users upgrading.
+            throw new NotImplementedException($"{nameof(CosmosDiagnostics)}.{nameof(GetStartTimeUtc)})");
+        }
+
+        /// <summary>
+        ///  This represents the count of failed requests.
+        /// </summary>
+        /// <returns>The count of failed requests with cosmos service.</returns>
+        public virtual int GetFailedRequestCount()
+        {
+            // Default implementation avoids breaking change for users upgrading.
+            throw new NotImplementedException($"{nameof(CosmosDiagnostics)}.{nameof(GetFailedRequestCount)}");
+        }
+
         /// <summary>
         /// Gets the string field <see cref="CosmosDiagnostics"/> instance in the Azure CosmosDB database service.
         /// </summary>

Microsoft.Azure.Cosmos/src/Diagnostics/CosmosTraceDiagnostics.cs

@@ -98,5 +98,25 @@ private ReadOnlyMemory<byte> WriteTraceToJsonWriter(JsonSerializationFormat json
             TraceWriter.WriteTrace(jsonTextWriter, this.Value);
             return jsonTextWriter.GetResult();
         }
+
+        public override DateTime? GetStartTimeUtc()
+        {
+            if (this.Value == null || this.Value.StartTime == null)
+            {
+                return null;
+            }
+
+            return this.Value.StartTime;
+        }
+
+        public override int GetFailedRequestCount()
+        {
+            if (this.Value == null || this.Value.Summary == null)
+            {
+                return 0;
+            }
+
+            return this.Value.Summary.GetFailedCount();
+       }
     }
-}
+}

Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs

@@ -1,4 +1,4 @@
-//------------------------------------------------------------
+//------------------------------------------------------------
 // Copyright (c) Microsoft Corporation.  All rights reserved.
 //------------------------------------------------------------
 
@@ -47,33 +47,29 @@ await this.cosmosAuthorization.AddAuthorizationHeaderAsync(
                 HttpConstants.HttpMethods.Get,
                 AuthorizationTokenType.PrimaryMasterKey);
 
-            IClientSideRequestStatistics stats = new ClientSideRequestStatisticsTraceDatum(DateTime.UtcNow);
-            try
+            using (ITrace trace = Trace.GetRootTrace("Account Read", TraceComponent.Transport, TraceLevel.Info))
             {
-                using (HttpResponseMessage responseMessage = await this.httpClient.GetAsync(
-                    uri: serviceEndpoint,
-                    additionalHeaders: headers,
-                    resourceType: ResourceType.DatabaseAccount,
-                    timeoutPolicy: HttpTimeoutPolicyControlPlaneRead.Instance,
-                    clientSideRequestStatistics: stats,
-                    cancellationToken: this.cancellationToken))
+                IClientSideRequestStatistics stats = new ClientSideRequestStatisticsTraceDatum(DateTime.UtcNow, trace.Summary);
+
+                try
                 {
+                    using (HttpResponseMessage responseMessage = await this.httpClient.GetAsync(
+                        uri: serviceEndpoint,
+                        additionalHeaders: headers,
+                        resourceType: ResourceType.DatabaseAccount,
+                        timeoutPolicy: HttpTimeoutPolicyControlPlaneRead.Instance,
+                        clientSideRequestStatistics: stats,
+                        cancellationToken: default))
                     using (DocumentServiceResponse documentServiceResponse = await ClientExtensions.ParseResponseAsync(responseMessage))
                     {
                         return CosmosResource.FromStream<AccountProperties>(documentServiceResponse);
                     }
                 }
-            }
-            catch (ObjectDisposedException) when (this.cancellationToken.IsCancellationRequested)
-            {
-                throw new OperationCanceledException($"Client is being disposed for {serviceEndpoint} at {DateTime.UtcNow}, cancelling further operations.");
-            }
-            catch (OperationCanceledException ex)
-            // this is the DocumentClient token, which only gets cancelled upon disposal
-            when (!this.cancellationToken.IsCancellationRequested) 
-            {
-                // Catch Operation Cancelled Exception and convert to Timeout 408 if the user did not cancel it.
-                using (ITrace trace = Trace.GetRootTrace("Account Read Exception", TraceComponent.Transport, TraceLevel.Info))
+                catch (ObjectDisposedException) when (this.cancellationToken.IsCancellationRequested)
+                {
+                    throw new OperationCanceledException($"Client is being disposed for {serviceEndpoint} at {DateTime.UtcNow}, cancelling further operations.");
+                }
+                catch (OperationCanceledException ex)
                 {
                     trace.AddDatum("Client Side Request Stats", stats);
                     throw CosmosExceptionFactory.CreateRequestTimeoutException(

Microsoft.Azure.Cosmos/src/Handler/TransportHandler.cs

@@ -93,7 +93,7 @@ internal async Task<ResponseMessage> ProcessMessageAsync(
 
             DocumentServiceRequest serviceRequest = request.ToDocumentServiceRequest();
 
-            ClientSideRequestStatisticsTraceDatum clientSideRequestStatisticsTraceDatum = new ClientSideRequestStatisticsTraceDatum(DateTime.UtcNow);
+            ClientSideRequestStatisticsTraceDatum clientSideRequestStatisticsTraceDatum = new ClientSideRequestStatisticsTraceDatum(DateTime.UtcNow, request.Trace.Summary);
             serviceRequest.RequestContext.ClientRequestStatistics = clientSideRequestStatisticsTraceDatum;
 
             //TODO: extrace auth into a separate handler

Microsoft.Azure.Cosmos/src/Routing/ClientCollectionCache.cs

@@ -26,7 +26,7 @@ internal class ClientCollectionCache : CollectionCache
         public ClientCollectionCache(
             ISessionContainer sessionContainer,
             IStoreModel storeModel,
-            ICosmosAuthorizationTokenProvider tokenProvider, 
+            ICosmosAuthorizationTokenProvider tokenProvider,
             IRetryPolicyFactory retryPolicy)
         {
             this.storeModel = storeModel ?? throw new ArgumentNullException("storeModel");
@@ -35,8 +35,8 @@ public ClientCollectionCache(
             this.sessionContainer = sessionContainer;
         }
 
-        protected override Task<ContainerProperties> GetByRidAsync(string apiVersion, 
-                                                    string collectionRid, 
+        protected override Task<ContainerProperties> GetByRidAsync(string apiVersion,
+                                                    string collectionRid,
                                                     ITrace trace,
                                                     IClientSideRequestStatistics clientSideRequestStatistics,
                                                     CancellationToken cancellationToken)
@@ -46,16 +46,16 @@ protected override Task<ContainerProperties> GetByRidAsync(string apiVersion,
                 this.sessionContainer, this.retryPolicy.GetRequestPolicy());
             return TaskHelper.InlineIfPossible(
                   () => this.ReadCollectionAsync(
-                      PathsHelper.GeneratePath(ResourceType.Collection, collectionRid, false), 
-                      retryPolicyInstance, 
+                      PathsHelper.GeneratePath(ResourceType.Collection, collectionRid, false),
+                      retryPolicyInstance,
                       trace,
-                      clientSideRequestStatistics, 
+                      clientSideRequestStatistics,
                       cancellationToken),
                   retryPolicyInstance,
                   cancellationToken);
         }
 
-        protected override Task<ContainerProperties> GetByNameAsync(string apiVersion, 
+        protected override Task<ContainerProperties> GetByNameAsync(string apiVersion,
                                                 string resourceAddress,
                                                 ITrace trace,
                                                 IClientSideRequestStatistics clientSideRequestStatistics,
@@ -150,7 +150,7 @@ private async Task<ContainerProperties> ResolveCollectionWithSessionContainerCle
             ContainerProperties properties = await resolveContainerProvider();
 
             if (this.sessionContainer != null &&
-                previouslyResolvedCollectionRid != null && 
+                previouslyResolvedCollectionRid != null &&
                 previouslyResolvedCollectionRid != properties.ResourceId)
             {
                 this.sessionContainer.ClearTokenByResourceId(previouslyResolvedCollectionRid);
@@ -167,7 +167,7 @@ private async Task<ContainerProperties> ReadCollectionAsync(
             CancellationToken cancellationToken)
         {
             using (ITrace childTrace = trace.StartChild("Read Collection", TraceComponent.Transport, TraceLevel.Info))
-            { 
+            {
                 cancellationToken.ThrowIfCancellationRequested();
 
                 RequestNameValueCollection headers = new RequestNameValueCollection();
@@ -180,8 +180,7 @@ private async Task<ContainerProperties> ReadCollectionAsync(
                 {
                     headers.XDate = Rfc1123DateTimeCache.UtcNow();
 
-                    request.RequestContext.ClientRequestStatistics = 
-                        clientSideRequestStatistics ?? new ClientSideRequestStatisticsTraceDatum(DateTime.UtcNow);
+                    request.RequestContext.ClientRequestStatistics = clientSideRequestStatistics ?? new ClientSideRequestStatisticsTraceDatum(DateTime.UtcNow, trace.Summary);
                     if (clientSideRequestStatistics == null)
                     {
                         childTrace.AddDatum(
@@ -221,4 +220,4 @@ await this.storeModel.ProcessMessageAsync(request))
             }
         }
     }
-}
+}

Microsoft.Azure.Cosmos/src/Routing/PartitionKeyRangeCache.cs

@@ -316,7 +316,7 @@ private async Task<DocumentServiceResponse> ExecutePartitionKeyRangeReadChangeFe
                     }
 
                     request.Headers[HttpConstants.HttpHeaders.Authorization] = authorizationToken;
-                    request.RequestContext.ClientRequestStatistics = clientSideRequestStatistics ?? new ClientSideRequestStatisticsTraceDatum(DateTime.UtcNow);
+                    request.RequestContext.ClientRequestStatistics = clientSideRequestStatistics ?? new ClientSideRequestStatisticsTraceDatum(DateTime.UtcNow, trace.Summary);
                     if (clientSideRequestStatistics == null)
                     {
                         childTrace.AddDatum("Client Side Request Stats", request.RequestContext.ClientRequestStatistics);

Microsoft.Azure.Cosmos/src/Tracing/ITrace.cs

@@ -47,6 +47,11 @@ interface ITrace : IDisposable
         /// Gets the component that governs this trace.
         /// </summary>
         TraceComponent Component { get; }
+
+        /// <summary>
+        /// Gets the summary of this trace.
+        /// </summary>
+        TraceSummary Summary { get; }
 
         /// <summary>
         /// Gets the parent node of this trace.

Microsoft.Azure.Cosmos/src/Tracing/NoOpTrace.cs

@@ -30,6 +30,8 @@ private NoOpTrace()
 
         public TraceLevel Level => default;
 
+        public TraceSummary Summary => default;
+
         public TraceComponent Component => default;
 
         public ITrace Parent => null;

Microsoft.Azure.Cosmos/src/Tracing/Trace.cs

@@ -25,7 +25,8 @@ private Trace(
             TraceLevel level,
             TraceComponent component,
             Trace parent,
-            ISet<(string, Uri)> regionContactedInternal)
+            ISet<(string, Uri)> regionContactedInternal,
+            TraceSummary summary)
         {
             this.Name = name ?? throw new ArgumentNullException(nameof(name));
             this.Id = Guid.NewGuid();
@@ -38,6 +39,7 @@ private Trace(
             this.data = new Lazy<Dictionary<string, object>>();
 
             this.regionContactedInternal = regionContactedInternal;
+            this.Summary = summary ?? throw new ArgumentNullException(nameof(summary));
         }
 
         public string Name { get; }
@@ -52,6 +54,8 @@ private Trace(
 
         public TraceComponent Component { get; }
 
+        public TraceSummary Summary { get; }
+
         public ITrace Parent { get; }
 
         public IReadOnlyList<ITrace> Children => this.children;
@@ -119,8 +123,9 @@ public ITrace StartChild(
                 level: level,
                 component: component,
                 parent: this,
-                regionContactedInternal: this.regionContactedInternal);           
-
+                regionContactedInternal: this.regionContactedInternal,
+                summary: this.Summary);
+
             this.AddChild(child);
             return child;
         }
@@ -151,7 +156,8 @@ public static Trace GetRootTrace(
                 level: level,
                 component: component,
                 parent: null,
-                regionContactedInternal: new HashSet<(string, Uri)>());
+                regionContactedInternal: new HashSet<(string, Uri)>(),
+                summary: new TraceSummary());
         }
 
         public void AddDatum(string key, TraceDatum traceDatum)

Microsoft.Azure.Cosmos/src/Tracing/TraceData/ClientSideRequestStatisticsTraceDatum.cs

@@ -7,6 +7,7 @@ namespace Microsoft.Azure.Cosmos.Tracing.TraceData
     using System;
     using System.Collections.Generic;
     using System.Diagnostics;
+    using System.Net;
     using System.Net.Http;
     using System.Text;
     using Microsoft.Azure.Cosmos.Handler;
@@ -32,8 +33,9 @@ internal sealed class ClientSideRequestStatisticsTraceDatum : TraceDatum, IClien
         private IReadOnlyList<StoreResponseStatistics> shallowCopyOfStoreResponseStatistics = null;
         private IReadOnlyList<HttpResponseStatistics> shallowCopyOfHttpResponseStatistics = null;
         private SystemUsageHistory systemUsageHistory = null;
+        public TraceSummary TraceSummary = null;
 
-        public ClientSideRequestStatisticsTraceDatum(DateTime startTime)
+        public ClientSideRequestStatisticsTraceDatum(DateTime startTime, TraceSummary summary)
         {
             this.RequestStartTimeUtc = startTime;
             this.RequestEndTimeUtc = null;
@@ -43,6 +45,7 @@ public ClientSideRequestStatisticsTraceDatum(DateTime startTime)
             this.FailedReplicas = new HashSet<TransportAddressUri>();
             this.RegionsContacted = new HashSet<(string, Uri)>();
             this.httpResponseStatistics = new List<HttpResponseStatistics>();
+            this.TraceSummary = summary;
         }
 
         public DateTime RequestStartTimeUtc { get; }
@@ -263,6 +266,17 @@ public void RecordResponse(
                     this.RegionsContacted.Add((regionName, locationEndpoint));
                 }
 
+                if (responseStatistics.StoreResult != null && !((HttpStatusCode)responseStatistics.StoreResult.StatusCode).IsSuccess()
+                    && !(responseStatistics.StoreResult.StatusCode == StatusCodes.NotFound && responseStatistics.StoreResult.SubStatusCode == SubStatusCodes.Unknown)
+                    && !(responseStatistics.StoreResult.StatusCode == StatusCodes.Conflict && responseStatistics.StoreResult.SubStatusCode == SubStatusCodes.Unknown)
+                    && !(responseStatistics.StoreResult.StatusCode == StatusCodes.PreconditionFailed && responseStatistics.StoreResult.SubStatusCode == SubStatusCodes.Unknown))
+                {
+                    if (this.TraceSummary != null)
+                    {
+                        this.TraceSummary.IncrementFailedCount();
+                    }
+                }
+
                 // Reset the shallow copy
                 this.shallowCopyOfStoreResponseStatistics = null;
                 this.storeResponseStatistics.Add(responseStatistics);

Microsoft.Azure.Cosmos/src/Tracing/TraceSummary.cs

@@ -0,0 +1,45 @@
+// ------------------------------------------------------------
+// Copyright (c) Microsoft Corporation.  All rights reserved.
+// ------------------------------------------------------------
+
+namespace Microsoft.Azure.Cosmos.Tracing
+{
+    using System;
+    using System.Collections.Generic;
+    using System.Text;
+    using System.Threading;
+
+    /// <summary>
+    /// The total count of failed requests for an <see cref="ITrace"/>.
+    /// </summary>
+#if INTERNAL
+    public
+#else
+    internal
+#endif 
+    class TraceSummary
+    {
+        /// <summary>
+        ///  The total count of failed requests for an <see cref="ITrace"/>
+        /// </summary>
+        private int failedRequestCount = 0;
+
+        /// <summary>
+        ///  The increment of failed requests with thread safe for an <see cref="ITrace"/>
+        /// </summary>
+        public void IncrementFailedCount()
+        {
+            Interlocked.Increment(ref this.failedRequestCount);
+        }
+
+        /// <summary>
+        ///  The return the count of failed requests for an <see cref="ITrace"/>
+        /// </summary>
+        /// <returns>The value of failed requests count</returns>
+        public int GetFailedCount()
+        {
+            return this.failedRequestCount;
+        }
+
+    }
+}