Merge branch 'master' into users/jawilley/query/optimize_stream

Azure · Oct 16, 2019 · 6613d34 · 6613d34
2 parents e06d7a5 + 5ef49f4
commit 6613d34
Show file tree

Hide file tree

Showing 104 changed files with 6,574,940 additions and 361 deletions.
diff --git a/.gitattributes b/.gitattributes
diff --git a/Microsoft.Azure.Cosmos.Samples/Usage/AzureFunctions/AzureFunctions.csproj b/Microsoft.Azure.Cosmos.Samples/Usage/AzureFunctions/AzureFunctions.csproj
@@ -13,7 +13,7 @@
   <ItemGroup>
     <PackageReference Include="Microsoft.NET.Sdk.Functions" Version="1.0.27" />
     <PackageReference Include="Microsoft.Azure.Functions.Extensions" Version="1.0.0" />
-    <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.3.0" />
+    <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.3.1" />
   </ItemGroup>
   <ItemGroup>
     <None Update="host.json">

diff --git a/Microsoft.Azure.Cosmos.Samples/Usage/ChangeFeed/ChangeFeed.csproj b/Microsoft.Azure.Cosmos.Samples/Usage/ChangeFeed/ChangeFeed.csproj
@@ -8,7 +8,7 @@
     <LangVersion>latest</LangVersion>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.3.0" />
+    <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.3.1" />
     <PackageReference Include="Microsoft.Azure.DocumentDB.ChangeFeedProcessor" Version="2.2.7" />
     <PackageReference Include="Microsoft.Extensions.Configuration" Version="2.2.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="2.2.0" />

diff --git a/Microsoft.Azure.Cosmos.Samples/Usage/ContainerManagement/ContainerManagement.csproj b/Microsoft.Azure.Cosmos.Samples/Usage/ContainerManagement/ContainerManagement.csproj
@@ -7,7 +7,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.3.0" />
+    <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.3.1" />
     <PackageReference Include="Microsoft.Extensions.Configuration" Version="2.2.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="2.2.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.2.0" />

diff --git a/Microsoft.Azure.Cosmos.Samples/Usage/CustomSerialization/CustomSerialization.csproj b/Microsoft.Azure.Cosmos.Samples/Usage/CustomSerialization/CustomSerialization.csproj
@@ -6,7 +6,7 @@
     <LangVersion>latest</LangVersion>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.3.0" />
+    <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.3.1" />
     <PackageReference Include="Microsoft.Extensions.Configuration" Version="2.2.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="2.2.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.2.0" />

diff --git a/Microsoft.Azure.Cosmos.Samples/Usage/DatabaseManagement/DatabaseManagement.csproj b/Microsoft.Azure.Cosmos.Samples/Usage/DatabaseManagement/DatabaseManagement.csproj
@@ -7,7 +7,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.3.0" />
+    <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.3.1" />
     <PackageReference Include="Microsoft.Extensions.Configuration" Version="2.2.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="2.2.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.2.0" />

diff --git a/Microsoft.Azure.Cosmos.Samples/Usage/Handlers/HandlerSample.csproj b/Microsoft.Azure.Cosmos.Samples/Usage/Handlers/HandlerSample.csproj
@@ -16,7 +16,7 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.ApplicationInsights" Version="2.10.0" />
-    <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.3.0" />
+    <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.3.1" />
     <PackageReference Include="Microsoft.Extensions.Configuration" Version="2.2.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.2.0" />
     <PackageReference Include="Newtonsoft.Json" Version="12.0.2" />

diff --git a/Microsoft.Azure.Cosmos.Samples/Usage/IndexManagement/IndexManagement.csproj b/Microsoft.Azure.Cosmos.Samples/Usage/IndexManagement/IndexManagement.csproj
@@ -7,7 +7,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.3.0" />
+    <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.3.1" />
     <PackageReference Include="Microsoft.Extensions.Configuration" Version="2.2.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="2.2.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.2.0" />

diff --git a/Microsoft.Azure.Cosmos.Samples/Usage/ItemManagement/ItemManagement.csproj b/Microsoft.Azure.Cosmos.Samples/Usage/ItemManagement/ItemManagement.csproj
@@ -6,7 +6,7 @@
     <LangVersion>latest</LangVersion>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.3.0" />
+    <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.3.1" />
     <PackageReference Include="Microsoft.Extensions.Configuration" Version="2.2.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="2.2.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.2.0" />

diff --git a/...Cosmos.Samples/Usage/NonPartitionContainerMigration/NonPartitionContainerMigration.csproj b/...Cosmos.Samples/Usage/NonPartitionContainerMigration/NonPartitionContainerMigration.csproj
@@ -6,7 +6,7 @@
     <LangVersion>latest</LangVersion>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.3.0" />
+    <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.3.1" />
     <PackageReference Include="Microsoft.Extensions.Configuration" Version="2.2.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="2.2.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.2.0" />

diff --git a/Microsoft.Azure.Cosmos.Samples/Usage/Queries/Queries.csproj b/Microsoft.Azure.Cosmos.Samples/Usage/Queries/Queries.csproj
@@ -6,7 +6,7 @@
     <LangVersion>latest</LangVersion>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.3.0" />
+    <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.3.1" />
     <PackageReference Include="Microsoft.Extensions.Configuration" Version="2.2.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="2.2.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.2.0" />

diff --git a/Microsoft.Azure.Cosmos.Samples/Usage/ServerSideScripts/ServerSideScripts.csproj b/Microsoft.Azure.Cosmos.Samples/Usage/ServerSideScripts/ServerSideScripts.csproj
@@ -7,7 +7,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.3.0" />
+    <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.3.1" />
     <PackageReference Include="Microsoft.Extensions.Configuration" Version="2.2.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="2.2.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.2.0" />

diff --git a/Microsoft.Azure.Cosmos/Directory.Build.props b/Microsoft.Azure.Cosmos/Directory.Build.props
@@ -1,8 +1,8 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <PropertyGroup>
-    <ClientVersion>3.3.1</ClientVersion>
-    <DirectVersion>3.3.0</DirectVersion>
+    <ClientVersion>3.3.2</ClientVersion>
+    <DirectVersion>3.4.0</DirectVersion>
     <HybridRowVersion>1.0.0-preview</HybridRowVersion>
     <AboveDirBuildProps>$([MSBuild]::GetPathOfFileAbove('Directory.Build.props', '$(MSBuildThisFileDirectory)../'))</AboveDirBuildProps>
   </PropertyGroup>

diff --git a/Microsoft.Azure.Cosmos/src/Linq/ExpressionToSQL.cs b/Microsoft.Azure.Cosmos/src/Linq/ExpressionToSQL.cs
@@ -86,8 +86,8 @@ public static class LinqMethods
         /// <returns>The corresponding SQL query.</returns>
         public static SqlQuery TranslateQuery(
             Expression inputExpression,
-            IDictionary<object, string> parameters = null,
-            CosmosSerializationOptions serializationOptions = null)
+            IDictionary<object, string> parameters,
+            CosmosSerializationOptions serializationOptions)
         {
             TranslationContext context = new TranslationContext(serializationOptions, parameters);
             ExpressionToSql.Translate(inputExpression, context); // ignore result here

diff --git a/Microsoft.Azure.Cosmos/src/Linq/SQLTranslator.cs b/Microsoft.Azure.Cosmos/src/Linq/SQLTranslator.cs
@@ -42,11 +42,11 @@ internal static string TranslateExpressionOld(
 
         internal static SqlQuerySpec TranslateQuery(
             Expression inputExpression,
-            CosmosSerializationOptions serializationOptions = null,
-            IDictionary<object, string> parameters = null)
+            CosmosSerializationOptions serializationOptions,
+            IDictionary<object, string> parameters)
         {
             inputExpression = ConstantEvaluator.PartialEval(inputExpression);
-            SqlQuery query = ExpressionToSql.TranslateQuery(inputExpression, parameters);
+            SqlQuery query = ExpressionToSql.TranslateQuery(inputExpression, parameters, serializationOptions);
             string queryText = null;
             SqlParameterCollection sqlParameters = new SqlParameterCollection();
             if (parameters != null && parameters.Count > 0)

diff --git a/Microsoft.Azure.Cosmos/src/Resource/QueryResponses/ChangeFeedResultSetIteratorCore.cs b/Microsoft.Azure.Cosmos/src/Resource/QueryResponses/ChangeFeedResultSetIteratorCore.cs
@@ -16,14 +16,10 @@ namespace Microsoft.Azure.Cosmos
     /// </summary>
     internal class ChangeFeedResultSetIteratorCore : FeedIterator
     {
-        private const int DefaultMaxItemCount = 100;
-        private const string PageSizeErrorOnChangeFeedText = "Reduce page size and try again.";
-
         internal StandByFeedContinuationToken compositeContinuationToken;
 
         private readonly CosmosClientContext clientContext;
         private readonly ContainerCore container;
-        private readonly int? originalMaxItemCount;
         private string containerRid;
         private string continuationToken;
         private int? maxItemCount;
@@ -41,7 +37,6 @@ internal ChangeFeedResultSetIteratorCore(
             this.container = container;
             this.changeFeedOptions = options;
             this.maxItemCount = maxItemCount;
-            this.originalMaxItemCount = maxItemCount;
             this.continuationToken = continuationToken;
         }
 
@@ -129,11 +124,6 @@ internal async Task<bool> ShouldRetryFailureAsync(
         {
             if (response.IsSuccessStatusCode || response.StatusCode == HttpStatusCode.NotModified)
             {
-                if (this.maxItemCount != this.originalMaxItemCount)
-                {
-                    this.maxItemCount = this.originalMaxItemCount;   // Reset after successful execution.
-                }
-
                 return false;
             }
 
@@ -146,22 +136,6 @@ internal async Task<bool> ShouldRetryFailureAsync(
                 return true;
             }
 
-            bool pageSizeError = response.ErrorMessage.Contains(ChangeFeedResultSetIteratorCore.PageSizeErrorOnChangeFeedText);
-            if (pageSizeError)
-            {
-                if (!this.maxItemCount.HasValue)
-                {
-                    this.maxItemCount = ChangeFeedResultSetIteratorCore.DefaultMaxItemCount;
-                }
-                else if (this.maxItemCount <= 1)
-                {
-                    return false;
-                }
-
-                this.maxItemCount /= 2;
-                return true;
-            }
-
             return false;
         }
 

diff --git a/Microsoft.Azure.Cosmos/src/Resource/Settings/CosmosClientSideRequestStatistics.cs b/Microsoft.Azure.Cosmos/src/Resource/Settings/CosmosClientSideRequestStatistics.cs
@@ -29,10 +29,6 @@ internal sealed class CosmosClientSideRequestStatistics : IClientSideRequestStat
         public List<StoreResponseStatistics> supplementalResponseStatisticsList { get; internal set; }
         public Dictionary<string, AddressResolutionStatistics> addressResolutionStatistics { get; private set; }
 
-        public List<Uri> ContactedReplicas { get; set; }
-        public HashSet<Uri> FailedReplicas { get; private set; }
-        public HashSet<Uri> RegionsContacted { get; private set; }
-
         public CosmosClientSideRequestStatistics()
         {
             this.requestStartTime = DateTime.UtcNow;
@@ -45,6 +41,12 @@ public CosmosClientSideRequestStatistics()
             this.RegionsContacted = new HashSet<Uri>();
         }
 
+        public List<Uri> ContactedReplicas { get; set; }
+
+        public HashSet<Uri> FailedReplicas { get; private set; }
+
+        public HashSet<Uri> RegionsContacted { get; private set; }
+
         public TimeSpan RequestLatency
         {
             get
@@ -154,16 +156,68 @@ public void RecordAddressResolutionEnd(string identifier)
 
         public override string ToString()
         {
-            if (this.supplementalResponseStatisticsList != null)
+            StringBuilder sb = new StringBuilder();
+            this.AppendToBuilder(sb);
+            return sb.ToString();
+        }
+
+        public void AppendToBuilder(StringBuilder stringBuilder)
+        {
+            if (stringBuilder == null)
+            {
+                throw new ArgumentNullException(nameof(stringBuilder));
+            }
+
+            //need to lock in case of concurrent operations. this should be extremely rare since ToString()
+            //should only be called at the end of request.
+            lock (this.lockObject)
             {
+                stringBuilder.AppendLine();
+
+                //first trace request start time, as well as total non-head/headfeed requests made.
+                stringBuilder.AppendFormat(
+                   CultureInfo.InvariantCulture,
+                   "RequestStartTime: {0}, RequestEndTime: {1},  Number of regions attempted:{2}",
+                   this.requestStartTime.ToString("o", CultureInfo.InvariantCulture),
+                   this.requestEndTime.ToString("o", CultureInfo.InvariantCulture),
+                   this.RegionsContacted.Count == 0 ? 1 : this.RegionsContacted.Count);
+                stringBuilder.AppendLine();
+
+                //take all responses here - this should be limited in number and each one contains relevant information.
+                foreach (StoreResponseStatistics item in this.responseStatisticsList)
+                {
+                    item.AppendToBuilder(stringBuilder);
+                    stringBuilder.AppendLine();
+                }
+
+                //take all responses here - this should be limited in number and each one is important.
+                foreach (AddressResolutionStatistics item in this.addressResolutionStatistics.Values)
+                {
+                    item.AppendToBuilder(stringBuilder);
+                    stringBuilder.AppendLine();
+                }
+
+                //only take last 10 responses from this list - this has potential of having large number of entries. 
+                //since this is for establishing consistency, we can make do with the last responses to paint a meaningful picture.
                 int supplementalResponseStatisticsListCount = this.supplementalResponseStatisticsList.Count;
-                int countToRemove = Math.Max(supplementalResponseStatisticsListCount - CosmosClientSideRequestStatistics.MaxSupplementalRequestsForToString, 0);
-                if (countToRemove > 0)
+                int initialIndex = Math.Max(supplementalResponseStatisticsListCount - CosmosClientSideRequestStatistics.MaxSupplementalRequestsForToString, 0);
+
+                if (initialIndex != 0)
+                {
+                    stringBuilder.AppendFormat(
+                        CultureInfo.InvariantCulture,
+                        "  -- Displaying only the last {0} head/headfeed requests. Total head/headfeed requests: {1}",
+                        CosmosClientSideRequestStatistics.MaxSupplementalRequestsForToString,
+                        supplementalResponseStatisticsListCount);
+                    stringBuilder.AppendLine();
+                }
+
+                for (int i = initialIndex; i < supplementalResponseStatisticsListCount; i++)
                 {
-                    this.supplementalResponseStatisticsList.RemoveRange(0, countToRemove);
+                    this.supplementalResponseStatisticsList[i].AppendToBuilder(stringBuilder);
+                    stringBuilder.AppendLine();
                 }
             }
-            return JsonConvert.SerializeObject(this, CosmosClientSideRequestStatistics.SerializerSettings);
         }
 
         internal struct StoreResponseStatistics
@@ -175,10 +229,31 @@ internal struct StoreResponseStatistics
 
             public override string ToString()
             {
-                return String.Format(CultureInfo.InvariantCulture, "ResponseTime: {0}, StoreResult: {1}, ResourceType: {2}, OperationType: {3}",
-                    this.RequestResponseTime.ToString("o", System.Globalization.CultureInfo.InvariantCulture),
-                    this.StoreResult != null ? this.StoreResult.ToString() : string.Empty,
-                    this.RequestResourceType, this.RequestOperationType);
+                StringBuilder stringBuilder = new StringBuilder();
+                this.AppendToBuilder(stringBuilder);
+                return stringBuilder.ToString();
+            }
+
+            public void AppendToBuilder(StringBuilder stringBuilder)
+            {
+                if (stringBuilder == null)
+                {
+                    throw new ArgumentNullException(nameof(stringBuilder));
+                }
+
+                stringBuilder.Append($"ResponseTime: {this.RequestResponseTime.ToString("o", CultureInfo.InvariantCulture)}, ");
+
+                stringBuilder.Append("StoreResult: ");
+                if (this.StoreResult != null)
+                {
+                    this.StoreResult.AppendToBuilder(stringBuilder);
+                }
+
+                stringBuilder.AppendFormat(
+                    CultureInfo.InvariantCulture,
+                    ", ResourceType: {0}, OperationType: {1}",
+                    this.RequestResourceType,
+                    this.RequestOperationType);
             }
         }
 
@@ -190,10 +265,24 @@ internal class AddressResolutionStatistics
 
             public override string ToString()
             {
-                return string.Format(CultureInfo.InvariantCulture, "AddressResolution - StartTime: {0}, EndTime: {1}, TargetEndpoint: {2}",
-                    this.StartTime.ToString("o", System.Globalization.CultureInfo.InvariantCulture),
-                    this.EndTime.ToString("o", System.Globalization.CultureInfo.InvariantCulture),
-                    this.TargetEndpoint);
+                StringBuilder stringBuilder = new StringBuilder();
+                this.AppendToBuilder(stringBuilder);
+                return stringBuilder.ToString();
+            }
+
+            public void AppendToBuilder(StringBuilder stringBuilder)
+            {
+                if (stringBuilder == null)
+                {
+                    throw new ArgumentNullException(nameof(stringBuilder));
+                }
+
+                stringBuilder
+                    .Append($"AddressResolution - StartTime: {this.StartTime.ToString("o", CultureInfo.InvariantCulture)}, ")
+                    .Append($"EndTime: {this.EndTime.ToString("o", CultureInfo.InvariantCulture)}, ")
+                    .Append("TargetEndpoint: ")
+                    .Append(this.TargetEndpoint);
+
             }
         }
     }

diff --git a/Microsoft.Azure.Cosmos/src/Routing/GatewayAddressCache.cs b/Microsoft.Azure.Cosmos/src/Routing/GatewayAddressCache.cs
@@ -106,11 +106,19 @@ public async Task OpenAsync(
             int batchSize = GatewayAddressCache.DefaultBatchSize;
 
 #if !(NETSTANDARD15 || NETSTANDARD16)
-            int userSpecifiedBatchSize = 0;
-            if (int.TryParse(System.Configuration.ConfigurationManager.AppSettings[GatewayAddressCache.AddressResolutionBatchSize], out userSpecifiedBatchSize))
+#if NETSTANDARD20
+            // GetEntryAssembly returns null when loaded from native netstandard2.0
+            if (System.Reflection.Assembly.GetEntryAssembly() != null)
             {
-                batchSize = userSpecifiedBatchSize;
+#endif
+                int userSpecifiedBatchSize = 0;
+                if (int.TryParse(System.Configuration.ConfigurationManager.AppSettings[GatewayAddressCache.AddressResolutionBatchSize], out userSpecifiedBatchSize))
+                {
+                    batchSize = userSpecifiedBatchSize;
+                }
+#if NETSTANDARD20
             }
+#endif  
 #endif
 
             string collectionAltLink = string.Format(CultureInfo.InvariantCulture, "{0}/{1}/{2}/{3}", Paths.DatabasesPathSegment, Uri.EscapeUriString(databaseName),