merged

Azure · Apr 1, 2021 · c972026 · c972026
2 parents b5b82b1 + 95793b5
commit c972026
Show file tree

Hide file tree

Showing 84 changed files with 14,960 additions and 7,328 deletions.
diff --git a/Directory.Build.props b/Directory.Build.props
@@ -1,14 +1,14 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
 	<PropertyGroup>
-		<ClientOfficialVersion>3.17.0</ClientOfficialVersion>
-		<ClientPreviewVersion>3.17.0</ClientPreviewVersion>
-		<ClientPreviewSuffixVersion>preview1</ClientPreviewSuffixVersion>
-		<DirectVersion>3.17.1</DirectVersion>
-		<EncryptionVersion>1.0.0-previewV12</EncryptionVersion>
-		<HybridRowVersion>1.1.0-preview1</HybridRowVersion>
+		<ClientOfficialVersion>3.17.1</ClientOfficialVersion>
+		<ClientPreviewVersion>3.18.0</ClientPreviewVersion>
+		<ClientPreviewSuffixVersion>preview</ClientPreviewSuffixVersion>
+		<DirectVersion>3.17.2</DirectVersion>
+		<EncryptionVersion>1.0.0-previewV13</EncryptionVersion>
+		<HybridRowVersion>1.1.0-preview3</HybridRowVersion>
 		<AboveDirBuildProps>$([MSBuild]::GetPathOfFileAbove('Directory.Build.props', '$(MSBuildThisFileDirectory)../'))</AboveDirBuildProps>
 		<DefineConstants Condition=" '$(IsNightly)' == 'true' or '$(IsPreview)' == 'true' ">$(DefineConstants);PREVIEW</DefineConstants>
 	</PropertyGroup>
 	<Import Project="$(AboveDirBuildProps)" Condition=" '$(AboveDirBuildProps)' != '' " />
-</Project>
+</Project>
diff --git a/Microsoft.Azure.Cosmos.Encryption/changelog.md b/Microsoft.Azure.Cosmos.Encryption/changelog.md
@@ -3,6 +3,11 @@ Preview features are treated as a separate branch and will not be included in th
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+### <a name="1.0.0-previewV13"/> [1.0.0-previewV13](https://www.nuget.org/packages/Microsoft.Azure.Cosmos.Encryption/1.0.0-previewV13) - 2021-03-26
+
+#### Added 
+- [#2340](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/2340) Adds integration with latest CosmosDb Preview Package - 3.18.0-preview.
+
 ### <a name="1.0.0-previewV12"/> [1.0.0-previewV12](https://www.nuget.org/packages/Microsoft.Azure.Cosmos.Encryption/1.0.0-previewV12) - 2021-03-15
 
 #### Added 

diff --git a/Microsoft.Azure.Cosmos.Encryption/src/Custom/EncryptionContainer.cs b/Microsoft.Azure.Cosmos.Encryption/src/Custom/EncryptionContainer.cs
@@ -873,5 +873,25 @@ public override FeedIterator<T> GetChangeFeedIterator<T>(
                     changeFeedRequestOptions),
                 this.ResponseFactory);
         }
+
+        public override Task<ItemResponse<T>> PatchItemAsync<T>(
+            string id,
+            PartitionKey partitionKey,
+            IReadOnlyList<PatchOperation> patchOperations,
+            PatchItemRequestOptions requestOptions = null,
+            CancellationToken cancellationToken = default)
+        {
+            throw new NotImplementedException();
+        }
+
+        public override Task<ResponseMessage> PatchItemStreamAsync(
+            string id,
+            PartitionKey partitionKey,
+            IReadOnlyList<PatchOperation> patchOperations,
+            PatchItemRequestOptions requestOptions = null,
+            CancellationToken cancellationToken = default)
+        {
+            throw new NotImplementedException();
+        }
     }
 }
diff --git a/Microsoft.Azure.Cosmos.Encryption/src/Custom/EncryptionTransactionalBatch.cs b/Microsoft.Azure.Cosmos.Encryption/src/Custom/EncryptionTransactionalBatch.cs
@@ -252,5 +252,13 @@ private async Task<TransactionalBatchResponse> DecryptTransactionalBatchResponse
                 response,
                 this.cosmosSerializer);
         }
+
+        public override TransactionalBatch PatchItem(
+            string id,
+            IReadOnlyList<PatchOperation> patchOperations,
+            TransactionalBatchPatchItemRequestOptions requestOptions = null)
+        {
+            throw new NotImplementedException();
+        }
     }
 }
diff --git a/Microsoft.Azure.Cosmos.Encryption/src/EncryptionContainer.cs b/Microsoft.Azure.Cosmos.Encryption/src/EncryptionContainer.cs
@@ -741,5 +741,25 @@ public override FeedIterator<T> GetChangeFeedIterator<T>(
                     changeFeedRequestOptions),
                 this.ResponseFactory);
         }
+
+        public override Task<ItemResponse<T>> PatchItemAsync<T>(
+            string id,
+            PartitionKey partitionKey,
+            IReadOnlyList<PatchOperation> patchOperations,
+            PatchItemRequestOptions requestOptions = null,
+            CancellationToken cancellationToken = default)
+        {
+            throw new NotImplementedException();
+        }
+
+        public override Task<ResponseMessage> PatchItemStreamAsync(
+            string id,
+            PartitionKey partitionKey,
+            IReadOnlyList<PatchOperation> patchOperations,
+            PatchItemRequestOptions requestOptions = null,
+            CancellationToken cancellationToken = default)
+        {
+            throw new NotImplementedException();
+        }
     }
 }
diff --git a/Microsoft.Azure.Cosmos.Encryption/src/EncryptionTransactionalBatch.cs b/Microsoft.Azure.Cosmos.Encryption/src/EncryptionTransactionalBatch.cs
@@ -202,5 +202,13 @@ private async Task<TransactionalBatchResponse> DecryptTransactionalBatchResponse
                 response,
                 this.cosmosSerializer);
         }
+
+        public override TransactionalBatch PatchItem(
+            string id,
+            IReadOnlyList<PatchOperation> patchOperations,
+            TransactionalBatchPatchItemRequestOptions requestOptions = null)
+        {
+            throw new NotImplementedException();
+        }
     }
 }
diff --git a/Microsoft.Azure.Cosmos.Encryption/src/Microsoft.Azure.Cosmos.Encryption.csproj b/Microsoft.Azure.Cosmos.Encryption/src/Microsoft.Azure.Cosmos.Encryption.csproj
@@ -26,7 +26,7 @@
 
   <ItemGroup>
      <PackageReference Include="Azure.Security.KeyVault.Keys" Version="4.0.3" />
-     <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.17.0-preview1" />
+     <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.18.0-preview" />
      <PackageReference Include="Azure.Core" Version="1.3.0" />
      <PackageReference Include="Azure.Identity" Version="1.1.1" />
      <PackageReference Include="Microsoft.Data.Encryption.Cryptography" Version="0.1.0-pre" />

diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/CosmosBenchmark.csproj b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/CosmosBenchmark.csproj
@@ -33,6 +33,6 @@
   <ItemGroup Condition=" '$(OSSProjectRef)' == 'True' ">
     <ProjectReference Include="..\..\..\Microsoft.Azure.Cosmos\src\Microsoft.Azure.Cosmos.csproj" />
     <PackageReference Include="Microsoft.Azure.Cosmos.Direct" Version="[$(DirectVersion)]" />
-    <PackageReference Include="Microsoft.Azure.Cosmos.Serialization.HybridRow" Version="[$(HybridRowVersion)]" />
+    <PackageReference Include="Microsoft.HybridRow" Version="[$(HybridRowVersion)]" />
   </ItemGroup>
 </Project>
diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs
@@ -129,13 +129,16 @@ private async Task<RunSummary> LogOutputStats(IExecutor[] executors)
                     runSummary.Top80PercentAverageRps = Math.Round(summaryCounters.Take((int)(0.8 * summaryCounters.Length)).Average(), 0);
                     runSummary.Top90PercentAverageRps = Math.Round(summaryCounters.Take((int)(0.9 * summaryCounters.Length)).Average(), 0);
                     runSummary.Top95PercentAverageRps = Math.Round(summaryCounters.Take((int)(0.95 * summaryCounters.Length)).Average(), 0);
+                    runSummary.Top99PercentAverageRps = Math.Round(summaryCounters.Take((int)(0.99 * summaryCounters.Length)).Average(), 0);
                     runSummary.AverageRps = Math.Round(summaryCounters.Average(), 0);
 
                     runSummary.Top50PercentLatencyInMs = TelemetrySpan.GetLatencyPercentile(50);
                     runSummary.Top75PercentLatencyInMs = TelemetrySpan.GetLatencyPercentile(75);
                     runSummary.Top90PercentLatencyInMs = TelemetrySpan.GetLatencyPercentile(90);
                     runSummary.Top95PercentLatencyInMs = TelemetrySpan.GetLatencyPercentile(95);
+                    runSummary.Top98PercentLatencyInMs = TelemetrySpan.GetLatencyPercentile(98);
                     runSummary.Top99PercentLatencyInMs = TelemetrySpan.GetLatencyPercentile(99);
+                    runSummary.MaxLatencyInMs = TelemetrySpan.GetLatencyPercentile(100);
 
                     string summary = JsonConvert.SerializeObject(runSummary);
                     Utility.TeeTraceInformation(summary);

diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/README.md b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/README.md
@@ -36,7 +36,7 @@ export ACCOUNT_KEY=<KEY>
 export RESULTS_PK="runs-summary" #For test runs use different one
 export PL=18
 
-dotnet run -c Release  -- -e $ACCOUNT_ENDPOINT -k $ACCOUNT_KEY --publishresults --resultspartitionkeyvalue $RESULTS_PK -commitid $(git log -1 | head -n 1 | cut -d ' ' -f 2) --commitdate $(git log -1 --date=format:'%Y-%m-%d %H:%M:%S' | grep Date | cut -f 2- -d ':' | sed 's/^[ \t]*//;s/[ \t]*$//' | cut -f 1 -d ' ') --committime $(git log -1 --date=format:'%Y-%m-%d %H:%M:%S' | grep Date | cut -f 2- -d ':' | sed 's/^[ \t]*//;s/[ \t]*$//' | cut -f 2 -d ' ') --branchname $(git rev-parse --abbrev-ref HEAD)  --database testdb --container testcol --partitionkeypath /pk -n 500000 -w ReadStreamExistsV3 --pl $PL 
+dotnet run -c Release  -- -e $ACCOUNT_ENDPOINT -k $ACCOUNT_KEY --tcp 10 --enablelatencypercentiles --disablecoresdklogging --publishresults --resultspartitionkeyvalue $RESULTS_PK --commitid $(git log -1 | head -n 1 | cut -d ' ' -f 2) --commitdate $(git log -1 --date=format:'%Y-%m-%d %H:%M:%S' | grep Date | cut -f 2- -d ':' | sed 's/^[ \t]*//;s/[ \t]*$//' | cut -f 1 -d ' ') --committime $(git log -1 --date=format:'%Y-%m-%d %H:%M:%S' | grep Date | cut -f 2- -d ':' | sed 's/^[ \t]*//;s/[ \t]*$//' | cut -f 2 -d ' ') --branchname $(git rev-parse --abbrev-ref HEAD)  --database testdb --container testcol --partitionkeypath /pk -n 2000000 -w ReadStreamExistsV3 --pl $PL 
 ```
 
 ![image](https://user-images.githubusercontent.com/6880899/61565403-8e41bd00-aa96-11e9-9996-b7fc77c3aed3.png)

diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/RunSummary.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/RunSummary.cs
@@ -57,12 +57,15 @@ public class RunSummary
         public double Top80PercentAverageRps { get; set; }
         public double Top90PercentAverageRps { get; set; }
         public double Top95PercentAverageRps { get; set; }
+        public double Top99PercentAverageRps { get; set; }
 
         public double? Top50PercentLatencyInMs { get; set; }
         public double? Top75PercentLatencyInMs { get; set; }
         public double? Top90PercentLatencyInMs { get; set; }
         public double? Top95PercentLatencyInMs { get; set; }
+        public double? Top98PercentLatencyInMs { get; set; }
         public double? Top99PercentLatencyInMs { get; set; }
+        public double? MaxLatencyInMs { get; set; }
 
         public double AverageRps { get; set; }
 

diff --git a/....Cosmos.Samples/Tools/Benchmark/v3/ReadStreamExistsWithDiagnosticsV3BenchmarkOperation.cs b/....Cosmos.Samples/Tools/Benchmark/v3/ReadStreamExistsWithDiagnosticsV3BenchmarkOperation.cs
@@ -0,0 +1,99 @@
+//------------------------------------------------------------
+// Copyright (c) Microsoft Corporation.  All rights reserved.
+//------------------------------------------------------------
+
+namespace CosmosBenchmark
+{
+    using System;
+    using System.Collections.Generic;
+    using System.IO;
+    using System.Net;
+    using System.Threading.Tasks;
+    using Microsoft.Azure.Cosmos;
+
+    internal class ReadStreamExistsWithDiagnosticsV3BenchmarkOperation : IBenchmarkOperation
+    {
+        private readonly Container container;
+        private readonly string partitionKeyPath;
+        private readonly Dictionary<string, object> sampleJObject;
+
+        private readonly string databsaeName;
+        private readonly string containerName;
+
+        private string nextExecutionItemPartitionKey;
+        private string nextExecutionItemId;
+
+        public ReadStreamExistsWithDiagnosticsV3BenchmarkOperation(
+            CosmosClient cosmosClient,
+            string dbName,
+            string containerName,
+            string partitionKeyPath,
+            string sampleJson)
+        {
+            this.databsaeName = dbName;
+            this.containerName = containerName;
+
+            this.container = cosmosClient.GetContainer(this.databsaeName, this.containerName);
+            this.partitionKeyPath = partitionKeyPath.Replace("/", "");
+
+            this.sampleJObject = JsonHelper.Deserialize<Dictionary<string, object>>(sampleJson);
+        }
+
+        public async Task<OperationResult> ExecuteOnceAsync()
+        {
+            using (ResponseMessage itemResponse = await this.container.ReadItemStreamAsync(
+                        this.nextExecutionItemId,
+                        new PartitionKey(this.nextExecutionItemPartitionKey)))
+            {
+                if (itemResponse.StatusCode != HttpStatusCode.OK)
+                {
+                    throw new Exception($"ReadItem failed wth {itemResponse.StatusCode}");
+                }
+
+                string diagnostics = itemResponse.Diagnostics.ToString();
+                if (string.IsNullOrEmpty(diagnostics))
+                {
+                    throw new Exception();
+                }
+
+                return new OperationResult()
+                {
+                    DatabseName = databsaeName,
+                    ContainerName = containerName,
+                    RuCharges = itemResponse.Headers.RequestCharge,
+                    CosmosDiagnostics = itemResponse.Diagnostics,
+                    LazyDiagnostics = () => itemResponse.Diagnostics.ToString(),
+                };
+            }
+        }
+
+        public async Task PrepareAsync()
+        {
+            if (string.IsNullOrEmpty(this.nextExecutionItemId) ||
+                string.IsNullOrEmpty(this.nextExecutionItemPartitionKey))
+            {
+                this.nextExecutionItemId = Guid.NewGuid().ToString();
+                this.nextExecutionItemPartitionKey = Guid.NewGuid().ToString();
+
+                this.sampleJObject["id"] = this.nextExecutionItemId;
+                this.sampleJObject[this.partitionKeyPath] = this.nextExecutionItemPartitionKey;
+
+                using (MemoryStream inputStream = JsonHelper.ToStream(this.sampleJObject))
+                {
+                    using (ResponseMessage itemResponse = await this.container.CreateItemStreamAsync(
+                            inputStream,
+                            new PartitionKey(this.nextExecutionItemPartitionKey)))
+                    {
+
+                        System.Buffers.ArrayPool<byte>.Shared.Return(inputStream.GetBuffer());
+
+                        if (itemResponse.StatusCode != HttpStatusCode.Created)
+                        {
+                            throw new Exception($"Create failed with statuscode: {itemResponse.StatusCode}");
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/CTL/CTLConfig.cs b/Microsoft.Azure.Cosmos.Samples/Tools/CTL/CTLConfig.cs
@@ -75,6 +75,9 @@ public string DiagnosticsThresholdDuration
         [Option("ctl_output_event_traces", Required = false, HelpText = "Outputs TraceSource to console")]
         public bool OutputEventTraces { get; set; } = false;
 
+        [Option("ctl_gateway_mode", Required = false, HelpText = "Uses gateway mode")]
+        public bool UseGatewayMode { get; set; } = false;
+
         [Option("ctl_reporting_interval", Required = false, HelpText = "Reporting interval")]
         public int ReportingIntervalInSeconds { get; set; } = 10;
 
@@ -113,6 +116,11 @@ internal CosmosClient CreateCosmosClient()
                 ApplicationName = CTLConfig.UserAgentSuffix
             };
 
+            if (this.UseGatewayMode)
+            {
+                clientOptions.ConnectionMode = ConnectionMode.Gateway;
+            }
+
             if (!string.IsNullOrWhiteSpace(this.ConsistencyLevel))
             {
                 if (Enum.TryParse(this.ConsistencyLevel, out ConsistencyLevel consistencyLevel))

diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/CTL/CTLOperationHandler.cs b/Microsoft.Azure.Cosmos.Samples/Tools/CTL/CTLOperationHandler.cs
@@ -5,7 +5,6 @@
 namespace CosmosCTL
 {
     using System;
-    using System.Diagnostics;
     using System.Threading;
     using System.Threading.Tasks;
     using App.Metrics.Timer;
@@ -18,7 +17,6 @@ internal static class CTLOperationHandler<T>
         /// <summary>
         /// Waits until the synchronization semaphore is available, creates a new operation and handles resolution.
         /// </summary>
-        /// <param name="semaphoreSlim">Synchronization semaphore that defines maximum degree of parallelism.</param>
         /// <param name="diagnosticsLoggingThreshold">Latency threshold above which <paramref name="logDiagnostics"/> will be called.</param>
         /// <param name="createTimerContext">Creates a <see cref="TimerContext"/> to measure operation latency.</param>
         /// <param name="resultProducer">Producer to generate operation calls as a producer-consumer.</param>
@@ -28,7 +26,6 @@ internal static class CTLOperationHandler<T>
         /// <param name="cancellationToken"></param>
         /// <returns></returns>
         public static async Task PerformOperationAsync(
-            SemaphoreSlim semaphoreSlim,
             long diagnosticsLoggingThreshold,
             Func<TimerContext> createTimerContext,
             ICTLResultProducer<T> resultProducer,
@@ -39,12 +36,10 @@ public static async Task PerformOperationAsync(
         {
             while (resultProducer.HasMoreResults)
             {
-                await semaphoreSlim.WaitAsync(cancellationToken);
                 using (TimerContext timerContext = createTimerContext())
                 {
                     await resultProducer.GetNextAsync().ContinueWith(task =>
                     {
-                        semaphoreSlim.Release();
                         long latency = (long)timerContext.Elapsed.TotalMilliseconds;
                         if (task.IsCompletedSuccessfully)
                         {
@@ -53,7 +48,10 @@ await resultProducer.GetNextAsync().ContinueWith(task =>
                                 logDiagnostics(task.Result);
                             }
 
-                            onSuccess();
+                            if (!resultProducer.HasMoreResults)
+                            {
+                                onSuccess();
+                            }
                         }
                         else
                         {