diff --git a/Directory.Build.props b/Directory.Build.props index 12639600f6..412e7a1712 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -7,7 +7,7 @@ 2.0.4 2.1.0 preview4 - 1.0.0-preview06 + 1.0.0-preview07 1.1.0-preview3 10.0 $([MSBuild]::GetPathOfFileAbove('Directory.Build.props', '$(MSBuildThisFileDirectory)../')) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/changelog.md b/Microsoft.Azure.Cosmos.Encryption.Custom/changelog.md index a96c9c325b..7fc52287ce 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/changelog.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/changelog.md @@ -3,6 +3,11 @@ Preview features are treated as a separate branch and will not be included in th The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +### [1.0.0-preview07](https://www.nuget.org/packages/Microsoft.Azure.Cosmos.Encryption.Custom/1.0.0-preview07) - 2024-06-12 + +#### Fixes +- [#4546](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/4546) Updates package reference Microsoft.Azure.Cosmos to version 3.41.0-preview and 3.40.0 for preview and stable version support. + ### [1.0.0-preview06](https://www.nuget.org/packages/Microsoft.Azure.Cosmos.Encryption.Custom/1.0.0-preview06) - 2023-06-28 #### Fixes diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs index 10496f704a..b898a77179 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs @@ -1020,12 +1020,8 @@ public override Task DeleteAllItemsByPartitionKeyStreamAsync( requestOptions, cancellationToken); } -#endif -#if SDKPROJECTREF - public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilderWithAllVersionsAndDeletes( - string processorName, - ChangeFeedHandler> onChangesDelegate) + public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilderWithAllVersionsAndDeletes(string processorName, ChangeFeedHandler> onChangesDelegate) { return this.container.GetChangeFeedProcessorBuilderWithAllVersionsAndDeletes( processorName, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj index b7c5eb3955..ed4cca0596 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj @@ -23,11 +23,11 @@ - + - + @@ -36,7 +36,7 @@ - + diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs index 2785f04815..fd83ef528d 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs @@ -308,11 +308,14 @@ public async Task EncryptionCreateItem() TestDoc expectedDoc = new TestDoc(testDoc); +#if SDKPROJECTREF + // FIXME Remove the above once the binary encoding issue is fixed. // Read feed (null query) await LegacyEncryptionTests.ValidateQueryResultsAsync( LegacyEncryptionTests.encryptionContainer, query: null, expectedDoc); +#endif await LegacyEncryptionTests.ValidateQueryResultsAsync( LegacyEncryptionTests.encryptionContainer, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs index e100268ea0..85b7bf3f36 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs @@ -441,11 +441,14 @@ public async Task EncryptionCreateItem() TestDoc expectedDoc = new TestDoc(testDoc); +#if SDKPROJECTREF + // FIXME Remove the above once the binary encoding issue is fixed. // Read feed (null query) await MdeCustomEncryptionTests.ValidateQueryResultsAsync( MdeCustomEncryptionTests.encryptionContainer, query: null, expectedDoc); +#endif await MdeCustomEncryptionTests.ValidateQueryResultsAsync( MdeCustomEncryptionTests.encryptionContainer, @@ -1043,7 +1046,7 @@ public async Task VerifyDekOperationWithSystemTextSerializer() DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull }; - CosmosSystemTextJsonSerializer cosmosSystemTextJsonSerializer = new CosmosSystemTextJsonSerializer(jsonSerializerOptions); + LegacyEncryptionTests.CosmosSystemTextJsonSerializer cosmosSystemTextJsonSerializer = new (jsonSerializerOptions); CosmosClient clientWithCosmosSystemTextJsonSerializer = TestCommon.CreateCosmosClient(builder => builder .WithCustomSerializer(cosmosSystemTextJsonSerializer) @@ -2263,7 +2266,7 @@ public override async Task EncryptAsync( #region Legacy - #pragma warning disable CS0618 // Type or member is obsolete +#pragma warning disable CS0618 // Type or member is obsolete [TestMethod] public async Task EncryptionCreateDekWithDualDekProvider() { @@ -2477,12 +2480,15 @@ public async Task ReadLegacyEncryptedDataWithMdeProcessor() TestDoc expectedDoc = new TestDoc(testDoc); +#if SDKPROJECTREF + // FIXME Remove the above once the binary encoding issue is fixed. // Read feed (null query) await MdeCustomEncryptionTests.ValidateQueryResultsAsync( MdeCustomEncryptionTests.encryptionContainer, query: null, expectedDoc, legacyAlgo: true); +#endif await MdeCustomEncryptionTests.ValidateQueryResultsAsync( MdeCustomEncryptionTests.encryptionContainer, @@ -2651,7 +2657,7 @@ public override Task WrapKeyAsync(byte[] key, Encryptio } } - #pragma warning restore CS0618 // Type or member is obsolete - #endregion +#pragma warning restore CS0618 // Type or member is obsolete +#endregion } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/CosmosEncryptorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/CosmosEncryptorTests.cs index 63c5b370d1..2ef5af58ac 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/CosmosEncryptorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/CosmosEncryptorTests.cs @@ -55,7 +55,7 @@ await CosmosEncryptorTests.cosmosEncryptor.EncryptAsync( } catch (InvalidOperationException ex) { - Assert.AreEqual("Null DataEncryptionKey returned from FetchDataEncryptionKeyAsync.", ex.Message); + Assert.AreEqual("Null DataEncryptionKey returned from FetchDataEncryptionKeyWithoutRawKeyAsync.", ex.Message); } } diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/FaultInjection/src/implementataion/ChaosInterceptor.cs b/Microsoft.Azure.Cosmos.Samples/Tools/FaultInjection/src/implementataion/ChaosInterceptor.cs index 34e3211fee..17a7a547ed 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/FaultInjection/src/implementataion/ChaosInterceptor.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/FaultInjection/src/implementataion/ChaosInterceptor.cs @@ -237,5 +237,20 @@ internal FaultInjectionDynamicChannelStore GetChannelStore() { return this.channelStore; } + + public Task<(bool, StoreResponse)> OnHttpRequestCallAsync(DocumentServiceRequest request) + { + throw new NotImplementedException(); + } + + public Task OnBeforeHttpSendAsync(DocumentServiceRequest request) + { + throw new NotImplementedException(); + } + + public Task OnAfterHttpSendAsync(DocumentServiceRequest request) + { + throw new NotImplementedException(); + } } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Samples/Usage/ReEncryption/ReEncryption.csproj b/Microsoft.Azure.Cosmos.Samples/Usage/ReEncryption/ReEncryption.csproj index 2677c74426..bd49c1df40 100644 --- a/Microsoft.Azure.Cosmos.Samples/Usage/ReEncryption/ReEncryption.csproj +++ b/Microsoft.Azure.Cosmos.Samples/Usage/ReEncryption/ReEncryption.csproj @@ -8,7 +8,7 @@ latest - + diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CrossPartition/Parallel/ParallelCrossPartitionQueryPipelineStage.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CrossPartition/Parallel/ParallelCrossPartitionQueryPipelineStage.cs index 7f1d1079c1..fb5f87db73 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CrossPartition/Parallel/ParallelCrossPartitionQueryPipelineStage.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CrossPartition/Parallel/ParallelCrossPartitionQueryPipelineStage.cs @@ -78,14 +78,12 @@ public async ValueTask MoveNextAsync(ITrace trace, CancellationToken cance else { // left most and any non null continuations - IOrderedEnumerable> feedRangeStates = crossPartitionState - .Value - .ToArray() - .OrderBy(tuple => ((FeedRangeEpk)tuple.FeedRange).Range.Min); + FeedRangeState[] feedRangeStates = crossPartitionState.Value.ToArray(); + Array.Sort>(feedRangeStates, (x, y) => string.CompareOrdinal(((FeedRangeEpk)x.FeedRange).Range.Min, ((FeedRangeEpk)y.FeedRange).Range.Min)); List activeParallelContinuationTokens = new List(); { - FeedRangeState firstState = feedRangeStates.First(); + FeedRangeState firstState = feedRangeStates[0]; ParallelContinuationToken firstParallelContinuationToken = new ParallelContinuationToken( token: firstState.State != null ? ((CosmosString)firstState.State.Value).Value : null, range: ((FeedRangeEpk)firstState.FeedRange).Range); @@ -93,10 +91,11 @@ public async ValueTask MoveNextAsync(ITrace trace, CancellationToken cance activeParallelContinuationTokens.Add(firstParallelContinuationToken); } - foreach (FeedRangeState feedRangeState in feedRangeStates.Skip(1)) + for (int i = 1; i < feedRangeStates.Length; i++) { cancellationToken.ThrowIfCancellationRequested(); + FeedRangeState feedRangeState = feedRangeStates[i]; if (feedRangeState.State != null) { ParallelContinuationToken parallelContinuationToken = new ParallelContinuationToken( diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/QueryPlan/QueryPartitionProvider.cs b/Microsoft.Azure.Cosmos/src/Query/Core/QueryPlan/QueryPartitionProvider.cs index 6bec60605e..b326c47579 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/QueryPlan/QueryPartitionProvider.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/QueryPlan/QueryPartitionProvider.cs @@ -270,13 +270,15 @@ internal TryCatch TryGetPartitionedQueryE fixed (byte* bytePtr2 = buffer) { - errorCode = ServiceInteropWrapper.GetPartitionKeyRangesFromQuery3( + errorCode = ServiceInteropWrapper.GetPartitionKeyRangesFromQuery4( this.serviceProvider, querySpecJsonString, partitionKeyRangesApiOptions, allParts, partsLengths, (uint)partitionKeyDefinition.Paths.Count, + vectorEmbeddingPolicyString, + vectorEmbeddingPolicyString?.Length ?? 0, new IntPtr(bytePtr2), (uint)buffer.Length, out serializedQueryExecutionInfoResultLength); diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs index c5bd4642fa..38bf63c79b 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs @@ -19,55 +19,36 @@ namespace Microsoft.Azure.Cosmos #endif class ChangeFeedMetadata { - /// - /// New instance of meta data for created. - /// - /// - /// - /// - /// - public ChangeFeedMetadata( - DateTime conflictResolutionTimestamp, - long lsn, - ChangeFeedOperationType operationType, - long previousLsn) - { - this.ConflictResolutionTimestamp = conflictResolutionTimestamp; - this.Lsn = lsn; - this.OperationType = operationType; - this.PreviousLsn = previousLsn; - } - /// /// The conflict resolution timestamp. /// [JsonProperty(PropertyName = "crts", NullValueHandling = NullValueHandling.Ignore)] [JsonConverter(typeof(UnixDateTimeConverter))] - public DateTime ConflictResolutionTimestamp { get; } + public DateTime ConflictResolutionTimestamp { get; internal set; } /// /// The current logical sequence number. /// [JsonProperty(PropertyName = "lsn", NullValueHandling = NullValueHandling.Ignore)] - public long Lsn { get; } + public long Lsn { get; internal set; } /// /// The change feed operation type. /// [JsonProperty(PropertyName = "operationType")] [JsonConverter(typeof(StringEnumConverter))] - public ChangeFeedOperationType OperationType { get; } + public ChangeFeedOperationType OperationType { get; internal set; } /// /// The previous logical sequence number. /// [JsonProperty(PropertyName = "previousImageLSN", NullValueHandling = NullValueHandling.Ignore)] - public long PreviousLsn { get; } + public long PreviousLsn { get; internal set; } /// /// Used to distinquish explicit deletes (e.g. via DeleteItem) from deletes caused by TTL expiration (a collection may define time-to-live policy for documents). /// [JsonProperty(PropertyName = "timeToLiveExpired", NullValueHandling= NullValueHandling.Ignore)] - public bool IsTimeToLiveExpired { get; } + public bool IsTimeToLiveExpired { get; internal set; } } } diff --git a/Microsoft.Azure.Cosmos/src/Serializer/CosmosSystemTextJsonSerializer.cs b/Microsoft.Azure.Cosmos/src/Serializer/CosmosSystemTextJsonSerializer.cs new file mode 100644 index 0000000000..ac5550b128 --- /dev/null +++ b/Microsoft.Azure.Cosmos/src/Serializer/CosmosSystemTextJsonSerializer.cs @@ -0,0 +1,104 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos +{ + using System; + using System.IO; + using System.Reflection; + using System.Text.Json; + using System.Text.Json.Serialization; + + /// + /// This class provides a default implementation of System.Text.Json Cosmos Linq Serializer. + /// + internal class CosmosSystemTextJsonSerializer : CosmosLinqSerializer + { + /// + /// A read-only instance of . + /// + private readonly JsonSerializerOptions jsonSerializerOptions; + + /// + /// Creates an instance of + /// with the default values for the Cosmos SDK + /// + /// An instance of containing the json serialization options. + internal CosmosSystemTextJsonSerializer( + JsonSerializerOptions jsonSerializerOptions) + { + this.jsonSerializerOptions = jsonSerializerOptions; + } + + /// + public override T FromStream(Stream stream) + { + if (stream == null) + throw new ArgumentNullException(nameof(stream)); + + if (typeof(Stream).IsAssignableFrom(typeof(T))) + { + return (T)(object)stream; + } + + if (stream.CanSeek && stream.Length == 0) + { + return default; + } + + using (stream) + { + using StreamReader reader = new (stream); + return JsonSerializer.Deserialize(reader.ReadToEnd(), this.jsonSerializerOptions); + } + } + + /// + public override Stream ToStream(T input) + { + MemoryStream streamPayload = new (); + using Utf8JsonWriter writer = new (streamPayload); + + JsonSerializer.Serialize(writer, input, this.jsonSerializerOptions); + + streamPayload.Position = 0; + return streamPayload; + } + + /// + /// Convert a MemberInfo to a string for use in LINQ query translation. + /// + /// Any MemberInfo used in the query. + /// A serialized representation of the member. + /// + /// Note that this is just a default implementation which handles the basic scenarios.To handle any special cases, + /// please create a custom serializer which inherits from the and overrides the + /// SerializeMemberName() method. + /// + public override string SerializeMemberName(MemberInfo memberInfo) + { + JsonExtensionDataAttribute jsonExtensionDataAttribute = + memberInfo.GetCustomAttribute(true); + + if (jsonExtensionDataAttribute != null) + { + return null; + } + + JsonPropertyNameAttribute jsonPropertyNameAttribute = memberInfo.GetCustomAttribute(true); + + if (!string.IsNullOrEmpty(jsonPropertyNameAttribute?.Name)) + { + return jsonPropertyNameAttribute.Name; + } + + if (this.jsonSerializerOptions.PropertyNamingPolicy != null) + { + return this.jsonSerializerOptions.PropertyNamingPolicy.ConvertName(memberInfo.Name); + } + + return memberInfo.Name; + } + } +} diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqAggregateCustomSerializationBaseline.TestAggregateQueriesWithCustomSerializer.xml b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqAggregateCustomSerializationBaseline.TestAggregateQueriesWithCustomSerializer.xml index 1a7337ba81..10b5e673b6 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqAggregateCustomSerializationBaseline.TestAggregateQueriesWithCustomSerializer.xml +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqAggregateCustomSerializationBaseline.TestAggregateQueriesWithCustomSerializer.xml @@ -1,7 +1,7 @@  - + doc.NumericField), Object)]]> @@ -12,7 +12,7 @@ FROM root]]> - + doc.NumericField), Object)]]> @@ -23,7 +23,7 @@ FROM root]]> - Filter -> Select -> Average, Custom serializer: True]]> + Filter -> Select -> Average, Serializer Name: SystemTextJsonLinqSerializer]]> doc.ArrayField.Where(m => ((m % 3) == 0)).Select(m => m)).Average(), Object)]]> @@ -36,7 +36,7 @@ WHERE ((m0 % 3) = 0)]]> - Skip -> Count, Custom serializer: True]]> + Skip -> Count, Serializer Name: SystemTextJsonLinqSerializer]]> f.NumericField).Skip(2).Count(), Object)]]> @@ -63,7 +63,7 @@ FROM root]]> - + doc.NumericField), Object)]]> @@ -74,7 +74,7 @@ FROM root]]> - + doc.NumericField), Object)]]> @@ -85,7 +85,7 @@ FROM root]]> - Filter -> Select -> Average, Custom serializer: False]]> + Filter -> Select -> Average, Serializer Name: SystemTextJsonSerializer]]> doc.ArrayField.Where(m => ((m % 3) == 0)).Select(m => m)).Average(), Object)]]> @@ -98,7 +98,7 @@ WHERE ((m0 % 3) = 0)]]> - Skip -> Count, Custom serializer: False]]> + Skip -> Count, Serializer Name: SystemTextJsonSerializer]]> f.NumericField).Skip(2).Count(), Object)]]> @@ -120,6 +120,68 @@ FROM ( + + + + + + doc.NumericField), Object)]]> + + + + + + + + + doc.NumericField), Object)]]> + + + + + + + + Filter -> Select -> Average, Serializer Name: CosmosSystemTextJsonSerializer]]> + doc.ArrayField.Where(m => ((m % 3) == 0)).Select(m => m)).Average(), Object)]]> + + + + + + + + Skip -> Count, Serializer Name: CosmosSystemTextJsonSerializer]]> + f.NumericField).Skip(2).Count(), Object)]]> + + + + + + + + + Min w/ mapping]]> + doc.NumericField).Min(num => num), Object)]]> + + + diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqTranslationWithCustomSerializerBaseline.TestMemberInitializerDotNetDefaultSerializer.xml b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqTranslationWithCustomSerializerBaseline.TestMemberInitializerDotNetDefaultSerializer.xml new file mode 100644 index 0000000000..ca941f5bd0 --- /dev/null +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqTranslationWithCustomSerializerBaseline.TestMemberInitializerDotNetDefaultSerializer.xml @@ -0,0 +1,237 @@ + + + + + (doc.NumericField == 1))]]> + + + + + + + + + + + (doc == new DataObjectDotNet() {NumericField = 1, StringField = "1"}))]]> + + + + + + + + + + + new DataObjectDotNet() {NumericField = 1, StringField = "1"})]]> + + + + + + + + + + + IIF((doc.NumericField > 1), new DataObjectDotNet() {NumericField = 1, StringField = "1"}, new DataObjectDotNet() {NumericField = 1, StringField = "1"}))]]> + + + 1) ? {"NumberValueDotNet": 1, "StringValueDotNet": "1", "id": null, "Pk": null, "DateTimeFieldDotNet": null, "DataTypeField": null} : {"NumberValueDotNet": 1, "StringValueDotNet": "1", "id": null, "Pk": null, "DateTimeFieldDotNet": null, "DataTypeField": null}) +FROM root]]> + + + + + + + + (doc == new DataObjectDotNet() {NumericField = doc.NumericField, StringField = doc.StringField})).Select(b => "A")]]> + + + + + + + + + + + x).OrderBy(x => x.NumericField).Take(5)]]> + + + + + + + + + + + IIF((c.NumericField > 1), "true", "false"))]]> + + + 1) ? "true" : "false") +FROM root]]> + + + + + + + + (doc.DateTimeField != null))]]> + + + + + + + + + + + (doc.DataTypeField != null))]]> + + + + + + + + + + + (doc.DateTimeField == Convert(new DateTime(1970, 1, 1, 0, 0, 0, 0, Utc), Nullable`1)))]]> + + + + + + + + + + + (Convert(doc.DataTypeField, Nullable`1) == Convert(Point, Nullable`1)))]]> + + + + + + + + + + + (doc.StringField != null))]]> + + + + + + + + \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/ChangeFeed/GetChangeFeedProcessorBuilderWithAllVersionsAndDeletesTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/ChangeFeed/GetChangeFeedProcessorBuilderWithAllVersionsAndDeletesTests.cs index 7217993fd5..0479bbb353 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/ChangeFeed/GetChangeFeedProcessorBuilderWithAllVersionsAndDeletesTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/ChangeFeed/GetChangeFeedProcessorBuilderWithAllVersionsAndDeletesTests.cs @@ -6,11 +6,14 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests.ChangeFeed { using System; using System.Collections.Generic; + using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.Cosmos.ChangeFeed.Utils; + using Microsoft.Azure.Cosmos.Services.Management.Tests; using Microsoft.VisualStudio.TestTools.UnitTesting; + using Newtonsoft.Json; using Newtonsoft.Json.Linq; [TestClass] @@ -29,6 +32,115 @@ public async Task Cleanup() await base.TestCleanup(); } + [TestMethod] + [Timeout(300000)] + [TestCategory("LongRunning")] + [Owner("philipthomas-MSFT")] + [Description("Scenario: When a document is created with ttl set, there should be 1 create and 1 delete that will appear for that " + + "document when using ChangeFeedProcessor with AllVersionsAndDeletes set as the ChangeFeedMode.")] + public async Task WhenADocumentIsCreatedWithTtlSetThenTheDocumentIsDeletedTestsAsync() + { + ContainerInternal monitoredContainer = await this.CreateMonitoredContainer(ChangeFeedMode.AllVersionsAndDeletes); + Exception exception = default; + int ttlInSeconds = 5; + Stopwatch stopwatch = new(); + ManualResetEvent allDocsProcessed = new ManualResetEvent(false); + + ChangeFeedProcessor processor = monitoredContainer + .GetChangeFeedProcessorBuilderWithAllVersionsAndDeletes(processorName: "processor", onChangesDelegate: (ChangeFeedProcessorContext context, IReadOnlyCollection> docs, CancellationToken token) => + { + // NOTE(philipthomas-MSFT): Please allow these Logger.LogLine because TTL on items will purge at random times so I am using this to test when ran locally using emulator. + + Logger.LogLine($"@ {DateTime.Now}, {nameof(stopwatch)} -> CFP AVAD took '{stopwatch.ElapsedMilliseconds}' to read document CRUD in feed."); + Logger.LogLine($"@ {DateTime.Now}, {nameof(docs)} -> {JsonConvert.SerializeObject(docs)}"); + + foreach (ChangeFeedItem change in docs) + { + if (change.Metadata.OperationType == ChangeFeedOperationType.Create) + { + // current + Assert.AreEqual(expected: "1", actual: change.Current.id.ToString()); + Assert.AreEqual(expected: "1", actual: change.Current.pk.ToString()); + Assert.AreEqual(expected: "Testing TTL on CFP.", actual: change.Current.description.ToString()); + Assert.AreEqual(expected: ttlInSeconds, actual: change.Current.ttl.ToObject()); + + // metadata + Assert.IsTrue(DateTime.TryParse(s: change.Metadata.ConflictResolutionTimestamp.ToString(), out _), message: "Invalid csrt must be a datetime value."); + Assert.IsTrue(change.Metadata.Lsn > 0, message: "Invalid lsn must be a long value."); + Assert.IsFalse(change.Metadata.IsTimeToLiveExpired); + + // previous + Assert.IsNull(change.Previous); + } + else if (change.Metadata.OperationType == ChangeFeedOperationType.Delete) + { + // current + Assert.IsNull(change.Current.id); + + // metadata + Assert.IsTrue(DateTime.TryParse(s: change.Metadata.ConflictResolutionTimestamp.ToString(), out _), message: "Invalid csrt must be a datetime value."); + Assert.IsTrue(change.Metadata.Lsn > 0, message: "Invalid lsn must be a long value."); + Assert.IsTrue(change.Metadata.IsTimeToLiveExpired); + + // previous + Assert.AreEqual(expected: "1", actual: change.Previous.id.ToString()); + Assert.AreEqual(expected: "1", actual: change.Previous.pk.ToString()); + Assert.AreEqual(expected: "Testing TTL on CFP.", actual: change.Previous.description.ToString()); + Assert.AreEqual(expected: ttlInSeconds, actual: change.Previous.ttl.ToObject()); + + // stop after reading delete since it is the last document in feed. + stopwatch.Stop(); + allDocsProcessed.Set(); + } + else + { + Assert.Fail("Invalid operation."); + } + } + + return Task.CompletedTask; + }) + .WithInstanceName(Guid.NewGuid().ToString()) + .WithLeaseContainer(this.LeaseContainer) + .WithErrorNotification((leaseToken, error) => + { + exception = error.InnerException; + + return Task.CompletedTask; + }) + .Build(); + + stopwatch.Start(); + + // NOTE(philipthomas-MSFT): Please allow these Logger.LogLine because TTL on items will purge at random times so I am using this to test when ran locally using emulator. + + Logger.LogLine($"@ {DateTime.Now}, CFProcessor starting..."); + + await processor.StartAsync(); + + try + { + await Task.Delay(GetChangeFeedProcessorBuilderWithAllVersionsAndDeletesTests.ChangeFeedSetupTime); + await monitoredContainer.CreateItemAsync(new { id = "1", pk = "1", description = "Testing TTL on CFP.", ttl = ttlInSeconds }, partitionKey: new PartitionKey("1")); + + // NOTE(philipthomas-MSFT): Please allow these Logger.LogLine because TTL on items will purge at random times so I am using this to test when ran locally using emulator. + + Logger.LogLine($"@ {DateTime.Now}, Document created."); + + bool receivedDelete = allDocsProcessed.WaitOne(250000); + Assert.IsTrue(receivedDelete, "Timed out waiting for docs to process"); + + if (exception != default) + { + Assert.Fail(exception.ToString()); + } + } + finally + { + await processor.StopAsync(); + } + } + [TestMethod] [Owner("philipthomas-MSFT")] [Description("Scenario: When a document is created, then updated, and finally deleted, there should be 3 changes that will appear for that " + @@ -467,6 +579,7 @@ private async Task CreateMonitoredContainer(ChangeFeedMode ch if (changeFeedMode == ChangeFeedMode.AllVersionsAndDeletes) { properties.ChangeFeedPolicy.FullFidelityRetention = TimeSpan.FromMinutes(5); + properties.DefaultTimeToLive = -1; } ContainerResponse response = await this.database.CreateContainerAsync(properties, diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/ClientTelemetryReleaseTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/ClientTelemetryReleaseTests.cs index 07230e63a8..2cf15bbf56 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/ClientTelemetryReleaseTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/ClientTelemetryReleaseTests.cs @@ -14,6 +14,7 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests /// If you are making changes in this file please make sure you are adding similar test in also. /// [TestClass] + [TestCategory("Quarantine") /* Release pipelines failing to connect to telemetry service */] [TestCategory("ClientTelemetryRelease")] public class ClientTelemetryReleaseTests : ClientTelemetryTestsBase { diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Linq/LinqAggregateCustomSerializationBaseline.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Linq/LinqAggregateCustomSerializationBaseline.cs index f7be777aee..ac686b31e1 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Linq/LinqAggregateCustomSerializationBaseline.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Linq/LinqAggregateCustomSerializationBaseline.cs @@ -29,6 +29,11 @@ public class LinqAggregateCustomSerializationBaseline : BaselineTests lastExecutedScalarQuery = q; - string dbName = $"{nameof(LinqAggregateCustomSerializationBaseline)}-{Guid.NewGuid().ToString("N")}"; + string dbName = $"{nameof(LinqAggregateCustomSerializationBaseline)}-{Guid.NewGuid():N}"; testDbLinq = await clientLinq.CreateDatabaseAsync(dbName); testContainerLinq = testDbLinq.CreateContainerAsync(new ContainerProperties(id: Guid.NewGuid().ToString(), partitionKeyPath: "/Pk")).Result; @@ -55,9 +60,23 @@ public async static Task Initialize(TestContext textContext) // to verify the translated LINQ directly as other queries type. client.DocumentClient.OnExecuteScalarQueryCallback = q => lastExecutedScalarQuery = q; - dbName = $"{nameof(LinqAggregateCustomSerializationBaseline)}-{Guid.NewGuid().ToString("N")}"; + dbName = $"{nameof(LinqAggregateCustomSerializationBaseline)}-{Guid.NewGuid():N}"; testDb = await client.CreateDatabaseAsync(dbName); testContainer = testDb.CreateContainerAsync(new ContainerProperties(id: Guid.NewGuid().ToString(), partitionKeyPath: "/Pk")).Result; + + stjCosmosSerializer = new CosmosSystemTextJsonSerializer(new JsonSerializerOptions()); + + stjClient = TestCommon.CreateCosmosClient((cosmosClientBuilder) + => cosmosClientBuilder.WithCustomSerializer(stjCosmosSerializer)); + + // Set a callback to get the handle of the last executed query to do the verification + // This is neede because aggregate queries return type is a scalar so it can't be used + // to verify the translated LINQ directly as other queries type. + stjClient.DocumentClient.OnExecuteScalarQueryCallback = q => lastExecutedScalarQuery = q; + + dbName = $"{nameof(LinqAggregateCustomSerializationBaseline)}-{Guid.NewGuid():N}"; + testDbSTJ = await stjClient.CreateDatabaseAsync(dbName); + testContainerSTJ = testDbSTJ.CreateContainerAsync(new ContainerProperties(id: Guid.NewGuid().ToString(), partitionKeyPath: "/Pk")).Result; } [ClassCleanup] @@ -98,29 +117,37 @@ static DataObjectDotNet createDataObj(int index, bool camelCase) List>> getQueryList = new List>> { LinqTestsCommon.GenerateSerializationTestCosmosData(createDataObj, 5, testContainerLinq, new CosmosLinqSerializerOptions()), - LinqTestsCommon.GenerateSerializationTestCosmosData(createDataObj, 5, testContainer, new CosmosLinqSerializerOptions()) + LinqTestsCommon.GenerateSerializationTestCosmosData(createDataObj, 5, testContainer, new CosmosLinqSerializerOptions()), + LinqTestsCommon.GenerateSerializationTestCosmosData(createDataObj, 5, testContainerSTJ, new CosmosLinqSerializerOptions()) + }; + + Dictionary serializerIndexes = new() + { + { nameof(SystemTextJsonLinqSerializer), 0 }, + { nameof(SystemTextJsonSerializer), 1}, + { nameof(CosmosSystemTextJsonSerializer), 2} }; List inputs = new List(); - foreach (bool applyCustomSerializer in new List{ true, false }) + foreach (KeyValuePair entry in serializerIndexes) { - Func> getQuery = getQueryList[applyCustomSerializer ? 0 : 1]; + Func> getQuery = getQueryList[entry.Value]; inputs.Add(new LinqAggregateInput( - "Avg, Custom serializer: " + applyCustomSerializer, b => getQuery(b) + "Avg, Serializer Name: " + entry.Key, b => getQuery(b) .Average(doc => doc.NumericField))); inputs.Add(new LinqAggregateInput( - "Sum, Custom serializer: " + applyCustomSerializer, b => getQuery(b) + "Sum, Serializer Name: " + entry.Key, b => getQuery(b) .Sum(doc => doc.NumericField))); inputs.Add(new LinqAggregateInput( - "Select many -> Filter -> Select -> Average, Custom serializer: " + applyCustomSerializer, b => getQuery(b) + "Select many -> Filter -> Select -> Average, Serializer Name: " + entry.Key, b => getQuery(b) .SelectMany(doc => doc.ArrayField.Where(m => (m % 3) == 0).Select(m => m)).Average())); inputs.Add(new LinqAggregateInput( - "Select number -> Skip -> Count, Custom serializer: " + applyCustomSerializer, b => getQuery(b) + "Select number -> Skip -> Count, Serializer Name: " + entry.Key, b => getQuery(b) .Select(f => f.NumericField).Skip(2).Count())); inputs.Add(new LinqAggregateInput( diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/LinqTranslationWithCustomSerializerBaseline.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/LinqTranslationWithCustomSerializerBaseline.cs index 91a537522c..1224f48074 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/LinqTranslationWithCustomSerializerBaseline.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/LinqTranslationWithCustomSerializerBaseline.cs @@ -32,6 +32,10 @@ public class LinqTranslationWithCustomSerializerBaseline : BaselineTests cosmosClientBuilder.WithCustomSerializer(new CosmosSystemTextJsonSerializer(new JsonSerializerOptions()))); + + string dbNameSTJ = $"{nameof(LinqTranslationBaselineTests)}-{Guid.NewGuid():N}"; + TestDbSTJDefault = await CosmosDefaultSTJClient.CreateDatabaseAsync(dbNameSTJ); } [ClassCleanup] @@ -65,11 +75,17 @@ public async static Task Cleanup() { await TestDb.DeleteStreamAsync(); } + + if (TestDbSTJDefault != null) + { + await TestDbSTJDefault.DeleteStreamAsync(); + } } [TestInitialize] public async Task TestInitialize() { + TestSTJContainer = await TestDbSTJDefault.CreateContainerAsync(new ContainerProperties(id: Guid.NewGuid().ToString(), partitionKeyPath: "/Pk")); TestLinqContainer = await TestDbLinq.CreateContainerAsync(new ContainerProperties(id: Guid.NewGuid().ToString(), partitionKeyPath: "/Pk")); TestContainer = await TestDb.CreateContainerAsync(new ContainerProperties(id: Guid.NewGuid().ToString(), partitionKeyPath: "/Pk")); } @@ -78,6 +94,7 @@ public async Task TestInitialize() public async Task TestCleanup() { await TestLinqContainer.DeleteContainerStreamAsync(); + await TestSTJContainer.DeleteContainerStreamAsync(); await TestContainer.DeleteContainerStreamAsync(); } @@ -147,6 +164,33 @@ public void TestMemberInitializerNewtonsoft() this.ExecuteTestSuite(inputs); } + [TestMethod] + public void TestMemberInitializerDotNetDefaultSerializer() + { + Func> getQuery; + (_, getQuery) = this.InsertDataAndGetQueryables(true, TestSTJContainer); + + string insertedData = this.GetInsertedData(TestSTJContainer).Result; + + List inputs = new List + { + new LinqTestInput("Filter w/ constant value", b => getQuery(b).Where(doc => doc.NumericField == 1), skipVerification : true, inputData: insertedData), + new LinqTestInput("Filter w/ DataObject initializer with constant value", b => getQuery(b).Where(doc => doc == new DataObjectDotNet() { NumericField = 1, StringField = "1" }), skipVerification : true, inputData: insertedData), + new LinqTestInput("Select w/ DataObject initializer", b => getQuery(b).Select(doc => new DataObjectDotNet() { NumericField = 1, StringField = "1" }), skipVerification : true, inputData: insertedData), + new LinqTestInput("Deeper than top level reference", b => getQuery(b).Select(doc => doc.NumericField > 1 ? new DataObjectDotNet() { NumericField = 1, StringField = "1" } : new DataObjectDotNet() { NumericField = 1, StringField = "1" }), skipVerification : true, inputData: insertedData), + new LinqTestInput("Filter w/ DataObject initializer with member initialization", b => getQuery(b).Where(doc => doc == new DataObjectDotNet() { NumericField = doc.NumericField, StringField = doc.StringField }).Select(b => "A"), skipVerification : true, inputData: insertedData), + new LinqTestInput("OrderBy query", b => getQuery(b).Select(x => x).OrderBy(x => x.NumericField).Take(5), skipVerification : true, inputData: insertedData), + new LinqTestInput("Conditional", b => getQuery(b).Select(c => c.NumericField > 1 ? "true" : "false"), skipVerification : true, inputData: insertedData), + new LinqTestInput("Filter w/ nullable property", b => getQuery(b).Where(doc => doc.DateTimeField != null), skipVerification : true, inputData: insertedData), + new LinqTestInput("Filter w/ nullable enum", b => getQuery(b).Where(doc => doc.DataTypeField != null), skipVerification : true, inputData: insertedData), + new LinqTestInput("Filter w/ non-null nullable property", b => getQuery(b).Where(doc => doc.DateTimeField == new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc)), skipVerification : true, inputData: insertedData), + new LinqTestInput("Filter w/ non-null nullable enum", b => getQuery(b).Where(doc => doc.DataTypeField == DataType.Point), skipVerification : true, inputData: insertedData), + new LinqTestInput("Filter w/ string null comparison", b => getQuery(b).Where(doc => doc.StringField != null), skipVerification : true, inputData: insertedData), + }; + + this.ExecuteTestSuite(inputs); + } + [TestMethod] public void TestMemberInitializerDataMember() { diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Microsoft.Azure.Cosmos.EmulatorTests.csproj b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Microsoft.Azure.Cosmos.EmulatorTests.csproj index 345d61ed83..ede84818e9 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Microsoft.Azure.Cosmos.EmulatorTests.csproj +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Microsoft.Azure.Cosmos.EmulatorTests.csproj @@ -43,6 +43,7 @@ + @@ -286,6 +287,9 @@ PreserveNewest + + PreserveNewest + PreserveNewest diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BaselineTest/TestBaseline/QueryPlanBaselineTests.VectorSearch.xml b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BaselineTest/TestBaseline/QueryPlanBaselineTests.VectorSearch.xml index 0b3074ec9d..c17a92d4b8 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BaselineTest/TestBaseline/QueryPlanBaselineTests.VectorSearch.xml +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BaselineTest/TestBaseline/QueryPlanBaselineTests.VectorSearch.xml @@ -1,7 +1,7 @@  - Euclidean Distance + Euclidean Distance with query parameter SELECT TOP 10 c.title AS Title, VectorDistance(c.embedding, @vectorEmbedding, true) AS SimilarityScore FROM c ORDER BY VectorDistance(c.embedding, @vectorEmbedding, true) @@ -59,7 +59,7 @@ ORDER BY VectorDistance(c.embedding, @vectorEmbedding, true)]]> - Cosine Similarity + Cosine Similarity with query parameter SELECT TOP 10 c.title AS Title, VectorDistance(c.embedding, @vectorEmbedding, true) AS SimilarityScore FROM c ORDER BY VectorDistance(c.embedding, @vectorEmbedding, true) @@ -117,7 +117,7 @@ ORDER BY VectorDistance(c.embedding, @vectorEmbedding, true)]]> - Dot Product + Dot Product with query parameter SELECT TOP 10 c.title AS Title, VectorDistance(c.embedding, @vectorEmbedding, true) AS SimilarityScore FROM c ORDER BY VectorDistance(c.embedding, @vectorEmbedding, true) @@ -173,4 +173,133 @@ ORDER BY VectorDistance(c.embedding, @vectorEmbedding, true)]]> + + + Euclidean Distance with inline vector + SELECT TOP 10 c.title AS Title, VectorDistance(c.embedding, [0.0039695268496870995, 0.027338456362485886, -0.005676387343555689, -0.013547309674322605, -0.002445543883368373, 0.01579204574227333, -0.016796082258224487, -0.012471556663513184], true) AS SimilarityScore + FROM c + ORDER BY VectorDistance(c.embedding, [0.0039695268496870995, 0.027338456362485886, -0.005676387343555689, -0.013547309674322605, -0.002445543883368373, 0.01579204574227333, -0.016796082258224487, -0.012471556663513184], true) + + /PartitionKey + + Hash + Geography + + + + + None + 10 + + + + + Ascending + + + VectorDistance(c.embedding, [0.0039695268496870995, 0.027338456362485886, (- 0.0056763873435556889), (- 0.013547309674322605), (- 0.0024455438833683729), 0.015792045742273331, (- 0.016796082258224487), (- 0.012471556663513184)], true) + + + + + False + + + + [[],"Infinity") + + + + + + + + + Cosine Similarity with inline vector + SELECT TOP 10 c.title AS Title, VectorDistance(c.embedding, [0.0039695268496870995, 0.027338456362485886, -0.005676387343555689, -0.013547309674322605, -0.002445543883368373, 0.01579204574227333, -0.016796082258224487, -0.012471556663513184], true) AS SimilarityScore + FROM c + ORDER BY VectorDistance(c.embedding, [0.0039695268496870995, 0.027338456362485886, -0.005676387343555689, -0.013547309674322605, -0.002445543883368373, 0.01579204574227333, -0.016796082258224487, -0.012471556663513184], true) + + /PartitionKey + + Hash + Geography + + + + + None + 10 + + + + + Descending + + + VectorDistance(c.embedding, [0.0039695268496870995, 0.027338456362485886, (- 0.0056763873435556889), (- 0.013547309674322605), (- 0.0024455438833683729), 0.015792045742273331, (- 0.016796082258224487), (- 0.012471556663513184)], true) + + + + + False + + + + [[],"Infinity") + + + + + + + + + Dot Product with inline vector + SELECT TOP 10 c.title AS Title, VectorDistance(c.embedding, [0.0039695268496870995, 0.027338456362485886, -0.005676387343555689, -0.013547309674322605, -0.002445543883368373, 0.01579204574227333, -0.016796082258224487, -0.012471556663513184], true) AS SimilarityScore + FROM c + ORDER BY VectorDistance(c.embedding, [0.0039695268496870995, 0.027338456362485886, -0.005676387343555689, -0.013547309674322605, -0.002445543883368373, 0.01579204574227333, -0.016796082258224487, -0.012471556663513184], true) + + /PartitionKey + + Hash + Geography + + + + + None + 10 + + + + + Descending + + + VectorDistance(c.embedding, [0.0039695268496870995, 0.027338456362485886, (- 0.0056763873435556889), (- 0.013547309674322605), (- 0.0024455438833683729), 0.015792045742273331, (- 0.016796082258224487), (- 0.012471556663513184)], true) + + + + + False + + + + [[],"Infinity") + + + + + + \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json index b43ef3d0e7..4a4b7b08dc 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json @@ -89,7 +89,7 @@ "Attributes": [ "JsonPropertyAttribute" ], - "MethodInfo": "Boolean IsTimeToLiveExpired;CanRead:True;CanWrite:False;Boolean get_IsTimeToLiveExpired();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + "MethodInfo": "Boolean IsTimeToLiveExpired;CanRead:True;CanWrite:True;Boolean get_IsTimeToLiveExpired();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, "Int64 get_Lsn()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { "Type": "Method", @@ -110,14 +110,14 @@ "Attributes": [ "JsonPropertyAttribute" ], - "MethodInfo": "Int64 Lsn;CanRead:True;CanWrite:False;Int64 get_Lsn();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + "MethodInfo": "Int64 Lsn;CanRead:True;CanWrite:True;Int64 get_Lsn();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, "Int64 PreviousLsn[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = 1, PropertyName = \"previousImageLSN\")]": { "Type": "Property", "Attributes": [ "JsonPropertyAttribute" ], - "MethodInfo": "Int64 PreviousLsn;CanRead:True;CanWrite:False;Int64 get_PreviousLsn();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + "MethodInfo": "Int64 PreviousLsn;CanRead:True;CanWrite:True;Int64 get_PreviousLsn();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, "Microsoft.Azure.Cosmos.ChangeFeedOperationType get_OperationType()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { "Type": "Method", @@ -132,7 +132,7 @@ "JsonConverterAttribute", "JsonPropertyAttribute" ], - "MethodInfo": "Microsoft.Azure.Cosmos.ChangeFeedOperationType OperationType;CanRead:True;CanWrite:False;Microsoft.Azure.Cosmos.ChangeFeedOperationType get_OperationType();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + "MethodInfo": "Microsoft.Azure.Cosmos.ChangeFeedOperationType OperationType;CanRead:True;CanWrite:True;Microsoft.Azure.Cosmos.ChangeFeedOperationType get_OperationType();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, "System.DateTime ConflictResolutionTimestamp[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = 1, PropertyName = \"crts\")]-[Newtonsoft.Json.JsonConverterAttribute(typeof(Microsoft.Azure.Documents.UnixDateTimeConverter))]": { "Type": "Property", @@ -140,7 +140,7 @@ "JsonConverterAttribute", "JsonPropertyAttribute" ], - "MethodInfo": "System.DateTime ConflictResolutionTimestamp;CanRead:True;CanWrite:False;System.DateTime get_ConflictResolutionTimestamp();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + "MethodInfo": "System.DateTime ConflictResolutionTimestamp;CanRead:True;CanWrite:True;System.DateTime get_ConflictResolutionTimestamp();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, "System.DateTime get_ConflictResolutionTimestamp()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { "Type": "Method", @@ -149,10 +149,10 @@ ], "MethodInfo": "System.DateTime get_ConflictResolutionTimestamp();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "Void .ctor(System.DateTime, Int64, Microsoft.Azure.Cosmos.ChangeFeedOperationType, Int64)": { + "Void .ctor()": { "Type": "Constructor", "Attributes": [], - "MethodInfo": "[Void .ctor(System.DateTime, Int64, Microsoft.Azure.Cosmos.ChangeFeedOperationType, Int64), Void .ctor(System.DateTime, Int64, Microsoft.Azure.Cosmos.ChangeFeedOperationType, Int64)]" + "MethodInfo": "[Void .ctor(), Void .ctor()]" } }, "NestedTypes": {} diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/CosmosSystemTextJsonSerializerTest.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/CosmosSystemTextJsonSerializerTest.cs new file mode 100644 index 0000000000..147c747809 --- /dev/null +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/CosmosSystemTextJsonSerializerTest.cs @@ -0,0 +1,180 @@ +namespace Microsoft.Azure.Cosmos.Tests.Json +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Reflection; + using Microsoft.Azure.Cosmos.Tests.Poco.STJ; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public sealed class CosmosSystemTextJsonSerializerTest + { + CosmosSystemTextJsonSerializer stjSerializer; + + [TestInitialize] + public void SetUp() + { + this.stjSerializer = new( + new System.Text.Json.JsonSerializerOptions()); + } + + [TestMethod] + public void TestPoco() + { + // Arrange. + Cars car = Cars.GetRandomCar(); + + // Act. + Stream serializedStream = this.stjSerializer.ToStream(car); + Cars deserializedValue = this.stjSerializer.FromStream(serializedStream); + + // Assert. + Assert.AreEqual(car.Name, deserializedValue.Name); + Assert.AreEqual(car.ColorCode, deserializedValue.ColorCode); + Assert.AreEqual(car.CustomFeatures.Count, deserializedValue.CustomFeatures.Count); + } + + [TestMethod] + public void TestList() + { + // Arrange. + List list = new List { 1, 2, 3 }; + + // Act. + Stream serializedStream = this.stjSerializer.ToStream>(list); + List deserializedList = this.stjSerializer.FromStream>(serializedStream); + + // Assert. + Assert.AreEqual(list[0], deserializedList[0]); + Assert.AreEqual(list[1], deserializedList[1]); + Assert.AreEqual(list[2], deserializedList[2]); + } + + [TestMethod] + [DataRow(true)] + [DataRow(false)] + public void TestBool(bool booleanValue) + { + // Arrange and Act. + Stream serializedStream = this.stjSerializer.ToStream(booleanValue); + bool deserializedValue= this.stjSerializer.FromStream(serializedStream); + + // Assert. + if (booleanValue) + { + Assert.IsTrue(deserializedValue); + } + else + { + Assert.IsFalse(deserializedValue); + } + } + + [TestMethod] + public void TestNumber() + { + { + short value = 32; + Stream serializedStream = this.stjSerializer.ToStream(value); + short int16Value = this.stjSerializer.FromStream(serializedStream); + Assert.AreEqual(expected: value, actual: int16Value); + } + + { + int value = 32; + Stream serializedStream = this.stjSerializer.ToStream(value); + int int32Value = this.stjSerializer.FromStream(serializedStream); + Assert.AreEqual(expected: value, actual: int32Value); + } + + { + long value = 32; + Stream serializedStream = this.stjSerializer.ToStream(value); + long int64Value = this.stjSerializer.FromStream(serializedStream); + Assert.AreEqual(expected: value, actual: int64Value); + } + + { + uint value = 32; + Stream serializedStream = this.stjSerializer.ToStream(value); + uint uintValue = this.stjSerializer.FromStream(serializedStream); + Assert.AreEqual(expected: value, actual: uintValue); + } + + { + float value = 32.1337f; + Stream serializedStream = this.stjSerializer.ToStream(value); + float floatValue = this.stjSerializer.FromStream(serializedStream); + Assert.AreEqual(expected: value, actual: floatValue); + } + + { + double value = 32.1337; + Stream serializedStream = this.stjSerializer.ToStream(value); + double doubleValue = this.stjSerializer.FromStream(serializedStream); + Assert.AreEqual(expected: value, actual: doubleValue); + } + } + + [TestMethod] + public void TestString() + { + // Arrange. + string value = "asdf"; + + // Act. + Stream result = this.stjSerializer.ToStream(value); + string cosmosString = this.stjSerializer.FromStream(result); + + // Assert. + Assert.AreEqual(expected: value, actual: cosmosString.ToString()); + } + + [TestMethod] + public void TestBinary() + { + // Arrange. + byte[] value = new byte[] { 1, 2, 3 }; + + // Act. + Stream serializedStream = this.stjSerializer.ToStream(value); + byte[] array = this.stjSerializer.FromStream(serializedStream); + + // Assert. + Assert.IsTrue(array.SequenceEqual(value.ToArray())); + } + + [TestMethod] + public void TestGuid() + { + // Arrange. + Guid value = Guid.NewGuid(); + + // Act. + Stream serializedStream = this.stjSerializer.ToStream(value); + Guid guidValue = this.stjSerializer.FromStream(serializedStream); + + // Assert. + Assert.AreEqual(expected: value, actual: guidValue); + } + + [TestMethod] + public void TestSerializeMemberName() + { + // Arrange. + Cars car = Cars.GetRandomCar(); + + MemberInfo[] memberInfoArray = car + .GetType() + .GetMembers(bindingAttr: BindingFlags.Instance | BindingFlags.NonPublic); + + // Act and Assert. + foreach (MemberInfo member in memberInfoArray) + { + Assert.AreEqual(member.Name, this.stjSerializer.SerializeMemberName(member)); + } + } + } +} diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Poco/STJ/Cars.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Poco/STJ/Cars.cs new file mode 100644 index 0000000000..55e58cd32b --- /dev/null +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Poco/STJ/Cars.cs @@ -0,0 +1,101 @@ +namespace Microsoft.Azure.Cosmos.Tests.Poco.STJ +{ + using System; + using System.Collections.Generic; + using System.Text.Json.Serialization; + + public sealed class Cars + { + private static readonly Random random = new(); + private static readonly string[] names = new string[] + { + "Toyota 4Runner", + "Toyota Camry", + "Toyota Prius", + "Toyota Rav4", + "Toyota Avalon", + "Honda Civic", + "Honda Accord", + "Honda CRV", + "Honda MRV", + "BMW C18", + "BMW 328i", + "BMW 530i", + }; + + private static readonly string[] features = new string[] + { + "all wheel drive", + "abs", + "apple car play", + "power window", + "heated seating", + "steering mounted control", + }; + + public Cars( + string name, + int colorCode, + double vin, + List customFeatures) + { + this.Name = name; + this.ColorCode = colorCode; + this.Vin = vin; + this.CustomFeatures = customFeatures; + } + + [JsonPropertyName("name")] + public string Name { get; } + + [JsonPropertyName("colorCode")] + public int ColorCode { get; } + + [JsonPropertyName("vin")] + public double Vin { get; } + + [JsonPropertyName("customFeatures")] + public List CustomFeatures { get; } + + public override bool Equals(object obj) + { + if (!(obj is Cars car)) + { + return false; + } + + return this.Equals(car); + } + + public bool Equals(Cars other) + { + return this.Name == other.Name && this.ColorCode == other.ColorCode; + } + + public override int GetHashCode() + { + return 0; + } + + public static Cars GetRandomCar() + { + string name = names[random.Next(0, names.Length)]; + int colorCode = random.Next(0, 100); + double vin = random.NextDouble() * 10000000; + List customFeatures = new(); + + int numOfFeatures = random.Next(0, 3); + for (int i = 0; i < numOfFeatures; i++) + { + customFeatures.Add(GetRandomFeature()); + } + + return new Cars(name, colorCode, vin, customFeatures); + } + + public static string GetRandomFeature() + { + return features[random.Next(0, features.Length)]; + } + } +} diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/QueryPlanBaselineTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/QueryPlanBaselineTests.cs index 149a4010d4..ef68af02bc 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/QueryPlanBaselineTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/QueryPlanBaselineTests.cs @@ -1369,28 +1369,62 @@ public void VectorSearch() { List testCases = new List { - MakeVectorTest("Euclidean Distance", Cosmos.DistanceFunction.Euclidean), - MakeVectorTest("Cosine Similarity", Cosmos.DistanceFunction.Cosine), - MakeVectorTest("Dot Product", Cosmos.DistanceFunction.DotProduct), + MakeVectorTest("Euclidean Distance with query parameter", Cosmos.DistanceFunction.Euclidean), + MakeVectorTest("Cosine Similarity with query parameter", Cosmos.DistanceFunction.Cosine), + MakeVectorTest("Dot Product with query parameter", Cosmos.DistanceFunction.DotProduct), + MakeInlineVectorQueryTest("Euclidean Distance with inline vector", Cosmos.DistanceFunction.Euclidean), + MakeInlineVectorQueryTest("Cosine Similarity with inline vector", Cosmos.DistanceFunction.Cosine), + MakeInlineVectorQueryTest("Dot Product with inline vector", Cosmos.DistanceFunction.DotProduct), }; this.ExecuteTestSuite(testCases); } + private static QueryPlanBaselineTestInput MakeInlineVectorQueryTest(string description, Cosmos.DistanceFunction distanceFunction) + { + PartitionKeyDefinition partitionKeyDefinition = CreateHashPartitionKey("/PartitionKey"); + + Cosmos.VectorEmbeddingPolicy vectorEmbeddingPolicy = new Cosmos.VectorEmbeddingPolicy(new Collection + { + new Cosmos.Embedding + { + Path = "/embedding", + DataType = Cosmos.VectorDataType.Float32, + Dimensions = 8, + DistanceFunction = distanceFunction + } + }); + + string queryText = @"SELECT TOP 10 c.title AS Title, VectorDistance(c.embedding, [0.0039695268496870995, 0.027338456362485886, -0.005676387343555689, -0.013547309674322605, -0.002445543883368373, 0.01579204574227333, -0.016796082258224487, -0.012471556663513184], true) AS SimilarityScore + FROM c + ORDER BY VectorDistance(c.embedding, [0.0039695268496870995, 0.027338456362485886, -0.005676387343555689, -0.013547309674322605, -0.002445543883368373, 0.01579204574227333, -0.016796082258224487, -0.012471556663513184], true)"; + + SqlQuerySpec sqlQuerySpec = new SqlQuerySpec(queryText); + + QueryPlanBaselineTestInput testInput = new QueryPlanBaselineTestInput( + description, + partitionKeyDefinition, + vectorEmbeddingPolicy, + sqlQuerySpec, + Cosmos.GeospatialType.Geography); + + return testInput; + } + private static QueryPlanBaselineTestInput MakeVectorTest(string description, Cosmos.DistanceFunction distanceFunction) { PartitionKeyDefinition partitionKeyDefinition = CreateHashPartitionKey("/PartitionKey"); Cosmos.VectorEmbeddingPolicy vectorEmbeddingPolicy = new Cosmos.VectorEmbeddingPolicy(new Collection + { + new Cosmos.Embedding { - new Cosmos.Embedding - { - Path = "/embedding", - DataType = Cosmos.VectorDataType.Float32, - Dimensions = 8, - DistanceFunction = distanceFunction - } - }); + Path = "/embedding", + DataType = Cosmos.VectorDataType.Float32, + Dimensions = 8, + DistanceFunction = distanceFunction + } + }); string queryText = @"SELECT TOP 10 c.title AS Title, VectorDistance(c.embedding, @vectorEmbedding, true) AS SimilarityScore FROM c diff --git a/azure-pipelines-official.yml b/azure-pipelines-official.yml index 3081a1aa03..af2683c31e 100644 --- a/azure-pipelines-official.yml +++ b/azure-pipelines-official.yml @@ -47,7 +47,7 @@ stages: inputs: command: test projects: 'Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/*.csproj' - arguments: --filter "TestCategory=ClientTelemetryRelease" --verbosity normal --configuration ${{ variables.BuildConfiguration }} /p:OS=Windows + arguments: --filter "TestCategory=ClientTelemetryRelease & TestCategory!=Quarantine" --verbosity normal --configuration ${{ variables.BuildConfiguration }} /p:OS=Windows nugetConfigPath: NuGet.config publishTestResults: true testRunTitle: Microsoft.Azure.Cosmos.EmulatorTests