From ede4d3369695e0f9b1609745d96d26fbc38d26dd Mon Sep 17 00:00:00 2001 From: Arthur Vickers Date: Fri, 14 Jun 2024 11:19:03 +0100 Subject: [PATCH] Support vector search on Cosmos DB Fixes #33783 This PR introduces: - `IsVector()` to configure a property to be configured as a vector (embedding) in the document. - The distance function and dimensions are specified. - The data type can be specified, or otherwise is inferred. - `HasIndex().ForVectors()` to configure a vector index over a vector property. - `VectorDistance()` which translates to the Cosmos `VectorDistance` function - The distance function and data type are taken from the property mapping, or can be overridden. Known issues: - Float16 (Half) is not working in Cosmos--needs investigation - Exception on int array case--could be EF or Cosmos--needs investigation - Owned types mess up the materialization--this will be fixed by the ReadItem improvements I am working on --- src/EFCore.Cosmos/EFCore.Cosmos.csproj | 2 +- .../Extensions/CosmosDbFunctionsExtensions.cs | 46 ++ .../CosmosIndexBuilderExtensions.cs | 94 +++ .../Extensions/CosmosIndexExtensions.cs | 59 ++ .../CosmosPropertyBuilderExtensions.cs | 120 ++++ .../Extensions/CosmosPropertyExtensions.cs | 43 +- .../Internal/CosmosModelValidator.cs | 100 +++- .../Internal/CosmosAnnotationNames.cs | 16 + .../Metadata/Internal/CosmosVectorType.cs | 31 + .../Properties/CosmosStrings.Designer.cs | 31 + .../Properties/CosmosStrings.resx | 12 + .../CosmosMethodCallTranslatorProvider.cs | 4 +- .../Query/Internal/CosmosQuerySqlGenerator.cs | 38 +- .../Query/Internal/CosmosQueryUtils.cs | 3 +- ...yableMethodTranslatingExpressionVisitor.cs | 15 +- ...ionBindingRemovingExpressionVisitorBase.cs | 6 +- .../CosmosSqlTranslatingExpressionVisitor.cs | 3 +- .../Query/Internal/ExpressionExtensions.cs | 4 +- .../Expressions/ArrayConstantExpression.cs | 5 +- .../Internal/Expressions/ArrayExpression.cs | 5 +- .../Internal/Expressions/ExistsExpression.cs | 5 +- .../Internal/Expressions/InExpression.cs | 9 +- .../Expressions/KeyAccessExpression.cs | 4 +- .../Expressions/ScalarReferenceExpression.cs | 4 +- .../Expressions/ScalarSubqueryExpression.cs | 6 +- .../Internal/Expressions/SelectExpression.cs | 3 +- .../Expressions/SqlBinaryExpression.cs | 3 +- .../Expressions/SqlConstantExpression.cs | 6 +- .../Internal/Expressions/SqlExpression.cs | 5 +- .../Expressions/SqlFunctionExpression.cs | 6 +- .../Expressions/SqlParameterExpression.cs | 6 +- .../Expressions/SqlUnaryExpression.cs | 3 +- .../Query/Internal/ISqlExpressionFactory.cs | 29 +- .../Query/Internal/SqlExpressionFactory.cs | 63 +- .../CosmosVectorSearchTranslator.cs | 94 +++ .../Storage/Internal/ContainerProperties.cs | 92 +-- .../Storage/Internal/CosmosClientWrapper.cs | 53 +- .../Storage/Internal/CosmosDatabaseCreator.cs | 23 +- .../Storage/Internal/CosmosTypeMapping.cs | 44 ++ .../Internal/CosmosTypeMappingSource.cs | 48 +- ...mosVectorDistanceCalculationTypeMapping.cs | 83 +++ .../Internal/CosmosVectorTypeMapping.cs | 126 ++++ .../TestUtilities/CosmosTestStore.cs | 17 +- .../VectorSearchCosmosTest.cs | 537 ++++++++++++++++++ .../CosmosModelValidatorTest.cs | 27 + 45 files changed, 1694 insertions(+), 239 deletions(-) create mode 100644 src/EFCore.Cosmos/Extensions/CosmosIndexBuilderExtensions.cs create mode 100644 src/EFCore.Cosmos/Extensions/CosmosIndexExtensions.cs create mode 100644 src/EFCore.Cosmos/Metadata/Internal/CosmosVectorType.cs create mode 100644 src/EFCore.Cosmos/Query/Internal/Translators/CosmosVectorSearchTranslator.cs create mode 100644 src/EFCore.Cosmos/Storage/Internal/CosmosVectorDistanceCalculationTypeMapping.cs create mode 100644 src/EFCore.Cosmos/Storage/Internal/CosmosVectorTypeMapping.cs create mode 100644 test/EFCore.Cosmos.FunctionalTests/VectorSearchCosmosTest.cs diff --git a/src/EFCore.Cosmos/EFCore.Cosmos.csproj b/src/EFCore.Cosmos/EFCore.Cosmos.csproj index ca236f7f5bf..c0ad8eeed9e 100644 --- a/src/EFCore.Cosmos/EFCore.Cosmos.csproj +++ b/src/EFCore.Cosmos/EFCore.Cosmos.csproj @@ -46,7 +46,7 @@ - + diff --git a/src/EFCore.Cosmos/Extensions/CosmosDbFunctionsExtensions.cs b/src/EFCore.Cosmos/Extensions/CosmosDbFunctionsExtensions.cs index 94a5b2d50af..5f62fa33277 100644 --- a/src/EFCore.Cosmos/Extensions/CosmosDbFunctionsExtensions.cs +++ b/src/EFCore.Cosmos/Extensions/CosmosDbFunctionsExtensions.cs @@ -47,4 +47,50 @@ public static T CoalesceUndefined( T expression1, T expression2) => throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(CoalesceUndefined))); + + /// + /// Returns the distance between two vectors, using the distance function and data type defined using + /// . + /// + /// The instance. + /// The first vector. + /// The second vector. + public static double VectorDistance(this DbFunctions _, IEnumerable vector1, IEnumerable vector2) + => throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(VectorDistance))); + + /// + /// Returns the distance between two vectors, given a distance function (aka similarity measure). + /// + /// The instance. + /// The first vector. + /// The second vector. + /// A specifying how the computed value is used in an ORDER BY + /// expression. If , then brute force is used, otherwise any index defined on the vector + /// property is leveraged. + public static double VectorDistance( + this DbFunctions _, + IEnumerable vector1, + IEnumerable vector2, + [NotParameterized] bool useBruteForce) + => throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(VectorDistance))); + + /// + /// Returns the distance between two vectors, given a distance function (aka similarity measure). + /// + /// The instance. + /// The first vector. + /// The second vector. + /// The distance function to use. + /// The vector data type to use. + /// A specifying how the computed value is used in an ORDER BY + /// expression. If , then brute force is used, otherwise any index defined on the vector + /// property is leveraged. + public static double VectorDistance( + this DbFunctions _, + IEnumerable vector1, + IEnumerable vector2, + [NotParameterized] bool useBruteForce, + [NotParameterized] DistanceFunction distanceFunction, + [NotParameterized] VectorDataType dataType) + => throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(VectorDistance))); } diff --git a/src/EFCore.Cosmos/Extensions/CosmosIndexBuilderExtensions.cs b/src/EFCore.Cosmos/Extensions/CosmosIndexBuilderExtensions.cs new file mode 100644 index 00000000000..843967838ee --- /dev/null +++ b/src/EFCore.Cosmos/Extensions/CosmosIndexBuilderExtensions.cs @@ -0,0 +1,94 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.EntityFrameworkCore.Cosmos.Metadata.Internal; + +// ReSharper disable once CheckNamespace +namespace Microsoft.EntityFrameworkCore; + +/// +/// Azure Cosmos DB-specific extension methods for . +/// +/// +/// See Modeling entity types and relationships, and +/// Accessing Azure Cosmos DB with EF Core for more information and examples. +/// +public static class CosmosIndexBuilderExtensions +{ + /// + /// Configures whether the index as a vector index with the given . + /// + /// + /// See Modeling entity types and relationships, and + /// Accessing Azure Cosmos DB with EF Core for more information and examples. + /// + /// The builder for the index being configured. + /// The type of vector index to create. + /// A builder to further configure the index. + public static IndexBuilder ForVectors(this IndexBuilder indexBuilder, VectorIndexType? indexType) + { + indexBuilder.Metadata.SetVectorIndexType(indexType); + + return indexBuilder; + } + + /// + /// Configures whether the index as a vector index with the given . + /// + /// + /// See Modeling entity types and relationships, and + /// Accessing Azure Cosmos DB with EF Core for more information and examples. + /// + /// The builder for the index being configured. + /// The type of vector index to create. + /// A builder to further configure the index. + public static IndexBuilder ForVectors( + this IndexBuilder indexBuilder, + VectorIndexType? indexType) + => (IndexBuilder)ForVectors((IndexBuilder)indexBuilder, indexType); + + /// + /// Configures whether the index as a vector index with the given . + /// + /// + /// See Modeling entity types and relationships, and + /// Accessing Azure Cosmos DB with EF Core for more information and examples. + /// + /// The builder for the index being configured. + /// The type of vector index to create. + /// Indicates whether the configuration was specified using a data annotation. + /// + /// The same builder instance if the configuration was applied, + /// otherwise. + /// + public static IConventionIndexBuilder? ForVectors( + this IConventionIndexBuilder indexBuilder, + VectorIndexType? indexType, + bool fromDataAnnotation = false) + { + if (indexBuilder.CanSetVectorIndexType(indexType, fromDataAnnotation)) + { + indexBuilder.Metadata.SetVectorIndexType(indexType, fromDataAnnotation); + return indexBuilder; + } + + return null; + } + + /// + /// Returns a value indicating whether the index can be configured for vectors. + /// + /// + /// See Modeling entity types and relationships, and + /// Accessing Azure Cosmos DB with EF Core for more information and examples. + /// + /// The builder for the index being configured. + /// The index type to use. + /// Indicates whether the configuration was specified using a data annotation. + /// if the index can be configured for vectors. + public static bool CanSetVectorIndexType( + this IConventionIndexBuilder indexBuilder, + VectorIndexType? indexType, + bool fromDataAnnotation = false) + => indexBuilder.CanSetAnnotation(CosmosAnnotationNames.VectorIndexType, indexType, fromDataAnnotation); +} diff --git a/src/EFCore.Cosmos/Extensions/CosmosIndexExtensions.cs b/src/EFCore.Cosmos/Extensions/CosmosIndexExtensions.cs new file mode 100644 index 00000000000..d8096da1394 --- /dev/null +++ b/src/EFCore.Cosmos/Extensions/CosmosIndexExtensions.cs @@ -0,0 +1,59 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.EntityFrameworkCore.Cosmos.Metadata.Internal; + +// ReSharper disable once CheckNamespace +namespace Microsoft.EntityFrameworkCore; + +/// +/// Index extension methods for Azure Cosmos DB-specific metadata. +/// +/// +/// See Modeling entity types and relationships, and +/// Accessing Azure Cosmos DB with EF Core for more information and examples. +/// +public static class CosmosIndexExtensions +{ + /// + /// Returns the to use for this index. + /// + /// The index. + /// The index type to use, or if none is set. + public static VectorIndexType? GetVectorIndexType(this IReadOnlyIndex index) + => (index is RuntimeIndex) + ? throw new InvalidOperationException(CoreStrings.RuntimeModelMissingData) + : (VectorIndexType?)index[CosmosAnnotationNames.VectorIndexType]; + + /// + /// Sets the to use for this index. + /// + /// The index type to use. + /// The index. + public static void SetVectorIndexType(this IMutableIndex index, VectorIndexType? indexType) + => index.SetAnnotation(CosmosAnnotationNames.VectorIndexType, indexType); + + /// + /// Sets the to use for this index. + /// + /// The index type to use. + /// The index. + /// Indicates whether the configuration was specified using a data annotation. + /// The configured value. + public static VectorIndexType? SetVectorIndexType( + this IConventionIndex index, + VectorIndexType? indexType, + bool fromDataAnnotation = false) + => (VectorIndexType?)index.SetAnnotation( + CosmosAnnotationNames.VectorIndexType, + indexType, + fromDataAnnotation)?.Value; + + /// + /// Returns the for whether the . + /// + /// The property. + /// The for whether the index is clustered. + public static ConfigurationSource? GetVectorIndexTypeConfigurationSource(this IConventionIndex property) + => property.FindAnnotation(CosmosAnnotationNames.VectorIndexType)?.GetConfigurationSource(); +} diff --git a/src/EFCore.Cosmos/Extensions/CosmosPropertyBuilderExtensions.cs b/src/EFCore.Cosmos/Extensions/CosmosPropertyBuilderExtensions.cs index 61da371ef3a..6e84693d63b 100644 --- a/src/EFCore.Cosmos/Extensions/CosmosPropertyBuilderExtensions.cs +++ b/src/EFCore.Cosmos/Extensions/CosmosPropertyBuilderExtensions.cs @@ -104,6 +104,109 @@ public static bool CanSetJsonProperty( bool fromDataAnnotation = false) => propertyBuilder.CanSetAnnotation(CosmosAnnotationNames.PropertyName, name, fromDataAnnotation); + /// + /// Configures the property as a vector for Azure Cosmos DB. + /// + /// + /// See Modeling entity types and relationships, and + /// Accessing Azure Cosmos DB with EF Core for more information and examples. + /// + /// The builder for the property being configured. + /// The distance function for a vector comparisons. + /// The number of dimensions in the vector. + /// The vector data type, or to choose the data type automatically. + /// The same builder instance so that multiple calls can be chained. + public static PropertyBuilder IsVector( + this PropertyBuilder propertyBuilder, + DistanceFunction distanceFunction, + ulong dimensions, + VectorDataType? dataType = null) + { + propertyBuilder.Metadata.SetVectorType(CreateVectorType(distanceFunction, dimensions, dataType)); + return propertyBuilder; + } + + /// + /// Configures the property as a vector for Azure Cosmos DB. + /// + /// + /// See Modeling entity types and relationships, and + /// Accessing Azure Cosmos DB with EF Core for more information and examples. + /// + /// + /// See Modeling entity types and relationships, and + /// Accessing Azure Cosmos DB with EF Core for more information and examples. + /// + /// The type of the property being configured. + /// The builder for the property being configured. + /// The distance function for a vector comparisons. + /// The number of dimensions in the vector. + /// The vector data type, or to choose the data type automatically. + /// The same builder instance so that multiple calls can be chained. + public static PropertyBuilder IsVector( + this PropertyBuilder propertyBuilder, + DistanceFunction distanceFunction, + ulong dimensions, + VectorDataType? dataType = null) + => (PropertyBuilder)IsVector((PropertyBuilder)propertyBuilder, distanceFunction, dimensions, dataType); + + /// + /// Configures the property as a vector for Azure Cosmos DB. + /// + /// + /// See Modeling entity types and relationships, and + /// Accessing Azure Cosmos DB with EF Core for more information and examples. + /// + /// The builder for the property being configured. + /// The distance function for a vector comparisons. + /// The number of dimensions in the vector. + /// The vector data type, or to choose the data type automatically. + /// Indicates whether the configuration was specified using a data annotation. + /// + /// The same builder instance if the configuration was applied, + /// otherwise. + /// + public static IConventionPropertyBuilder? IsVector( + this IConventionPropertyBuilder propertyBuilder, + DistanceFunction distanceFunction, + ulong dimensions, + VectorDataType? dataType = null, + bool fromDataAnnotation = false) + { + if (!propertyBuilder.CanSetIsVector(distanceFunction, dimensions, dataType, fromDataAnnotation)) + { + return null; + } + + propertyBuilder.Metadata.SetVectorType(CreateVectorType(distanceFunction, dimensions, dataType), fromDataAnnotation); + + return propertyBuilder; + } + + /// + /// Returns a value indicating whether the vector type can be set. + /// + /// + /// See Modeling entity types and relationships, and + /// Accessing Azure Cosmos DB with EF Core for more information and examples. + /// + /// The builder for the property being configured. + /// The distance function for a vector comparisons. + /// The number of dimensions in the vector. + /// The vector data type, or to choose the data type automatically. + /// Indicates whether the configuration was specified using a data annotation. + /// if the vector type can be set. + public static bool CanSetIsVector( + this IConventionPropertyBuilder propertyBuilder, + DistanceFunction distanceFunction, + ulong dimensions, + VectorDataType? dataType = null, + bool fromDataAnnotation = false) + => propertyBuilder.CanSetAnnotation( + CosmosAnnotationNames.VectorType, + CreateVectorType(distanceFunction, dimensions, dataType), + fromDataAnnotation); + /// /// Configures this property to be the etag concurrency token. /// @@ -136,4 +239,21 @@ public static PropertyBuilder IsETagConcurrency(this PropertyBuilder propertyBui public static PropertyBuilder IsETagConcurrency( this PropertyBuilder propertyBuilder) => (PropertyBuilder)IsETagConcurrency((PropertyBuilder)propertyBuilder); + + private static CosmosVectorType CreateVectorType(DistanceFunction distanceFunction, ulong dimensions, VectorDataType? dataType) + { + if (!Enum.IsDefined(distanceFunction)) + { + throw new ArgumentException(CoreStrings.InvalidEnumValue(distanceFunction, nameof(distanceFunction), typeof(DistanceFunction))); + } + + if (dataType.HasValue + && !Enum.IsDefined(dataType.Value)) + { + throw new ArgumentException(CoreStrings.InvalidEnumValue(dataType.Value, nameof(dataType), typeof(VectorDataType))); + } + + var vectorType = new CosmosVectorType(distanceFunction, dimensions, dataType); + return vectorType; + } } diff --git a/src/EFCore.Cosmos/Extensions/CosmosPropertyExtensions.cs b/src/EFCore.Cosmos/Extensions/CosmosPropertyExtensions.cs index a397598c506..6574e03f7a7 100644 --- a/src/EFCore.Cosmos/Extensions/CosmosPropertyExtensions.cs +++ b/src/EFCore.Cosmos/Extensions/CosmosPropertyExtensions.cs @@ -73,12 +73,51 @@ public static void SetJsonPropertyName(this IMutableProperty property, string? n fromDataAnnotation)?.Value; /// - /// Gets the the property name that the property is mapped to when targeting Cosmos. + /// Gets the for the property name that the property is mapped to when targeting Cosmos. /// /// The property. /// - /// The the property name that the property is mapped to when targeting Cosmos. + /// The for the property name that the property is mapped to when targeting Cosmos. /// public static ConfigurationSource? GetJsonPropertyNameConfigurationSource(this IConventionProperty property) => property.FindAnnotation(CosmosAnnotationNames.PropertyName)?.GetConfigurationSource(); + + /// + /// Returns the definition of the vector stored in this property. + /// + /// The property. + /// Returns the definition of the vector stored in this property. + public static CosmosVectorType? GetVectorType(this IReadOnlyProperty property) + => (CosmosVectorType?)property[CosmosAnnotationNames.VectorType]; + + /// + /// Sets the definition of the vector stored in this property. + /// + /// The property. + /// The type of vector stored in the property. + public static void SetVectorType(this IMutableProperty property, CosmosVectorType? vectorType) + => property.SetOrRemoveAnnotation(CosmosAnnotationNames.VectorType, vectorType); + + /// + /// Sets the definition of the vector stored in this property. + /// + /// The property. + /// The type of vector stored in the property. + /// Indicates whether the configuration was specified using a data annotation. + /// The configured value. + public static CosmosVectorType? SetVectorType(this IConventionProperty property, CosmosVectorType? vectorType, bool fromDataAnnotation = false) + => (CosmosVectorType?)property.SetOrRemoveAnnotation( + CosmosAnnotationNames.VectorType, + vectorType, + fromDataAnnotation)?.Value; + + /// + /// Gets the for the definition of the vector stored in this property. + /// + /// The property. + /// + /// The for the definition of the vector stored in this property. + /// + public static ConfigurationSource? GetVectorTypeConfigurationSource(this IConventionProperty property) + => property.FindAnnotation(CosmosAnnotationNames.VectorType)?.GetConfigurationSource(); } diff --git a/src/EFCore.Cosmos/Infrastructure/Internal/CosmosModelValidator.cs b/src/EFCore.Cosmos/Infrastructure/Internal/CosmosModelValidator.cs index 59e148b1537..5727d34d147 100644 --- a/src/EFCore.Cosmos/Infrastructure/Internal/CosmosModelValidator.cs +++ b/src/EFCore.Cosmos/Infrastructure/Internal/CosmosModelValidator.cs @@ -40,6 +40,7 @@ public override void Validate(IModel model, IDiagnosticsLogger @@ -67,14 +68,18 @@ protected virtual void ValidateSharedContainerCompatibility( { throw new InvalidOperationException( CosmosStrings.OwnedTypeDifferentContainer( - entityType.DisplayName(), ownership.PrincipalEntityType.DisplayName(), container)); + entityType.DisplayName(), + ownership.PrincipalEntityType.DisplayName(), + container)); } if (entityType.GetContainingPropertyName() != null) { throw new InvalidOperationException( CosmosStrings.ContainerContainingPropertyConflict( - entityType.DisplayName(), container, entityType.GetContainingPropertyName())); + entityType.DisplayName(), + container, + entityType.GetContainingPropertyName())); } if (!containers.TryGetValue(container, out var mappedTypes)) @@ -141,8 +146,12 @@ protected virtual void ValidateSharedContainerCompatibility( { throw new InvalidOperationException( CosmosStrings.PartitionKeyStoreNameMismatch( - firstEntityType.GetPartitionKeyPropertyNames()[i], firstEntityType.DisplayName(), partitionKeyStoreNames[i], - entityType.GetPartitionKeyPropertyNames()[i], entityType.DisplayName(), storeNames[i])); + firstEntityType.GetPartitionKeyPropertyNames()[i], + firstEntityType.DisplayName(), + partitionKeyStoreNames[i], + entityType.GetPartitionKeyPropertyNames()[i], + entityType.DisplayName(), + storeNames[i])); } } } @@ -157,22 +166,23 @@ protected virtual void ValidateSharedContainerCompatibility( { if (entityType.FindDiscriminatorProperty() == null) { - throw new InvalidOperationException( - CosmosStrings.NoDiscriminatorProperty(entityType.DisplayName(), container)); + throw new InvalidOperationException(CosmosStrings.NoDiscriminatorProperty(entityType.DisplayName(), container)); } var discriminatorValue = entityType.GetDiscriminatorValue(); if (discriminatorValue == null) { - throw new InvalidOperationException( - CosmosStrings.NoDiscriminatorValue(entityType.DisplayName(), container)); + throw new InvalidOperationException(CosmosStrings.NoDiscriminatorValue(entityType.DisplayName(), container)); } if (discriminatorValues.TryGetValue(discriminatorValue, out var duplicateEntityType)) { throw new InvalidOperationException( CosmosStrings.DuplicateDiscriminatorValue( - entityType.DisplayName(), discriminatorValue, duplicateEntityType.DisplayName(), container)); + entityType.DisplayName(), + discriminatorValue, + duplicateEntityType.DisplayName(), + container)); } discriminatorValues[discriminatorValue] = entityType; @@ -190,7 +200,10 @@ protected virtual void ValidateSharedContainerCompatibility( var conflictingEntityType = mappedTypes.First(et => et.GetAnalyticalStoreTimeToLive() != null); throw new InvalidOperationException( CosmosStrings.AnalyticalTTLMismatch( - analyticalTtl, conflictingEntityType.DisplayName(), entityType.DisplayName(), currentAnalyticalTtl, + analyticalTtl, + conflictingEntityType.DisplayName(), + entityType.DisplayName(), + currentAnalyticalTtl, container)); } } @@ -207,7 +220,11 @@ protected virtual void ValidateSharedContainerCompatibility( var conflictingEntityType = mappedTypes.First(et => et.GetDefaultTimeToLive() != null); throw new InvalidOperationException( CosmosStrings.DefaultTTLMismatch( - defaultTtl, conflictingEntityType.DisplayName(), entityType.DisplayName(), currentDefaultTtl, container)); + defaultTtl, + conflictingEntityType.DisplayName(), + entityType.DisplayName(), + currentDefaultTtl, + container)); } } @@ -224,8 +241,10 @@ protected virtual void ValidateSharedContainerCompatibility( var conflictingEntityType = mappedTypes.First(et => et.GetThroughput() != null); throw new InvalidOperationException( CosmosStrings.ThroughputMismatch( - throughput.AutoscaleMaxThroughput ?? throughput.Throughput, conflictingEntityType.DisplayName(), - entityType.DisplayName(), currentThroughput.AutoscaleMaxThroughput ?? currentThroughput.Throughput, + throughput.AutoscaleMaxThroughput ?? throughput.Throughput, + conflictingEntityType.DisplayName(), + entityType.DisplayName(), + currentThroughput.AutoscaleMaxThroughput ?? currentThroughput.Throughput, container)); } else if ((throughput.AutoscaleMaxThroughput == null) @@ -240,8 +259,7 @@ protected virtual void ValidateSharedContainerCompatibility( : conflictingEntityType; throw new InvalidOperationException( - CosmosStrings.ThroughputTypeMismatch( - manualType.DisplayName(), autoscaleType.DisplayName(), container)); + CosmosStrings.ThroughputTypeMismatch(manualType.DisplayName(), autoscaleType.DisplayName(), container)); } } } @@ -266,16 +284,14 @@ protected virtual void ValidateOnlyETagConcurrencyToken( var storeName = property.GetJsonPropertyName(); if (storeName != "_etag") { - throw new InvalidOperationException( - CosmosStrings.NonETagConcurrencyToken(entityType.DisplayName(), storeName)); + throw new InvalidOperationException(CosmosStrings.NonETagConcurrencyToken(entityType.DisplayName(), storeName)); } var etagType = property.GetTypeMapping().Converter?.ProviderClrType ?? property.ClrType; if (etagType != typeof(string)) { throw new InvalidOperationException( - CosmosStrings.ETagNonStringStoreType( - property.Name, entityType.DisplayName(), etagType.ShortDisplayName())); + CosmosStrings.ETagNonStringStoreType(property.Name, entityType.DisplayName(), etagType.ShortDisplayName())); } } } @@ -313,8 +329,7 @@ protected virtual void ValidateKeys( if (idType != typeof(string)) { throw new InvalidOperationException( - CosmosStrings.IdNonStringStoreType( - idProperty.Name, entityType.DisplayName(), idType.ShortDisplayName())); + CosmosStrings.IdNonStringStoreType(idProperty.Name, entityType.DisplayName(), idType.ShortDisplayName())); } if (!idProperty.IsKey()) @@ -346,14 +361,15 @@ protected virtual void ValidateKeys( { throw new InvalidOperationException( CosmosStrings.PartitionKeyBadStoreType( - partitionKeyPropertyName, entityType.DisplayName(), partitionKeyType.ShortDisplayName())); + partitionKeyPropertyName, + entityType.DisplayName(), + partitionKeyType.ShortDisplayName())); } if (!partitionKey.GetContainingKeys().Any(k => k.Properties.Contains(idProperty))) { throw new InvalidOperationException( - CosmosStrings.NoPartitionKeyKey( - entityType.DisplayName(), partitionKeyPropertyName, idProperty.Name)); + CosmosStrings.NoPartitionKeyKey(entityType.DisplayName(), partitionKeyPropertyName, idProperty.Name)); } } } @@ -431,4 +447,40 @@ protected virtual void ValidateDatabaseProperties( } } } + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + protected virtual void ValidateIndexes( + IModel model, + IDiagnosticsLogger logger) + { + foreach (var entityType in model.GetEntityTypes()) + { + foreach (var index in entityType.GetIndexes()) + { + if (index.FindAnnotation(CosmosAnnotationNames.VectorIndexType) != null) + { + if (index.Properties.Count > 1) + { + throw new InvalidOperationException( + CosmosStrings.CompositeVectorIndex( + entityType.DisplayName(), + string.Join(",", index.Properties.Select(e => e.Name)))); + } + + if (index.Properties[0].FindAnnotation(CosmosAnnotationNames.VectorType) == null) + { + throw new InvalidOperationException( + CosmosStrings.VectorIndexOnNonVector( + entityType.DisplayName(), + index.Properties[0].Name)); + } + } + } + } + } } diff --git a/src/EFCore.Cosmos/Metadata/Internal/CosmosAnnotationNames.cs b/src/EFCore.Cosmos/Metadata/Internal/CosmosAnnotationNames.cs index e9eab69bbb5..b59a3348439 100644 --- a/src/EFCore.Cosmos/Metadata/Internal/CosmosAnnotationNames.cs +++ b/src/EFCore.Cosmos/Metadata/Internal/CosmosAnnotationNames.cs @@ -43,6 +43,22 @@ public static class CosmosAnnotationNames /// public const string PartitionKeyNames = Prefix + "PartitionKeyNames"; + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public const string VectorIndexType = Prefix + "VectorIndexType"; + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public const string VectorType = Prefix + "VectorType"; + /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to /// the same compatibility standards as public APIs. It may be changed or removed without notice in diff --git a/src/EFCore.Cosmos/Metadata/Internal/CosmosVectorType.cs b/src/EFCore.Cosmos/Metadata/Internal/CosmosVectorType.cs new file mode 100644 index 00000000000..7c552cbc971 --- /dev/null +++ b/src/EFCore.Cosmos/Metadata/Internal/CosmosVectorType.cs @@ -0,0 +1,31 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.EntityFrameworkCore.Cosmos.Metadata.Internal; + +/// +/// This is an internal API that supports the Entity Framework Core infrastructure and not subject to +/// the same compatibility standards as public APIs. It may be changed or removed without notice in +/// any release. You should only use it directly in your code with extreme caution and knowing that +/// doing so can result in application failures when updating to a new Entity Framework Core release. +/// +public sealed record class CosmosVectorType(DistanceFunction DistanceFunction, ulong Dimensions, VectorDataType? DataType) +{ + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public static VectorDataType CreateDefaultVectorDataType(Type clrType) + { + var elementType = clrType.TryGetElementType(typeof(IEnumerable<>))?.UnwrapNullableType(); + return elementType == typeof(sbyte) + ? VectorDataType.Int8 + : elementType == typeof(byte) + ? VectorDataType.Uint8 + : elementType == typeof(Half) + ? VectorDataType.Float16 + : VectorDataType.Float32; + } +} diff --git a/src/EFCore.Cosmos/Properties/CosmosStrings.Designer.cs b/src/EFCore.Cosmos/Properties/CosmosStrings.Designer.cs index 527c56ca5d4..4ade83281ec 100644 --- a/src/EFCore.Cosmos/Properties/CosmosStrings.Designer.cs +++ b/src/EFCore.Cosmos/Properties/CosmosStrings.Designer.cs @@ -31,12 +31,28 @@ public static string AnalyticalTTLMismatch(object? ttl1, object? entityType1, ob GetString("AnalyticalTTLMismatch", nameof(ttl1), nameof(entityType1), nameof(entityType2), nameof(ttl2), nameof(container)), ttl1, entityType1, entityType2, ttl2, container); + /// + /// The '{parameter}' value passed to '{methodName}' must be a constant. + /// + public static string ArgumentNotConstant(object? parameter, object? methodName) + => string.Format( + GetString("ArgumentNotConstant", nameof(parameter), nameof(methodName)), + parameter, methodName); + /// /// The Cosmos database does not support 'CanConnect' or 'CanConnectAsync'. /// public static string CanConnectNotSupported => GetString("CanConnectNotSupported"); + /// + /// A vector index on '{entityType}' is defined over properties `{properties}`. A vector index can only target a single property. + /// + public static string CompositeVectorIndex(object? entityType, object? properties) + => string.Format( + GetString("CompositeVectorIndex", nameof(entityType), nameof(properties)), + entityType, properties); + /// /// None of connection string, CredentialToken, account key or account endpoint were specified. Specify a set of connection details. /// @@ -403,6 +419,21 @@ public static string UpdateStoreException(object? itemId) GetString("UpdateStoreException", nameof(itemId)), itemId); + /// + /// A vector index is defined for `{entityType}.{property}`, but this property has not been configured as a vector. Use 'IsVector()' in 'OnModelCreating' to configure the property as a vector. + /// + public static string VectorIndexOnNonVector(object? entityType, object? property) + => string.Format( + GetString("VectorIndexOnNonVector", nameof(entityType), nameof(property)), + entityType, property); + + + /// + /// The 'VectorDistance' function can only be used with a property mapped as a vectors. Use 'IsVector()' in 'OnModelCreating' to configure the property as a vector. + /// + public static string VectorSearchRequiresVector + => GetString("VectorSearchRequiresVector"); + /// /// 'VisitChildren' must be overridden in the class deriving from 'SqlExpression'. /// diff --git a/src/EFCore.Cosmos/Properties/CosmosStrings.resx b/src/EFCore.Cosmos/Properties/CosmosStrings.resx index d15bd153984..8a4da29819b 100644 --- a/src/EFCore.Cosmos/Properties/CosmosStrings.resx +++ b/src/EFCore.Cosmos/Properties/CosmosStrings.resx @@ -120,9 +120,15 @@ The time to live for analytical store was configured to '{ttl1}' on '{entityType1}', but on '{entityType2}' it was configured to '{ttl2}'. All entity types mapped to the same container '{container}' must be configured with the same time to live for analytical store. + + The '{parameter}' value passed to '{methodName}' must be a constant. + The Cosmos database does not support 'CanConnect' or 'CanConnectAsync'. + + A vector index on '{entityType}' is defined over properties `{properties}`. A vector index can only target a single property. + None of connection string, CredentialToken, account key or account endpoint were specified. Specify a set of connection details. @@ -309,6 +315,12 @@ An error occurred while saving the item with id '{itemId}'. See the inner exception for details. + + A vector index is defined for `{entityType}.{property}`, but this property has not been configured as a vector. Use 'IsVector()' in 'OnModelCreating' to configure the property as a vector. + + + The 'VectorDistance' function can only be used with a property mapped as a vectors. Use 'IsVector()' in 'OnModelCreating' to configure the property as a vector. + 'VisitChildren' must be overridden in the class deriving from 'SqlExpression'. diff --git a/src/EFCore.Cosmos/Query/Internal/CosmosMethodCallTranslatorProvider.cs b/src/EFCore.Cosmos/Query/Internal/CosmosMethodCallTranslatorProvider.cs index 0d871e9230a..352e3d443e1 100644 --- a/src/EFCore.Cosmos/Query/Internal/CosmosMethodCallTranslatorProvider.cs +++ b/src/EFCore.Cosmos/Query/Internal/CosmosMethodCallTranslatorProvider.cs @@ -22,6 +22,7 @@ public class CosmosMethodCallTranslatorProvider : IMethodCallTranslatorProvider /// public CosmosMethodCallTranslatorProvider( ISqlExpressionFactory sqlExpressionFactory, + ITypeMappingSource typeMappingSource, IEnumerable plugins) { _plugins.AddRange(plugins.SelectMany(p => p.Translators)); @@ -34,7 +35,8 @@ public CosmosMethodCallTranslatorProvider( new CosmosRandomTranslator(sqlExpressionFactory), new CosmosRegexTranslator(sqlExpressionFactory), new CosmosStringMethodTranslator(sqlExpressionFactory), - new CosmosTypeCheckingTranslator(sqlExpressionFactory) + new CosmosTypeCheckingTranslator(sqlExpressionFactory), + new CosmosVectorSearchTranslator(sqlExpressionFactory, typeMappingSource) //new LikeTranslator(sqlExpressionFactory), //new EnumHasFlagTranslator(sqlExpressionFactory), //new GetValueOrDefaultTranslator(sqlExpressionFactory), diff --git a/src/EFCore.Cosmos/Query/Internal/CosmosQuerySqlGenerator.cs b/src/EFCore.Cosmos/Query/Internal/CosmosQuerySqlGenerator.cs index ec4b83b0552..4320fe6414f 100644 --- a/src/EFCore.Cosmos/Query/Internal/CosmosQuerySqlGenerator.cs +++ b/src/EFCore.Cosmos/Query/Internal/CosmosQuerySqlGenerator.cs @@ -352,9 +352,9 @@ when _parameterValues.TryGetValue(parameterExpression.Name, out var parameterVal for (var i = 0; i < constantValues.Length; i++) { var value = constantValues[i]; - var typeMapping = typeMappingSource.FindMapping(value.GetType()); + var typeMapping = (CosmosTypeMapping?)typeMappingSource.FindMapping(value.GetType()); Check.DebugAssert(typeMapping is not null, "Could not find type mapping for FromSql parameter"); - substitutions[i] = GenerateConstant(value, typeMapping); + substitutions[i] = typeMapping.GenerateConstant(value); } break; @@ -540,41 +540,11 @@ private void GenerateList( protected override Expression VisitSqlConstant(SqlConstantExpression sqlConstantExpression) { Check.DebugAssert(sqlConstantExpression.TypeMapping is not null, "SqlConstantExpression without a type mapping"); - _sqlBuilder.Append(GenerateConstant(sqlConstantExpression.Value, sqlConstantExpression.TypeMapping)); + _sqlBuilder.Append(sqlConstantExpression.TypeMapping.GenerateConstant(sqlConstantExpression.Value)); return sqlConstantExpression; } - private static string GenerateConstant(object? value, CoreTypeMapping typeMapping) - { - var jToken = GenerateJToken(value, typeMapping); - - return jToken is null ? "null" : jToken.ToString(Formatting.None); - } - - private static JToken? GenerateJToken(object? value, CoreTypeMapping typeMapping) - { - if (value?.GetType().IsInteger() == true) - { - var unwrappedType = typeMapping.ClrType.UnwrapNullableType(); - value = unwrappedType.IsEnum - ? Enum.ToObject(unwrappedType, value) - : unwrappedType == typeof(char) - ? Convert.ChangeType(value, unwrappedType) - : value; - } - - var converter = typeMapping.Converter; - if (converter != null) - { - value = converter.ConvertToProvider(value); - } - - return value == null - ? null - : (value as JToken) ?? JToken.FromObject(value, CosmosClientWrapper.Serializer); - } - /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to /// the same compatibility standards as public APIs. It may be changed or removed without notice in @@ -607,7 +577,7 @@ protected override Expression VisitSqlParameter(SqlParameterExpression sqlParame if (_sqlParameters.All(sp => sp.Name != parameterName)) { Check.DebugAssert(sqlParameterExpression.TypeMapping is not null, "SqlParameterExpression without a type mapping"); - var jToken = GenerateJToken(_parameterValues[sqlParameterExpression.Name], sqlParameterExpression.TypeMapping); + var jToken = sqlParameterExpression.TypeMapping.GenerateJToken(_parameterValues[sqlParameterExpression.Name]); _sqlParameters.Add(new SqlParameter(parameterName, jToken)); } diff --git a/src/EFCore.Cosmos/Query/Internal/CosmosQueryUtils.cs b/src/EFCore.Cosmos/Query/Internal/CosmosQueryUtils.cs index ce672f0098d..50ecfffa081 100644 --- a/src/EFCore.Cosmos/Query/Internal/CosmosQueryUtils.cs +++ b/src/EFCore.Cosmos/Query/Internal/CosmosQueryUtils.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Diagnostics.CodeAnalysis; +using Microsoft.EntityFrameworkCore.Cosmos.Storage.Internal; namespace Microsoft.EntityFrameworkCore.Cosmos.Query.Internal; @@ -55,7 +56,7 @@ public static bool TryConvertToArray( var arrayClrType = projection.Type.MakeArrayType(); // TODO: Temporary hack - need to perform proper derivation of the array type mapping from the element (e.g. for // value conversion). - var arrayTypeMapping = typeMappingSource.FindMapping(arrayClrType); + var arrayTypeMapping = (CosmosTypeMapping?)typeMappingSource.FindMapping(arrayClrType); array = new ArrayExpression(subquery, arrayClrType, arrayTypeMapping); return true; diff --git a/src/EFCore.Cosmos/Query/Internal/CosmosQueryableMethodTranslatingExpressionVisitor.cs b/src/EFCore.Cosmos/Query/Internal/CosmosQueryableMethodTranslatingExpressionVisitor.cs index 5f017ec2e84..9c1f0dd76f3 100644 --- a/src/EFCore.Cosmos/Query/Internal/CosmosQueryableMethodTranslatingExpressionVisitor.cs +++ b/src/EFCore.Cosmos/Query/Internal/CosmosQueryableMethodTranslatingExpressionVisitor.cs @@ -4,6 +4,7 @@ using System.Diagnostics.CodeAnalysis; using Microsoft.EntityFrameworkCore.Cosmos.Internal; using Microsoft.EntityFrameworkCore.Cosmos.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Cosmos.Storage.Internal; namespace Microsoft.EntityFrameworkCore.Cosmos.Query.Internal; @@ -1377,7 +1378,7 @@ when methodCallExpression.TryGetIndexerArguments(_queryCompilationContext.Model, && WrapPrimitiveCollectionAsShapedQuery( sqlExpression, sqlExpression.Type.GetSequenceType(), - sqlExpression.TypeMapping!.ElementTypeMapping!) is { } primitiveCollectionTranslation) + (CosmosTypeMapping?)sqlExpression.TypeMapping!.ElementTypeMapping!) is { } primitiveCollectionTranslation) { return primitiveCollectionTranslation; } @@ -1413,8 +1414,8 @@ when methodCallExpression.TryGetIndexerArguments(_queryCompilationContext.Model, // TODO: The following currently just gets the type mapping from the CLR type, which ignores e.g. value converters on // TODO: properties compared against var elementClrType = inlineQueryRootExpression.ElementType; - var elementTypeMapping = _typeMappingSource.FindMapping(elementClrType)!; - var arrayTypeMapping = _typeMappingSource.FindMapping(elementClrType.MakeArrayType()); // TODO: IEnumerable? + var elementTypeMapping = (CosmosTypeMapping?)_typeMappingSource.FindMapping(elementClrType)!; + var arrayTypeMapping = (CosmosTypeMapping?)_typeMappingSource.FindMapping(elementClrType.MakeArrayType()); // TODO: IEnumerable? var inlineArray = new ArrayConstantExpression(elementClrType, translatedItems, arrayTypeMapping); // Unfortunately, Cosmos doesn't support selecting directly from an inline array: SELECT i FROM i IN [1,2,3] (syntax error) @@ -1449,8 +1450,8 @@ [new ProjectionExpression(inlineArray, null!)], // TODO: The following currently just gets the type mapping from the CLR type, which ignores e.g. value converters on // TODO: properties compared against var elementClrType = parameterQueryRootExpression.ElementType; - var arrayTypeMapping = _typeMappingSource.FindMapping(elementClrType.MakeArrayType()); // TODO: IEnumerable? - var elementTypeMapping = _typeMappingSource.FindMapping(elementClrType)!; + var arrayTypeMapping = (CosmosTypeMapping?)_typeMappingSource.FindMapping(elementClrType.MakeArrayType()); // TODO: IEnumerable? + var elementTypeMapping = (CosmosTypeMapping?)_typeMappingSource.FindMapping(elementClrType)!; var sqlParameterExpression = new SqlParameterExpression(parameterQueryRootExpression.ParameterExpression, arrayTypeMapping); // Unfortunately, Cosmos doesn't support selecting directly from an inline array: SELECT i FROM i IN [1,2,3] (syntax error) @@ -1469,7 +1470,7 @@ [new ProjectionExpression(sqlParameterExpression, null!)], private ShapedQueryExpression WrapPrimitiveCollectionAsShapedQuery( Expression array, Type elementClrType, - CoreTypeMapping elementTypeMapping) + CosmosTypeMapping elementTypeMapping) { // TODO: Do proper alias management: #33894 var select = SelectExpression.CreateForPrimitiveCollection( @@ -1501,7 +1502,7 @@ private ShapedQueryExpression WrapPrimitiveCollectionAsShapedQuery( if (CosmosQueryUtils.TryConvertToArray(source1, _typeMappingSource, out var array1, out var projection1, ignoreOrderings) && CosmosQueryUtils.TryConvertToArray(source2, _typeMappingSource, out var array2, out var projection2, ignoreOrderings) && projection1.Type == projection2.Type - && (projection1.TypeMapping ?? projection2.TypeMapping) is CoreTypeMapping typeMapping) + && (projection1.TypeMapping ?? projection2.TypeMapping) is { } typeMapping) { var translation = _sqlExpressionFactory.Function(functionName, [array1, array2], projection1.Type, typeMapping); var select = SelectExpression.CreateForPrimitiveCollection( diff --git a/src/EFCore.Cosmos/Query/Internal/CosmosShapedQueryCompilingExpressionVisitor.CosmosProjectionBindingRemovingExpressionVisitorBase.cs b/src/EFCore.Cosmos/Query/Internal/CosmosShapedQueryCompilingExpressionVisitor.CosmosProjectionBindingRemovingExpressionVisitorBase.cs index fb9ab2e96f5..c62e2676d5d 100644 --- a/src/EFCore.Cosmos/Query/Internal/CosmosShapedQueryCompilingExpressionVisitor.CosmosProjectionBindingRemovingExpressionVisitorBase.cs +++ b/src/EFCore.Cosmos/Query/Internal/CosmosShapedQueryCompilingExpressionVisitor.CosmosProjectionBindingRemovingExpressionVisitorBase.cs @@ -653,7 +653,7 @@ private Expression CreateGetValueExpression( && !property.IsShadowProperty()) { var readExpression = CreateGetValueExpression( - jObjectExpression, storeName, type.MakeNullable(), property.GetTypeMapping()); + jObjectExpression, storeName, type.MakeNullable(), (CosmosTypeMapping)property.GetTypeMapping()); var nonNullReadExpression = readExpression; if (nonNullReadExpression.Type != type) @@ -674,7 +674,7 @@ private Expression CreateGetValueExpression( } return Convert( - CreateGetValueExpression(jObjectExpression, storeName, type.MakeNullable(), property.GetTypeMapping()), + CreateGetValueExpression(jObjectExpression, storeName, type.MakeNullable(), (CosmosTypeMapping)property.GetTypeMapping()), type); } @@ -682,7 +682,7 @@ private Expression CreateGetValueExpression( Expression jObjectExpression, string storeName, Type type, - CoreTypeMapping typeMapping = null) + CosmosTypeMapping typeMapping = null) { Check.DebugAssert(type.IsNullableType(), "Must read nullable type from JObject."); diff --git a/src/EFCore.Cosmos/Query/Internal/CosmosSqlTranslatingExpressionVisitor.cs b/src/EFCore.Cosmos/Query/Internal/CosmosSqlTranslatingExpressionVisitor.cs index c582467bc70..0b84e0d27d0 100644 --- a/src/EFCore.Cosmos/Query/Internal/CosmosSqlTranslatingExpressionVisitor.cs +++ b/src/EFCore.Cosmos/Query/Internal/CosmosSqlTranslatingExpressionVisitor.cs @@ -4,6 +4,7 @@ using System.Collections; using System.Diagnostics.CodeAnalysis; using Microsoft.EntityFrameworkCore.Cosmos.Internal; +using Microsoft.EntityFrameworkCore.Cosmos.Storage.Internal; namespace Microsoft.EntityFrameworkCore.Cosmos.Query.Internal; @@ -770,7 +771,7 @@ protected override Expression VisitNewArray(NewArrayExpression newArrayExpressio translatedItems[i] = translatedItem; } - var arrayTypeMapping = typeMappingSource.FindMapping(newArrayExpression.Type); + var arrayTypeMapping = (CosmosTypeMapping?)typeMappingSource.FindMapping(newArrayExpression.Type); var elementClrType = newArrayExpression.Type.GetElementType()!; var inlineArray = new ArrayConstantExpression(elementClrType, translatedItems, arrayTypeMapping); diff --git a/src/EFCore.Cosmos/Query/Internal/ExpressionExtensions.cs b/src/EFCore.Cosmos/Query/Internal/ExpressionExtensions.cs index 56b797680d9..9445bf5344c 100644 --- a/src/EFCore.Cosmos/Query/Internal/ExpressionExtensions.cs +++ b/src/EFCore.Cosmos/Query/Internal/ExpressionExtensions.cs @@ -1,6 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using Microsoft.EntityFrameworkCore.Cosmos.Storage.Internal; + namespace Microsoft.EntityFrameworkCore.Cosmos.Query.Internal; /// @@ -17,7 +19,7 @@ public static class ExpressionExtensions /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// - public static CoreTypeMapping? InferTypeMapping(params Expression[] expressions) + public static CosmosTypeMapping? InferTypeMapping(params Expression[] expressions) { for (var i = 0; i < expressions.Length; i++) { diff --git a/src/EFCore.Cosmos/Query/Internal/Expressions/ArrayConstantExpression.cs b/src/EFCore.Cosmos/Query/Internal/Expressions/ArrayConstantExpression.cs index 05c0700da2a..c09dbb4e67f 100644 --- a/src/EFCore.Cosmos/Query/Internal/Expressions/ArrayConstantExpression.cs +++ b/src/EFCore.Cosmos/Query/Internal/Expressions/ArrayConstantExpression.cs @@ -2,6 +2,9 @@ // The .NET Foundation licenses this file to you under the MIT license. // ReSharper disable once CheckNamespace + +using Microsoft.EntityFrameworkCore.Cosmos.Storage.Internal; + namespace Microsoft.EntityFrameworkCore.Cosmos.Query.Internal; /// @@ -15,7 +18,7 @@ namespace Microsoft.EntityFrameworkCore.Cosmos.Query.Internal; /// doing so can result in application failures when updating to a new Entity Framework Core release. /// [DebuggerDisplay("{Microsoft.EntityFrameworkCore.Query.ExpressionPrinter.Print(this), nq}")] -public class ArrayConstantExpression(Type elementClrType, IReadOnlyList items, CoreTypeMapping? typeMapping = null) +public class ArrayConstantExpression(Type elementClrType, IReadOnlyList items, CosmosTypeMapping? typeMapping = null) : SqlExpression(typeof(IEnumerable<>).MakeGenericType(elementClrType), typeMapping) { /// diff --git a/src/EFCore.Cosmos/Query/Internal/Expressions/ArrayExpression.cs b/src/EFCore.Cosmos/Query/Internal/Expressions/ArrayExpression.cs index fc93e4305bc..2a6f17507f0 100644 --- a/src/EFCore.Cosmos/Query/Internal/Expressions/ArrayExpression.cs +++ b/src/EFCore.Cosmos/Query/Internal/Expressions/ArrayExpression.cs @@ -2,6 +2,9 @@ // The .NET Foundation licenses this file to you under the MIT license. // ReSharper disable once CheckNamespace + +using Microsoft.EntityFrameworkCore.Cosmos.Storage.Internal; + namespace Microsoft.EntityFrameworkCore.Cosmos.Query.Internal; /// @@ -18,7 +21,7 @@ namespace Microsoft.EntityFrameworkCore.Cosmos.Query.Internal; /// doing so can result in application failures when updating to a new Entity Framework Core release. /// [DebuggerDisplay("{Microsoft.EntityFrameworkCore.Query.ExpressionPrinter.Print(this), nq}")] -public class ArrayExpression(SelectExpression subquery, Type arrayClrType, CoreTypeMapping? arrayTypeMapping = null) +public class ArrayExpression(SelectExpression subquery, Type arrayClrType, CosmosTypeMapping? arrayTypeMapping = null) : SqlExpression(arrayClrType, arrayTypeMapping) { /// diff --git a/src/EFCore.Cosmos/Query/Internal/Expressions/ExistsExpression.cs b/src/EFCore.Cosmos/Query/Internal/Expressions/ExistsExpression.cs index 2548d510b5b..a44468d31c2 100644 --- a/src/EFCore.Cosmos/Query/Internal/Expressions/ExistsExpression.cs +++ b/src/EFCore.Cosmos/Query/Internal/Expressions/ExistsExpression.cs @@ -2,6 +2,9 @@ // The .NET Foundation licenses this file to you under the MIT license. // ReSharper disable once CheckNamespace + +using Microsoft.EntityFrameworkCore.Cosmos.Storage.Internal; + namespace Microsoft.EntityFrameworkCore.Cosmos.Query.Internal; /// @@ -28,7 +31,7 @@ public class ExistsExpression : SqlExpression /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// - public ExistsExpression(SelectExpression subquery, CoreTypeMapping? typeMapping) + public ExistsExpression(SelectExpression subquery, CosmosTypeMapping? typeMapping) : base(typeof(bool), typeMapping) { Subquery = subquery; diff --git a/src/EFCore.Cosmos/Query/Internal/Expressions/InExpression.cs b/src/EFCore.Cosmos/Query/Internal/Expressions/InExpression.cs index 74cde489d33..a0a933839c5 100644 --- a/src/EFCore.Cosmos/Query/Internal/Expressions/InExpression.cs +++ b/src/EFCore.Cosmos/Query/Internal/Expressions/InExpression.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using Microsoft.EntityFrameworkCore.Cosmos.Internal; +using Microsoft.EntityFrameworkCore.Cosmos.Storage.Internal; // ReSharper disable once CheckNamespace namespace Microsoft.EntityFrameworkCore.Cosmos.Query.Internal; @@ -23,7 +24,7 @@ public class InExpression : SqlExpression public InExpression( SqlExpression item, IReadOnlyList values, - CoreTypeMapping typeMapping) + CosmosTypeMapping typeMapping) : this(item, values, valuesParameter: null, typeMapping) { } @@ -37,7 +38,7 @@ public InExpression( public InExpression( SqlExpression item, SqlParameterExpression valuesParameter, - CoreTypeMapping typeMapping) + CosmosTypeMapping typeMapping) : this(item, values: null, valuesParameter, typeMapping) { } @@ -46,7 +47,7 @@ private InExpression( SqlExpression item, IReadOnlyList? values, SqlParameterExpression? valuesParameter, - CoreTypeMapping? typeMapping) + CosmosTypeMapping? typeMapping) : base(typeof(bool), typeMapping) { Item = item; @@ -119,7 +120,7 @@ protected override Expression VisitChildren(ExpressionVisitor visitor) /// /// A relational type mapping to apply. /// A new expression which has supplied type mapping. - public virtual InExpression ApplyTypeMapping(CoreTypeMapping? typeMapping) + public virtual InExpression ApplyTypeMapping(CosmosTypeMapping? typeMapping) => new(Item, Values, ValuesParameter, typeMapping); /// diff --git a/src/EFCore.Cosmos/Query/Internal/Expressions/KeyAccessExpression.cs b/src/EFCore.Cosmos/Query/Internal/Expressions/KeyAccessExpression.cs index 3a15ab61823..495d1199a5e 100644 --- a/src/EFCore.Cosmos/Query/Internal/Expressions/KeyAccessExpression.cs +++ b/src/EFCore.Cosmos/Query/Internal/Expressions/KeyAccessExpression.cs @@ -1,6 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using Microsoft.EntityFrameworkCore.Cosmos.Storage.Internal; + // ReSharper disable once CheckNamespace namespace Microsoft.EntityFrameworkCore.Cosmos.Query.Internal; @@ -11,7 +13,7 @@ namespace Microsoft.EntityFrameworkCore.Cosmos.Query.Internal; /// doing so can result in application failures when updating to a new Entity Framework Core release. /// public class KeyAccessExpression(IProperty property, Expression accessExpression) - : SqlExpression(property.ClrType, property.GetTypeMapping()), IAccessExpression + : SqlExpression(property.ClrType, (CosmosTypeMapping)property.GetTypeMapping()), IAccessExpression { /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to diff --git a/src/EFCore.Cosmos/Query/Internal/Expressions/ScalarReferenceExpression.cs b/src/EFCore.Cosmos/Query/Internal/Expressions/ScalarReferenceExpression.cs index f4c60141a3b..da47d31f7c3 100644 --- a/src/EFCore.Cosmos/Query/Internal/Expressions/ScalarReferenceExpression.cs +++ b/src/EFCore.Cosmos/Query/Internal/Expressions/ScalarReferenceExpression.cs @@ -1,6 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using Microsoft.EntityFrameworkCore.Cosmos.Storage.Internal; + // ReSharper disable once CheckNamespace namespace Microsoft.EntityFrameworkCore.Cosmos.Query.Internal; @@ -14,7 +16,7 @@ namespace Microsoft.EntityFrameworkCore.Cosmos.Query.Internal; /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// -public class ScalarReferenceExpression(string name, Type clrType, CoreTypeMapping? typeMapping = null) +public class ScalarReferenceExpression(string name, Type clrType, CosmosTypeMapping? typeMapping = null) : SqlExpression(clrType, typeMapping), IAccessExpression { /// diff --git a/src/EFCore.Cosmos/Query/Internal/Expressions/ScalarSubqueryExpression.cs b/src/EFCore.Cosmos/Query/Internal/Expressions/ScalarSubqueryExpression.cs index 01ccf47b853..670295cc510 100644 --- a/src/EFCore.Cosmos/Query/Internal/Expressions/ScalarSubqueryExpression.cs +++ b/src/EFCore.Cosmos/Query/Internal/Expressions/ScalarSubqueryExpression.cs @@ -1,6 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using Microsoft.EntityFrameworkCore.Cosmos.Storage.Internal; + // ReSharper disable once CheckNamespace namespace Microsoft.EntityFrameworkCore.Cosmos.Query.Internal; @@ -35,7 +37,7 @@ subquery.Projection[0].Expression is SqlExpression sqlExpression Subquery = subquery; } - private ScalarSubqueryExpression(SelectExpression subquery, CoreTypeMapping? typeMapping) + private ScalarSubqueryExpression(SelectExpression subquery, CosmosTypeMapping? typeMapping) : base(subquery.Projection[0].Type, typeMapping) { Subquery = subquery; @@ -57,7 +59,7 @@ private ScalarSubqueryExpression(SelectExpression subquery, CoreTypeMapping? typ /// /// A relational type mapping to apply. /// A new expression which has supplied type mapping. - public virtual SqlExpression ApplyTypeMapping(CoreTypeMapping? typeMapping) + public virtual SqlExpression ApplyTypeMapping(CosmosTypeMapping? typeMapping) => new ScalarSubqueryExpression(Subquery, typeMapping); /// diff --git a/src/EFCore.Cosmos/Query/Internal/Expressions/SelectExpression.cs b/src/EFCore.Cosmos/Query/Internal/Expressions/SelectExpression.cs index fdf006147c0..4b71f4f5c06 100644 --- a/src/EFCore.Cosmos/Query/Internal/Expressions/SelectExpression.cs +++ b/src/EFCore.Cosmos/Query/Internal/Expressions/SelectExpression.cs @@ -3,6 +3,7 @@ using Microsoft.EntityFrameworkCore.Internal; using Microsoft.EntityFrameworkCore.Cosmos.Internal; +using Microsoft.EntityFrameworkCore.Cosmos.Storage.Internal; // ReSharper disable once CheckNamespace namespace Microsoft.EntityFrameworkCore.Cosmos.Query.Internal; @@ -111,7 +112,7 @@ private SelectExpression() public static SelectExpression CreateForPrimitiveCollection( SourceExpression source, Type elementClrType, - CoreTypeMapping elementTypeMapping) + CosmosTypeMapping elementTypeMapping) => new() { _sources = { source }, diff --git a/src/EFCore.Cosmos/Query/Internal/Expressions/SqlBinaryExpression.cs b/src/EFCore.Cosmos/Query/Internal/Expressions/SqlBinaryExpression.cs index a0e3dc07408..cefc4c05cba 100644 --- a/src/EFCore.Cosmos/Query/Internal/Expressions/SqlBinaryExpression.cs +++ b/src/EFCore.Cosmos/Query/Internal/Expressions/SqlBinaryExpression.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using Microsoft.EntityFrameworkCore.Cosmos.Internal; +using Microsoft.EntityFrameworkCore.Cosmos.Storage.Internal; // ReSharper disable once CheckNamespace namespace Microsoft.EntityFrameworkCore.Cosmos.Query.Internal; @@ -25,7 +26,7 @@ public SqlBinaryExpression( SqlExpression left, SqlExpression right, Type type, - CoreTypeMapping? typeMapping) + CosmosTypeMapping? typeMapping) : base(type, typeMapping) { if (!IsValidOperator(operatorType)) diff --git a/src/EFCore.Cosmos/Query/Internal/Expressions/SqlConstantExpression.cs b/src/EFCore.Cosmos/Query/Internal/Expressions/SqlConstantExpression.cs index d24082fa82c..b03e41bdb8c 100644 --- a/src/EFCore.Cosmos/Query/Internal/Expressions/SqlConstantExpression.cs +++ b/src/EFCore.Cosmos/Query/Internal/Expressions/SqlConstantExpression.cs @@ -15,7 +15,7 @@ namespace Microsoft.EntityFrameworkCore.Cosmos.Query.Internal; /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// -public class SqlConstantExpression(ConstantExpression constantExpression, CoreTypeMapping? typeMapping) +public class SqlConstantExpression(ConstantExpression constantExpression, CosmosTypeMapping? typeMapping) : SqlExpression(constantExpression.Type, typeMapping) { /// @@ -33,7 +33,7 @@ public virtual object? Value /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// - public virtual SqlExpression ApplyTypeMapping(CoreTypeMapping? typeMapping) + public virtual SqlExpression ApplyTypeMapping(CosmosTypeMapping? typeMapping) => new SqlConstantExpression(constantExpression, typeMapping ?? TypeMapping); /// @@ -80,7 +80,7 @@ private void Print( } } - private JToken? GenerateJToken(object? value, CoreTypeMapping? typeMapping) + private JToken? GenerateJToken(object? value, CosmosTypeMapping? typeMapping) { var mappingClrType = typeMapping?.ClrType.UnwrapNullableType() ?? Type; if (value?.GetType().IsInteger() == true diff --git a/src/EFCore.Cosmos/Query/Internal/Expressions/SqlExpression.cs b/src/EFCore.Cosmos/Query/Internal/Expressions/SqlExpression.cs index 39bc4345c06..95b3396a4ab 100644 --- a/src/EFCore.Cosmos/Query/Internal/Expressions/SqlExpression.cs +++ b/src/EFCore.Cosmos/Query/Internal/Expressions/SqlExpression.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using Microsoft.EntityFrameworkCore.Cosmos.Internal; +using Microsoft.EntityFrameworkCore.Cosmos.Storage.Internal; // ReSharper disable once CheckNamespace namespace Microsoft.EntityFrameworkCore.Cosmos.Query.Internal; @@ -13,7 +14,7 @@ namespace Microsoft.EntityFrameworkCore.Cosmos.Query.Internal; /// doing so can result in application failures when updating to a new Entity Framework Core release. /// [DebuggerDisplay("{Microsoft.EntityFrameworkCore.Query.ExpressionPrinter.Print(this), nq}")] -public abstract class SqlExpression(Type type, CoreTypeMapping? typeMapping) +public abstract class SqlExpression(Type type, CosmosTypeMapping? typeMapping) : Expression, IPrintableExpression { /// @@ -30,7 +31,7 @@ public abstract class SqlExpression(Type type, CoreTypeMapping? typeMapping) /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// - public virtual CoreTypeMapping? TypeMapping { get; } = typeMapping; + public virtual CosmosTypeMapping? TypeMapping { get; } = typeMapping; /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to diff --git a/src/EFCore.Cosmos/Query/Internal/Expressions/SqlFunctionExpression.cs b/src/EFCore.Cosmos/Query/Internal/Expressions/SqlFunctionExpression.cs index d703d614937..2dc56848b20 100644 --- a/src/EFCore.Cosmos/Query/Internal/Expressions/SqlFunctionExpression.cs +++ b/src/EFCore.Cosmos/Query/Internal/Expressions/SqlFunctionExpression.cs @@ -1,6 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using Microsoft.EntityFrameworkCore.Cosmos.Storage.Internal; + // ReSharper disable once CheckNamespace namespace Microsoft.EntityFrameworkCore.Cosmos.Query.Internal; @@ -14,7 +16,7 @@ public class SqlFunctionExpression( string name, IEnumerable arguments, Type type, - CoreTypeMapping? typeMapping) + CosmosTypeMapping? typeMapping) : SqlExpression(type, typeMapping) { /// @@ -64,7 +66,7 @@ protected override Expression VisitChildren(ExpressionVisitor visitor) /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// - public virtual SqlFunctionExpression ApplyTypeMapping(CoreTypeMapping? typeMapping) + public virtual SqlFunctionExpression ApplyTypeMapping(CosmosTypeMapping? typeMapping) => new(Name, Arguments, Type, typeMapping ?? TypeMapping); /// diff --git a/src/EFCore.Cosmos/Query/Internal/Expressions/SqlParameterExpression.cs b/src/EFCore.Cosmos/Query/Internal/Expressions/SqlParameterExpression.cs index 32c5696961c..7b7bdf79b46 100644 --- a/src/EFCore.Cosmos/Query/Internal/Expressions/SqlParameterExpression.cs +++ b/src/EFCore.Cosmos/Query/Internal/Expressions/SqlParameterExpression.cs @@ -1,6 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using Microsoft.EntityFrameworkCore.Cosmos.Storage.Internal; + // ReSharper disable once CheckNamespace namespace Microsoft.EntityFrameworkCore.Cosmos.Query.Internal; @@ -21,7 +23,7 @@ public sealed class SqlParameterExpression : SqlExpression /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// - public SqlParameterExpression(ParameterExpression parameterExpression, CoreTypeMapping? typeMapping) + public SqlParameterExpression(ParameterExpression parameterExpression, CosmosTypeMapping? typeMapping) : base(parameterExpression.Type, typeMapping) { Check.DebugAssert(parameterExpression.Name != null, "Parameter must have name."); @@ -45,7 +47,7 @@ public string Name /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// - public SqlExpression ApplyTypeMapping(CoreTypeMapping? typeMapping) + public SqlExpression ApplyTypeMapping(CosmosTypeMapping? typeMapping) => new SqlParameterExpression(_parameterExpression, typeMapping ?? TypeMapping); /// diff --git a/src/EFCore.Cosmos/Query/Internal/Expressions/SqlUnaryExpression.cs b/src/EFCore.Cosmos/Query/Internal/Expressions/SqlUnaryExpression.cs index 56fc45ea240..adb4ac8f6c3 100644 --- a/src/EFCore.Cosmos/Query/Internal/Expressions/SqlUnaryExpression.cs +++ b/src/EFCore.Cosmos/Query/Internal/Expressions/SqlUnaryExpression.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using Microsoft.EntityFrameworkCore.Cosmos.Internal; +using Microsoft.EntityFrameworkCore.Cosmos.Storage.Internal; // ReSharper disable once CheckNamespace namespace Microsoft.EntityFrameworkCore.Cosmos.Query.Internal; @@ -34,7 +35,7 @@ public SqlUnaryExpression( ExpressionType operatorType, SqlExpression operand, Type type, - CoreTypeMapping? typeMapping) + CosmosTypeMapping? typeMapping) : base(type, typeMapping) { if (!IsValidOperator(operatorType)) diff --git a/src/EFCore.Cosmos/Query/Internal/ISqlExpressionFactory.cs b/src/EFCore.Cosmos/Query/Internal/ISqlExpressionFactory.cs index ea23e416025..a4b1bb8936f 100644 --- a/src/EFCore.Cosmos/Query/Internal/ISqlExpressionFactory.cs +++ b/src/EFCore.Cosmos/Query/Internal/ISqlExpressionFactory.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Diagnostics.CodeAnalysis; +using Microsoft.EntityFrameworkCore.Cosmos.Storage.Internal; namespace Microsoft.EntityFrameworkCore.Cosmos.Query.Internal; @@ -20,7 +21,7 @@ public interface ISqlExpressionFactory /// doing so can result in application failures when updating to a new Entity Framework Core release. /// [return: NotNullIfNotNull(nameof(sqlExpression))] - SqlExpression? ApplyTypeMapping(SqlExpression? sqlExpression, CoreTypeMapping? typeMapping); + SqlExpression? ApplyTypeMapping(SqlExpression? sqlExpression, CosmosTypeMapping? typeMapping); /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to @@ -49,7 +50,7 @@ public interface ISqlExpressionFactory ExpressionType operatorType, SqlExpression left, SqlExpression right, - CoreTypeMapping? typeMapping); + CosmosTypeMapping? typeMapping); /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to @@ -131,7 +132,7 @@ public interface ISqlExpressionFactory SqlBinaryExpression Add( SqlExpression left, SqlExpression right, - CoreTypeMapping? typeMapping = null); + CosmosTypeMapping? typeMapping = null); /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to @@ -142,7 +143,7 @@ SqlBinaryExpression Add( SqlBinaryExpression Subtract( SqlExpression left, SqlExpression right, - CoreTypeMapping? typeMapping = null); + CosmosTypeMapping? typeMapping = null); /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to @@ -153,7 +154,7 @@ SqlBinaryExpression Subtract( SqlBinaryExpression Multiply( SqlExpression left, SqlExpression right, - CoreTypeMapping? typeMapping = null); + CosmosTypeMapping? typeMapping = null); /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to @@ -164,7 +165,7 @@ SqlBinaryExpression Multiply( SqlBinaryExpression Divide( SqlExpression left, SqlExpression right, - CoreTypeMapping? typeMapping = null); + CosmosTypeMapping? typeMapping = null); /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to @@ -175,7 +176,7 @@ SqlBinaryExpression Divide( SqlBinaryExpression Modulo( SqlExpression left, SqlExpression right, - CoreTypeMapping? typeMapping = null); + CosmosTypeMapping? typeMapping = null); /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to @@ -186,7 +187,7 @@ SqlBinaryExpression Modulo( SqlBinaryExpression And( SqlExpression left, SqlExpression right, - CoreTypeMapping? typeMapping = null); + CosmosTypeMapping? typeMapping = null); /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to @@ -197,7 +198,7 @@ SqlBinaryExpression And( SqlBinaryExpression Or( SqlExpression left, SqlExpression right, - CoreTypeMapping? typeMapping = null); + CosmosTypeMapping? typeMapping = null); /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to @@ -205,7 +206,7 @@ SqlBinaryExpression Or( /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// - SqlExpression CoalesceUndefined(SqlExpression left, SqlExpression right, CoreTypeMapping? typeMapping = null); + SqlExpression CoalesceUndefined(SqlExpression left, SqlExpression right, CosmosTypeMapping? typeMapping = null); /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to @@ -229,7 +230,7 @@ SqlBinaryExpression Or( /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// - SqlBinaryExpression ArrayIndex(SqlExpression left, SqlExpression right, Type type, CoreTypeMapping? typeMapping = null); + SqlBinaryExpression ArrayIndex(SqlExpression left, SqlExpression right, Type type, CosmosTypeMapping? typeMapping = null); /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to @@ -240,7 +241,7 @@ SqlBinaryExpression Or( SqlUnaryExpression Convert( SqlExpression operand, Type type, - CoreTypeMapping? typeMapping = null); + CosmosTypeMapping? typeMapping = null); /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to @@ -268,7 +269,7 @@ SqlFunctionExpression Function( string functionName, IEnumerable arguments, Type returnType, - CoreTypeMapping? typeMapping = null); + CosmosTypeMapping? typeMapping = null); /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to @@ -303,7 +304,7 @@ SqlConditionalExpression Condition( /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// - SqlConstantExpression Constant(object? value, CoreTypeMapping? typeMapping = null); + SqlConstantExpression Constant(object? value, CosmosTypeMapping? typeMapping = null); /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to diff --git a/src/EFCore.Cosmos/Query/Internal/SqlExpressionFactory.cs b/src/EFCore.Cosmos/Query/Internal/SqlExpressionFactory.cs index 4a0f78f0d60..10b259d712d 100644 --- a/src/EFCore.Cosmos/Query/Internal/SqlExpressionFactory.cs +++ b/src/EFCore.Cosmos/Query/Internal/SqlExpressionFactory.cs @@ -3,6 +3,7 @@ using System.Diagnostics.CodeAnalysis; using Microsoft.EntityFrameworkCore.Cosmos.Internal; +using Microsoft.EntityFrameworkCore.Cosmos.Storage.Internal; using Microsoft.EntityFrameworkCore.Internal; namespace Microsoft.EntityFrameworkCore.Cosmos.Query.Internal; @@ -16,7 +17,7 @@ namespace Microsoft.EntityFrameworkCore.Cosmos.Query.Internal; public class SqlExpressionFactory(ITypeMappingSource typeMappingSource, IModel model) : ISqlExpressionFactory { - private readonly CoreTypeMapping _boolTypeMapping = typeMappingSource.FindMapping(typeof(bool), model)!; + private readonly CosmosTypeMapping _boolTypeMapping = (CosmosTypeMapping)typeMappingSource.FindMapping(typeof(bool), model)!; /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to @@ -28,7 +29,7 @@ public class SqlExpressionFactory(ITypeMappingSource typeMappingSource, IModel m public virtual SqlExpression? ApplyDefaultTypeMapping(SqlExpression? sqlExpression) => sqlExpression is not { TypeMapping: null } ? sqlExpression - : ApplyTypeMapping(sqlExpression, typeMappingSource.FindMapping(sqlExpression.Type, model)); + : ApplyTypeMapping(sqlExpression, (CosmosTypeMapping?)typeMappingSource.FindMapping(sqlExpression.Type, model)); /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to @@ -37,7 +38,7 @@ public class SqlExpressionFactory(ITypeMappingSource typeMappingSource, IModel m /// doing so can result in application failures when updating to a new Entity Framework Core release. /// [return: NotNullIfNotNull(nameof(sqlExpression))] - public virtual SqlExpression? ApplyTypeMapping(SqlExpression? sqlExpression, CoreTypeMapping? typeMapping) + public virtual SqlExpression? ApplyTypeMapping(SqlExpression? sqlExpression, CosmosTypeMapping? typeMapping) => sqlExpression switch { null or { TypeMapping: not null } => sqlExpression, @@ -55,7 +56,7 @@ public class SqlExpressionFactory(ITypeMappingSource typeMappingSource, IModel m private SqlExpression ApplyTypeMappingOnSqlConditional( SqlConditionalExpression sqlConditionalExpression, - CoreTypeMapping? typeMapping) + CosmosTypeMapping? typeMapping) => sqlConditionalExpression.Update( sqlConditionalExpression.Test, ApplyTypeMapping(sqlConditionalExpression.IfTrue, typeMapping), @@ -63,11 +64,11 @@ private SqlExpression ApplyTypeMappingOnSqlConditional( private SqlExpression ApplyTypeMappingOnSqlUnary( SqlUnaryExpression sqlUnaryExpression, - CoreTypeMapping? typeMapping) + CosmosTypeMapping? typeMapping) { SqlExpression operand; Type resultType; - CoreTypeMapping? resultTypeMapping; + CosmosTypeMapping? resultTypeMapping; switch (sqlUnaryExpression.OperatorType) { case ExpressionType.Equal: @@ -107,14 +108,14 @@ when sqlUnaryExpression.IsLogicalNot(): private SqlExpression ApplyTypeMappingOnSqlBinary( SqlBinaryExpression sqlBinaryExpression, - CoreTypeMapping? typeMapping) + CosmosTypeMapping? typeMapping) { var left = sqlBinaryExpression.Left; var right = sqlBinaryExpression.Right; Type resultType; - CoreTypeMapping? resultTypeMapping; - CoreTypeMapping? inferredTypeMapping; + CosmosTypeMapping? resultTypeMapping; + CosmosTypeMapping? inferredTypeMapping; switch (sqlBinaryExpression.OperatorType) { case ExpressionType.Equal: @@ -127,8 +128,8 @@ private SqlExpression ApplyTypeMappingOnSqlBinary( inferredTypeMapping = ExpressionExtensions.InferTypeMapping(left, right) // We avoid object here since the result does not get typeMapping from outside. ?? (left.Type != typeof(object) - ? typeMappingSource.FindMapping(left.Type, model) - : typeMappingSource.FindMapping(right.Type, model)); + ? (CosmosTypeMapping?)typeMappingSource.FindMapping(left.Type, model) + : (CosmosTypeMapping?)typeMappingSource.FindMapping(right.Type, model)); resultType = typeof(bool); resultTypeMapping = _boolTypeMapping; break; @@ -165,7 +166,7 @@ private SqlExpression ApplyTypeMappingOnSqlBinary( // TODO: This infers based on the CLR type; need to properly infer based on the element type mapping // TODO: being applied here (e.g. WHERE @p[1] = c.PropertyWithValueConverter) var arrayTypeMapping = left.TypeMapping - ?? (typeMapping is null ? null : typeMappingSource.FindMapping(typeMapping.ClrType.MakeArrayType())); + ?? (typeMapping is null ? null : (CosmosTypeMapping?)typeMappingSource.FindMapping(typeMapping.ClrType.MakeArrayType())); return new SqlBinaryExpression( ExpressionType.ArrayIndex, ApplyTypeMapping(left, arrayTypeMapping), @@ -192,7 +193,7 @@ private InExpression ApplyTypeMappingOnIn(InExpression inExpression) { var missingTypeMappingInValues = false; - CoreTypeMapping? valuesTypeMapping = null; + CosmosTypeMapping? valuesTypeMapping = null; switch (inExpression) { case { ValuesParameter: SqlParameterExpression parameter }: @@ -221,7 +222,7 @@ private InExpression ApplyTypeMappingOnIn(InExpression inExpression) var item = ApplyTypeMapping( inExpression.Item, - valuesTypeMapping ?? typeMappingSource.FindMapping(inExpression.Item.Type, model)); + valuesTypeMapping ?? (CosmosTypeMapping?)typeMappingSource.FindMapping(inExpression.Item.Type, model)); switch (inExpression) { @@ -272,8 +273,8 @@ private InExpression ApplyTypeMappingOnIn(InExpression inExpression) ? unary.Operand.TypeMapping : null) // If we couldn't find a type mapping on the item, try inferring it from the array - ?? arrayMapping?.ElementTypeMapping - ?? typeMappingSource.FindMapping(itemExpression.Type, model); + ?? (CosmosTypeMapping?)arrayMapping?.ElementTypeMapping + ?? (CosmosTypeMapping?)typeMappingSource.FindMapping(itemExpression.Type, model); if (itemMapping is null) { @@ -297,7 +298,7 @@ var t when t.TryGetSequenceType() != typeof(object) => t, $"Can't construct generic primitive collection type for array type '{arrayExpression.Type}'") }; - arrayMapping = typeMappingSource.FindMapping(arrayClrType, model, itemMapping.ElementTypeMapping); + arrayMapping = (CosmosTypeMapping?)typeMappingSource.FindMapping(arrayClrType, model, itemMapping.ElementTypeMapping); if (arrayMapping is null) { @@ -318,7 +319,7 @@ var t when t.TryGetSequenceType() != typeof(object) => t, ExpressionType operatorType, SqlExpression left, SqlExpression right, - CoreTypeMapping? typeMapping) + CosmosTypeMapping? typeMapping) { if (!SqlBinaryExpression.IsValidOperator(operatorType)) { @@ -431,7 +432,7 @@ public virtual SqlBinaryExpression OrElse(SqlExpression left, SqlExpression righ /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// - public virtual SqlBinaryExpression Add(SqlExpression left, SqlExpression right, CoreTypeMapping? typeMapping = null) + public virtual SqlBinaryExpression Add(SqlExpression left, SqlExpression right, CosmosTypeMapping? typeMapping = null) => MakeBinary(ExpressionType.Add, left, right, typeMapping)!; /// @@ -440,7 +441,7 @@ public virtual SqlBinaryExpression Add(SqlExpression left, SqlExpression right, /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// - public virtual SqlBinaryExpression Subtract(SqlExpression left, SqlExpression right, CoreTypeMapping? typeMapping = null) + public virtual SqlBinaryExpression Subtract(SqlExpression left, SqlExpression right, CosmosTypeMapping? typeMapping = null) => MakeBinary(ExpressionType.Subtract, left, right, typeMapping)!; /// @@ -449,7 +450,7 @@ public virtual SqlBinaryExpression Subtract(SqlExpression left, SqlExpression ri /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// - public virtual SqlBinaryExpression Multiply(SqlExpression left, SqlExpression right, CoreTypeMapping? typeMapping = null) + public virtual SqlBinaryExpression Multiply(SqlExpression left, SqlExpression right, CosmosTypeMapping? typeMapping = null) => MakeBinary(ExpressionType.Multiply, left, right, typeMapping)!; /// @@ -458,7 +459,7 @@ public virtual SqlBinaryExpression Multiply(SqlExpression left, SqlExpression ri /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// - public virtual SqlBinaryExpression Divide(SqlExpression left, SqlExpression right, CoreTypeMapping? typeMapping = null) + public virtual SqlBinaryExpression Divide(SqlExpression left, SqlExpression right, CosmosTypeMapping? typeMapping = null) => MakeBinary(ExpressionType.Divide, left, right, typeMapping)!; /// @@ -467,7 +468,7 @@ public virtual SqlBinaryExpression Divide(SqlExpression left, SqlExpression righ /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// - public virtual SqlBinaryExpression Modulo(SqlExpression left, SqlExpression right, CoreTypeMapping? typeMapping = null) + public virtual SqlBinaryExpression Modulo(SqlExpression left, SqlExpression right, CosmosTypeMapping? typeMapping = null) => MakeBinary(ExpressionType.Modulo, left, right, typeMapping)!; /// @@ -476,7 +477,7 @@ public virtual SqlBinaryExpression Modulo(SqlExpression left, SqlExpression righ /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// - public virtual SqlBinaryExpression And(SqlExpression left, SqlExpression right, CoreTypeMapping? typeMapping = null) + public virtual SqlBinaryExpression And(SqlExpression left, SqlExpression right, CosmosTypeMapping? typeMapping = null) => MakeBinary(ExpressionType.And, left, right, typeMapping)!; /// @@ -485,14 +486,14 @@ public virtual SqlBinaryExpression And(SqlExpression left, SqlExpression right, /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// - public virtual SqlBinaryExpression Or(SqlExpression left, SqlExpression right, CoreTypeMapping? typeMapping = null) + public virtual SqlBinaryExpression Or(SqlExpression left, SqlExpression right, CosmosTypeMapping? typeMapping = null) => MakeBinary(ExpressionType.Or, left, right, typeMapping)!; private SqlUnaryExpression? MakeUnary( ExpressionType operatorType, SqlExpression operand, Type type, - CoreTypeMapping? typeMapping = null) + CosmosTypeMapping? typeMapping = null) => SqlUnaryExpression.IsValidOperator(operatorType) ? (SqlUnaryExpression)ApplyTypeMapping(new SqlUnaryExpression(operatorType, operand, type, null), typeMapping) : null; @@ -503,7 +504,7 @@ public virtual SqlBinaryExpression Or(SqlExpression left, SqlExpression right, C /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// - public virtual SqlExpression CoalesceUndefined(SqlExpression left, SqlExpression right, CoreTypeMapping? typeMapping = null) + public virtual SqlExpression CoalesceUndefined(SqlExpression left, SqlExpression right, CosmosTypeMapping? typeMapping = null) => MakeBinary(ExpressionType.Coalesce, left, right, typeMapping)!; /// @@ -530,7 +531,7 @@ public virtual SqlBinaryExpression IsNotNull(SqlExpression operand) /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// - public virtual SqlBinaryExpression ArrayIndex(SqlExpression left, SqlExpression right, Type type, CoreTypeMapping? typeMapping = null) + public virtual SqlBinaryExpression ArrayIndex(SqlExpression left, SqlExpression right, Type type, CosmosTypeMapping? typeMapping = null) => new(ExpressionType.ArrayIndex, left, right, type, typeMapping)!; /// @@ -539,7 +540,7 @@ public virtual SqlBinaryExpression ArrayIndex(SqlExpression left, SqlExpression /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// - public virtual SqlUnaryExpression Convert(SqlExpression operand, Type type, CoreTypeMapping? typeMapping = null) + public virtual SqlUnaryExpression Convert(SqlExpression operand, Type type, CosmosTypeMapping? typeMapping = null) => MakeUnary(ExpressionType.Convert, operand, type, typeMapping)!; /// @@ -570,7 +571,7 @@ public virtual SqlFunctionExpression Function( string functionName, IEnumerable arguments, Type returnType, - CoreTypeMapping? typeMapping = null) + CosmosTypeMapping? typeMapping = null) { var typeMappedArguments = new List(); @@ -626,7 +627,7 @@ public virtual InExpression In(SqlExpression item, SqlParameterExpression values /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// - public virtual SqlConstantExpression Constant(object? value, CoreTypeMapping? typeMapping = null) + public virtual SqlConstantExpression Constant(object? value, CosmosTypeMapping? typeMapping = null) => new(Expression.Constant(value), typeMapping); /// diff --git a/src/EFCore.Cosmos/Query/Internal/Translators/CosmosVectorSearchTranslator.cs b/src/EFCore.Cosmos/Query/Internal/Translators/CosmosVectorSearchTranslator.cs new file mode 100644 index 00000000000..819816224a6 --- /dev/null +++ b/src/EFCore.Cosmos/Query/Internal/Translators/CosmosVectorSearchTranslator.cs @@ -0,0 +1,94 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.EntityFrameworkCore.Cosmos.Extensions; +using Microsoft.EntityFrameworkCore.Cosmos.Internal; +using Microsoft.EntityFrameworkCore.Cosmos.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Cosmos.Storage.Internal; + +// ReSharper disable once CheckNamespace +namespace Microsoft.EntityFrameworkCore.Cosmos.Query.Internal; + +/// +/// This is an internal API that supports the Entity Framework Core infrastructure and not subject to +/// the same compatibility standards as public APIs. It may be changed or removed without notice in +/// any release. You should only use it directly in your code with extreme caution and knowing that +/// doing so can result in application failures when updating to a new Entity Framework Core release. +/// +public class CosmosVectorSearchTranslator(ISqlExpressionFactory sqlExpressionFactory, ITypeMappingSource typeMappingSource) + : IMethodCallTranslator +{ + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public SqlExpression? Translate( + SqlExpression? instance, + MethodInfo method, + IReadOnlyList arguments, + IDiagnosticsLogger logger) + { + if (method.DeclaringType != typeof(CosmosDbFunctionsExtensions) + && method.Name != nameof(CosmosDbFunctionsExtensions.VectorDistance)) + { + return null; + } + + var vectorMapping = arguments[1].TypeMapping as CosmosVectorTypeMapping + ?? arguments[2].TypeMapping as CosmosVectorTypeMapping + ?? throw new InvalidOperationException(CosmosStrings.VectorSearchRequiresVector); + + Check.DebugAssert(arguments.Count is 3 or 4 or 6, "Did you add a parameter?"); + + SqlConstantExpression bruteForce; + if (arguments.Count >= 4) + { + if (arguments[3] is not SqlConstantExpression { Value: bool }) + { + throw new InvalidOperationException( + CosmosStrings.ArgumentNotConstant("useBruteForce", nameof(CosmosDbFunctionsExtensions.VectorDistance))); + } + + bruteForce = (SqlConstantExpression)arguments[3]; + } + else + { + bruteForce = sqlExpressionFactory.Constant(false, (CosmosTypeMapping)typeMappingSource.FindMapping(typeof(bool))!); + } + + var vectorType = vectorMapping.VectorType; + if (arguments.Count == 6) + { + if (arguments[4] is not SqlConstantExpression { Value: DistanceFunction distanceFunction }) + { + throw new InvalidOperationException( + CosmosStrings.ArgumentNotConstant("distanceFunction", nameof(CosmosDbFunctionsExtensions.VectorDistance))); + } + vectorType = vectorType with { DistanceFunction = distanceFunction }; + + if (arguments[5] is not SqlConstantExpression { Value: VectorDataType vectorDataType }) + { + throw new InvalidOperationException( + CosmosStrings.ArgumentNotConstant("dataType", nameof(CosmosDbFunctionsExtensions.VectorDistance))); + } + vectorType = vectorType with { DataType = vectorDataType }; + } + + if (vectorType.DataType == null) + { + vectorType = vectorType with { DataType = CosmosVectorType.CreateDefaultVectorDataType(vectorMapping.ClrType) }; + } + + return sqlExpressionFactory.Function( + "VectorDistance", + [ + sqlExpressionFactory.ApplyTypeMapping(arguments[1], vectorMapping), + sqlExpressionFactory.ApplyTypeMapping(arguments[2], vectorMapping), + bruteForce, + sqlExpressionFactory.Constant(vectorType, (CosmosTypeMapping)typeMappingSource.FindMapping(typeof(CosmosVectorType))!) + ], + typeof(double)); + } +} diff --git a/src/EFCore.Cosmos/Storage/Internal/ContainerProperties.cs b/src/EFCore.Cosmos/Storage/Internal/ContainerProperties.cs index d0761cd9ecb..78965ed3073 100644 --- a/src/EFCore.Cosmos/Storage/Internal/ContainerProperties.cs +++ b/src/EFCore.Cosmos/Storage/Internal/ContainerProperties.cs @@ -1,6 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using Microsoft.EntityFrameworkCore.Cosmos.Metadata.Internal; + namespace Microsoft.EntityFrameworkCore.Cosmos.Storage.Internal; /// @@ -9,85 +11,11 @@ namespace Microsoft.EntityFrameworkCore.Cosmos.Storage.Internal; /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// -public readonly record struct ContainerProperties -{ - /// - /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to - /// the same compatibility standards as public APIs. It may be changed or removed without notice in - /// any release. You should only use it directly in your code with extreme caution and knowing that - /// doing so can result in application failures when updating to a new Entity Framework Core release. - /// - public readonly string Id; - - /// - /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to - /// the same compatibility standards as public APIs. It may be changed or removed without notice in - /// any release. You should only use it directly in your code with extreme caution and knowing that - /// doing so can result in application failures when updating to a new Entity Framework Core release. - /// - public readonly IReadOnlyList PartitionKeyStoreNames; - - /// - /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to - /// the same compatibility standards as public APIs. It may be changed or removed without notice in - /// any release. You should only use it directly in your code with extreme caution and knowing that - /// doing so can result in application failures when updating to a new Entity Framework Core release. - /// - public readonly int? AnalyticalStoreTimeToLiveInSeconds; - - /// - /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to - /// the same compatibility standards as public APIs. It may be changed or removed without notice in - /// any release. You should only use it directly in your code with extreme caution and knowing that - /// doing so can result in application failures when updating to a new Entity Framework Core release. - /// - public readonly int? DefaultTimeToLive; - - /// - /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to - /// the same compatibility standards as public APIs. It may be changed or removed without notice in - /// any release. You should only use it directly in your code with extreme caution and knowing that - /// doing so can result in application failures when updating to a new Entity Framework Core release. - /// - public readonly ThroughputProperties? Throughput; - - /// - /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to - /// the same compatibility standards as public APIs. It may be changed or removed without notice in - /// any release. You should only use it directly in your code with extreme caution and knowing that - /// doing so can result in application failures when updating to a new Entity Framework Core release. - /// - public ContainerProperties( - string containerId, - IReadOnlyList partitionKeyStoreNames, - int? analyticalTtl, - int? defaultTtl, - ThroughputProperties? throughput) - { - Id = containerId; - PartitionKeyStoreNames = partitionKeyStoreNames; - AnalyticalStoreTimeToLiveInSeconds = analyticalTtl; - DefaultTimeToLive = defaultTtl; - Throughput = throughput; - } - - /// - /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to - /// the same compatibility standards as public APIs. It may be changed or removed without notice in - /// any release. You should only use it directly in your code with extreme caution and knowing that - /// doing so can result in application failures when updating to a new Entity Framework Core release. - /// - public void Deconstruct( - out string containerId, - out IReadOnlyList partitionKeyStoreNames, - out int? analyticalTtl, - out int? defaultTtl, - out ThroughputProperties? throughput) - { - containerId = Id; - partitionKeyStoreNames = PartitionKeyStoreNames; - analyticalTtl = AnalyticalStoreTimeToLiveInSeconds; - defaultTtl = DefaultTimeToLive; - throughput = Throughput; - } -} +public readonly record struct ContainerProperties( + string Id, + IReadOnlyList PartitionKeyStoreNames, + int? AnalyticalStoreTimeToLiveInSeconds, + int? DefaultTimeToLive, + ThroughputProperties? Throughput, + IReadOnlyList Indexes, + IReadOnlyList<(IProperty Property, CosmosVectorType VectorType)> Vectors); diff --git a/src/EFCore.Cosmos/Storage/Internal/CosmosClientWrapper.cs b/src/EFCore.Cosmos/Storage/Internal/CosmosClientWrapper.cs index 62aa38750f8..cd22968535e 100644 --- a/src/EFCore.Cosmos/Storage/Internal/CosmosClientWrapper.cs +++ b/src/EFCore.Cosmos/Storage/Internal/CosmosClientWrapper.cs @@ -2,12 +2,14 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Collections; +using System.Collections.ObjectModel; using System.Diagnostics.CodeAnalysis; using System.Net; using System.Runtime.CompilerServices; using System.Text; using Microsoft.EntityFrameworkCore.Cosmos.Diagnostics.Internal; using Microsoft.EntityFrameworkCore.Cosmos.Infrastructure.Internal; +using Microsoft.EntityFrameworkCore.Cosmos.Metadata.Internal; using Microsoft.EntityFrameworkCore.Internal; using Newtonsoft.Json; using Newtonsoft.Json.Linq; @@ -203,13 +205,52 @@ private static async Task CreateContainerIfNotExistsOnceAsync( { var (parameters, wrapper) = parametersTuple; var partitionKeyPaths = parameters.PartitionKeyStoreNames.Select(e => "/" + e).ToList(); - var response = await wrapper.Client.GetDatabase(wrapper._databaseId).CreateContainerIfNotExistsAsync( - new Azure.Cosmos.ContainerProperties(parameters.Id, partitionKeyPaths) + + var vectorIndexes = new Collection(); + foreach (var index in parameters.Indexes) + { + var vectorIndexType = (VectorIndexType?)index.FindAnnotation(CosmosAnnotationNames.VectorIndexType)?.Value; + if (vectorIndexType.HasValue) + { + // Model validation will ensure there is only one property. + vectorIndexes.Add( + new VectorIndexPath { Path = "/" + index.Properties[0].GetJsonPropertyName(), Type = vectorIndexType.Value }); + } + } + + var embeddings = new Collection(); + foreach (var tuple in parameters.Vectors) + { + embeddings.Add( + new Embedding { - PartitionKeyDefinitionVersion = PartitionKeyDefinitionVersion.V2, - DefaultTimeToLive = parameters.DefaultTimeToLive, - AnalyticalStoreTimeToLiveInSeconds = parameters.AnalyticalStoreTimeToLiveInSeconds - }, + Path = "/" + tuple.Property.GetJsonPropertyName(), + DataType = tuple.VectorType.DataType ?? CosmosVectorType.CreateDefaultVectorDataType( + tuple.Property.GetValueConverter()?.ProviderClrType ?? tuple.Property.ClrType), + Dimensions = tuple.VectorType.Dimensions, + DistanceFunction = tuple.VectorType.DistanceFunction + }); + } + + var containerProperties = new Azure.Cosmos.ContainerProperties(parameters.Id, partitionKeyPaths) + { + PartitionKeyDefinitionVersion = PartitionKeyDefinitionVersion.V2, + DefaultTimeToLive = parameters.DefaultTimeToLive, + AnalyticalStoreTimeToLiveInSeconds = parameters.AnalyticalStoreTimeToLiveInSeconds, + }; + + if (embeddings.Any()) + { + containerProperties.VectorEmbeddingPolicy = new VectorEmbeddingPolicy(embeddings); + } + + if (vectorIndexes.Any()) + { + containerProperties.IndexingPolicy = new IndexingPolicy { VectorIndexes = vectorIndexes }; + } + + var response = await wrapper.Client.GetDatabase(wrapper._databaseId).CreateContainerIfNotExistsAsync( + containerProperties, throughput: parameters.Throughput?.Throughput, cancellationToken: cancellationToken) .ConfigureAwait(false); diff --git a/src/EFCore.Cosmos/Storage/Internal/CosmosDatabaseCreator.cs b/src/EFCore.Cosmos/Storage/Internal/CosmosDatabaseCreator.cs index e46852477e3..d55b7a63f9e 100644 --- a/src/EFCore.Cosmos/Storage/Internal/CosmosDatabaseCreator.cs +++ b/src/EFCore.Cosmos/Storage/Internal/CosmosDatabaseCreator.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using Microsoft.EntityFrameworkCore.Cosmos.Internal; +using Microsoft.EntityFrameworkCore.Cosmos.Metadata.Internal; namespace Microsoft.EntityFrameworkCore.Cosmos.Storage.Internal; @@ -112,6 +113,8 @@ private static IEnumerable GetContainersToCreate(IModel mod int? analyticalTtl = null; int? defaultTtl = null; ThroughputProperties? throughput = null; + var indexes = new List(); + var vectors = new List<(IProperty Property, CosmosVectorType VectorType)>(); foreach (var entityType in mappedTypes) { @@ -122,6 +125,15 @@ private static IEnumerable GetContainersToCreate(IModel mod analyticalTtl ??= entityType.GetAnalyticalStoreTimeToLive(); defaultTtl ??= entityType.GetDefaultTimeToLive(); throughput ??= entityType.GetThroughput(); + indexes.AddRange(entityType.GetIndexes()); + + foreach (var property in entityType.GetProperties()) + { + if (property.FindTypeMapping() is CosmosVectorTypeMapping vectorTypeMapping) + { + vectors.Add((property, vectorTypeMapping.VectorType)); + } + } } yield return new ContainerProperties( @@ -129,7 +141,9 @@ private static IEnumerable GetContainersToCreate(IModel mod partitionKeyStoreNames, analyticalTtl, defaultTtl, - throughput); + throughput, + indexes, + vectors); } } @@ -212,11 +226,16 @@ public virtual Task CanConnectAsync(CancellationToken cancellationToken = => throw new NotSupportedException(CosmosStrings.CanConnectNotSupported); /// + /// Returns the store names of the properties that is used to store the partition keys. + /// + /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to /// the same compatibility standards as public APIs. It may be changed or removed without notice in /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. - /// + /// + /// The entity type to get the partition key property names for. + /// The names of the partition key property. private static IReadOnlyList GetPartitionKeyStoreNames(IEntityType entityType) { var properties = entityType.GetPartitionKeyProperties(); diff --git a/src/EFCore.Cosmos/Storage/Internal/CosmosTypeMapping.cs b/src/EFCore.Cosmos/Storage/Internal/CosmosTypeMapping.cs index a7b1c835f3d..f0da0339f6c 100644 --- a/src/EFCore.Cosmos/Storage/Internal/CosmosTypeMapping.cs +++ b/src/EFCore.Cosmos/Storage/Internal/CosmosTypeMapping.cs @@ -2,6 +2,8 @@ // The .NET Foundation licenses this file to you under the MIT license. using Microsoft.EntityFrameworkCore.Storage.Json; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; namespace Microsoft.EntityFrameworkCore.Cosmos.Storage.Internal; @@ -77,4 +79,46 @@ public override CoreTypeMapping WithComposedConverter( /// protected override CoreTypeMapping Clone(CoreTypeMappingParameters parameters) => new CosmosTypeMapping(parameters); + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public virtual string GenerateConstant(object? value) + { + var jToken = GenerateJToken(value); + + return jToken is null ? "null" : jToken.ToString(Formatting.None); + } + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public virtual JToken? GenerateJToken(object? value) + { + if (value?.GetType().IsInteger() == true) + { + var unwrappedType = ClrType.UnwrapNullableType(); + value = unwrappedType.IsEnum + ? Enum.ToObject(unwrappedType, value) + : unwrappedType == typeof(char) + ? Convert.ChangeType(value, unwrappedType) + : value; + } + + var converter = Converter; + if (converter != null) + { + value = converter.ConvertToProvider(value); + } + + return value == null + ? null + : (value as JToken) ?? JToken.FromObject(value, CosmosClientWrapper.Serializer); + } } diff --git a/src/EFCore.Cosmos/Storage/Internal/CosmosTypeMappingSource.cs b/src/EFCore.Cosmos/Storage/Internal/CosmosTypeMappingSource.cs index 48a9227bcc3..4ffc063ecd5 100644 --- a/src/EFCore.Cosmos/Storage/Internal/CosmosTypeMappingSource.cs +++ b/src/EFCore.Cosmos/Storage/Internal/CosmosTypeMappingSource.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using Microsoft.EntityFrameworkCore.Cosmos.ChangeTracking.Internal; +using Microsoft.EntityFrameworkCore.Cosmos.Metadata.Internal; using Newtonsoft.Json.Linq; namespace Microsoft.EntityFrameworkCore.Cosmos.Storage.Internal; @@ -35,6 +36,31 @@ public CosmosTypeMappingSource(TypeMappingSourceDependencies dependencies) }; } + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public override CoreTypeMapping? FindMapping(IProperty property) + { + // A provider should typically not override this because using the property directly causes problems with Migrations where + // the property does not exist. However, since the Cosmos provider doesn't have Migrations, it should be okay to use the property + // directly. + var mapping = (CosmosTypeMapping?)base.FindMapping(property); + if (mapping != null) + { + var vectorType = (CosmosVectorType?)property.FindAnnotation(CosmosAnnotationNames.VectorType)?.Value; + if (vectorType != null) + { + mapping = new CosmosVectorTypeMapping(mapping, vectorType); + } + } + + return mapping; + } + /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to /// the same compatibility standards as public APIs. It may be changed or removed without notice in @@ -56,13 +82,19 @@ public CosmosTypeMappingSource(TypeMappingSourceDependencies dependencies) private CoreTypeMapping? FindPrimitiveMapping(in TypeMappingInfo mappingInfo) { var clrType = mappingInfo.ClrType!; + if (clrType.IsAssignableTo(typeof(CosmosVectorType))) + { + return new CosmosVectorDistanceCalculationTypeMapping(); + } + if ((clrType.IsValueType && clrType != typeof(Guid) && !clrType.IsEnum) || clrType == typeof(string)) { return new CosmosTypeMapping( - clrType, jsonValueReaderWriter: Dependencies.JsonValueReaderWriterSource.FindReaderWriter(clrType)); + clrType, + jsonValueReaderWriter: Dependencies.JsonValueReaderWriterSource.FindReaderWriter(clrType)); } return null; @@ -81,12 +113,19 @@ public CosmosTypeMappingSource(TypeMappingSourceDependencies dependencies) // First attempt to resolve this as a primitive collection (e.g. List). This does not handle Dictionary. if (TryFindJsonCollectionMapping( - mappingInfo, clrType, providerClrType: null, ref elementMapping, out var elementComparer, + mappingInfo, + clrType, + providerClrType: null, + ref elementMapping, + out var elementComparer, out var collectionReaderWriter) && elementMapping is not null) { return new CosmosTypeMapping( - clrType, elementComparer, elementMapping: elementMapping, jsonValueReaderWriter: collectionReaderWriter); + clrType, + elementComparer, + elementMapping: elementMapping, + jsonValueReaderWriter: collectionReaderWriter); } // Next, attempt to resolve this as a dictionary (e.g. Dictionary). @@ -123,7 +162,8 @@ public CosmosTypeMappingSource(TypeMappingSourceDependencies dependencies) return elementMapping == null ? null : new CosmosTypeMapping( - clrType, CreateStringDictionaryComparer(elementMapping, elementType, clrType), + clrType, + CreateStringDictionaryComparer(elementMapping, elementType, clrType), jsonValueReaderWriter: jsonValueReaderWriter); } } diff --git a/src/EFCore.Cosmos/Storage/Internal/CosmosVectorDistanceCalculationTypeMapping.cs b/src/EFCore.Cosmos/Storage/Internal/CosmosVectorDistanceCalculationTypeMapping.cs new file mode 100644 index 00000000000..ab1462b7d96 --- /dev/null +++ b/src/EFCore.Cosmos/Storage/Internal/CosmosVectorDistanceCalculationTypeMapping.cs @@ -0,0 +1,83 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.EntityFrameworkCore.Cosmos.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Storage.Json; + +namespace Microsoft.EntityFrameworkCore.Cosmos.Storage.Internal; + +/// +/// This is an internal API that supports the Entity Framework Core infrastructure and not subject to +/// the same compatibility standards as public APIs. It may be changed or removed without notice in +/// any release. You should only use it directly in your code with extreme caution and knowing that +/// doing so can result in application failures when updating to a new Entity Framework Core release. +/// +public class CosmosVectorDistanceCalculationTypeMapping : CosmosTypeMapping +{ + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public static new CosmosVectorDistanceCalculationTypeMapping Default { get; } + = new(); + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public CosmosVectorDistanceCalculationTypeMapping() + : base(new CoreTypeMappingParameters(typeof(CosmosVectorType))) + { + } + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + protected CosmosVectorDistanceCalculationTypeMapping(CoreTypeMappingParameters parameters) + : base(parameters) + { + } + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public override CoreTypeMapping WithComposedConverter( + ValueConverter? converter, + ValueComparer? comparer = null, + ValueComparer? keyComparer = null, + CoreTypeMapping? elementMapping = null, + JsonValueReaderWriter? jsonValueReaderWriter = null) + => new CosmosVectorDistanceCalculationTypeMapping( + Parameters.WithComposedConverter(converter, comparer, keyComparer, elementMapping, jsonValueReaderWriter)); + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + protected override CoreTypeMapping Clone(CoreTypeMappingParameters parameters) + => new CosmosVectorDistanceCalculationTypeMapping(parameters); + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public override string GenerateConstant(object? value) + { + var vectorType = (CosmosVectorType)value!; + return $"{{'distanceFunction':'{vectorType.DistanceFunction.ToString().ToLower()}', 'dataType':'{vectorType.DataType?.ToString().ToLower()}'}}"; + } +} diff --git a/src/EFCore.Cosmos/Storage/Internal/CosmosVectorTypeMapping.cs b/src/EFCore.Cosmos/Storage/Internal/CosmosVectorTypeMapping.cs new file mode 100644 index 00000000000..b95cb99250c --- /dev/null +++ b/src/EFCore.Cosmos/Storage/Internal/CosmosVectorTypeMapping.cs @@ -0,0 +1,126 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.EntityFrameworkCore.Cosmos.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Storage.Json; +using Newtonsoft.Json.Linq; + +namespace Microsoft.EntityFrameworkCore.Cosmos.Storage.Internal; + +/// +/// This is an internal API that supports the Entity Framework Core infrastructure and not subject to +/// the same compatibility standards as public APIs. It may be changed or removed without notice in +/// any release. You should only use it directly in your code with extreme caution and knowing that +/// doing so can result in application failures when updating to a new Entity Framework Core release. +/// +public class CosmosVectorTypeMapping : CosmosTypeMapping +{ + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public static new CosmosVectorTypeMapping Default { get; } + = new(typeof(byte[]), new CosmosVectorType(DistanceFunction.Cosine, 0, VectorDataType.Int8)); + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public CosmosVectorTypeMapping( + Type clrType, + CosmosVectorType vectorType, + ValueComparer? comparer = null, + ValueComparer? keyComparer = null, + CoreTypeMapping? elementMapping = null, + JsonValueReaderWriter? jsonValueReaderWriter = null) + : this( + new CoreTypeMappingParameters( + clrType, + converter: null, + comparer, + keyComparer, + elementMapping: elementMapping, + jsonValueReaderWriter: jsonValueReaderWriter), + vectorType) + { + } + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public CosmosVectorTypeMapping(CosmosTypeMapping mapping, CosmosVectorType vectorType) + : this( + new CoreTypeMappingParameters( + mapping.ClrType, + converter: mapping.Converter, + mapping.Comparer, + mapping.KeyComparer, + elementMapping: mapping.ElementTypeMapping, + jsonValueReaderWriter: mapping.JsonValueReaderWriter), + vectorType) + { + } + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + protected CosmosVectorTypeMapping(CoreTypeMappingParameters parameters, CosmosVectorType vectorType) + : base(parameters) + { + VectorType = vectorType; + } + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public virtual CosmosVectorType VectorType { get; } + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public override CoreTypeMapping WithComposedConverter( + ValueConverter? converter, + ValueComparer? comparer = null, + ValueComparer? keyComparer = null, + CoreTypeMapping? elementMapping = null, + JsonValueReaderWriter? jsonValueReaderWriter = null) + => new CosmosVectorTypeMapping( + Parameters.WithComposedConverter(converter, comparer, keyComparer, elementMapping, jsonValueReaderWriter), + VectorType); + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + protected override CoreTypeMapping Clone(CoreTypeMappingParameters parameters) + => new CosmosVectorTypeMapping(parameters, VectorType); + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public override JToken? GenerateJToken(object? value) + => value == null + ? null + : (value as JToken) ?? JToken.FromObject(value, CosmosClientWrapper.Serializer); +} diff --git a/test/EFCore.Cosmos.FunctionalTests/TestUtilities/CosmosTestStore.cs b/test/EFCore.Cosmos.FunctionalTests/TestUtilities/CosmosTestStore.cs index d49b0ed01f4..d8a0ed9c76a 100644 --- a/test/EFCore.Cosmos.FunctionalTests/TestUtilities/CosmosTestStore.cs +++ b/test/EFCore.Cosmos.FunctionalTests/TestUtilities/CosmosTestStore.cs @@ -3,13 +3,13 @@ using System.Net; using System.Net.Sockets; -using System.Threading; using Azure; using Azure.Core; using Azure.ResourceManager; using Azure.ResourceManager.CosmosDB; using Azure.ResourceManager.CosmosDB.Models; using Microsoft.Azure.Cosmos; +using Microsoft.EntityFrameworkCore.Cosmos.Metadata.Internal; using Microsoft.EntityFrameworkCore.Cosmos.Storage.Internal; using Newtonsoft.Json; using Newtonsoft.Json.Linq; @@ -384,6 +384,8 @@ await database.Value.GetCosmosDBSqlContainers().CreateOrUpdateAsync( int? analyticalTtl = null; int? defaultTtl = null; ThroughputProperties? throughput = null; + var indexes = new List(); + var vectors = new List<(IProperty Property, CosmosVectorType VectorType)>(); foreach (var entityType in mappedTypes) { @@ -394,6 +396,15 @@ await database.Value.GetCosmosDBSqlContainers().CreateOrUpdateAsync( analyticalTtl ??= entityType.GetAnalyticalStoreTimeToLive(); defaultTtl ??= entityType.GetDefaultTimeToLive(); throughput ??= entityType.GetThroughput(); + indexes.AddRange(entityType.GetIndexes()); + + foreach (var property in entityType.GetProperties()) + { + if (property.FindTypeMapping() is CosmosVectorTypeMapping vectorTypeMapping) + { + vectors.Add((property, vectorTypeMapping.VectorType)); + } + } } yield return new( @@ -401,7 +412,9 @@ await database.Value.GetCosmosDBSqlContainers().CreateOrUpdateAsync( partitionKeyStoreNames, analyticalTtl, defaultTtl, - throughput); + throughput, + indexes, + vectors); } } diff --git a/test/EFCore.Cosmos.FunctionalTests/VectorSearchCosmosTest.cs b/test/EFCore.Cosmos.FunctionalTests/VectorSearchCosmosTest.cs new file mode 100644 index 00000000000..3932682a098 --- /dev/null +++ b/test/EFCore.Cosmos.FunctionalTests/VectorSearchCosmosTest.cs @@ -0,0 +1,537 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.ComponentModel.DataAnnotations.Schema; +using Microsoft.Azure.Cosmos; +using Microsoft.EntityFrameworkCore.Cosmos.Extensions; +using Microsoft.EntityFrameworkCore.Cosmos.Internal; + +namespace Microsoft.EntityFrameworkCore; + +public class VectorSearchCosmosTest : IClassFixture +{ + public VectorSearchCosmosTest(ReadItemFixture fixture, ITestOutputHelper testOutputHelper) + { + Fixture = fixture; + this.testOutputHelper = testOutputHelper; + fixture.TestSqlLoggerFactory.Clear(); + } + + protected ReadItemFixture Fixture { get; } + + private readonly ITestOutputHelper testOutputHelper; + + [ConditionalFact] + public virtual async Task Query_for_vector_distance_sbyte_array() + { + await using var context = CreateContext(); + var inputVector = new sbyte[] { 2, 1, 4, 6, 5, 2, 5, 7, 3, 1 }; + + var booksFromStore = await context + .Set() + .Select(e => EF.Functions.VectorDistance(e.SByteArray, inputVector)) + .ToListAsync(); + + Assert.Equal(3, booksFromStore.Count); + Assert.All(booksFromStore, s => Assert.NotEqual(0.0, s)); + + AssertSql( + """ +@__inputVector_1='[2,1,4,6,5,2,5,7,3,1]' + +SELECT VectorDistance(c["SByteArray"], @__inputVector_1, false, {'distanceFunction':'cosine', 'dataType':'int8'}) AS c +FROM root c +WHERE (c["Discriminator"] = "Book") +"""); + } + + [ConditionalFact] + public virtual async Task Query_for_vector_distance_byte_array() + { + await using var context = CreateContext(); + var inputVector = new byte[] { 2, 1, 4, 6, 5, 2, 5, 7, 3, 1 }; + + var message = (await Assert.ThrowsAsync( + async () => await context + .Set() + .Select(e => EF.Functions.VectorDistance(e.ByteArray, inputVector)) + .ToListAsync())).Message; + + testOutputHelper.WriteLine(message); + + AssertSql( + """ +@__inputVector_1='[2,1,4,6,5,2,5,7,3,1]' + +SELECT VectorDistance(c["ByteArray"], @__inputVector_1, false, {'distanceFunction':'cosine', 'dataType':'uint8'}) AS c +FROM root c +WHERE (c["Discriminator"] = "Book") +"""); + } + + [ConditionalFact] + public virtual async Task Query_for_vector_distance_double_array() + { + await using var context = CreateContext(); + var inputVector = new[] { 0.33, -0.52, 0.45, -0.67, 0.89, -0.34, 0.86, -0.78 }; + + var booksFromStore = await context + .Set() + .Select( + e => EF.Functions.VectorDistance(e.DoubleArray, inputVector, false, DistanceFunction.DotProduct, VectorDataType.Float32)) + .ToListAsync(); + + Assert.Equal(3, booksFromStore.Count); + Assert.All(booksFromStore, s => Assert.NotEqual(0.0, s)); + + AssertSql( + """ +@__inputVector_1='[0.33,-0.52,0.45,-0.67,0.89,-0.34,0.86,-0.78]' + +SELECT VectorDistance(c["DoubleArray"], @__inputVector_1, false, {'distanceFunction':'dotproduct', 'dataType':'float32'}) AS c +FROM root c +WHERE (c["Discriminator"] = "Book") +"""); + } + + [ConditionalFact] + public virtual async Task Query_for_vector_distance_int_list() + { + await using var context = CreateContext(); + var inputVector = new List + { + 2, + 1, + 4, + 6, + 5, + 2, + 5, + 7, + 3, + 1 + }; + + var message = (await Assert.ThrowsAsync( + async () => await context + .Set() + .Select(e => EF.Functions.VectorDistance(e.IntList, inputVector)) + .ToListAsync())).Message; + + testOutputHelper.WriteLine(message); + + // Assert.Equal(3, booksFromStore.Count); + // Assert.All(booksFromStore, s => Assert.NotEqual(0.0, s)); + + AssertSql( + """ +@__inputVector_1='[2,1,4,6,5,2,5,7,3,1]' + +SELECT VectorDistance(c["IntList"], @__inputVector_1, false, {'distanceFunction':'cosine', 'dataType':'uint8'}) AS c +FROM root c +WHERE (c["Discriminator"] = "Book") +"""); + } + + + [ConditionalFact] + public virtual async Task Query_for_vector_distance_float_enumerable() + { + await using var context = CreateContext(); + var inputVector = new List + { + 0.33f, + -0.52f, + 0.45f, + -0.67f, + 0.89f, + -0.34f, + 0.86f, + -0.78f + }; + + var booksFromStore = await context + .Set() + .Select(e => EF.Functions.VectorDistance(e.FloatEnumerable, inputVector)) + .ToListAsync(); + + Assert.Equal(3, booksFromStore.Count); + Assert.All(booksFromStore, s => Assert.NotEqual(0.0, s)); + + AssertSql( + """ +@__inputVector_1='[0.33,-0.52,0.45,-0.67,0.89,-0.34,0.86,-0.78]' + +SELECT VectorDistance(c["FloatEnumerable"], @__inputVector_1, false, {'distanceFunction':'cosine', 'dataType':'float32'}) AS c +FROM root c +WHERE (c["Discriminator"] = "Book") +"""); + } + + [ConditionalFact] + public virtual async Task Vector_distance_sbyte_array_in_OrderBy() + { + await using var context = CreateContext(); + var inputVector = new sbyte[] { 2, 1, 4, 6, 5, 2, 5, 7, 3, 1 }; + + var booksFromStore = await context + .Set() + .OrderBy(e => EF.Functions.VectorDistance(e.SByteArray, inputVector, false, DistanceFunction.DotProduct, VectorDataType.Uint8)) + .ToListAsync(); + + Assert.Equal(3, booksFromStore.Count); + + AssertSql( + """ +@__inputVector_1='[2,1,4,6,5,2,5,7,3,1]' + +SELECT c +FROM root c +WHERE (c["Discriminator"] = "Book") +ORDER BY VectorDistance(c["SByteArray"], @__inputVector_1, false, {'distanceFunction':'dotproduct', 'dataType':'uint8'}) +"""); + } + + + [ConditionalFact] + public virtual async Task Vector_distance_byte_array_in_OrderBy() + { + await using var context = CreateContext(); + var inputVector = new byte[] { 2, 1, 4, 6, 5, 2, 5, 7, 3, 1 }; + + var booksFromStore = await context + .Set() + .OrderBy(e => EF.Functions.VectorDistance(e.ByteArray, inputVector)) + .ToListAsync(); + + Assert.Equal(3, booksFromStore.Count); + + AssertSql( + """ +@__inputVector_1='[2,1,4,6,5,2,5,7,3,1]' + +SELECT c +FROM root c +WHERE (c["Discriminator"] = "Book") +ORDER BY VectorDistance(c["ByteArray"], @__inputVector_1, false, {'distanceFunction':'cosine', 'dataType':'uint8'}) +"""); + } + + [ConditionalFact] + public virtual async Task Vector_distance_double_array_in_OrderBy() + { + await using var context = CreateContext(); + var inputVector = new[] { 0.33, -0.52, 0.45, -0.67, 0.89, -0.34, 0.86, -0.78 }; + + var booksFromStore = await context + .Set() + .OrderBy(e => EF.Functions.VectorDistance(e.DoubleArray, inputVector)) + .ToListAsync(); + + Assert.Equal(3, booksFromStore.Count); + AssertSql( + """ +@__inputVector_1='[0.33,-0.52,0.45,-0.67,0.89,-0.34,0.86,-0.78]' + +SELECT c +FROM root c +WHERE (c["Discriminator"] = "Book") +ORDER BY VectorDistance(c["DoubleArray"], @__inputVector_1, false, {'distanceFunction':'cosine', 'dataType':'float32'}) +"""); + } + + [ConditionalFact] + public virtual async Task Vector_distance_int_list_in_OrderBy() + { + await using var context = CreateContext(); + var inputVector = new List + { + 2, + 1, + 4, + 6, + 5, + 2, + 5, + 7, + 3, + 1 + }; + + var booksFromStore = await context + .Set() + .OrderBy(e => EF.Functions.VectorDistance(e.IntList, inputVector)) + .ToListAsync(); + + Assert.Equal(3, booksFromStore.Count); + + AssertSql( + """ +@__inputVector_1='[2,1,4,6,5,2,5,7,3,1]' + +SELECT c +FROM root c +WHERE (c["Discriminator"] = "Book") +ORDER BY VectorDistance(c["IntList"], @__inputVector_1, false, {'distanceFunction':'cosine', 'dataType':'uint8'}) +"""); + } + + + [ConditionalFact] + public virtual async Task Vector_distance_float_enumerable_in_OrderBy() + { + await using var context = CreateContext(); + var inputVector = new List + { + 0.33f, + -0.52f, + 0.45f, + -0.67f, + 0.89f, + -0.34f, + 0.86f, + -0.78f + }; + + var booksFromStore = await context + .Set() + .Select(e => EF.Functions.VectorDistance(inputVector, e.FloatEnumerable, true)) + .ToListAsync(); + + Assert.Equal(3, booksFromStore.Count); + Assert.All(booksFromStore, s => Assert.NotEqual(0.0, s)); + + AssertSql( + """ +@__inputVector_1='[0.33,-0.52,0.45,-0.67,0.89,-0.34,0.86,-0.78]' + +SELECT VectorDistance(@__inputVector_1, c["FloatEnumerable"], true, {'distanceFunction':'cosine', 'dataType':'float32'}) AS c +FROM root c +WHERE (c["Discriminator"] = "Book") +"""); + } + + [ConditionalFact] + public virtual async Task VectorDistance_throws_when_used_on_non_vector() + { + await using var context = CreateContext(); + var inputVector = "WhatsOccuring"; + + Assert.Equal( + CosmosStrings.VectorSearchRequiresVector, + (await Assert.ThrowsAsync( + async () => await context + .Set() + .OrderBy(e => EF.Functions.VectorDistance(e.Isbn, inputVector)) + .ToListAsync())).Message); + + Assert.Equal( + CosmosStrings.VectorSearchRequiresVector, + (await Assert.ThrowsAsync( + async () => await context + .Set() + .OrderBy(e => EF.Functions.VectorDistance(inputVector, e.Isbn)) + .ToListAsync())).Message); + } + + + [ConditionalFact] + public virtual async Task VectorDistance_throws_when_used_with_non_const_args() + { + await using var context = CreateContext(); + var inputVector = new List + { + 0.33f, + -0.52f, + 0.45f, + -0.67f, + 0.89f, + -0.34f, + 0.86f, + -0.78f + }; + + Assert.Equal( + CosmosStrings.ArgumentNotConstant("useBruteForce", nameof(CosmosDbFunctionsExtensions.VectorDistance)), + (await Assert.ThrowsAsync( + async () => await context + .Set() + .OrderBy(e => EF.Functions.VectorDistance(e.FloatEnumerable, inputVector, e.IsPublished)) + .ToListAsync())).Message); + + Assert.Equal( + CosmosStrings.ArgumentNotConstant("distanceFunction", nameof(CosmosDbFunctionsExtensions.VectorDistance)), + (await Assert.ThrowsAsync( + async () => await context + .Set() + .OrderBy( + e => EF.Functions.VectorDistance(e.FloatEnumerable, inputVector, false, e.DistanceFunction, VectorDataType.Float32)) + .ToListAsync())).Message); + + Assert.Equal( + CosmosStrings.ArgumentNotConstant("dataType", nameof(CosmosDbFunctionsExtensions.VectorDistance)), + (await Assert.ThrowsAsync( + async () => await context + .Set() + .OrderBy( + e => EF.Functions.VectorDistance(e.FloatEnumerable, inputVector, false, DistanceFunction.Cosine, e.VectorDataType)) + .ToListAsync())).Message); + } + + private class Book + { + public Guid Id { get; set; } + + public string Publisher { get; set; } = null!; + + public string Title { get; set; } = null!; + + public string Author { get; set; } = null!; + + public string Isbn { get; set; } = null!; + + public bool IsPublished { get; set; } + + public DistanceFunction DistanceFunction { get; set; } // Not meaningful; used for exception testing. + + public VectorDataType VectorDataType { get; set; } // Not meaningful; used for exception testing. + + public byte[] ByteArray { get; set; } = null!; + + public double[] DoubleArray { get; set; } = null!; + + public sbyte[] SByteArray { get; set; } = null!; + + public List IntList { get; set; } = null!; + + public IEnumerable FloatEnumerable { get; set; } = null!; + + // public Owned1 OwnedReference { get; set; } = null!; + // public List OwnedCollection { get; set; } = null!; + } + + // [Owned] + // protected class Owned1 + // { + // public int Prop { get; set; } + // public Owned2 NestedOwned { get; set; } = null!; + // public List NestedOwnedCollection { get; set; } = null!; + // } + // + // [Owned] + // protected class Owned2 + // { + // public string Prop { get; set; } = null!; + // } + + protected DbContext CreateContext() + => Fixture.CreateContext(); + + private void AssertSql(params string[] expected) + => Fixture.TestSqlLoggerFactory.AssertBaseline(expected); + + public class ReadItemFixture : SharedStoreFixtureBase + { + protected override string StoreName + => "VectorSearchTest"; + + protected override void OnModelCreating(ModelBuilder modelBuilder, DbContext context) + { + modelBuilder.Entity( + b => + { + b.Property(e => e.Id).ValueGeneratedOnAdd(); + b.HasKey(e => e.Id); + b.HasPartitionKey(e => e.Publisher); + + b.HasIndex(e => e.ByteArray).ForVectors(VectorIndexType.Flat); + b.HasIndex(e => e.SByteArray).ForVectors(VectorIndexType.Flat); + b.HasIndex(e => e.FloatEnumerable).ForVectors(VectorIndexType.Flat); + b.HasIndex(e => e.IntList).ForVectors(VectorIndexType.Flat); + b.HasIndex(e => e.DoubleArray).ForVectors(VectorIndexType.Flat); + + b.Property(e => e.ByteArray).IsVector(DistanceFunction.Cosine, 10); + b.Property(e => e.SByteArray).IsVector(DistanceFunction.Cosine, 10); + b.Property(e => e.FloatEnumerable).IsVector(DistanceFunction.Cosine, 10); + b.Property(e => e.IntList).IsVector(DistanceFunction.Cosine, 10, VectorDataType.Uint8); + b.Property(e => e.DoubleArray).IsVector(DistanceFunction.Cosine, 10, VectorDataType.Float32); + }); + } + + protected override Task SeedAsync(PoolableDbContext context) + { + var book1 = new Book + { + Publisher = "Manning", + Author = "Jon P Smith", + Title = "Entity Framework Core in Action", + Isbn = "978-1617298363", + ByteArray = [2, 1, 4, 3, 5, 2, 5, 7, 3, 1], + SByteArray = [2, -1, 4, 3, 5, -2, 5, -7, 3, 1], + IntList = [2, -1, 4, 3, 5, -2, 5, -7, 3, 1], + FloatEnumerable = new[] { 0.33f, -0.52f, 0.45f, -0.67f, 0.89f, -0.34f, 0.86f, -0.78f }, + DoubleArray = [0.33, -0.52, 0.45, -0.67, 0.89, -0.34, 0.86, -0.78], + + // OwnedReference = new() + // { + // Prop = 7, + // NestedOwned = new() { Prop = "7" }, + // NestedOwnedCollection = new() { new() { Prop = "71" }, new() { Prop = "72" } } + // }, + // OwnedCollection = new() { new() { Prop = 71 }, new() { Prop = 72 } } + }; + + var book2 = new Book + { + Publisher = "O'Reilly", + Author = "Julie Lerman", + Title = "Programming Entity Framework: DbContext", + Isbn = "978-1449312961", + ByteArray = [2, 1, 4, 3, 5, 2, 5, 7, 3, 1], + SByteArray = [2, -1, 4, 3, 5, -2, 5, -7, 3, 1], + IntList = [2, -1, 4, 3, 5, -2, 5, -7, 3, 1], + FloatEnumerable = new[] { 0.33f, -0.52f, 0.45f, -0.67f, 0.89f, -0.34f, 0.86f, -0.78f }, + DoubleArray = [0.33, -0.52, 0.45, -0.67, 0.89, -0.34, 0.86, -0.78], + + // OwnedReference = new() + // { + // Prop = 7, + // NestedOwned = new() { Prop = "7" }, + // NestedOwnedCollection = new() { new() { Prop = "71" }, new() { Prop = "72" } } + // }, + // OwnedCollection = new() { new() { Prop = 71 }, new() { Prop = 72 } } + }; + + var book3 = new Book + { + Publisher = "O'Reilly", + Author = "Julie Lerman", + Title = "Programming Entity Framework", + Isbn = "978-0596807269", + ByteArray = [2, 1, 4, 3, 5, 2, 5, 7, 3, 1], + SByteArray = [2, -1, 4, 3, 5, -2, 5, -7, 3, 1], + IntList = [2, -1, 4, 3, 5, -2, 5, -7, 3, 1], + FloatEnumerable = new[] { 0.33f, -0.52f, 0.45f, -0.67f, 0.89f, -0.34f, 0.86f, -0.78f }, + DoubleArray = [0.33, -0.52, 0.45, -0.67, 0.89, -0.34, 0.86, -0.78], + + // OwnedReference = new() + // { + // Prop = 7, + // NestedOwned = new() { Prop = "7" }, + // NestedOwnedCollection = new() { new() { Prop = "71" }, new() { Prop = "72" } } + // }, + // OwnedCollection = new() { new() { Prop = 71 }, new() { Prop = 72 } } + }; + + context.AddRange(book1, book2, book3); + + return context.SaveChangesAsync(); + } + + public TestSqlLoggerFactory TestSqlLoggerFactory + => (TestSqlLoggerFactory)ListLoggerFactory; + + protected override ITestStoreFactory TestStoreFactory + => CosmosTestStoreFactory.Instance; + } +} diff --git a/test/EFCore.Cosmos.Tests/Infrastructure/CosmosModelValidatorTest.cs b/test/EFCore.Cosmos.Tests/Infrastructure/CosmosModelValidatorTest.cs index 4ca8bf18436..b47181ad7ba 100644 --- a/test/EFCore.Cosmos.Tests/Infrastructure/CosmosModelValidatorTest.cs +++ b/test/EFCore.Cosmos.Tests/Infrastructure/CosmosModelValidatorTest.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using Microsoft.Azure.Cosmos; using Microsoft.EntityFrameworkCore.Cosmos.Internal; using Newtonsoft.Json.Linq; @@ -413,6 +414,32 @@ public virtual void Detects_nonString_concurrency_token() VerifyError(CosmosStrings.ETagNonStringStoreType("_etag", nameof(Customer), "int"), modelBuilder); } + [ConditionalFact] + public virtual void Detects_multi_property_vector_index() + { + var modelBuilder = CreateConventionModelBuilder(); + modelBuilder.Entity( + b => + { + b.HasIndex(e => new { e.Name, e.OtherName }).ForVectors(VectorIndexType.Flat); + }); + + VerifyError(CosmosStrings.CompositeVectorIndex(nameof(Customer), "Name,OtherName"), modelBuilder); + } + + [ConditionalFact] + public virtual void Detects_vector_index_on_non_vector_property() + { + var modelBuilder = CreateConventionModelBuilder(); + modelBuilder.Entity( + b => + { + b.HasIndex(e => new { e.Name }).ForVectors(VectorIndexType.Flat); + }); + + VerifyError(CosmosStrings.VectorIndexOnNonVector(nameof(Customer), "Name"), modelBuilder); + } + protected override TestHelpers TestHelpers => CosmosTestHelpers.Instance; }