Skip to content

Commit

Permalink
feat: Add FindNearest API to the stable branch (#1333)
Browse files Browse the repository at this point in the history 
* feat: Support for field update operators in the Datastore API and resolution strategies when there is a conflict at write time

PiperOrigin-RevId: 683253625

Source-Link: googleapis/googleapis@3effbf2

Source-Link: googleapis/googleapis-gen@5dd983c
Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiNWRkOTgzYzc2NDE3ZjJhZDg4ZjlkNDc0MzhjNDhjMjdkNWFjMGUyNyJ9

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* feat: Add FindNearest API to the stable branch

PiperOrigin-RevId: 684905940

Source-Link: googleapis/googleapis@2196d48

Source-Link: googleapis/googleapis-gen@05df6fa
Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiMDVkZjZmYTE2YWI5M2JkOWRhMTdiNTZlZGQzNzliNDM5NjNkZTE2NyJ9

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

---------

Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
Co-authored-by: danieljbruce <[email protected]>
  • Loading branch information
@gcf-owl-bot @danieljbruce
3 people authored Oct 23, 2024
1 parent 519e33b commit 1d56433
Show file tree
Hide file tree
Showing 5 changed files with 1,698 additions and 11 deletions.
141 changes: 133 additions & 8 deletions protos/google/datastore/v1/datastore.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -501,6 +501,18 @@ message ReserveIdsResponse {}

// A mutation to apply to an entity.
message Mutation {
// The possible ways to resolve a conflict detected in a mutation.
enum ConflictResolutionStrategy {
// Unspecified. Defaults to `SERVER_VALUE`.
STRATEGY_UNSPECIFIED = 0;

// The server entity is kept.
SERVER_VALUE = 1;

// The whole commit request fails.
FAIL = 3;
}

// The mutation operation.
//
// For `insert`, `update`, and `upsert`:
Expand Down Expand Up @@ -542,6 +554,11 @@ message Mutation {
google.protobuf.Timestamp update_time = 11;
}

// The strategy to use when a conflict is detected. Defaults to
// `SERVER_VALUE`.
// If this is set, then `conflict_detection_strategy` must also be set.
ConflictResolutionStrategy conflict_resolution_strategy = 10;

// The properties to write in this mutation.
// None of the properties in the mask may have a reserved name, except for
// `__key__`.
Expand All @@ -551,6 +568,112 @@ message Mutation {
// updated, others are left untouched.
// Properties referenced in the mask but not in the entity are deleted.
PropertyMask property_mask = 9;

// Optional. The transforms to perform on the entity.
//
// This field can be set only when the operation is `insert`, `update`,
// or `upsert`. If present, the transforms are be applied to the entity
// regardless of the property mask, in order, after the operation.
repeated PropertyTransform property_transforms = 12
[(google.api.field_behavior) = OPTIONAL];
}

// A transformation of an entity property.
message PropertyTransform {
// A value that is calculated by the server.
enum ServerValue {
// Unspecified. This value must not be used.
SERVER_VALUE_UNSPECIFIED = 0;

// The time at which the server processed the request, with millisecond
// precision. If used on multiple properties (same or different entities)
// in a transaction, all the properties will get the same server timestamp.
REQUEST_TIME = 1;
}

// Optional. The name of the property.
//
// Property paths (a list of property names separated by dots (`.`)) may be
// used to refer to properties inside entity values. For example `foo.bar`
// means the property `bar` inside the entity property `foo`.
//
// If a property name contains a dot `.` or a backlslash `\`, then that name
// must be escaped.
string property = 1 [(google.api.field_behavior) = OPTIONAL];

// The transformation to apply to the property.
oneof transform_type {
// Sets the property to the given server value.
ServerValue set_to_server_value = 2;

// Adds the given value to the property's current value.
//
// This must be an integer or a double value.
// If the property is not an integer or double, or if the property does not
// yet exist, the transformation will set the property to the given value.
// If either of the given value or the current property value are doubles,
// both values will be interpreted as doubles. Double arithmetic and
// representation of double values follows IEEE 754 semantics.
// If there is positive/negative integer overflow, the property is resolved
// to the largest magnitude positive/negative integer.
Value increment = 3;

// Sets the property to the maximum of its current value and the given
// value.
//
// This must be an integer or a double value.
// If the property is not an integer or double, or if the property does not
// yet exist, the transformation will set the property to the given value.
// If a maximum operation is applied where the property and the input value
// are of mixed types (that is - one is an integer and one is a double)
// the property takes on the type of the larger operand. If the operands are
// equivalent (e.g. 3 and 3.0), the property does not change.
// 0, 0.0, and -0.0 are all zero. The maximum of a zero stored value and
// zero input value is always the stored value.
// The maximum of any numeric value x and NaN is NaN.
Value maximum = 4;

// Sets the property to the minimum of its current value and the given
// value.
//
// This must be an integer or a double value.
// If the property is not an integer or double, or if the property does not
// yet exist, the transformation will set the property to the input value.
// If a minimum operation is applied where the property and the input value
// are of mixed types (that is - one is an integer and one is a double)
// the property takes on the type of the smaller operand. If the operands
// are equivalent (e.g. 3 and 3.0), the property does not change. 0, 0.0,
// and -0.0 are all zero. The minimum of a zero stored value and zero input
// value is always the stored value. The minimum of any numeric value x and
// NaN is NaN.
Value minimum = 5;

// Appends the given elements in order if they are not already present in
// the current property value.
// If the property is not an array, or if the property does not yet exist,
// it is first set to the empty array.
//
// Equivalent numbers of different types (e.g. 3L and 3.0) are
// considered equal when checking if a value is missing.
// NaN is equal to NaN, and the null value is equal to the null value.
// If the input contains multiple equivalent values, only the first will
// be considered.
//
// The corresponding transform result will be the null value.
ArrayValue append_missing_elements = 6;

// Removes all of the given elements from the array in the property.
// If the property is not an array, or if the property does not yet exist,
// it is set to the empty array.
//
// Equivalent numbers of different types (e.g. 3L and 3.0) are
// considered equal when deciding whether an element should be removed.
// NaN is equal to NaN, and the null value is equal to the null value.
// This will remove all equivalent values if there are duplicates.
//
// The corresponding transform result will be the null value.
ArrayValue remove_all_from_array = 7;
}
}

// The result of applying a mutation.
Expand Down Expand Up @@ -578,6 +701,11 @@ message MutationResult {
// Whether a conflict was detected for this mutation. Always false when a
// conflict detection strategy field is not set in the mutation.
bool conflict_detected = 5;

// The results of applying each
// [PropertyTransform][google.datastore.v1.PropertyTransform], in the same
// order of the request.
repeated Value transform_results = 8;
}

// The set of arbitrarily nested property paths used to restrict an operation to
Expand Down Expand Up @@ -611,16 +739,13 @@ message ReadOptions {
EVENTUAL = 2;
}

// For Cloud Datastore, if read_consistency is not specified, then lookups and
// ancestor queries default to `read_consistency`=`STRONG`, global queries
// default to `read_consistency`=`EVENTUAL`.
//
// For Cloud Firestore in Datastore mode, if read_consistency is not specified
// then lookups and all queries default to `read_consistency`=`STRONG`.
// For Cloud Firestore in Datastore mode, if you don't specify
// read_consistency then all lookups and queries default to
// `read_consistency`=`STRONG`. Note that, in Cloud Datastore, global queries
// defaulted to `read_consistency`=`EVENTUAL`.
//
// Explicitly setting `read_consistency`=`EVENTUAL` will result in eventually
// consistent lookups & queries in both Cloud Datastore & Cloud Firestore in
// Datastore mode.
// consistent lookups and queries.
oneof consistency_type {
// The non-transactional read consistency to use.
ReadConsistency read_consistency = 1;
Expand Down
80 changes: 80 additions & 0 deletions protos/google/datastore/v1/query.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,15 @@ message EntityResult {
}

// A query for entities.
//
// The query stages are executed in the following order:
// 1. kind
// 2. filter
// 3. projection
// 4. order + start_cursor + end_cursor
// 5. offset
// 6. limit
// 7. find_nearest
message Query {
// The projection to return. Defaults to returning all properties.
repeated Projection projection = 2;
Expand Down Expand Up @@ -127,6 +136,13 @@ message Query {
// Unspecified is interpreted as no limit.
// Must be >= 0 if specified.
google.protobuf.Int32Value limit = 12;

// Optional. A potential Nearest Neighbors Search.
//
// Applies after all other filters and ordering.
//
// Finds the closest vector embeddings to the given query vector.
FindNearest find_nearest = 13 [(google.api.field_behavior) = OPTIONAL];
}

// Datastore query for running an aggregation over a
Expand Down Expand Up @@ -436,6 +452,70 @@ message PropertyFilter {
Value value = 3;
}

// Nearest Neighbors search config. The ordering provided by FindNearest
// supersedes the order_by stage. If multiple documents have the same vector
// distance, the returned document order is not guaranteed to be stable between
// queries.
message FindNearest {
// The distance measure to use when comparing vectors.
enum DistanceMeasure {
// Should not be set.
DISTANCE_MEASURE_UNSPECIFIED = 0;

// Measures the EUCLIDEAN distance between the vectors. See
// [Euclidean](https://en.wikipedia.org/wiki/Euclidean_distance) to learn
// more. The resulting distance decreases the more similar two vectors are.
EUCLIDEAN = 1;

// COSINE distance compares vectors based on the angle between them, which
// allows you to measure similarity that isn't based on the vectors
// magnitude. We recommend using DOT_PRODUCT with unit normalized vectors
// instead of COSINE distance, which is mathematically equivalent with
// better performance. See [Cosine
// Similarity](https://en.wikipedia.org/wiki/Cosine_similarity) to learn
// more about COSINE similarity and COSINE distance. The resulting COSINE
// distance decreases the more similar two vectors are.
COSINE = 2;

// Similar to cosine but is affected by the magnitude of the vectors. See
// [Dot Product](https://en.wikipedia.org/wiki/Dot_product) to learn more.
// The resulting distance increases the more similar two vectors are.
DOT_PRODUCT = 3;
}

// Required. An indexed vector property to search upon. Only documents which
// contain vectors whose dimensionality match the query_vector can be
// returned.
PropertyReference vector_property = 1
[(google.api.field_behavior) = REQUIRED];

// Required. The query vector that we are searching on. Must be a vector of no
// more than 2048 dimensions.
Value query_vector = 2 [(google.api.field_behavior) = REQUIRED];

// Required. The Distance Measure to use, required.
DistanceMeasure distance_measure = 3 [(google.api.field_behavior) = REQUIRED];

// Required. The number of nearest neighbors to return. Must be a positive
// integer of no more than 100.
google.protobuf.Int32Value limit = 4 [(google.api.field_behavior) = REQUIRED];

// Optional. Optional name of the field to output the result of the vector
// distance calculation. Must conform to [entity
// property][google.datastore.v1.Entity.properties] limitations.
string distance_result_property = 5 [(google.api.field_behavior) = OPTIONAL];

// Optional. Option to specify a threshold for which no less similar documents
// will be returned. The behavior of the specified `distance_measure` will
// affect the meaning of the distance threshold. Since DOT_PRODUCT distances
// increase when the vectors are more similar, the comparison is inverted.
//
// For EUCLIDEAN, COSINE: WHERE distance <= distance_threshold
// For DOT_PRODUCT: WHERE distance >= distance_threshold
google.protobuf.DoubleValue distance_threshold = 6
[(google.api.field_behavior) = OPTIONAL];
}

// A [GQL
// query](https://cloud.google.com/datastore/docs/apis/gql/gql_reference).
message GqlQuery {
Expand Down
Loading

0 comments on commit 1d56433

Please sign in to comment.