feat: Add BigQuery Metastore Partition Service API version v1alpha (#471

)

* docs: A comment for message `StreamMetastorePartitionsRequest` is changed
docs: A comment for message `StreamMetastorePartitionsResponse` is changed
docs: A comment for field `location_uri` in message `.google.cloud.bigquery.storage.v1alpha.StorageDescriptor` is changed

PiperOrigin-RevId: 670602530

Source-Link: googleapis/googleapis@9c6ceea

Source-Link: googleapis/googleapis-gen@1ab1ed5
Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiMWFiMWVkNTljZTQzYmE5NmU3MWNiNTg5NzA0MzM5ZjNiOGM2ZTUyNCJ9

feat: add documentation for partition value limit

BREAKING CHANGE: make the client library gRPC only

PiperOrigin-RevId: 666551276

Source-Link: googleapis/googleapis@6f3c628

Source-Link: googleapis/googleapis-gen@9c59969
Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiOWM1OTk2OThmOTBhZTU2YTYxZTM4YTI2NmJkNzcwNTYxNGM1ZGZmNiJ9

feat: Support for a custom error message for BatchSizeTooLargeError

PiperOrigin-RevId: 665560115

Source-Link: googleapis/googleapis@869c2e1

Source-Link: googleapis/googleapis-gen@cb7632b
Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiY2I3NjMyYjU2MTlmZGU4ZTI0ZTQ3YTMwYThiNWE3MjBhYzk4ODZlMiJ9

feat: Add BigQuery Metastore Partition Service API version v1alpha

PiperOrigin-RevId: 662212485

Source-Link: googleapis/googleapis@456a812

Source-Link: googleapis/googleapis-gen@2ec266e
Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiMmVjMjY2ZTZkYTAzMjA4YTc2YjBmZDYwMDFiYTdkZjkzZGFlNDRlNiJ9

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* fix: Increase method timeout to 240s for BigQuery Metastore Partition Service API version v1alpha

PiperOrigin-RevId: 676173688

Source-Link: googleapis/googleapis@02f1184

Source-Link: googleapis/googleapis-gen@93f225b
Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiOTNmMjI1YjFlNWM4OTcxMmZhMTdkYzM5OGY5OTBiYjFjZDkyNzAyNSJ9

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* fix: export v1alpha api

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

---------

Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
Co-authored-by: Alvaro Viebrantz <[email protected]>

protos/google/cloud/bigquery/storage/v1alpha/metastore_partition.proto

@@ -0,0 +1,282 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package google.cloud.bigquery.storage.v1alpha;
+
+import "google/api/annotations.proto";
+import "google/api/client.proto";
+import "google/api/field_behavior.proto";
+import "google/api/resource.proto";
+import "google/cloud/bigquery/storage/v1alpha/partition.proto";
+import "google/protobuf/empty.proto";
+import "google/protobuf/field_mask.proto";
+
+option csharp_namespace = "Google.Cloud.BigQuery.Storage.V1Alpha";
+option go_package = "cloud.google.com/go/bigquery/storage/apiv1alpha/storagepb;storagepb";
+option java_multiple_files = true;
+option java_outer_classname = "MetastorePartitionServiceProto";
+option java_package = "com.google.cloud.bigquery.storage.v1alpha";
+option php_namespace = "Google\\Cloud\\BigQuery\\Storage\\V1alpha";
+option (google.api.resource_definition) = {
+  type: "bigquery.googleapis.com/Table"
+  pattern: "projects/{project}/datasets/{dataset}/tables/{table}"
+};
+
+// BigQuery Metastore Partition Service API.
+// This service is used for managing metastore partitions in BigQuery metastore.
+// The service supports only batch operations for write.
+service MetastorePartitionService {
+  option (google.api.default_host) = "bigquerystorage.googleapis.com";
+  option (google.api.oauth_scopes) =
+      "https://www.googleapis.com/auth/bigquery,"
+      "https://www.googleapis.com/auth/cloud-platform";
+
+  // Adds metastore partitions to a table.
+  rpc BatchCreateMetastorePartitions(BatchCreateMetastorePartitionsRequest)
+      returns (BatchCreateMetastorePartitionsResponse) {
+    option (google.api.http) = {
+      post: "/v1alpha/{parent=projects/*/locations/*/datasets/*/tables/*}/partitions:batchCreate"
+      body: "*"
+    };
+  }
+
+  // Deletes metastore partitions from a table.
+  rpc BatchDeleteMetastorePartitions(BatchDeleteMetastorePartitionsRequest)
+      returns (google.protobuf.Empty) {
+    option (google.api.http) = {
+      post: "/v1alpha/{parent=projects/*/locations/*/datasets/*/tables/*}/partitions:batchDelete"
+      body: "*"
+    };
+  }
+
+  // Updates metastore partitions in a table.
+  rpc BatchUpdateMetastorePartitions(BatchUpdateMetastorePartitionsRequest)
+      returns (BatchUpdateMetastorePartitionsResponse) {
+    option (google.api.http) = {
+      post: "/v1alpha/{parent=projects/*/locations/*/datasets/*/tables/*}/partitions:batchUpdate"
+      body: "*"
+    };
+  }
+
+  // Gets metastore partitions from a table.
+  rpc ListMetastorePartitions(ListMetastorePartitionsRequest)
+      returns (ListMetastorePartitionsResponse) {
+    option (google.api.http) = {
+      get: "/v1alpha/{parent=projects/*/locations/*/datasets/*/tables/*}/partitions:list"
+    };
+    option (google.api.method_signature) = "parent";
+  }
+
+  // This is a bi-di streaming rpc method that allows the client to send
+  // a stream of partitions and commit all of them atomically at the end.
+  // If the commit is successful, the server will return a
+  // response and close the stream. If the commit fails (due to duplicate
+  // partitions or other reason), the server will close the stream with an
+  // error. This method is only available via the gRPC API (not REST).
+  rpc StreamMetastorePartitions(stream StreamMetastorePartitionsRequest)
+      returns (stream StreamMetastorePartitionsResponse) {}
+}
+
+// Request message for CreateMetastorePartition. The MetastorePartition is
+// uniquely identified by values, which is an ordered list. Hence, there is no
+// separate name or partition id field.
+message CreateMetastorePartitionRequest {
+  // Required. Reference to the table to where the metastore partition to be
+  // added, in the format of
+  // projects/{project}/databases/{databases}/tables/{table}.
+  string parent = 1 [
+    (google.api.field_behavior) = REQUIRED,
+    (google.api.resource_reference) = { type: "bigquery.googleapis.com/Table" }
+  ];
+
+  // Required. The metastore partition to be added.
+  MetastorePartition metastore_partition = 2
+      [(google.api.field_behavior) = REQUIRED];
+}
+
+// Request message for BatchCreateMetastorePartitions.
+message BatchCreateMetastorePartitionsRequest {
+  // Required. Reference to the table to where the metastore partitions to be
+  // added, in the format of
+  // projects/{project}/locations/{location}/datasets/{dataset}/tables/{table}.
+  string parent = 1 [
+    (google.api.field_behavior) = REQUIRED,
+    (google.api.resource_reference) = { type: "bigquery.googleapis.com/Table" }
+  ];
+
+  // Required. Requests to add metastore partitions to the table.
+  repeated CreateMetastorePartitionRequest requests = 2
+      [(google.api.field_behavior) = REQUIRED];
+
+  // Optional. Mimics the ifNotExists flag in IMetaStoreClient
+  // add_partitions(..). If the flag is set to false, the server will return
+  // ALREADY_EXISTS if any partition already exists. If the flag is set to true,
+  // the server will skip existing partitions and insert only the non-existing
+  // partitions.
+  bool skip_existing_partitions = 3 [(google.api.field_behavior) = OPTIONAL];
+}
+
+// Response message for BatchCreateMetastorePartitions.
+message BatchCreateMetastorePartitionsResponse {
+  // The list of metastore partitions that have been created.
+  repeated MetastorePartition partitions = 1;
+}
+
+// Request message for BatchDeleteMetastorePartitions. The MetastorePartition is
+// uniquely identified by values, which is an ordered list. Hence, there is no
+// separate name or partition id field.
+message BatchDeleteMetastorePartitionsRequest {
+  // Required. Reference to the table to which these metastore partitions
+  // belong, in the format of
+  // projects/{project}/locations/{location}/datasets/{dataset}/tables/{table}.
+  string parent = 1 [
+    (google.api.field_behavior) = REQUIRED,
+    (google.api.resource_reference) = { type: "bigquery.googleapis.com/Table" }
+  ];
+
+  // Required. The list of metastore partitions (identified by its values) to be
+  // deleted. A maximum of 100 partitions can be deleted in a batch.
+  repeated MetastorePartitionValues partition_values = 2
+      [(google.api.field_behavior) = REQUIRED];
+}
+
+// Request message for UpdateMetastorePartition.
+message UpdateMetastorePartitionRequest {
+  // Required. The metastore partition to be updated.
+  MetastorePartition metastore_partition = 1
+      [(google.api.field_behavior) = REQUIRED];
+
+  // Optional. The list of fields to update.
+  google.protobuf.FieldMask update_mask = 2
+      [(google.api.field_behavior) = OPTIONAL];
+}
+
+// Request message for BatchUpdateMetastorePartitions.
+message BatchUpdateMetastorePartitionsRequest {
+  // Required. Reference to the table to which these metastore partitions
+  // belong, in the format of
+  // projects/{project}/locations/{location}/datasets/{dataset}/tables/{table}.
+  string parent = 1 [
+    (google.api.field_behavior) = REQUIRED,
+    (google.api.resource_reference) = { type: "bigquery.googleapis.com/Table" }
+  ];
+
+  // Required. Requests to update metastore partitions in the table.
+  repeated UpdateMetastorePartitionRequest requests = 2
+      [(google.api.field_behavior) = REQUIRED];
+}
+
+// Response message for BatchUpdateMetastorePartitions.
+message BatchUpdateMetastorePartitionsResponse {
+  // The list of metastore partitions that have been updated.
+  repeated MetastorePartition partitions = 1;
+}
+
+// Request message for ListMetastorePartitions.
+message ListMetastorePartitionsRequest {
+  // Required. Reference to the table to which these metastore partitions
+  // belong, in the format of
+  // projects/{project}/locations/{location}/datasets/{dataset}/tables/{table}.
+  string parent = 1 [
+    (google.api.field_behavior) = REQUIRED,
+    (google.api.resource_reference) = { type: "bigquery.googleapis.com/Table" }
+  ];
+
+  // Optional. SQL text filtering statement, similar to a WHERE clause in a
+  // query. Only supports single-row expressions.  Aggregate functions are not
+  // supported.
+  //
+  // Examples: "int_field > 5"
+  //           "date_field = CAST('2014-9-27' as DATE)"
+  //           "nullable_field is not NULL"
+  //           "st_equals(geo_field, st_geofromtext("POINT(2, 2)"))"
+  //           "numeric_field BETWEEN 1.0 AND 5.0"
+  // Restricted to a maximum length for 1 MB.
+  string filter = 2 [(google.api.field_behavior) = OPTIONAL];
+}
+
+// Response message for ListMetastorePartitions.
+message ListMetastorePartitionsResponse {
+  // The response depends on the number of metastore partitions to be returned;
+  // it can be a list of partitions or a list of
+  // [ReadStream]((https://cloud.google.com/bigquery/docs/reference/storage/rpc/google.cloud.bigquery.storage.v1#readstream))
+  // objects. For the second situation, the BigQuery [Read API
+  // ReadRows](https://cloud.google.com/bigquery/docs/reference/storage#read_from_a_session_stream)
+  // method must be used to stream the data and convert it into a list of
+  // partitions.
+  oneof response {
+    // The list of partitions.
+    MetastorePartitionList partitions = 1;
+
+    // The list of streams.
+    StreamList streams = 2;
+  }
+}
+
+// The top-level message sent by the client to the
+// [Partitions.StreamMetastorePartitions][] method.
+// Follows the default gRPC streaming maximum size of 4 MB.
+message StreamMetastorePartitionsRequest {
+  // Required. Reference to the table to where the partition to be added, in the
+  // format of
+  // projects/{project}/locations/{location}/datasets/{dataset}/tables/{table}.
+  string parent = 1 [
+    (google.api.field_behavior) = REQUIRED,
+    (google.api.resource_reference) = { type: "bigquery.googleapis.com/Table" }
+  ];
+
+  // Optional. A list of metastore partitions to be added to the table.
+  repeated MetastorePartition metastore_partitions = 2
+      [(google.api.field_behavior) = OPTIONAL];
+
+  // Optional. Mimics the ifNotExists flag in IMetaStoreClient
+  // add_partitions(..). If the flag is set to false, the server will return
+  // ALREADY_EXISTS on commit if any partition already exists. If the flag is
+  // set to true:
+  //  1) the server will skip existing partitions
+  //  insert only the non-existing partitions as part of the commit.
+  //  2) The client must set the `skip_existing_partitions` field to true for
+  //  all requests in the stream.
+  bool skip_existing_partitions = 3 [(google.api.field_behavior) = OPTIONAL];
+}
+
+// This is the response message sent by the server
+// to the client for the [Partitions.StreamMetastorePartitions][] method when
+// the commit is successful. Server will close the stream after sending this
+// message.
+message StreamMetastorePartitionsResponse {
+  // Total count of partitions streamed by the client during the lifetime of the
+  // stream. This is only set in the final response message before closing the
+  // stream.
+  int64 total_partitions_streamed_count = 2;
+
+  // Total count of partitions inserted by the server during the lifetime of the
+  // stream. This is only set in the final response message before closing the
+  // stream.
+  int64 total_partitions_inserted_count = 3;
+}
+
+// Structured custom error message for batch size too large error.
+// The error can be attached as error details in the returned rpc Status for
+// more structured error handling in the client.
+message BatchSizeTooLargeError {
+  // The maximum number of items that are supported in a single batch. This is
+  // returned as a hint to the client to adjust the batch size.
+  int64 max_batch_size = 1;
+
+  // Optional. The error message that is returned to the client.
+  string error_message = 2 [(google.api.field_behavior) = OPTIONAL];
+}