From 3ce00fb39d1c55c03b036ff64b0d5cdef8eac018 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Wed, 29 Jun 2022 12:40:42 -0400 Subject: [PATCH] fix: Modify client lib retry policy for CreateWriteStream with longer backoff, more error code and longer overall time (#279) feat: add fields to eventually contain row level errors Committer: @gnanda PiperOrigin-RevId: 456324780 Source-Link: https://github.com/googleapis/googleapis/commit/f24b37a351260ddce8208edae50d637fa0b88d6b Source-Link: https://github.com/googleapis/googleapis-gen/commit/33f9d814082117116c4b68a6f5aac3f42bec35c2 Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiMzNmOWQ4MTQwODIxMTcxMTZjNGI2OGE2ZjVhYWMzZjQyYmVjMzVjMiJ9 feat: support regapic LRO PiperOrigin-RevId: 456946341 Source-Link: https://github.com/googleapis/googleapis/commit/88fd18d9d3b872b3d06a3d9392879f50b5bf3ce5 Source-Link: https://github.com/googleapis/googleapis-gen/commit/accfa371f667439313335c64042b063c1c53102e Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiYWNjZmEzNzFmNjY3NDM5MzEzMzM1YzY0MDQyYjA2M2MxYzUzMTAyZSJ9 See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md fix: Modify client lib retry policy for CreateWriteStream with longer backoff, more error code and longer overall time PiperOrigin-RevId: 457061436 Source-Link: https://github.com/googleapis/googleapis/commit/8ff130bc81fa1d175e410d14a300caa18d5ebf80 Source-Link: https://github.com/googleapis/googleapis-gen/commit/2eb0faca717d9cf44b838b7db5e862451b8a86ef Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiMmViMGZhY2E3MTdkOWNmNDRiODM4YjdkYjVlODYyNDUxYjhhODZlZiJ9 See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot --- .../cloud/bigquery/storage/v1/storage.proto | 38 ++- .../cloud/bigquery/storage/v1/stream.proto | 3 +- .../cloud/bigquery/storage/v1/table.proto | 4 +- .../protos/protos.d.ts | 117 +++++++ .../protos/protos.js | 317 ++++++++++++++++++ .../protos/protos.json | 29 ++ .../v1/big_query_read.create_read_session.js | 10 +- ...data.google.cloud.bigquery.storage.v1.json | 2 +- .../src/v1/big_query_read_client.ts | 21 +- .../src/v1/big_query_write_client.ts | 11 +- .../src/v1/big_query_write_client_config.json | 20 +- .../src/v1beta1/big_query_storage_client.ts | 11 +- 12 files changed, 546 insertions(+), 37 deletions(-) diff --git a/packages/google-cloud-bigquery-storage/protos/google/cloud/bigquery/storage/v1/storage.proto b/packages/google-cloud-bigquery-storage/protos/google/cloud/bigquery/storage/v1/storage.proto index 67c6c8a0295..f3c974c6461 100644 --- a/packages/google-cloud-bigquery-storage/protos/google/cloud/bigquery/storage/v1/storage.proto +++ b/packages/google-cloud-bigquery-storage/protos/google/cloud/bigquery/storage/v1/storage.proto @@ -248,11 +248,13 @@ message CreateReadSessionRequest { // Max initial number of streams. If unset or zero, the server will // provide a value of streams so as to produce reasonable throughput. Must be // non-negative. The number of streams may be lower than the requested number, - // depending on the amount parallelism that is reasonable for the table. Error - // will be returned if the max count is greater than the current system - // max limit of 1,000. + // depending on the amount parallelism that is reasonable for the table. + // There is a default system max limit of 1,000. // - // Streams must be read starting from offset 0. + // This must be greater than or equal to preferred_min_stream_count. + // Typically, clients should either leave this unset to let the system to + // determine an upper bound OR set this a size for the maximum "units of work" + // it can gracefully handle. int32 max_stream_count = 3; } @@ -329,7 +331,7 @@ message ReadRowsResponse { // The schema for the read. If read_options.selected_fields is set, the // schema may be different from the table schema as it will only contain - // the selected fields. This schema is equivelant to the one returned by + // the selected fields. This schema is equivalent to the one returned by // CreateSession. This field is only populated in the first ReadRowsResponse // RPC. oneof schema { @@ -488,6 +490,11 @@ message AppendRowsResponse { // use it to input new type of message. It will be empty when no schema // updates have occurred. TableSchema updated_schema = 3; + + // If a request failed due to corrupted rows, no rows in the batch will be + // appended. The API will return row level error info, so that the caller can + // remove the bad rows and retry the request. + repeated RowError row_errors = 4; } // Request message for `GetWriteStreamRequest`. @@ -622,3 +629,24 @@ message StorageError { // Message that describes the error. string error_message = 3; } + +// The message that presents row level error info in a request. +message RowError { + // Error code for `RowError`. + enum RowErrorCode { + // Default error. + ROW_ERROR_CODE_UNSPECIFIED = 0; + + // One or more fields in the row has errors. + FIELDS_ERROR = 1; + } + + // Index of the malformed row in the request. + int64 index = 1; + + // Structured error reason for a row error. + RowErrorCode code = 2; + + // Description of the issue encountered when processing the row. + string message = 3; +} diff --git a/packages/google-cloud-bigquery-storage/protos/google/cloud/bigquery/storage/v1/stream.proto b/packages/google-cloud-bigquery-storage/protos/google/cloud/bigquery/storage/v1/stream.proto index bd1fa2ce98a..fd1e25b65fd 100644 --- a/packages/google-cloud-bigquery-storage/protos/google/cloud/bigquery/storage/v1/stream.proto +++ b/packages/google-cloud-bigquery-storage/protos/google/cloud/bigquery/storage/v1/stream.proto @@ -32,6 +32,7 @@ option php_namespace = "Google\\Cloud\\BigQuery\\Storage\\V1"; // Data format for input or output data. enum DataFormat { + // Data format is unspecified. DATA_FORMAT_UNSPECIFIED = 0; // Avro is a standard open source row based file format. @@ -91,7 +92,7 @@ message ReadSession { // automatically assigned and currently cannot be specified or updated. google.protobuf.Timestamp expire_time = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; - // Immutable. Data format of the output data. + // Immutable. Data format of the output data. DATA_FORMAT_UNSPECIFIED not supported. DataFormat data_format = 3 [(google.api.field_behavior) = IMMUTABLE]; // The schema for the read. If read_options.selected_fields is set, the diff --git a/packages/google-cloud-bigquery-storage/protos/google/cloud/bigquery/storage/v1/table.proto b/packages/google-cloud-bigquery-storage/protos/google/cloud/bigquery/storage/v1/table.proto index 545f6292712..fa4f840c580 100644 --- a/packages/google-cloud-bigquery-storage/protos/google/cloud/bigquery/storage/v1/table.proto +++ b/packages/google-cloud-bigquery-storage/protos/google/cloud/bigquery/storage/v1/table.proto @@ -25,7 +25,9 @@ option java_outer_classname = "TableProto"; option java_package = "com.google.cloud.bigquery.storage.v1"; option php_namespace = "Google\\Cloud\\BigQuery\\Storage\\V1"; -// Schema of a table. +// Schema of a table. This schema is a subset of +// google.cloud.bigquery.v2.TableSchema containing information necessary to +// generate valid message to write to BigQuery. message TableSchema { // Describes the fields in a table. repeated TableFieldSchema fields = 1; diff --git a/packages/google-cloud-bigquery-storage/protos/protos.d.ts b/packages/google-cloud-bigquery-storage/protos/protos.d.ts index a82f40762cd..b111ca1f564 100644 --- a/packages/google-cloud-bigquery-storage/protos/protos.d.ts +++ b/packages/google-cloud-bigquery-storage/protos/protos.d.ts @@ -2036,6 +2036,9 @@ export namespace google { /** AppendRowsResponse updatedSchema */ updatedSchema?: (google.cloud.bigquery.storage.v1.ITableSchema|null); + + /** AppendRowsResponse rowErrors */ + rowErrors?: (google.cloud.bigquery.storage.v1.IRowError[]|null); } /** Represents an AppendRowsResponse. */ @@ -2056,6 +2059,9 @@ export namespace google { /** AppendRowsResponse updatedSchema. */ public updatedSchema?: (google.cloud.bigquery.storage.v1.ITableSchema|null); + /** AppendRowsResponse rowErrors. */ + public rowErrors: google.cloud.bigquery.storage.v1.IRowError[]; + /** AppendRowsResponse response. */ public response?: ("appendResult"|"error"); @@ -2990,6 +2996,117 @@ export namespace google { } } + /** Properties of a RowError. */ + interface IRowError { + + /** RowError index */ + index?: (number|Long|string|null); + + /** RowError code */ + code?: (google.cloud.bigquery.storage.v1.RowError.RowErrorCode|keyof typeof google.cloud.bigquery.storage.v1.RowError.RowErrorCode|null); + + /** RowError message */ + message?: (string|null); + } + + /** Represents a RowError. */ + class RowError implements IRowError { + + /** + * Constructs a new RowError. + * @param [properties] Properties to set + */ + constructor(properties?: google.cloud.bigquery.storage.v1.IRowError); + + /** RowError index. */ + public index: (number|Long|string); + + /** RowError code. */ + public code: (google.cloud.bigquery.storage.v1.RowError.RowErrorCode|keyof typeof google.cloud.bigquery.storage.v1.RowError.RowErrorCode); + + /** RowError message. */ + public message: string; + + /** + * Creates a new RowError instance using the specified properties. + * @param [properties] Properties to set + * @returns RowError instance + */ + public static create(properties?: google.cloud.bigquery.storage.v1.IRowError): google.cloud.bigquery.storage.v1.RowError; + + /** + * Encodes the specified RowError message. Does not implicitly {@link google.cloud.bigquery.storage.v1.RowError.verify|verify} messages. + * @param message RowError message or plain object to encode + * @param [writer] Writer to encode to + * @returns Writer + */ + public static encode(message: google.cloud.bigquery.storage.v1.IRowError, writer?: $protobuf.Writer): $protobuf.Writer; + + /** + * Encodes the specified RowError message, length delimited. Does not implicitly {@link google.cloud.bigquery.storage.v1.RowError.verify|verify} messages. + * @param message RowError message or plain object to encode + * @param [writer] Writer to encode to + * @returns Writer + */ + public static encodeDelimited(message: google.cloud.bigquery.storage.v1.IRowError, writer?: $protobuf.Writer): $protobuf.Writer; + + /** + * Decodes a RowError message from the specified reader or buffer. + * @param reader Reader or buffer to decode from + * @param [length] Message length if known beforehand + * @returns RowError + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + public static decode(reader: ($protobuf.Reader|Uint8Array), length?: number): google.cloud.bigquery.storage.v1.RowError; + + /** + * Decodes a RowError message from the specified reader or buffer, length delimited. + * @param reader Reader or buffer to decode from + * @returns RowError + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + public static decodeDelimited(reader: ($protobuf.Reader|Uint8Array)): google.cloud.bigquery.storage.v1.RowError; + + /** + * Verifies a RowError message. + * @param message Plain object to verify + * @returns `null` if valid, otherwise the reason why it is not + */ + public static verify(message: { [k: string]: any }): (string|null); + + /** + * Creates a RowError message from a plain object. Also converts values to their respective internal types. + * @param object Plain object + * @returns RowError + */ + public static fromObject(object: { [k: string]: any }): google.cloud.bigquery.storage.v1.RowError; + + /** + * Creates a plain object from a RowError message. Also converts values to other types if specified. + * @param message RowError + * @param [options] Conversion options + * @returns Plain object + */ + public static toObject(message: google.cloud.bigquery.storage.v1.RowError, options?: $protobuf.IConversionOptions): { [k: string]: any }; + + /** + * Converts this RowError to JSON. + * @returns JSON object + */ + public toJSON(): { [k: string]: any }; + } + + namespace RowError { + + /** RowErrorCode enum. */ + enum RowErrorCode { + ROW_ERROR_CODE_UNSPECIFIED = 0, + FIELDS_ERROR = 1 + } + } + /** DataFormat enum. */ enum DataFormat { DATA_FORMAT_UNSPECIFIED = 0, diff --git a/packages/google-cloud-bigquery-storage/protos/protos.js b/packages/google-cloud-bigquery-storage/protos/protos.js index 1d0989c2671..ea81e6448f4 100644 --- a/packages/google-cloud-bigquery-storage/protos/protos.js +++ b/packages/google-cloud-bigquery-storage/protos/protos.js @@ -4532,6 +4532,7 @@ * @property {google.cloud.bigquery.storage.v1.AppendRowsResponse.IAppendResult|null} [appendResult] AppendRowsResponse appendResult * @property {google.rpc.IStatus|null} [error] AppendRowsResponse error * @property {google.cloud.bigquery.storage.v1.ITableSchema|null} [updatedSchema] AppendRowsResponse updatedSchema + * @property {Array.|null} [rowErrors] AppendRowsResponse rowErrors */ /** @@ -4543,6 +4544,7 @@ * @param {google.cloud.bigquery.storage.v1.IAppendRowsResponse=} [properties] Properties to set */ function AppendRowsResponse(properties) { + this.rowErrors = []; if (properties) for (var keys = Object.keys(properties), i = 0; i < keys.length; ++i) if (properties[keys[i]] != null) @@ -4573,6 +4575,14 @@ */ AppendRowsResponse.prototype.updatedSchema = null; + /** + * AppendRowsResponse rowErrors. + * @member {Array.} rowErrors + * @memberof google.cloud.bigquery.storage.v1.AppendRowsResponse + * @instance + */ + AppendRowsResponse.prototype.rowErrors = $util.emptyArray; + // OneOf field names bound to virtual getters and setters var $oneOfFields; @@ -4617,6 +4627,9 @@ $root.google.rpc.Status.encode(message.error, writer.uint32(/* id 2, wireType 2 =*/18).fork()).ldelim(); if (message.updatedSchema != null && Object.hasOwnProperty.call(message, "updatedSchema")) $root.google.cloud.bigquery.storage.v1.TableSchema.encode(message.updatedSchema, writer.uint32(/* id 3, wireType 2 =*/26).fork()).ldelim(); + if (message.rowErrors != null && message.rowErrors.length) + for (var i = 0; i < message.rowErrors.length; ++i) + $root.google.cloud.bigquery.storage.v1.RowError.encode(message.rowErrors[i], writer.uint32(/* id 4, wireType 2 =*/34).fork()).ldelim(); return writer; }; @@ -4660,6 +4673,11 @@ case 3: message.updatedSchema = $root.google.cloud.bigquery.storage.v1.TableSchema.decode(reader, reader.uint32()); break; + case 4: + if (!(message.rowErrors && message.rowErrors.length)) + message.rowErrors = []; + message.rowErrors.push($root.google.cloud.bigquery.storage.v1.RowError.decode(reader, reader.uint32())); + break; default: reader.skipType(tag & 7); break; @@ -4719,6 +4737,15 @@ if (error) return "updatedSchema." + error; } + if (message.rowErrors != null && message.hasOwnProperty("rowErrors")) { + if (!Array.isArray(message.rowErrors)) + return "rowErrors: array expected"; + for (var i = 0; i < message.rowErrors.length; ++i) { + var error = $root.google.cloud.bigquery.storage.v1.RowError.verify(message.rowErrors[i]); + if (error) + return "rowErrors." + error; + } + } return null; }; @@ -4749,6 +4776,16 @@ throw TypeError(".google.cloud.bigquery.storage.v1.AppendRowsResponse.updatedSchema: object expected"); message.updatedSchema = $root.google.cloud.bigquery.storage.v1.TableSchema.fromObject(object.updatedSchema); } + if (object.rowErrors) { + if (!Array.isArray(object.rowErrors)) + throw TypeError(".google.cloud.bigquery.storage.v1.AppendRowsResponse.rowErrors: array expected"); + message.rowErrors = []; + for (var i = 0; i < object.rowErrors.length; ++i) { + if (typeof object.rowErrors[i] !== "object") + throw TypeError(".google.cloud.bigquery.storage.v1.AppendRowsResponse.rowErrors: object expected"); + message.rowErrors[i] = $root.google.cloud.bigquery.storage.v1.RowError.fromObject(object.rowErrors[i]); + } + } return message; }; @@ -4765,6 +4802,8 @@ if (!options) options = {}; var object = {}; + if (options.arrays || options.defaults) + object.rowErrors = []; if (options.defaults) object.updatedSchema = null; if (message.appendResult != null && message.hasOwnProperty("appendResult")) { @@ -4779,6 +4818,11 @@ } if (message.updatedSchema != null && message.hasOwnProperty("updatedSchema")) object.updatedSchema = $root.google.cloud.bigquery.storage.v1.TableSchema.toObject(message.updatedSchema, options); + if (message.rowErrors && message.rowErrors.length) { + object.rowErrors = []; + for (var j = 0; j < message.rowErrors.length; ++j) + object.rowErrors[j] = $root.google.cloud.bigquery.storage.v1.RowError.toObject(message.rowErrors[j], options); + } return object; }; @@ -6756,6 +6800,279 @@ return StorageError; })(); + v1.RowError = (function() { + + /** + * Properties of a RowError. + * @memberof google.cloud.bigquery.storage.v1 + * @interface IRowError + * @property {number|Long|null} [index] RowError index + * @property {google.cloud.bigquery.storage.v1.RowError.RowErrorCode|null} [code] RowError code + * @property {string|null} [message] RowError message + */ + + /** + * Constructs a new RowError. + * @memberof google.cloud.bigquery.storage.v1 + * @classdesc Represents a RowError. + * @implements IRowError + * @constructor + * @param {google.cloud.bigquery.storage.v1.IRowError=} [properties] Properties to set + */ + function RowError(properties) { + if (properties) + for (var keys = Object.keys(properties), i = 0; i < keys.length; ++i) + if (properties[keys[i]] != null) + this[keys[i]] = properties[keys[i]]; + } + + /** + * RowError index. + * @member {number|Long} index + * @memberof google.cloud.bigquery.storage.v1.RowError + * @instance + */ + RowError.prototype.index = $util.Long ? $util.Long.fromBits(0,0,false) : 0; + + /** + * RowError code. + * @member {google.cloud.bigquery.storage.v1.RowError.RowErrorCode} code + * @memberof google.cloud.bigquery.storage.v1.RowError + * @instance + */ + RowError.prototype.code = 0; + + /** + * RowError message. + * @member {string} message + * @memberof google.cloud.bigquery.storage.v1.RowError + * @instance + */ + RowError.prototype.message = ""; + + /** + * Creates a new RowError instance using the specified properties. + * @function create + * @memberof google.cloud.bigquery.storage.v1.RowError + * @static + * @param {google.cloud.bigquery.storage.v1.IRowError=} [properties] Properties to set + * @returns {google.cloud.bigquery.storage.v1.RowError} RowError instance + */ + RowError.create = function create(properties) { + return new RowError(properties); + }; + + /** + * Encodes the specified RowError message. Does not implicitly {@link google.cloud.bigquery.storage.v1.RowError.verify|verify} messages. + * @function encode + * @memberof google.cloud.bigquery.storage.v1.RowError + * @static + * @param {google.cloud.bigquery.storage.v1.IRowError} message RowError message or plain object to encode + * @param {$protobuf.Writer} [writer] Writer to encode to + * @returns {$protobuf.Writer} Writer + */ + RowError.encode = function encode(message, writer) { + if (!writer) + writer = $Writer.create(); + if (message.index != null && Object.hasOwnProperty.call(message, "index")) + writer.uint32(/* id 1, wireType 0 =*/8).int64(message.index); + if (message.code != null && Object.hasOwnProperty.call(message, "code")) + writer.uint32(/* id 2, wireType 0 =*/16).int32(message.code); + if (message.message != null && Object.hasOwnProperty.call(message, "message")) + writer.uint32(/* id 3, wireType 2 =*/26).string(message.message); + return writer; + }; + + /** + * Encodes the specified RowError message, length delimited. Does not implicitly {@link google.cloud.bigquery.storage.v1.RowError.verify|verify} messages. + * @function encodeDelimited + * @memberof google.cloud.bigquery.storage.v1.RowError + * @static + * @param {google.cloud.bigquery.storage.v1.IRowError} message RowError message or plain object to encode + * @param {$protobuf.Writer} [writer] Writer to encode to + * @returns {$protobuf.Writer} Writer + */ + RowError.encodeDelimited = function encodeDelimited(message, writer) { + return this.encode(message, writer).ldelim(); + }; + + /** + * Decodes a RowError message from the specified reader or buffer. + * @function decode + * @memberof google.cloud.bigquery.storage.v1.RowError + * @static + * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from + * @param {number} [length] Message length if known beforehand + * @returns {google.cloud.bigquery.storage.v1.RowError} RowError + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + RowError.decode = function decode(reader, length) { + if (!(reader instanceof $Reader)) + reader = $Reader.create(reader); + var end = length === undefined ? reader.len : reader.pos + length, message = new $root.google.cloud.bigquery.storage.v1.RowError(); + while (reader.pos < end) { + var tag = reader.uint32(); + switch (tag >>> 3) { + case 1: + message.index = reader.int64(); + break; + case 2: + message.code = reader.int32(); + break; + case 3: + message.message = reader.string(); + break; + default: + reader.skipType(tag & 7); + break; + } + } + return message; + }; + + /** + * Decodes a RowError message from the specified reader or buffer, length delimited. + * @function decodeDelimited + * @memberof google.cloud.bigquery.storage.v1.RowError + * @static + * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from + * @returns {google.cloud.bigquery.storage.v1.RowError} RowError + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + RowError.decodeDelimited = function decodeDelimited(reader) { + if (!(reader instanceof $Reader)) + reader = new $Reader(reader); + return this.decode(reader, reader.uint32()); + }; + + /** + * Verifies a RowError message. + * @function verify + * @memberof google.cloud.bigquery.storage.v1.RowError + * @static + * @param {Object.} message Plain object to verify + * @returns {string|null} `null` if valid, otherwise the reason why it is not + */ + RowError.verify = function verify(message) { + if (typeof message !== "object" || message === null) + return "object expected"; + if (message.index != null && message.hasOwnProperty("index")) + if (!$util.isInteger(message.index) && !(message.index && $util.isInteger(message.index.low) && $util.isInteger(message.index.high))) + return "index: integer|Long expected"; + if (message.code != null && message.hasOwnProperty("code")) + switch (message.code) { + default: + return "code: enum value expected"; + case 0: + case 1: + break; + } + if (message.message != null && message.hasOwnProperty("message")) + if (!$util.isString(message.message)) + return "message: string expected"; + return null; + }; + + /** + * Creates a RowError message from a plain object. Also converts values to their respective internal types. + * @function fromObject + * @memberof google.cloud.bigquery.storage.v1.RowError + * @static + * @param {Object.} object Plain object + * @returns {google.cloud.bigquery.storage.v1.RowError} RowError + */ + RowError.fromObject = function fromObject(object) { + if (object instanceof $root.google.cloud.bigquery.storage.v1.RowError) + return object; + var message = new $root.google.cloud.bigquery.storage.v1.RowError(); + if (object.index != null) + if ($util.Long) + (message.index = $util.Long.fromValue(object.index)).unsigned = false; + else if (typeof object.index === "string") + message.index = parseInt(object.index, 10); + else if (typeof object.index === "number") + message.index = object.index; + else if (typeof object.index === "object") + message.index = new $util.LongBits(object.index.low >>> 0, object.index.high >>> 0).toNumber(); + switch (object.code) { + case "ROW_ERROR_CODE_UNSPECIFIED": + case 0: + message.code = 0; + break; + case "FIELDS_ERROR": + case 1: + message.code = 1; + break; + } + if (object.message != null) + message.message = String(object.message); + return message; + }; + + /** + * Creates a plain object from a RowError message. Also converts values to other types if specified. + * @function toObject + * @memberof google.cloud.bigquery.storage.v1.RowError + * @static + * @param {google.cloud.bigquery.storage.v1.RowError} message RowError + * @param {$protobuf.IConversionOptions} [options] Conversion options + * @returns {Object.} Plain object + */ + RowError.toObject = function toObject(message, options) { + if (!options) + options = {}; + var object = {}; + if (options.defaults) { + if ($util.Long) { + var long = new $util.Long(0, 0, false); + object.index = options.longs === String ? long.toString() : options.longs === Number ? long.toNumber() : long; + } else + object.index = options.longs === String ? "0" : 0; + object.code = options.enums === String ? "ROW_ERROR_CODE_UNSPECIFIED" : 0; + object.message = ""; + } + if (message.index != null && message.hasOwnProperty("index")) + if (typeof message.index === "number") + object.index = options.longs === String ? String(message.index) : message.index; + else + object.index = options.longs === String ? $util.Long.prototype.toString.call(message.index) : options.longs === Number ? new $util.LongBits(message.index.low >>> 0, message.index.high >>> 0).toNumber() : message.index; + if (message.code != null && message.hasOwnProperty("code")) + object.code = options.enums === String ? $root.google.cloud.bigquery.storage.v1.RowError.RowErrorCode[message.code] : message.code; + if (message.message != null && message.hasOwnProperty("message")) + object.message = message.message; + return object; + }; + + /** + * Converts this RowError to JSON. + * @function toJSON + * @memberof google.cloud.bigquery.storage.v1.RowError + * @instance + * @returns {Object.} JSON object + */ + RowError.prototype.toJSON = function toJSON() { + return this.constructor.toObject(this, $protobuf.util.toJSONOptions); + }; + + /** + * RowErrorCode enum. + * @name google.cloud.bigquery.storage.v1.RowError.RowErrorCode + * @enum {number} + * @property {number} ROW_ERROR_CODE_UNSPECIFIED=0 ROW_ERROR_CODE_UNSPECIFIED value + * @property {number} FIELDS_ERROR=1 FIELDS_ERROR value + */ + RowError.RowErrorCode = (function() { + var valuesById = {}, values = Object.create(valuesById); + values[valuesById[0] = "ROW_ERROR_CODE_UNSPECIFIED"] = 0; + values[valuesById[1] = "FIELDS_ERROR"] = 1; + return values; + })(); + + return RowError; + })(); + /** * DataFormat enum. * @name google.cloud.bigquery.storage.v1.DataFormat diff --git a/packages/google-cloud-bigquery-storage/protos/protos.json b/packages/google-cloud-bigquery-storage/protos/protos.json index a892cd68782..d796f2c8745 100644 --- a/packages/google-cloud-bigquery-storage/protos/protos.json +++ b/packages/google-cloud-bigquery-storage/protos/protos.json @@ -523,6 +523,11 @@ "updatedSchema": { "type": "TableSchema", "id": 3 + }, + "rowErrors": { + "rule": "repeated", + "type": "RowError", + "id": 4 } }, "nested": { @@ -657,6 +662,30 @@ } } }, + "RowError": { + "fields": { + "index": { + "type": "int64", + "id": 1 + }, + "code": { + "type": "RowErrorCode", + "id": 2 + }, + "message": { + "type": "string", + "id": 3 + } + }, + "nested": { + "RowErrorCode": { + "values": { + "ROW_ERROR_CODE_UNSPECIFIED": 0, + "FIELDS_ERROR": 1 + } + } + } + }, "DataFormat": { "values": { "DATA_FORMAT_UNSPECIFIED": 0, diff --git a/packages/google-cloud-bigquery-storage/samples/generated/v1/big_query_read.create_read_session.js b/packages/google-cloud-bigquery-storage/samples/generated/v1/big_query_read.create_read_session.js index f4d2f95025e..c5cc160a217 100644 --- a/packages/google-cloud-bigquery-storage/samples/generated/v1/big_query_read.create_read_session.js +++ b/packages/google-cloud-bigquery-storage/samples/generated/v1/big_query_read.create_read_session.js @@ -38,10 +38,12 @@ function main(parent, readSession) { * Max initial number of streams. If unset or zero, the server will * provide a value of streams so as to produce reasonable throughput. Must be * non-negative. The number of streams may be lower than the requested number, - * depending on the amount parallelism that is reasonable for the table. Error - * will be returned if the max count is greater than the current system - * max limit of 1,000. - * Streams must be read starting from offset 0. + * depending on the amount parallelism that is reasonable for the table. + * There is a default system max limit of 1,000. + * This must be greater than or equal to preferred_min_stream_count. + * Typically, clients should either leave this unset to let the system to + * determine an upper bound OR set this a size for the maximum "units of work" + * it can gracefully handle. */ // const maxStreamCount = 1234 diff --git a/packages/google-cloud-bigquery-storage/samples/generated/v1/snippet_metadata.google.cloud.bigquery.storage.v1.json b/packages/google-cloud-bigquery-storage/samples/generated/v1/snippet_metadata.google.cloud.bigquery.storage.v1.json index 93aaa2255ba..90db40853ef 100644 --- a/packages/google-cloud-bigquery-storage/samples/generated/v1/snippet_metadata.google.cloud.bigquery.storage.v1.json +++ b/packages/google-cloud-bigquery-storage/samples/generated/v1/snippet_metadata.google.cloud.bigquery.storage.v1.json @@ -22,7 +22,7 @@ "segments": [ { "start": 25, - "end": 66, + "end": 68, "type": "FULL" } ], diff --git a/packages/google-cloud-bigquery-storage/src/v1/big_query_read_client.ts b/packages/google-cloud-bigquery-storage/src/v1/big_query_read_client.ts index 525177ca372..553fe889849 100644 --- a/packages/google-cloud-bigquery-storage/src/v1/big_query_read_client.ts +++ b/packages/google-cloud-bigquery-storage/src/v1/big_query_read_client.ts @@ -70,7 +70,7 @@ export class BigQueryReadClient { * * @param {object} [options] - The configuration object. * The options accepted by the constructor are described in detail - * in [this document](https://github.com/googleapis/gax-nodejs/blob/master/client-libraries.md#creating-the-client-instance). + * in [this document](https://github.com/googleapis/gax-nodejs/blob/main/client-libraries.md#creating-the-client-instance). * The common options are: * @param {object} [options.credentials] - Credentials object. * @param {string} [options.credentials.client_email] @@ -93,11 +93,10 @@ export class BigQueryReadClient { * API remote host. * @param {gax.ClientConfig} [options.clientConfig] - Client configuration override. * Follows the structure of {@link gapicConfig}. - * @param {boolean} [options.fallback] - Use HTTP fallback mode. - * In fallback mode, a special browser-compatible transport implementation is used - * instead of gRPC transport. In browser context (if the `window` object is defined) - * the fallback mode is enabled automatically; set `options.fallback` to `false` - * if you need to override this behavior. + * @param {boolean | "rest"} [options.fallback] - Use HTTP fallback mode. + * Pass "rest" to use HTTP/1.1 REST API instead of gRPC. + * For more information, please check the + * {@link https://github.com/googleapis/gax-nodejs/blob/main/client-libraries.md#http11-rest-api-mode documentation}. */ constructor(opts?: ClientOptions) { // Ensure that options include all the required fields. @@ -370,11 +369,13 @@ export class BigQueryReadClient { * Max initial number of streams. If unset or zero, the server will * provide a value of streams so as to produce reasonable throughput. Must be * non-negative. The number of streams may be lower than the requested number, - * depending on the amount parallelism that is reasonable for the table. Error - * will be returned if the max count is greater than the current system - * max limit of 1,000. + * depending on the amount parallelism that is reasonable for the table. + * There is a default system max limit of 1,000. * - * Streams must be read starting from offset 0. + * This must be greater than or equal to preferred_min_stream_count. + * Typically, clients should either leave this unset to let the system to + * determine an upper bound OR set this a size for the maximum "units of work" + * it can gracefully handle. * @param {object} [options] * Call options. See {@link https://googleapis.dev/nodejs/google-gax/latest/interfaces/CallOptions.html|CallOptions} for more details. * @returns {Promise} - The promise which resolves to an array. diff --git a/packages/google-cloud-bigquery-storage/src/v1/big_query_write_client.ts b/packages/google-cloud-bigquery-storage/src/v1/big_query_write_client.ts index 2ab96c5d6e9..1e0569e1f7e 100644 --- a/packages/google-cloud-bigquery-storage/src/v1/big_query_write_client.ts +++ b/packages/google-cloud-bigquery-storage/src/v1/big_query_write_client.ts @@ -73,7 +73,7 @@ export class BigQueryWriteClient { * * @param {object} [options] - The configuration object. * The options accepted by the constructor are described in detail - * in [this document](https://github.com/googleapis/gax-nodejs/blob/master/client-libraries.md#creating-the-client-instance). + * in [this document](https://github.com/googleapis/gax-nodejs/blob/main/client-libraries.md#creating-the-client-instance). * The common options are: * @param {object} [options.credentials] - Credentials object. * @param {string} [options.credentials.client_email] @@ -96,11 +96,10 @@ export class BigQueryWriteClient { * API remote host. * @param {gax.ClientConfig} [options.clientConfig] - Client configuration override. * Follows the structure of {@link gapicConfig}. - * @param {boolean} [options.fallback] - Use HTTP fallback mode. - * In fallback mode, a special browser-compatible transport implementation is used - * instead of gRPC transport. In browser context (if the `window` object is defined) - * the fallback mode is enabled automatically; set `options.fallback` to `false` - * if you need to override this behavior. + * @param {boolean | "rest"} [options.fallback] - Use HTTP fallback mode. + * Pass "rest" to use HTTP/1.1 REST API instead of gRPC. + * For more information, please check the + * {@link https://github.com/googleapis/gax-nodejs/blob/main/client-libraries.md#http11-rest-api-mode documentation}. */ constructor(opts?: ClientOptions) { // Ensure that options include all the required fields. diff --git a/packages/google-cloud-bigquery-storage/src/v1/big_query_write_client_config.json b/packages/google-cloud-bigquery-storage/src/v1/big_query_write_client_config.json index 67eb3165cb6..4b7f4b0657b 100644 --- a/packages/google-cloud-bigquery-storage/src/v1/big_query_write_client_config.json +++ b/packages/google-cloud-bigquery-storage/src/v1/big_query_write_client_config.json @@ -7,6 +7,11 @@ "DEADLINE_EXCEEDED", "UNAVAILABLE" ], + "deadline_exceeded_resource_exhausted_unavailable": [ + "DEADLINE_EXCEEDED", + "RESOURCE_EXHAUSTED", + "UNAVAILABLE" + ], "unavailable": [ "UNAVAILABLE" ] @@ -20,13 +25,22 @@ "rpc_timeout_multiplier": 1, "max_rpc_timeout_millis": 60000, "total_timeout_millis": 600000 + }, + "ec82364a95d03873ac5f61710bb6b9b42e40f31d": { + "initial_retry_delay_millis": 10000, + "retry_delay_multiplier": 1.3, + "max_retry_delay_millis": 120000, + "initial_rpc_timeout_millis": 60000, + "rpc_timeout_multiplier": 1, + "max_rpc_timeout_millis": 60000, + "total_timeout_millis": 600000 } }, "methods": { "CreateWriteStream": { - "timeout_millis": 600000, - "retry_codes_name": "idempotent", - "retry_params_name": "default" + "timeout_millis": 1200000, + "retry_codes_name": "deadline_exceeded_resource_exhausted_unavailable", + "retry_params_name": "ec82364a95d03873ac5f61710bb6b9b42e40f31d" }, "AppendRows": { "timeout_millis": 86400000, diff --git a/packages/google-cloud-bigquery-storage/src/v1beta1/big_query_storage_client.ts b/packages/google-cloud-bigquery-storage/src/v1beta1/big_query_storage_client.ts index 544d6d476ce..e1c27a9e683 100644 --- a/packages/google-cloud-bigquery-storage/src/v1beta1/big_query_storage_client.ts +++ b/packages/google-cloud-bigquery-storage/src/v1beta1/big_query_storage_client.ts @@ -70,7 +70,7 @@ export class BigQueryStorageClient { * * @param {object} [options] - The configuration object. * The options accepted by the constructor are described in detail - * in [this document](https://github.com/googleapis/gax-nodejs/blob/master/client-libraries.md#creating-the-client-instance). + * in [this document](https://github.com/googleapis/gax-nodejs/blob/main/client-libraries.md#creating-the-client-instance). * The common options are: * @param {object} [options.credentials] - Credentials object. * @param {string} [options.credentials.client_email] @@ -93,11 +93,10 @@ export class BigQueryStorageClient { * API remote host. * @param {gax.ClientConfig} [options.clientConfig] - Client configuration override. * Follows the structure of {@link gapicConfig}. - * @param {boolean} [options.fallback] - Use HTTP fallback mode. - * In fallback mode, a special browser-compatible transport implementation is used - * instead of gRPC transport. In browser context (if the `window` object is defined) - * the fallback mode is enabled automatically; set `options.fallback` to `false` - * if you need to override this behavior. + * @param {boolean | "rest"} [options.fallback] - Use HTTP fallback mode. + * Pass "rest" to use HTTP/1.1 REST API instead of gRPC. + * For more information, please check the + * {@link https://github.com/googleapis/gax-nodejs/blob/main/client-libraries.md#http11-rest-api-mode documentation}. */ constructor(opts?: ClientOptions) { // Ensure that options include all the required fields.