From 6c3c397d080750b696024266a114e12b2db8ab04 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Wed, 6 Apr 2022 17:28:56 -0400 Subject: [PATCH] feat: enable channel pooling (#1065) * feat: enable channel pooling * fix unit tests * bump grpc-gcp version and tweak the channel pool config * inline json to avoid compliants from pack-n-play. Also only install the channel pool on data connection. And improve error reporting for pack n play * fixes * typo * style Co-authored-by: Mattie Fu --- package.json | 1 + src/index.ts | 98 +++++++++++++++++++++--------------------- system-test/install.ts | 10 ++++- test/index.ts | 44 ++++++++++++++++--- 4 files changed, 98 insertions(+), 55 deletions(-) diff --git a/package.json b/package.json index 298f6f6e6..46485426e 100644 --- a/package.json +++ b/package.json @@ -54,6 +54,7 @@ "escape-string-regexp": "^4.0.0", "extend": "^3.0.2", "google-gax": "^2.29.5", + "grpc-gcp": "0.4.1", "is": "^3.0.1", "is-utf8": "^0.2.1", "lodash.snakecase": "^4.1.1", diff --git a/src/index.ts b/src/index.ts index 506c072ea..6dc3a892f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -16,7 +16,7 @@ import {replaceProjectIdToken} from '@google-cloud/projectify'; import {promisifyAll} from '@google-cloud/promisify'; import arrify = require('arrify'); import * as extend from 'extend'; -import {GoogleAuth, CallOptions} from 'google-gax'; +import {GoogleAuth, CallOptions, grpc as gaxVendoredGrpc} from 'google-gax'; import * as gax from 'google-gax'; import * as protos from '../protos/protos'; @@ -33,6 +33,7 @@ import {google} from '../protos/protos'; import {ServiceError} from 'google-gax'; import * as v2 from './v2'; import {PassThrough, Duplex} from 'stream'; +import grpcGcpModule = require('grpc-gcp'); // eslint-disable-next-line @typescript-eslint/no-var-requires const streamEvents = require('stream-events'); @@ -42,6 +43,9 @@ const PKG = require('../../package.json'); const {grpc} = new gax.GrpcClient(); +// Enable channel pooling +const grpcGcp = grpcGcpModule(gaxVendoredGrpc); + export interface GetInstancesCallback { ( err: ServiceError | null, @@ -408,17 +412,6 @@ export class Bigtable { } } - options = Object.assign( - { - libName: 'gccl', - libVersion: PKG.version, - scopes, - 'grpc.keepalive_time_ms': 30000, - 'grpc.keepalive_timeout_ms': 10000, - }, - options - ); - const defaultBaseUrl = 'bigtable.googleapis.com'; const defaultAdminBaseUrl = 'bigtableadmin.googleapis.com'; @@ -428,52 +421,61 @@ export class Bigtable { let customEndpointBaseUrl; let customEndpointPort; + let sslCreds; if (customEndpoint) { const customEndpointParts = customEndpoint.split(':'); customEndpointBaseUrl = customEndpointParts[0]; - customEndpointPort = customEndpointParts[1]; + customEndpointPort = Number(customEndpointParts[1]); + sslCreds = grpc.credentials.createInsecure(); } + const baseOptions = Object.assign({ + libName: 'gccl', + libVersion: PKG.version, + port: customEndpointPort || 443, + sslCreds, + scopes, + 'grpc.keepalive_time_ms': 30000, + 'grpc.keepalive_timeout_ms': 10000, + }) as gax.ClientOptions; + + const dataOptions = Object.assign( + {}, + baseOptions, + { + servicePath: customEndpointBaseUrl || defaultBaseUrl, + 'grpc.callInvocationTransformer': grpcGcp.gcpCallInvocationTransformer, + 'grpc.channelFactoryOverride': grpcGcp.gcpChannelFactoryOverride, + 'grpc.gcpApiConfig': grpcGcp.createGcpApiConfig({ + channelPool: { + minSize: 2, + maxSize: 4, + maxConcurrentStreamsLowWatermark: 10, + debugHeaderIntervalSecs: 600, + }, + }), + }, + options + ) as gax.ClientOptions; + + const adminOptions = Object.assign( + {}, + baseOptions, + { + servicePath: customEndpointBaseUrl || defaultAdminBaseUrl, + }, + options + ); + this.options = { - BigtableClient: Object.assign( - { - servicePath: customEndpoint ? customEndpointBaseUrl : defaultBaseUrl, - port: customEndpoint ? Number(customEndpointPort) : 443, - sslCreds: customEndpoint - ? grpc.credentials.createInsecure() - : undefined, - }, - options - ) as gax.ClientOptions, - BigtableInstanceAdminClient: Object.assign( - { - servicePath: customEndpoint - ? customEndpointBaseUrl - : defaultAdminBaseUrl, - port: customEndpoint ? Number(customEndpointPort) : 443, - sslCreds: customEndpoint - ? grpc.credentials.createInsecure() - : undefined, - }, - options - ) as gax.ClientOptions, - BigtableTableAdminClient: Object.assign( - { - servicePath: customEndpoint - ? customEndpointBaseUrl - : defaultAdminBaseUrl, - port: customEndpoint ? Number(customEndpointPort) : 443, - sslCreds: customEndpoint - ? grpc.credentials.createInsecure() - : undefined, - }, - options - ) as gax.ClientOptions, + BigtableClient: dataOptions, + BigtableInstanceAdminClient: adminOptions, + BigtableTableAdminClient: adminOptions, }; this.api = {}; - this.auth = new GoogleAuth(options); + this.auth = new GoogleAuth(Object.assign({}, baseOptions, options)); this.projectId = options.projectId || '{{projectId}}'; this.appProfileId = options.appProfileId; this.projectName = `projects/${this.projectId}`; diff --git a/system-test/install.ts b/system-test/install.ts index 6dd1eaada..5ff2e2ad9 100644 --- a/system-test/install.ts +++ b/system-test/install.ts @@ -46,6 +46,14 @@ describe('📦 pack-n-play test', () => { ).toString(), }, }; - await packNTest(options); + try { + await packNTest(options); + } catch (e) { + // all of the actionable information is on the output attribute + if (e.output) { + e.message += 'output: ' + e.output; + } + throw e; + } }); }); diff --git a/test/index.ts b/test/index.ts index 147a76001..76d28e20b 100644 --- a/test/index.ts +++ b/test/index.ts @@ -190,6 +190,8 @@ describe('Bigtable', () => { assert.deepStrictEqual( options_, Object.assign( + {}, + options_, { libName: 'gccl', libVersion: PKG.version, @@ -233,18 +235,24 @@ describe('Bigtable', () => { assert.deepStrictEqual(bigtable.options, { BigtableClient: Object.assign( + {}, + bigtable.options['BigtableClient'], { servicePath: 'bigtable.googleapis.com', }, expectedOptions ), BigtableInstanceAdminClient: Object.assign( + {}, + bigtable.options['BigtableInstanceAdminClient'], { servicePath: 'bigtableadmin.googleapis.com', }, expectedOptions ), BigtableTableAdminClient: Object.assign( + {}, + bigtable.options['BigtableTableAdminClient'], { servicePath: 'bigtableadmin.googleapis.com', }, @@ -283,9 +291,21 @@ describe('Bigtable', () => { ); assert.deepStrictEqual(bigtable.options, { - BigtableClient: expectedOptions, - BigtableInstanceAdminClient: expectedOptions, - BigtableTableAdminClient: expectedOptions, + BigtableClient: Object.assign( + {}, + bigtable.options['BigtableClient'], + expectedOptions + ), + BigtableInstanceAdminClient: Object.assign( + {}, + bigtable.options['BigtableInstanceAdminClient'], + expectedOptions + ), + BigtableTableAdminClient: Object.assign( + {}, + bigtable.options['BigtableTableAdminClient'], + expectedOptions + ), }); }); @@ -315,9 +335,21 @@ describe('Bigtable', () => { assert.strictEqual(bigtable.customEndpoint, options.apiEndpoint); assert.deepStrictEqual(bigtable.options, { - BigtableClient: expectedOptions, - BigtableInstanceAdminClient: expectedOptions, - BigtableTableAdminClient: expectedOptions, + BigtableClient: Object.assign( + {}, + bigtable.options['BigtableClient'], + expectedOptions + ), + BigtableInstanceAdminClient: Object.assign( + {}, + bigtable.options['BigtableInstanceAdminClient'], + expectedOptions + ), + BigtableTableAdminClient: Object.assign( + {}, + bigtable.options['BigtableTableAdminClient'], + expectedOptions + ), }); });