feat: enable channel pooling (#1065)

* feat: enable channel pooling * fix unit tests * bump grpc-gcp version and tweak the channel pool config * inline json to avoid compliants from pack-n-play. Also only install the channel pool on data connection. And improve error reporting for pack n play * fixes * typo * style Co-authored-by: Mattie Fu <[email protected]>
googleapis · Apr 6, 2022 · 6c3c397 · 6c3c397
1 parent 9951e75
commit 6c3c397
Show file tree

Hide file tree

Showing 4 changed files with 98 additions and 55 deletions.
diff --git a/package.json b/package.json
@@ -54,6 +54,7 @@
     "escape-string-regexp": "^4.0.0",
     "extend": "^3.0.2",
     "google-gax": "^2.29.5",
+    "grpc-gcp": "0.4.1",
     "is": "^3.0.1",
     "is-utf8": "^0.2.1",
     "lodash.snakecase": "^4.1.1",

diff --git a/src/index.ts b/src/index.ts
@@ -16,7 +16,7 @@ import {replaceProjectIdToken} from '@google-cloud/projectify';
 import {promisifyAll} from '@google-cloud/promisify';
 import arrify = require('arrify');
 import * as extend from 'extend';
-import {GoogleAuth, CallOptions} from 'google-gax';
+import {GoogleAuth, CallOptions, grpc as gaxVendoredGrpc} from 'google-gax';
 import * as gax from 'google-gax';
 import * as protos from '../protos/protos';
 
@@ -33,6 +33,7 @@ import {google} from '../protos/protos';
 import {ServiceError} from 'google-gax';
 import * as v2 from './v2';
 import {PassThrough, Duplex} from 'stream';
+import grpcGcpModule = require('grpc-gcp');
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 const streamEvents = require('stream-events');
@@ -42,6 +43,9 @@ const PKG = require('../../package.json');
 
 const {grpc} = new gax.GrpcClient();
 
+// Enable channel pooling
+const grpcGcp = grpcGcpModule(gaxVendoredGrpc);
+
 export interface GetInstancesCallback {
   (
     err: ServiceError | null,
@@ -408,17 +412,6 @@ export class Bigtable {
       }
     }
 
-    options = Object.assign(
-      {
-        libName: 'gccl',
-        libVersion: PKG.version,
-        scopes,
-        'grpc.keepalive_time_ms': 30000,
-        'grpc.keepalive_timeout_ms': 10000,
-      },
-      options
-    );
-
     const defaultBaseUrl = 'bigtable.googleapis.com';
     const defaultAdminBaseUrl = 'bigtableadmin.googleapis.com';
 
@@ -428,52 +421,61 @@ export class Bigtable {
 
     let customEndpointBaseUrl;
     let customEndpointPort;
+    let sslCreds;
 
     if (customEndpoint) {
       const customEndpointParts = customEndpoint.split(':');
       customEndpointBaseUrl = customEndpointParts[0];
-      customEndpointPort = customEndpointParts[1];
+      customEndpointPort = Number(customEndpointParts[1]);
+      sslCreds = grpc.credentials.createInsecure();
     }
 
+    const baseOptions = Object.assign({
+      libName: 'gccl',
+      libVersion: PKG.version,
+      port: customEndpointPort || 443,
+      sslCreds,
+      scopes,
+      'grpc.keepalive_time_ms': 30000,
+      'grpc.keepalive_timeout_ms': 10000,
+    }) as gax.ClientOptions;
+
+    const dataOptions = Object.assign(
+      {},
+      baseOptions,
+      {
+        servicePath: customEndpointBaseUrl || defaultBaseUrl,
+        'grpc.callInvocationTransformer': grpcGcp.gcpCallInvocationTransformer,
+        'grpc.channelFactoryOverride': grpcGcp.gcpChannelFactoryOverride,
+        'grpc.gcpApiConfig': grpcGcp.createGcpApiConfig({
+          channelPool: {
+            minSize: 2,
+            maxSize: 4,
+            maxConcurrentStreamsLowWatermark: 10,
+            debugHeaderIntervalSecs: 600,
+          },
+        }),
+      },
+      options
+    ) as gax.ClientOptions;
+
+    const adminOptions = Object.assign(
+      {},
+      baseOptions,
+      {
+        servicePath: customEndpointBaseUrl || defaultAdminBaseUrl,
+      },
+      options
+    );
+
     this.options = {
-      BigtableClient: Object.assign(
-        {
-          servicePath: customEndpoint ? customEndpointBaseUrl : defaultBaseUrl,
-          port: customEndpoint ? Number(customEndpointPort) : 443,
-          sslCreds: customEndpoint
-            ? grpc.credentials.createInsecure()
-            : undefined,
-        },
-        options
-      ) as gax.ClientOptions,
-      BigtableInstanceAdminClient: Object.assign(
-        {
-          servicePath: customEndpoint
-            ? customEndpointBaseUrl
-            : defaultAdminBaseUrl,
-          port: customEndpoint ? Number(customEndpointPort) : 443,
-          sslCreds: customEndpoint
-            ? grpc.credentials.createInsecure()
-            : undefined,
-        },
-        options
-      ) as gax.ClientOptions,
-      BigtableTableAdminClient: Object.assign(
-        {
-          servicePath: customEndpoint
-            ? customEndpointBaseUrl
-            : defaultAdminBaseUrl,
-          port: customEndpoint ? Number(customEndpointPort) : 443,
-          sslCreds: customEndpoint
-            ? grpc.credentials.createInsecure()
-            : undefined,
-        },
-        options
-      ) as gax.ClientOptions,
+      BigtableClient: dataOptions,
+      BigtableInstanceAdminClient: adminOptions,
+      BigtableTableAdminClient: adminOptions,
     };
 
     this.api = {};
-    this.auth = new GoogleAuth(options);
+    this.auth = new GoogleAuth(Object.assign({}, baseOptions, options));
     this.projectId = options.projectId || '{{projectId}}';
     this.appProfileId = options.appProfileId;
     this.projectName = `projects/${this.projectId}`;

diff --git a/system-test/install.ts b/system-test/install.ts
@@ -46,6 +46,14 @@ describe('📦 pack-n-play test', () => {
         ).toString(),
       },
     };
-    await packNTest(options);
+    try {
+      await packNTest(options);
+    } catch (e) {
+      // all of the actionable information is on the output attribute
+      if (e.output) {
+        e.message += 'output: ' + e.output;
+      }
+      throw e;
+    }
   });
 });
diff --git a/test/index.ts b/test/index.ts
@@ -190,6 +190,8 @@ describe('Bigtable', () => {
         assert.deepStrictEqual(
           options_,
           Object.assign(
+            {},
+            options_,
             {
               libName: 'gccl',
               libVersion: PKG.version,
@@ -233,18 +235,24 @@ describe('Bigtable', () => {
 
       assert.deepStrictEqual(bigtable.options, {
         BigtableClient: Object.assign(
+          {},
+          bigtable.options['BigtableClient'],
           {
             servicePath: 'bigtable.googleapis.com',
           },
           expectedOptions
         ),
         BigtableInstanceAdminClient: Object.assign(
+          {},
+          bigtable.options['BigtableInstanceAdminClient'],
           {
             servicePath: 'bigtableadmin.googleapis.com',
           },
           expectedOptions
         ),
         BigtableTableAdminClient: Object.assign(
+          {},
+          bigtable.options['BigtableTableAdminClient'],
           {
             servicePath: 'bigtableadmin.googleapis.com',
           },
@@ -283,9 +291,21 @@ describe('Bigtable', () => {
       );
 
       assert.deepStrictEqual(bigtable.options, {
-        BigtableClient: expectedOptions,
-        BigtableInstanceAdminClient: expectedOptions,
-        BigtableTableAdminClient: expectedOptions,
+        BigtableClient: Object.assign(
+          {},
+          bigtable.options['BigtableClient'],
+          expectedOptions
+        ),
+        BigtableInstanceAdminClient: Object.assign(
+          {},
+          bigtable.options['BigtableInstanceAdminClient'],
+          expectedOptions
+        ),
+        BigtableTableAdminClient: Object.assign(
+          {},
+          bigtable.options['BigtableTableAdminClient'],
+          expectedOptions
+        ),
       });
     });
 
@@ -315,9 +335,21 @@ describe('Bigtable', () => {
       assert.strictEqual(bigtable.customEndpoint, options.apiEndpoint);
 
       assert.deepStrictEqual(bigtable.options, {
-        BigtableClient: expectedOptions,
-        BigtableInstanceAdminClient: expectedOptions,
-        BigtableTableAdminClient: expectedOptions,
+        BigtableClient: Object.assign(
+          {},
+          bigtable.options['BigtableClient'],
+          expectedOptions
+        ),
+        BigtableInstanceAdminClient: Object.assign(
+          {},
+          bigtable.options['BigtableInstanceAdminClient'],
+          expectedOptions
+        ),
+        BigtableTableAdminClient: Object.assign(
+          {},
+          bigtable.options['BigtableTableAdminClient'],
+          expectedOptions
+        ),
       });
     });