From e5f6fdb5345a19055be812c59796a2e08ea45345 Mon Sep 17 00:00:00 2001 From: danieljbruce Date: Tue, 31 May 2022 11:57:08 -0400 Subject: [PATCH] feat: Autoscaler (#1077) * work in progress - broken * Add autoscaling options to interface * Work in progress on function for setting metadata * Fixed some tests that were broken due to the chang * Cluster.ts in test file * Update cluster creation to accept new parameters * Fix for failing test * Enhance system tests for cluster * Add another test * condense tests * Added more tests * Test improvements * Change update mask to get manual scaling right * Added validation for cluster creation configs * Refactor cluster object in tests * Another cluster id refactor * Remove TODO that no longer applies * Code reorganization for test cases * Fix the test so that it breaks for good reason * update mask push fix * Add nodes in all calls to update and create cluste * Add license headers * correct copyright year * Add singlequote * Add a validation * Update test descriptions * refactor to use one check metadata function * Fix tests as specifying nodes is required anyway * should added to test case descriptions * PR updates * validation error * Remove call to get metadata * PR updates --- src/cluster.ts | 31 ++-- src/index.ts | 14 +- src/instance.ts | 22 ++- src/utils/cluster.ts | 153 +++++++++++++++++++ system-test/cluster.ts | 339 +++++++++++++++++++++++++++++++++++++++++ test/cluster.ts | 35 +++-- test/index.ts | 2 + test/instance.ts | 20 ++- test/utils/cluster.ts | 101 ++++++++++++ 9 files changed, 667 insertions(+), 50 deletions(-) create mode 100644 src/utils/cluster.ts create mode 100644 system-test/cluster.ts create mode 100644 test/utils/cluster.ts diff --git a/src/cluster.ts b/src/cluster.ts index 9235bf7e8..2518e9aae 100644 --- a/src/cluster.ts +++ b/src/cluster.ts @@ -22,6 +22,7 @@ const pumpify = require('pumpify'); import {google} from '../protos/protos'; import {Bigtable} from '.'; import {Instance} from './instance'; +import {ClusterUtils} from './utils/cluster'; import { Backup, @@ -75,7 +76,10 @@ export type GetClustersCallback = ( apiResponse?: google.bigtable.admin.v2.IListClustersResponse ) => void; export interface SetClusterMetadataOptions { - nodes: number; + nodes?: number; + minServeNodes?: number; + maxServeNodes?: number; + cpuUtilizationPercent?: number; } export type SetClusterMetadataCallback = GenericOperationCallback< Operation | null | undefined @@ -84,8 +88,11 @@ export interface BasicClusterConfig { encryption?: google.bigtable.admin.v2.Cluster.IEncryptionConfig; key?: string; location: string; - nodes: number; + nodes?: number; storage?: string; + minServeNodes?: number; + maxServeNodes?: number; + cpuUtilizationPercent?: number; } export interface CreateBackupConfig extends ModifiableBackupFields { @@ -690,29 +697,23 @@ Please use the format 'my-cluster' or '${instance.name}/clusters/my-cluster'.`); gaxOptionsOrCallback?: CallOptions | SetClusterMetadataCallback, cb?: SetClusterMetadataCallback ): void | Promise { + ClusterUtils.validateClusterMetadata(metadata); const callback = typeof gaxOptionsOrCallback === 'function' ? gaxOptionsOrCallback : cb!; const gaxOptions = typeof gaxOptionsOrCallback === 'object' ? gaxOptionsOrCallback : ({} as CallOptions); - - const reqOpts: ICluster = Object.assign( - {}, - { - name: this.name, - serveNodes: metadata.nodes, - }, - metadata + const reqOpts = ClusterUtils.getRequestFromMetadata( + metadata, + this?.metadata?.location, + this.name ); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - delete (reqOpts as any).nodes; - this.bigtable.request( { client: 'BigtableInstanceAdminClient', - method: 'updateCluster', - reqOpts, + method: 'partialUpdateCluster', + reqOpts: reqOpts, gaxOpts: gaxOptions, }, (err, resp) => { diff --git a/src/index.ts b/src/index.ts index 55a04bf2c..5d69a3429 100644 --- a/src/index.ts +++ b/src/index.ts @@ -34,6 +34,7 @@ import {ServiceError} from 'google-gax'; import * as v2 from './v2'; import {PassThrough, Duplex} from 'stream'; import grpcGcpModule = require('grpc-gcp'); +import {ClusterUtils} from './utils/cluster'; // eslint-disable-next-line @typescript-eslint/no-var-requires const streamEvents = require('stream-events'); @@ -610,12 +611,15 @@ export class Bigtable { 'A cluster was provided with both `encryption` and `key` defined.' ); } - - clusters[cluster.id!] = { - location: Cluster.getLocation_(this.projectId, cluster.location!), - serveNodes: cluster.nodes, + ClusterUtils.validateClusterMetadata(cluster); + clusters[cluster.id!] = ClusterUtils.getClusterBaseConfig( + cluster, + Cluster.getLocation_(this.projectId, cluster.location!), + undefined + ); + Object.assign(clusters[cluster.id!], { defaultStorageType: Cluster.getStorageType_(cluster.storage!), - }; + }); if (cluster.key) { clusters[cluster.id!].encryptionConfig = { diff --git a/src/instance.ts b/src/instance.ts index c54c59790..9c51dac63 100644 --- a/src/instance.ts +++ b/src/instance.ts @@ -67,6 +67,7 @@ import {ServiceError} from 'google-gax'; import {Bigtable} from '.'; import {google} from '../protos/protos'; import {Backup, RestoreTableCallback, RestoreTableResponse} from './backup'; +import {ClusterUtils} from './utils/cluster'; export interface ClusterInfo extends BasicClusterConfig { id: string; @@ -391,9 +392,15 @@ Please use the format 'my-instance' or '${bigtable.projectName}/instances/my-ins parent: this.name, clusterId: id, } as google.bigtable.admin.v2.CreateClusterRequest; - + ClusterUtils.validateClusterMetadata(options); if (!is.empty(options)) { - reqOpts.cluster = {}; + reqOpts.cluster = ClusterUtils.getClusterBaseConfig( + options, + options.location + ? Cluster.getLocation_(this.bigtable.projectId, options.location) + : undefined, + undefined + ); } if ( @@ -415,17 +422,6 @@ Please use the format 'my-instance' or '${bigtable.projectName}/instances/my-ins reqOpts.cluster!.encryptionConfig = options.encryption; } - if (options.location) { - reqOpts.cluster!.location = Cluster.getLocation_( - this.bigtable.projectId, - options.location - ); - } - - if (options.nodes) { - reqOpts.cluster!.serveNodes = options.nodes; - } - if (options.storage) { const storageType = Cluster.getStorageType_(options.storage); reqOpts.cluster!.defaultStorageType = storageType; diff --git a/src/utils/cluster.ts b/src/utils/cluster.ts new file mode 100644 index 000000000..e8c5697f6 --- /dev/null +++ b/src/utils/cluster.ts @@ -0,0 +1,153 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import * as protos from '../../protos/protos'; +import { + BasicClusterConfig, + ICluster, + SetClusterMetadataOptions, +} from '../cluster'; +import {google} from '../../protos/protos'; + +export class ClusterUtils { + static noConfigError = + 'Must specify either serve_nodes or all of the autoscaling configurations (min_serve_nodes, max_serve_nodes, and cpu_utilization_percent).'; + static allConfigError = + 'Cannot specify both serve_nodes and autoscaling configurations (min_serve_nodes, max_serve_nodes, and cpu_utilization_percent).'; + static incompleteConfigError = + 'All of autoscaling configurations must be specified at the same time (min_serve_nodes, max_serve_nodes, and cpu_utilization_percent).'; + + static validateClusterMetadata( + metadata: SetClusterMetadataOptions | BasicClusterConfig + ): void { + if (metadata.nodes) { + if ( + metadata.minServeNodes || + metadata.maxServeNodes || + metadata.cpuUtilizationPercent + ) { + throw new Error(this.allConfigError); + } + } else { + if ( + metadata.minServeNodes || + metadata.maxServeNodes || + metadata.cpuUtilizationPercent + ) { + if ( + !( + metadata.minServeNodes && + metadata.maxServeNodes && + metadata.cpuUtilizationPercent + ) + ) { + throw new Error(this.incompleteConfigError); + } + } else { + throw new Error(this.noConfigError); + } + } + } + static getUpdateMask(metadata: SetClusterMetadataOptions): string[] { + const updateMask: string[] = []; + if (metadata.nodes) { + updateMask.push('serve_nodes'); + if ( + !( + metadata.minServeNodes || + metadata.maxServeNodes || + metadata.cpuUtilizationPercent + ) + ) { + updateMask.push('cluster_config.cluster_autoscaling_config'); + } + } + if (metadata.minServeNodes) { + updateMask.push( + 'cluster_config.cluster_autoscaling_config.autoscaling_limits.min_serve_nodes' + ); + } + if (metadata.maxServeNodes) { + updateMask.push( + 'cluster_config.cluster_autoscaling_config.autoscaling_limits.max_serve_nodes' + ); + } + if (metadata.cpuUtilizationPercent) { + updateMask.push( + 'cluster_config.cluster_autoscaling_config.autoscaling_targets.cpu_utilization_percent' + ); + } + return updateMask; + } + + static getClusterBaseConfig( + metadata: SetClusterMetadataOptions | BasicClusterConfig, + location: string | undefined | null, + name: string | undefined + ): google.bigtable.admin.v2.ICluster { + let clusterConfig; + if ( + metadata.cpuUtilizationPercent || + metadata.minServeNodes || + metadata.maxServeNodes + ) { + clusterConfig = { + clusterAutoscalingConfig: { + autoscalingTargets: { + cpuUtilizationPercent: metadata.cpuUtilizationPercent, + }, + autoscalingLimits: { + minServeNodes: metadata.minServeNodes, + maxServeNodes: metadata.maxServeNodes, + }, + }, + }; + } + return Object.assign( + {}, + name ? {name} : null, + location ? {location} : null, + clusterConfig ? {clusterConfig} : null, + metadata.nodes ? {serveNodes: metadata.nodes} : null + ); + } + + static getClusterFromMetadata( + metadata: SetClusterMetadataOptions, + location: string | undefined | null, + name: string + ): google.bigtable.admin.v2.ICluster { + const cluster: ICluster | SetClusterMetadataOptions = Object.assign( + {}, + this.getClusterBaseConfig(metadata, location, name), + metadata + ); + delete (cluster as SetClusterMetadataOptions).nodes; + delete (cluster as SetClusterMetadataOptions).minServeNodes; + delete (cluster as SetClusterMetadataOptions).maxServeNodes; + delete (cluster as SetClusterMetadataOptions).cpuUtilizationPercent; + return cluster as ICluster; + } + + static getRequestFromMetadata( + metadata: SetClusterMetadataOptions, + location: string | undefined | null, + name: string + ): protos.google.bigtable.admin.v2.IPartialUpdateClusterRequest { + return { + cluster: this.getClusterFromMetadata(metadata, location, name), + updateMask: {paths: this.getUpdateMask(metadata)}, + }; + } +} diff --git a/system-test/cluster.ts b/system-test/cluster.ts new file mode 100644 index 000000000..f365d3610 --- /dev/null +++ b/system-test/cluster.ts @@ -0,0 +1,339 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {afterEach, beforeEach, describe, it} from 'mocha'; +import {generateId} from './common'; +import {Bigtable, ClusterInfo, Instance, Cluster} from '../src'; +import assert = require('assert'); +import {ClusterUtils} from '../src/utils/cluster'; +import {SetClusterMetadataOptions} from '../src/cluster'; + +export interface ValidationError { + message: string; +} + +function isValidationError(err: any): err is ValidationError { + return (err as ValidationError).message !== undefined; +} + +describe('Cluster', () => { + const bigtable = new Bigtable(); + let instance: Instance; + + async function checkMetadata( + cluster: Cluster, + compareValues: SetClusterMetadataOptions, + isConfigDefined: boolean + ): Promise { + // const cluster: Cluster = instance.cluster(clusterId); + const metadata = await cluster.getMetadata({}); + const {clusterConfig, serveNodes} = metadata[0]; + assert.strictEqual(serveNodes, compareValues.nodes); + if (clusterConfig) { + assert.equal(isConfigDefined, true); + assert.deepStrictEqual(clusterConfig, { + clusterAutoscalingConfig: { + autoscalingLimits: { + minServeNodes: compareValues.minServeNodes, + maxServeNodes: compareValues.maxServeNodes, + }, + autoscalingTargets: { + cpuUtilizationPercent: compareValues.cpuUtilizationPercent, + }, + }, + }); + } else { + assert.equal(isConfigDefined, false); + } + } + + async function createNewInstance(clusters: ClusterInfo[]): Promise { + const instanceId: string = generateId('instance'); + instance = bigtable.instance(instanceId); + const [, operation] = await instance.create({ + clusters, + labels: { + time_created: Date.now(), + }, + }); + await operation.promise(); + } + async function createStandardNewInstance( + clusterId: string, + nodes: number + ): Promise { + return await createNewInstance(standardCreationClusters(clusterId, nodes)); + } + function standardCreationClusters( + clusterId: string, + nodes: number + ): ClusterInfo[] { + return [ + { + id: clusterId, + location: 'us-east1-c', + nodes, + }, + ]; + } + afterEach(async () => { + await instance.delete(); + }); + describe('Create cluster', () => { + describe('With manual scaling', () => { + let clusterId: string; + let cluster: Cluster; + beforeEach(async () => { + clusterId = generateId('cluster'); + await createStandardNewInstance(clusterId, 2); + cluster = instance.cluster(clusterId); + }); + it('should create an instance with clusters for manual scaling', async () => { + await checkMetadata(cluster, {nodes: 2}, false); + }); + it('should create an instance and then create a cluster for manual scaling', async () => { + const clusterId2: string = generateId('cluster'); + const cluster2 = instance.cluster(clusterId2); + await cluster2.create({ + location: 'us-west1-c', + nodes: 3, + }); + await checkMetadata(cluster2, {nodes: 3}, false); + }); + describe('Using an incorrect configuration', () => { + let cluster2: Cluster; + beforeEach(async () => { + const clusterId2: string = generateId('cluster'); + cluster2 = instance.cluster(clusterId2); + }); + it('should throw an error when providing no cluster configuration', async () => { + try { + await cluster2.create({ + location: 'us-west1-c', + }); + assert.fail(); + } catch (e) { + assert.ok(isValidationError(e)); + assert.equal(e.message, ClusterUtils.noConfigError); + } + }); + it('should throw an error when providing manual and autoscaling configurations', async () => { + try { + await cluster2.create({ + location: 'us-west1-c', + nodes: 2, + minServeNodes: 3, + }); + assert.fail(); + } catch (e) { + assert.ok(isValidationError(e)); + assert.equal(e.message, ClusterUtils.allConfigError); + } + }); + it('should throw an error when missing all autoscaling configurations', async () => { + try { + await cluster2.create({ + location: 'us-west1-c', + minServeNodes: 3, + cpuUtilizationPercent: 51, + }); + assert.fail(); + } catch (e) { + assert.ok(isValidationError(e)); + assert.equal(e.message, ClusterUtils.incompleteConfigError); + } + }); + }); + }); + describe('With automatic scaling', () => { + const minServeNodes = 2; + const maxServeNodes = 4; + const cpuUtilizationPercent = 50; + + const createClusterOptions = { + location: 'us-west1-c', + minServeNodes, + maxServeNodes, + cpuUtilizationPercent, + }; + it('should create an instance with clusters for automatic scaling', async () => { + const clusterId = generateId('cluster'); + await createNewInstance([ + Object.assign({id: clusterId}, createClusterOptions), + ]); + const cluster: Cluster = instance.cluster(clusterId); + await checkMetadata( + cluster, + { + ...createClusterOptions, + nodes: minServeNodes, + }, + true + ); + }); + it('should create an instance and then create clusters for automatic scaling', async () => { + const clusterId: string = generateId('cluster'); + await createStandardNewInstance(clusterId, 2); + const clusterId2: string = generateId('cluster'); + const cluster: Cluster = instance.cluster(clusterId2); + await cluster.create(createClusterOptions); + await checkMetadata( + cluster, + { + ...createClusterOptions, + nodes: minServeNodes, + }, + true + ); + }); + }); + }); + describe('Update cluster', () => { + describe('Updating manual scaling for a cluster', () => { + let cluster: Cluster; + const startingNodes = 2; + + beforeEach(async () => { + const clusterId = generateId('cluster'); + await createStandardNewInstance(clusterId, startingNodes); + cluster = instance.cluster(clusterId); + }); + + it('should change nodes for manual scaling', async () => { + const updateNodes = 5; + await cluster.setMetadata({nodes: updateNodes}); + await checkMetadata( + cluster, + { + nodes: updateNodes, + }, + false + ); + }); + it('should change cluster to autoscaling', async () => { + const minServeNodes = 3; + const maxServeNodes = 4; + const cpuUtilizationPercent = 50; + await cluster.setMetadata({ + minServeNodes, + maxServeNodes, + cpuUtilizationPercent, + }); + await checkMetadata( + cluster, + { + nodes: startingNodes, + minServeNodes, + maxServeNodes, + cpuUtilizationPercent, + }, + true + ); + }); + describe('Using an incorrect configuration', () => { + it('should throw an error when providing no cluster configuration', async () => { + try { + await cluster.setMetadata({}); + assert.fail(); + } catch (e) { + assert.ok(isValidationError(e)); + assert.equal(e.message, ClusterUtils.noConfigError); + } + }); + it('should throw an error when providing manual and autoscaling configurations', async () => { + try { + await cluster.setMetadata({ + nodes: 2, + minServeNodes: 3, + }); + assert.fail(); + } catch (e) { + assert.ok(isValidationError(e)); + assert.equal(e.message, ClusterUtils.allConfigError); + } + }); + it('should throw an error when missing some autoscaling configurations', async () => { + try { + await cluster.setMetadata({ + minServeNodes: 3, + cpuUtilizationPercent: 51, + }); + assert.fail(); + } catch (e) { + assert.ok(isValidationError(e)); + assert.equal(e.message, ClusterUtils.incompleteConfigError); + } + }); + }); + }); + describe('Starting from autoscaling', () => { + let cluster: Cluster; + + const minServeNodes = 3; + const maxServeNodes = 4; + const cpuUtilizationPercent = 50; + const createClusterOptions = { + location: 'us-west1-c', + minServeNodes, + maxServeNodes, + cpuUtilizationPercent, + }; + + beforeEach(async () => { + const clusterId = generateId('cluster'); + await createNewInstance([ + Object.assign({id: clusterId}, createClusterOptions), + ]); + cluster = instance.cluster(clusterId); + }); + + it('should change cluster to manual scaling', async () => { + const updateNodes = 5; + await cluster.setMetadata({ + nodes: updateNodes, + }); + await checkMetadata( + cluster, + { + nodes: updateNodes, + }, + false + ); + }); + it('should change autoscaling properties', async () => { + const newMinServeNodes = 5; + const newMaxServeNodes = 6; + const newCpuUtilizationPercent = 53; + assert.notEqual(minServeNodes, newMinServeNodes); + assert.notEqual(maxServeNodes, newMaxServeNodes); + assert.notEqual(cpuUtilizationPercent, newCpuUtilizationPercent); + await cluster.setMetadata({ + minServeNodes: newMinServeNodes, + maxServeNodes: newMaxServeNodes, + cpuUtilizationPercent: newCpuUtilizationPercent, + }); + await checkMetadata( + cluster, + { + nodes: minServeNodes, + minServeNodes: newMinServeNodes, + maxServeNodes: newMaxServeNodes, + cpuUtilizationPercent: newCpuUtilizationPercent, + }, + true + ); + }); + }); + }); +}); diff --git a/test/cluster.ts b/test/cluster.ts index ebd96061d..240412b84 100644 --- a/test/cluster.ts +++ b/test/cluster.ts @@ -19,6 +19,7 @@ import * as proxyquire from 'proxyquire'; import {PassThrough, Readable} from 'stream'; import {CallOptions} from 'google-gax'; import {PreciseDate} from '@google-cloud/precise-date'; +import {ClusterUtils} from '../src/utils/cluster'; export interface Options { nodes?: Number; @@ -961,16 +962,23 @@ describe('Bigtable/Cluster', () => { }); describe('setMetadata', () => { + beforeEach(() => { + const metadata = { + location: 'projects/{{projectId}}/locations/us-east4-b', + }; + cluster.metadata = metadata; + }); + it('should provide the proper request options', done => { // eslint-disable-next-line @typescript-eslint/no-explicit-any cluster.bigtable.request = (config: any, callback: Function) => { assert.strictEqual(config.client, 'BigtableInstanceAdminClient'); - assert.strictEqual(config.method, 'updateCluster'); - assert.strictEqual(config.reqOpts.name, CLUSTER_NAME); + assert.strictEqual(config.method, 'partialUpdateCluster'); + assert.strictEqual(config.reqOpts.cluster.name, CLUSTER_NAME); callback(); // done() }; - cluster.setMetadata({}, done); + cluster.setMetadata({nodes: 2}, done); }); it('should respect the nodes option', done => { @@ -980,7 +988,7 @@ describe('Bigtable/Cluster', () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any cluster.bigtable.request = (config: any) => { - assert.strictEqual(config.reqOpts.serveNodes, options.nodes); + assert.strictEqual(config.reqOpts.cluster.serveNodes, options.nodes); done(); }; @@ -994,12 +1002,11 @@ describe('Bigtable/Cluster', () => { defaultStorageType: 'exellent_type', }; - const expectedReqOpts: Options = Object.assign( - {}, - {name: CLUSTER_NAME, serveNodes: options.nodes}, - options + const expectedReqOpts = ClusterUtils.getRequestFromMetadata( + options, + options.location, + CLUSTER_NAME ); - delete expectedReqOpts.nodes; // eslint-disable-next-line @typescript-eslint/no-explicit-any cluster.bigtable.request = (config: any) => { @@ -1018,7 +1025,7 @@ describe('Bigtable/Cluster', () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any cluster.bigtable.request = (config: any) => { - assert.strictEqual(config.reqOpts.serveNodes, options.nodes); + assert.strictEqual(config.reqOpts.cluster.serveNodes, options.nodes); assert.strictEqual(config.gaxOpts, gaxOptions); done(); }; @@ -1026,14 +1033,16 @@ describe('Bigtable/Cluster', () => { cluster.setMetadata(options, gaxOptions, assert.ifError); }); + // eslint-disable-next-line no-restricted-properties it('should execute callback with all arguments', done => { const args = [{}, {}]; - cluster.bigtable.request = (config: {}, callback: Function) => { callback(...args); }; - - cluster.setMetadata({}, (...argsies: Array<{}>) => { + const name = + 'projects/{{projectId}}/instances/fake-instance/clusters/fake-cluster'; + cluster.name = name; + cluster.setMetadata({nodes: 2}, (...argsies: Array<{}>) => { assert.deepStrictEqual([].slice.call(argsies), args); done(); }); diff --git a/test/index.ts b/test/index.ts index 76d28e20b..1e42c4660 100644 --- a/test/index.ts +++ b/test/index.ts @@ -515,6 +515,7 @@ describe('Bigtable', () => { { id: 'my-cluster', key, + nodes: 3, }, ], }, @@ -548,6 +549,7 @@ describe('Bigtable', () => { encryption: { kmsKeyName: key, }, + nodes: 2, }, ], }, diff --git a/test/instance.ts b/test/instance.ts index 163cf2706..6f578c0a1 100644 --- a/test/instance.ts +++ b/test/instance.ts @@ -357,11 +357,16 @@ describe('Bigtable/Instance', () => { assert.strictEqual(config.gaxOpts, undefined); done(); }; - instance.createCluster(CLUSTER_ID, assert.ifError); + instance.createCluster( + CLUSTER_ID, + {nodes: 2, location: 'us-central1-b'}, + assert.ifError + ); }); it('should accept gaxOptions', done => { const options = { + nodes: 2, gaxOptions: {}, } as CreateClusterOptions; // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -375,6 +380,7 @@ describe('Bigtable/Instance', () => { it('should respect the location option', done => { const options = { location: 'us-central1-b', + nodes: 2, } as CreateClusterOptions; const fakeLocation = 'a/b/c/d'; sandbox @@ -408,6 +414,7 @@ describe('Bigtable/Instance', () => { it('should respect the storage option', done => { const options = { storage: 'ssd', + nodes: 2, } as CreateClusterOptions; const fakeStorageType = 2; sandbox.stub(FakeCluster, 'getStorageType_').callsFake(type => { @@ -438,7 +445,7 @@ describe('Bigtable/Instance', () => { instance.createCluster( CLUSTER_ID, - {key} as CreateClusterOptions, + {key, nodes: 2} as CreateClusterOptions, assert.ifError ); }); @@ -456,7 +463,7 @@ describe('Bigtable/Instance', () => { instance.createCluster( CLUSTER_ID, - {encryption: {kmsKeyName: key}} as CreateClusterOptions, + {encryption: {kmsKeyName: key}, nodes: 2} as CreateClusterOptions, assert.ifError ); }); @@ -467,7 +474,11 @@ describe('Bigtable/Instance', () => { assert.throws(() => { instance.createCluster( CLUSTER_ID, - {encryption: {kmsKeyName: key}, key} as CreateClusterOptions, + { + encryption: {kmsKeyName: key}, + key, + nodes: 2, + } as CreateClusterOptions, assert.ifError ); }, /The cluster cannot have both `encryption` and `key` defined\./); @@ -485,6 +496,7 @@ describe('Bigtable/Instance', () => { }; (instance.createCluster as Function)( CLUSTER_ID, + {nodes: 2}, (err: Error, cluster: {}, apiResponse: {}) => { assert.ifError(err); assert.strictEqual(cluster, fakeCluster); diff --git a/test/utils/cluster.ts b/test/utils/cluster.ts new file mode 100644 index 000000000..ea6390a49 --- /dev/null +++ b/test/utils/cluster.ts @@ -0,0 +1,101 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {describe, it} from 'mocha'; +import {ClusterUtils} from '../../src/utils/cluster'; +import * as assert from 'assert'; + +describe('Bigtable/Utils/Cluster', () => { + describe('getRequestFromMetadata', () => { + it('should translate metadata for a full set of parameters', () => { + const metadata = { + nodes: 1, + minServeNodes: 2, + maxServeNodes: 3, + cpuUtilizationPercent: 50, + }; + const name = 'cluster1'; + const location = 'projects/{{projectId}}/locations/us-east4-b'; + const reqOpts = ClusterUtils.getRequestFromMetadata( + metadata, + location, + name + ); + assert.deepStrictEqual(reqOpts, { + cluster: { + name: name, + location: location, + serveNodes: metadata.nodes, + clusterConfig: { + clusterAutoscalingConfig: { + autoscalingLimits: { + minServeNodes: metadata.minServeNodes, + maxServeNodes: metadata.maxServeNodes, + }, + autoscalingTargets: { + cpuUtilizationPercent: metadata.cpuUtilizationPercent, + }, + }, + }, + }, + updateMask: { + paths: [ + 'serve_nodes', + 'cluster_config.cluster_autoscaling_config.autoscaling_limits.min_serve_nodes', + 'cluster_config.cluster_autoscaling_config.autoscaling_limits.max_serve_nodes', + 'cluster_config.cluster_autoscaling_config.autoscaling_targets.cpu_utilization_percent', + ], + }, + }); + }); + it('should translate metadata for autoscaling parameters', () => { + const metadata = { + minServeNodes: 2, + maxServeNodes: 3, + cpuUtilizationPercent: 50, + }; + const name = 'cluster1'; + const location = 'projects/{{projectId}}/locations/us-east4-b'; + const reqOpts = ClusterUtils.getRequestFromMetadata( + metadata, + location, + name + ); + assert.deepStrictEqual(reqOpts, { + cluster: { + name: name, + location: location, + clusterConfig: { + clusterAutoscalingConfig: { + autoscalingLimits: { + minServeNodes: metadata.minServeNodes, + maxServeNodes: metadata.maxServeNodes, + }, + autoscalingTargets: { + cpuUtilizationPercent: metadata.cpuUtilizationPercent, + }, + }, + }, + }, + updateMask: { + paths: [ + 'cluster_config.cluster_autoscaling_config.autoscaling_limits.min_serve_nodes', + 'cluster_config.cluster_autoscaling_config.autoscaling_limits.max_serve_nodes', + 'cluster_config.cluster_autoscaling_config.autoscaling_targets.cpu_utilization_percent', + ], + }, + }); + }); + }); +});