From 879377b4abcc3bed152fc8e4cfcdd3d09e0b9ffe Mon Sep 17 00:00:00 2001 From: jsetton Date: Mon, 6 Dec 2021 18:39:16 -0500 Subject: [PATCH] fix: wait for lambda function state updates --- .../lambda-deployer/helper.js | 115 +++++++++++++----- lib/utils/constants.js | 19 +++ .../builtins/lambda-deployer/helper-test.js | 113 ++++++++++------- 3 files changed, 171 insertions(+), 76 deletions(-) diff --git a/lib/builtins/deploy-delegates/lambda-deployer/helper.js b/lib/builtins/deploy-delegates/lambda-deployer/helper.js index c5399e7a..e710824c 100644 --- a/lib/builtins/deploy-delegates/lambda-deployer/helper.js +++ b/lib/builtins/deploy-delegates/lambda-deployer/helper.js @@ -189,28 +189,28 @@ function _createLambdaFunction(reporter, lambdaClient, options, callback) { }); }; const shouldRetryCondition = retryResponse => retryResponse === RETRY_MESSAGE; - retryUtils.retry(retryConfig, retryCall, shouldRetryCondition, (retryErr, lambdaData) => { + retryUtils.retry(retryConfig, retryCall, shouldRetryCondition, (retryErr, createData) => { if (retryErr) { return callback(retryErr); } - const arn = lambdaData.FunctionArn; - // 2. Grant permissions to use a Lambda function - _addEventPermissions(lambdaClient, skillId, arn, profile, (addErr) => { - if (addErr) { - return callback(addErr); + const functionArn = createData.FunctionArn; + // 2. Wait for created lambda function to be active + _waitForLambdaFunction(lambdaClient, functionArn, (waitErr) => { + if (waitErr) { + return callback(waitErr); } - // 3. Get the latest revisionId from getFunction API - lambdaClient.getFunction(arn, (revisionErr, revisionData) => { - if (revisionErr) { - return callback(revisionErr); + // 3. Grant permissions to use a Lambda function + _addEventPermissions(lambdaClient, skillId, functionArn, profile, (permErr, lambdaData) => { + if (permErr) { + return callback(permErr); } callback(null, { isAllStepSuccess: true, isCodeDeployed: true, lambdaResponse: { - arn, + arn: functionArn, lastModified: lambdaData.LastModified, - revisionId: revisionData.Configuration.RevisionId + revisionId: lambdaData.RevisionId } }); }); @@ -222,32 +222,23 @@ function _addEventPermissions(lambdaClient, skillId, functionArn, profile, callb const targetEndpoints = ResourcesConfig.getInstance().getTargetEndpoints(profile); // for backward compatibility, defaulting to api from skill manifest if targetEndpoints is not defined const domains = targetEndpoints.length ? targetEndpoints : Object.keys(Manifest.getInstance().getApis()); - async.forEach(domains, (domain, addCallback) => { - lambdaClient.addAlexaPermissionByDomain(domain, skillId, functionArn, (err) => { - if (err) { - return addCallback(err); - } - addCallback(); - }); - }, (error) => { - callback(error); + async.forEach(domains, (domain, permCallback) => { + lambdaClient.addAlexaPermissionByDomain(domain, skillId, functionArn, permCallback); + }, (permErr) => { + if (permErr) { + return callback(permErr); + } + _waitForLambdaFunction(lambdaClient, functionArn, callback); }); } function _updateLambdaFunction(reporter, lambdaClient, options, callback) { - const { zipFilePath, userConfig, deployState } = options; - const zipFile = fs.readFileSync(zipFilePath); - const functionName = deployState.lambda.arn; - let { revisionId } = deployState.lambda; - lambdaClient.updateFunctionCode(zipFile, functionName, revisionId, (codeErr, codeData) => { + _updateLambdaFunctionCode(reporter, lambdaClient, options, (codeErr, codeData) => { if (codeErr) { return callback(codeErr); } - reporter.updateStatus(`Update a lambda function (${functionName}) in progress...`); - const { runtime } = userConfig; - const { handler } = userConfig; - revisionId = codeData.RevisionId; - lambdaClient.updateFunctionConfiguration(functionName, runtime, handler, revisionId, (configErr, configData) => { + + _updateLambdaFunctionConfig(reporter, lambdaClient, codeData, (configErr, configData) => { if (configErr) { return callback(null, { isAllStepSuccess: false, @@ -272,3 +263,65 @@ function _updateLambdaFunction(reporter, lambdaClient, options, callback) { }); }); } + +function _updateLambdaFunctionCode(reporter, lambdaClient, options, callback) { + const { zipFilePath, deployState } = options; + const zipFile = fs.readFileSync(zipFilePath); + const functionName = deployState.lambda.arn; + const { revisionId } = deployState.lambda; + + lambdaClient.updateFunctionCode(zipFile, functionName, revisionId, (err) => { + if (err) { + return callback(err); + } + + reporter.updateStatus(`Update a lambda function code (${functionName}) in progress...`); + + _waitForLambdaFunction(lambdaClient, functionName, callback); + }); +} + +function _updateLambdaFunctionConfig(reporter, lambdaClient, lambdaConfig, callback) { + const { FunctionName: functionName, Runtime: runtime, Handler: handler, RevisionId: revisionId } = lambdaConfig; + + lambdaClient.updateFunctionConfiguration(functionName, runtime, handler, revisionId, (err) => { + if (err) { + return callback(err); + } + + reporter.updateStatus(`Update a lambda function configuration (${functionName}) in progress...`); + + _waitForLambdaFunction(lambdaClient, functionName, callback); + }); +} + +function _waitForLambdaFunction(lambdaClient, functionName, callback) { + const retryConfig = { + base: CONSTANTS.CONFIGURATION.RETRY.WAIT_LAMBDA_FUNCTION.BASE, + factor: CONSTANTS.CONFIGURATION.RETRY.WAIT_LAMBDA_FUNCTION.FACTOR, + maxRetry: CONSTANTS.CONFIGURATION.RETRY.WAIT_LAMBDA_FUNCTION.MAXRETRY + }; + const retryCall = (loopCallback) => { + lambdaClient.getFunction(functionName, (err, data) => { + if (err) { + return loopCallback(err); + } + loopCallback(null, data.Configuration); + }); + }; + const shouldRetryCondition = (retryResponse) => retryResponse.State === CONSTANTS.LAMBDA.FUNCTION_STATE.PENDING + || retryResponse.LastUpdateStatus === CONSTANTS.LAMBDA.LAST_UPDATE_STATUS.IN_PROGRESS; + + retryUtils.retry(retryConfig, retryCall, shouldRetryCondition, (retryErr, lambdaData) => { + if (retryErr) { + return callback(retryErr); + } + if (lambdaData.State !== CONSTANTS.LAMBDA.FUNCTION_STATE.ACTIVE) { + return callback(`Function [${functionName}] state is ${lambdaData.State}.`); + } + if (lambdaData.LastUpdateStatus !== CONSTANTS.LAMBDA.LAST_UPDATE_STATUS.SUCCESSFUL) { + return callback(`Function [${functionName}] last update status is ${lambdaData.LastUpdateStatus}.`); + } + callback(null, lambdaData); + }); +} diff --git a/lib/utils/constants.js b/lib/utils/constants.js index e8d1e24b..a239afc9 100644 --- a/lib/utils/constants.js +++ b/lib/utils/constants.js @@ -170,6 +170,11 @@ module.exports.CONFIGURATION = { BASE: 5000, FACTOR: 1.5, MAXRETRY: 3 + }, + WAIT_LAMBDA_FUNCTION: { + BASE: 1000, + FACTOR: 1.2, + MAXRETRY: 10 } }, S3: { @@ -296,6 +301,20 @@ module.exports.AWS = { } }; +module.exports.LAMBDA = { + FUNCTION_STATE: { + ACTIVE: 'Active', + INACTIVE: 'Inactive', + PENDING: 'Pending', + FAILED: 'Failed' + }, + LAST_UPDATE_STATUS: { + SUCCESSFUL: 'Successful', + FAILED: 'Failed', + IN_PROGRESS: 'InProgress' + } +}; + module.exports.PLACEHOLDER = { ENVIRONMENT_VAR: { AWS_CREDENTIALS: '__AWS_CREDENTIALS_IN_ENVIRONMENT_VARIABLE__', diff --git a/test/unit/builtins/lambda-deployer/helper-test.js b/test/unit/builtins/lambda-deployer/helper-test.js index 1cd01742..fec6538e 100644 --- a/test/unit/builtins/lambda-deployer/helper-test.js +++ b/test/unit/builtins/lambda-deployer/helper-test.js @@ -331,6 +331,8 @@ Please solve this revision mismatch and re-deploy again. To ignore this error ru const TEST_REVISION_ID = 'revision_id'; const TEST_UPDATED_REVISION_ID = 'revision id'; const TEST_FUNCTION_ARN = 'lambda function arn'; + const TEST_FUNCTION_STATE = 'Active'; + const TEST_LAST_UPDATE_STATUS = 'Successful'; const TEST_RUNTIME = 'runtime'; const TEST_HANDLER = 'handler'; const TEST_CREATE_OPTIONS = { @@ -358,11 +360,23 @@ Please solve this revision mismatch and re-deploy again. To ignore this error ru zipFilePath: TEST_ZIP_FILE_PATH, userConfig: { awsRegion: TEST_AWS_REGION, runtime: TEST_RUNTIME, handler: TEST_HANDLER }, deployState: { lambda: { - arn: TEST_LAMBDA_ARN, + arn: TEST_FUNCTION_ARN, lastModified: TEST_LAST_MODIFIED, revisionId: TEST_REVISION_ID } } }; + const TEST_CREATE_DATA = { + FunctionArn: TEST_FUNCTION_ARN + }; + const TEST_GET_DATA = { + Configuration: { + FunctionArn: TEST_FUNCTION_ARN, + LastModified: TEST_LAST_MODIFIED, + RevisionId: TEST_UPDATED_REVISION_ID, + State: TEST_FUNCTION_STATE, + LastUpdateStatus: TEST_LAST_UPDATE_STATUS + } + }; beforeEach(() => { new Manifest(FIXTURE_MANIFEST_FILE_PATH); @@ -391,19 +405,15 @@ Please solve this revision mismatch and re-deploy again. To ignore this error ru + ' add Alexa Permission fails, expect an error return.', (done) => { // setup const RETRY_MESSAGE = 'The role defined for the function cannot be assumed by Lambda.'; - const TEST_CREATE_FUNCTION_ERROR = { - message: RETRY_MESSAGE - }; - const TEST_LAMBDA_DATA = { - FunctionArn: TEST_FUNCTION_ARN - }; + const TEST_CREATE_FUNCTION_ERROR = { message: RETRY_MESSAGE }; const TEST_ADD_PERMISSION_ERROR = 'addAlexaPermissionByDomain error'; sinon.stub(fs, 'readFileSync').withArgs(TEST_ZIP_FILE_PATH).returns(TEST_ZIP_FILE); sinon.stub(aws, 'Lambda'); const stubTestFunc = sinon.stub(LambdaClient.prototype, 'createLambdaFunction'); stubTestFunc.onCall(0).callsArgWith(7, TEST_CREATE_FUNCTION_ERROR); - stubTestFunc.onCall(1).callsArgWith(7, null, TEST_LAMBDA_DATA); + stubTestFunc.onCall(1).callsArgWith(7, null, TEST_CREATE_DATA); sinon.stub(LambdaClient.prototype, 'addAlexaPermissionByDomain').callsArgWith(3, TEST_ADD_PERMISSION_ERROR); + sinon.stub(LambdaClient.prototype, 'getFunction').callsArgWith(1, null, TEST_GET_DATA); // call helper.deployLambdaFunction(REPORTER, TEST_CREATE_OPTIONS, (err) => { // verify @@ -412,16 +422,30 @@ Please solve this revision mismatch and re-deploy again. To ignore this error ru }); }).timeout(10000); + it('| no Lambda found, create Lambda function passes but not in active state, expect an error return.', (done) => { + // setup + const TEST_GET_STATE_DATA = { Configuration: { State: 'Inactive' } }; + const TEST_GET_STATE_ERROR = `Function [${TEST_FUNCTION_ARN}] state is Inactive.`; + sinon.stub(fs, 'readFileSync').withArgs(TEST_ZIP_FILE_PATH).returns(TEST_ZIP_FILE); + sinon.stub(aws, 'Lambda'); + sinon.stub(LambdaClient.prototype, 'createLambdaFunction').callsArgWith(7, null, TEST_CREATE_DATA); + sinon.stub(LambdaClient.prototype, 'getFunction').callsArgWith(1, null, TEST_GET_STATE_DATA); + // call + helper.deployLambdaFunction(REPORTER, TEST_CREATE_OPTIONS, (err) => { + // verify + expect(err).equal(TEST_GET_STATE_ERROR); + done(); + }); + }); + it('| no Lambda found, create Lambda function passes, add Alexa Permission fails, expect an error return.', (done) => { // setup - const TEST_LAMBDA_DATA = { - FunctionArn: TEST_FUNCTION_ARN - }; const TEST_ADD_PERMISSION_ERROR = 'addAlexaPermissionByDomain error'; sinon.stub(fs, 'readFileSync').withArgs(TEST_ZIP_FILE_PATH).returns(TEST_ZIP_FILE); sinon.stub(aws, 'Lambda'); - sinon.stub(LambdaClient.prototype, 'createLambdaFunction').callsArgWith(7, null, TEST_LAMBDA_DATA); + sinon.stub(LambdaClient.prototype, 'createLambdaFunction').callsArgWith(7, null, TEST_CREATE_DATA); sinon.stub(LambdaClient.prototype, 'addAlexaPermissionByDomain').callsArgWith(3, TEST_ADD_PERMISSION_ERROR); + sinon.stub(LambdaClient.prototype, 'getFunction').callsArgWith(1, null, TEST_GET_DATA); // call helper.deployLambdaFunction(REPORTER, TEST_CREATE_OPTIONS, (err) => { // verify @@ -433,15 +457,14 @@ Please solve this revision mismatch and re-deploy again. To ignore this error ru it('| no Lambda found, create Lambda function and add Alexa Permission pass, get Function revisionId fails,' + ' expect an error return.', (done) => { // setup - const TEST_LAMBDA_DATA = { - FunctionArn: TEST_FUNCTION_ARN - }; const TEST_REVISION_ID_ERROR = 'getFunctionRevisionId error'; sinon.stub(fs, 'readFileSync').withArgs(TEST_ZIP_FILE_PATH).returns(TEST_ZIP_FILE); sinon.stub(aws, 'Lambda'); - sinon.stub(LambdaClient.prototype, 'createLambdaFunction').callsArgWith(7, null, TEST_LAMBDA_DATA); + sinon.stub(LambdaClient.prototype, 'createLambdaFunction').callsArgWith(7, null, TEST_CREATE_DATA); sinon.stub(LambdaClient.prototype, 'addAlexaPermissionByDomain').callsArgWith(3, null); - sinon.stub(LambdaClient.prototype, 'getFunction').callsArgWith(1, TEST_REVISION_ID_ERROR); + const stubTestFunc = sinon.stub(LambdaClient.prototype, 'getFunction'); + stubTestFunc.onCall(0).callsArgWith(1, null, TEST_GET_DATA); + stubTestFunc.onCall(1).callsArgWith(1, TEST_REVISION_ID_ERROR); // call helper.deployLambdaFunction(REPORTER, TEST_CREATE_OPTIONS, (err) => { @@ -454,20 +477,11 @@ Please solve this revision mismatch and re-deploy again. To ignore this error ru it('| no Lambda found, create Lambda function, add Alexa Permission for target domains' + ' and get Function revisionId pass, expect Lambda data return.', (done) => { // setup - const TEST_LAMBDA_DATA = { - FunctionArn: TEST_FUNCTION_ARN, - LastModified: TEST_LAST_MODIFIED - }; - const TEST_GET_FUNCTION_RESPONSE = { - Configuration: { - RevisionId: TEST_UPDATED_REVISION_ID, - } - }; sinon.stub(fs, 'readFileSync').withArgs(TEST_ZIP_FILE_PATH).returns(TEST_ZIP_FILE); sinon.stub(aws, 'Lambda'); - sinon.stub(LambdaClient.prototype, 'createLambdaFunction').callsArgWith(7, null, TEST_LAMBDA_DATA); + sinon.stub(LambdaClient.prototype, 'createLambdaFunction').callsArgWith(7, null, TEST_CREATE_DATA); sinon.stub(LambdaClient.prototype, 'addAlexaPermissionByDomain').callsArgWith(3, null); - sinon.stub(LambdaClient.prototype, 'getFunction').callsArgWith(1, null, TEST_GET_FUNCTION_RESPONSE); + sinon.stub(LambdaClient.prototype, 'getFunction').callsArgWith(1, null, TEST_GET_DATA); // call helper.deployLambdaFunction(REPORTER, TEST_CREATE_OPTIONS, (err, res) => { @@ -488,21 +502,12 @@ Please solve this revision mismatch and re-deploy again. To ignore this error ru it('| no Lambda found, create Lambda function, add Alexa Permission for domains from skill ' + 'manifest and get Function revisionId pass, expect Lambda data return.', (done) => { // setup - const TEST_LAMBDA_DATA = { - FunctionArn: TEST_FUNCTION_ARN, - LastModified: TEST_LAST_MODIFIED - }; - const TEST_GET_FUNCTION_RESPONSE = { - Configuration: { - RevisionId: TEST_UPDATED_REVISION_ID, - } - }; sinon.stub(fs, 'readFileSync').withArgs(TEST_ZIP_FILE_PATH).returns(TEST_ZIP_FILE); sinon.stub(aws, 'Lambda'); sinon.stub(ResourcesConfig.prototype, 'getTargetEndpoints').returns([]); - sinon.stub(LambdaClient.prototype, 'createLambdaFunction').callsArgWith(7, null, TEST_LAMBDA_DATA); + sinon.stub(LambdaClient.prototype, 'createLambdaFunction').callsArgWith(7, null, TEST_CREATE_DATA); sinon.stub(LambdaClient.prototype, 'addAlexaPermissionByDomain').callsArgWith(3, null); - sinon.stub(LambdaClient.prototype, 'getFunction').callsArgWith(1, null, TEST_GET_FUNCTION_RESPONSE); + sinon.stub(LambdaClient.prototype, 'getFunction').callsArgWith(1, null, TEST_GET_DATA); // call helper.deployLambdaFunction(REPORTER, TEST_CREATE_OPTIONS, (err, res) => { @@ -534,6 +539,22 @@ Please solve this revision mismatch and re-deploy again. To ignore this error ru }); }); + it('| an existing Lambda found, update function code passes but last update status failed, expect an error return.', (done) => { + // setup + const TEST_GET_UPDATE_STATUS_DATA = { Configuration: { State: 'Active', LastUpdateStatus: 'Failed' } }; + const TEST_GET_UPDATE_STATUS_ERROR = `Function [${TEST_FUNCTION_ARN}] last update status is Failed.`; + sinon.stub(fs, 'readFileSync').withArgs(TEST_ZIP_FILE_PATH).returns(TEST_ZIP_FILE); + sinon.stub(aws, 'Lambda'); + sinon.stub(LambdaClient.prototype, 'updateFunctionCode').callsArgWith(3, null, { RevisionId: TEST_REVISION_ID }); + sinon.stub(LambdaClient.prototype, 'getFunction').callsArgWith(1, null, TEST_GET_UPDATE_STATUS_DATA); + // call + helper.deployLambdaFunction(REPORTER, TEST_UPDATE_OPTIONS, (err, res) => { + // verify + expect(err).equal(TEST_GET_UPDATE_STATUS_ERROR); + done(); + }); + }); + it('| an existing Lambda found, update function code passes, update function configuration fails, expect an error return.', (done) => { // setup const TEST_UPDATE_CONGIF_ERROR = 'updateFunctionConfiguration error'; @@ -541,6 +562,7 @@ Please solve this revision mismatch and re-deploy again. To ignore this error ru sinon.stub(aws, 'Lambda'); sinon.stub(LambdaClient.prototype, 'updateFunctionCode').callsArgWith(3, null, { RevisionId: TEST_REVISION_ID }); sinon.stub(LambdaClient.prototype, 'updateFunctionConfiguration').callsArgWith(4, TEST_UPDATE_CONGIF_ERROR); + sinon.stub(LambdaClient.prototype, 'getFunction').callsArgWith(1, null, TEST_GET_DATA); // call helper.deployLambdaFunction(REPORTER, TEST_UPDATE_OPTIONS, (err, res) => { // verify @@ -549,9 +571,9 @@ Please solve this revision mismatch and re-deploy again. To ignore this error ru isAllStepSuccess: false, isCodeDeployed: true, lambdaResponse: { - arn: undefined, - lastModified: undefined, - revisionId: TEST_REVISION_ID + arn: TEST_FUNCTION_ARN, + lastModified: TEST_LAST_MODIFIED, + revisionId: TEST_UPDATED_REVISION_ID }, resultMessage: TEST_UPDATE_CONGIF_ERROR }); @@ -562,7 +584,7 @@ Please solve this revision mismatch and re-deploy again. To ignore this error ru it('| an existing Lambda found, update function code and update function configuration pass, expect updated lambda data return.', (done) => { // setup const TEST_UPDATE_CONFIG_DATA = { - FunctionArn: TEST_LAMBDA_ARN, + FunctionArn: TEST_FUNCTION_ARN, LastModified: TEST_LAST_MODIFIED, RevisionId: TEST_REVISION_ID }; @@ -570,6 +592,7 @@ Please solve this revision mismatch and re-deploy again. To ignore this error ru sinon.stub(aws, 'Lambda'); sinon.stub(LambdaClient.prototype, 'updateFunctionCode').callsArgWith(3, null, { RevisionId: TEST_REVISION_ID }); sinon.stub(LambdaClient.prototype, 'updateFunctionConfiguration').callsArgWith(4, null, TEST_UPDATE_CONFIG_DATA); + sinon.stub(LambdaClient.prototype, 'getFunction').callsArgWith(1, null, TEST_GET_DATA); // call helper.deployLambdaFunction(REPORTER, TEST_UPDATE_OPTIONS, (err, data) => { // verify @@ -577,9 +600,9 @@ Please solve this revision mismatch and re-deploy again. To ignore this error ru isAllStepSuccess: true, isCodeDeployed: true, lambdaResponse: { - arn: TEST_LAMBDA_ARN, + arn: TEST_FUNCTION_ARN, lastModified: TEST_LAST_MODIFIED, - revisionId: TEST_REVISION_ID + revisionId: TEST_UPDATED_REVISION_ID } }); done();