diff --git a/src/__tests__/backend/release-notes/generate-release-notes.lambda.test.ts b/src/__tests__/backend/release-notes/generate-release-notes.lambda.test.ts index 61bcf215c..fcd48a535 100644 --- a/src/__tests__/backend/release-notes/generate-release-notes.lambda.test.ts +++ b/src/__tests__/backend/release-notes/generate-release-notes.lambda.test.ts @@ -1,10 +1,15 @@ +import { + GetObjectCommand, + PutObjectCommand, + S3Client, +} from '@aws-sdk/client-s3'; import { metricScope, MetricsLogger, Unit } from 'aws-embedded-metrics'; -import * as AWS from 'aws-sdk'; -import { AWSError } from 'aws-sdk'; -import * as AWSMock from 'aws-sdk-mock'; +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest'; import * as constants from '../../../backend/release-notes/constants'; import { generateReleaseNotes } from '../../../backend/release-notes/shared/github-changelog-fetcher.lambda-shared'; import { extractObjects } from '../../../backend/shared/tarball.lambda-shared'; +import { stringToStream } from '../../streams'; jest.mock('aws-embedded-metrics'); jest.mock('../../../backend/shared/tarball.lambda-shared'); @@ -14,7 +19,8 @@ jest.mock( const MOCK_BUCKET_NAME = 'package-data-bucket'; const PACKAGE_TGZ = 'data/@aws-cdk/aws-amplify/v1.144.0/package.tgz'; -const FAKE_TAR_GZ = Buffer.from('fake-tarball-content[gzipped]'); +const FAKE_TAR_DATA = 'fake-tarball-content[gzipped]'; +const FAKE_TAR_GZ = Buffer.from(FAKE_TAR_DATA); const MOCK_TARBALL_URI = `s3://${MOCK_BUCKET_NAME}.test-bermuda-2.s3.amazonaws.com/${PACKAGE_TGZ}`; const MOCK_RELEASE_NOTES_KEY = 'data/@aws-cdk/aws-amplify/v1.144.0/release-notes.md'; @@ -45,6 +51,8 @@ const MOCKED_RELEASE_NOTES = ` * Some other bug `; +const mockS3 = mockClient(S3Client); + const mockPutMetric = jest .fn() .mockName('MetricsLogger.putMetric') as jest.MockedFunction< @@ -58,8 +66,6 @@ const mockSetNamespace = jest >; let handler: any; -let s3GetObjSpy: jest.Mock; -let s3PutObjSpy: jest.Mock; const extractObjectMock = >( extractObjects ); @@ -97,9 +103,9 @@ beforeEach(async () => { jest.resetAllMocks(); process.env.BUCKET_NAME = MOCK_BUCKET_NAME; - AWSMock.setSDKInstance(AWS); - s3GetObjSpy = setupPkgTarS3GetObjectMock(); - s3PutObjSpy = setupReleaseNotesS3PutObjectMock(); + mockS3.reset(); + setupPkgTarS3GetObjectMock(); + setupReleaseNotesS3PutObjectMock(); extractObjectMock.mockResolvedValue({ packageJson: Buffer.from(JSON.stringify(MOCKED_PACKAGE_JSON)), @@ -110,12 +116,8 @@ beforeEach(async () => { afterEach(async () => { // clean up the env vars process.env.BUCKET_NAME = undefined; - - AWSMock.restore(); }); -type Response = (err: AWS.AWSError | null, data?: T) => void; - test('happy case', async () => { await expect( handler({ tarballUri: MOCK_TARBALL_URI }, {} as any) @@ -125,8 +127,8 @@ test('happy case', async () => { packageJson: { path: 'package/package.json', required: true }, }); - expect(s3GetObjSpy).toHaveBeenCalledTimes(1); - expect(s3PutObjSpy).toHaveBeenCalledTimes(1); + expect(mockS3).toHaveReceivedCommandTimes(GetObjectCommand, 1); + expect(mockS3).toHaveReceivedCommandTimes(PutObjectCommand, 1); expect(generateReleaseNotes).toHaveBeenCalledTimes(1); expect(generateReleaseNotes).toHaveBeenCalledWith( @@ -160,8 +162,8 @@ test('When repository uses git@github.com url', async () => { packageJson: { path: 'package/package.json', required: true }, }); - expect(s3GetObjSpy).toHaveBeenCalledTimes(1); - expect(s3PutObjSpy).toHaveBeenCalledTimes(1); + expect(mockS3).toHaveReceivedCommandTimes(GetObjectCommand, 1); + expect(mockS3).toHaveReceivedCommandTimes(PutObjectCommand, 1); expect(generateReleaseNotes).toHaveBeenCalledTimes(1); expect(generateReleaseNotes).toHaveBeenCalledWith( @@ -187,8 +189,8 @@ test('When repository info is missing sends "UnSupportedRepo"', async () => { ).resolves.toEqual({ error: 'UnSupportedRepo' }); expect(generateReleaseNotes).not.toHaveBeenCalled(); - expect(s3GetObjSpy).toHaveBeenCalledTimes(1); - expect(s3PutObjSpy).not.toHaveBeenCalled(); + expect(mockS3).toHaveReceivedCommandTimes(GetObjectCommand, 1); + expect(mockS3).not.toHaveReceivedCommand(PutObjectCommand); const PKG_JSON_WITH_GITLAB_REPO = { ...MOCKED_PACKAGE_JSON, @@ -202,8 +204,8 @@ test('When repository info is missing sends "UnSupportedRepo"', async () => { handler({ tarballUri: MOCK_TARBALL_URI }, {} as any) ).resolves.toEqual({ error: 'UnSupportedRepo' }); expect(generateReleaseNotes).not.toHaveBeenCalled(); - expect(s3GetObjSpy).toHaveBeenCalledTimes(2); - expect(s3PutObjSpy).not.toHaveBeenCalled(); + expect(mockS3).toHaveReceivedCommandTimes(GetObjectCommand, 2); + expect(mockS3).not.toHaveReceivedCommand(PutObjectCommand); expect(mockPutMetric).toHaveBeenCalledWith( constants.UnSupportedRepo, 1, @@ -231,8 +233,8 @@ test('sends RequestQuotaExhausted error when GitHub sends error code 403', async packageJson: { path: 'package/package.json', required: true }, }); - expect(s3GetObjSpy).toHaveBeenCalledTimes(1); - expect(s3PutObjSpy).not.toHaveBeenCalled(); + expect(mockS3).toHaveReceivedCommandTimes(GetObjectCommand, 1); + expect(mockS3).not.toHaveReceivedCommand(PutObjectCommand); expect(mockPutMetric).toHaveBeenCalledWith( constants.RequestQuotaExhausted, 1, @@ -258,8 +260,8 @@ test('sends InvalidCredentials error when GitHub sends error code 401', async () packageJson: { path: 'package/package.json', required: true }, }); - expect(s3GetObjSpy).toHaveBeenCalledTimes(1); - expect(s3PutObjSpy).not.toHaveBeenCalled(); + expect(mockS3).toHaveReceivedCommandTimes(GetObjectCommand, 1); + expect(mockS3).not.toHaveReceivedCommand(PutObjectCommand); expect(mockPutMetric).toHaveBeenCalledWith( constants.InvalidCredentials, 1, @@ -283,8 +285,8 @@ test('When GH does not have any release notes', async () => { packageJson: { path: 'package/package.json', required: true }, }); - expect(s3GetObjSpy).toHaveBeenCalledTimes(1); - expect(s3PutObjSpy).not.toHaveBeenCalled(); + expect(mockS3).toHaveReceivedCommandTimes(GetObjectCommand, 1); + expect(mockS3).not.toHaveReceivedCommand(PutObjectCommand); }); test('when tarball is invalid send InvalidTarball error', async () => { @@ -301,8 +303,8 @@ test('when tarball is invalid send InvalidTarball error', async () => { packageJson: { path: 'package/package.json', required: true }, }); - expect(s3GetObjSpy).toHaveBeenCalledTimes(1); - expect(s3PutObjSpy).not.toHaveBeenCalled(); + expect(mockS3).toHaveReceivedCommandTimes(GetObjectCommand, 1); + expect(mockS3).not.toHaveReceivedCommand(PutObjectCommand); expect(mockPutMetric).toHaveBeenCalledWith( constants.InvalidTarball, 1, @@ -335,8 +337,8 @@ test('throw error when package.json is not valid', async () => { packageJson: { path: 'package/package.json', required: true }, }); - expect(s3GetObjSpy).toHaveBeenCalledTimes(1); - expect(s3PutObjSpy).not.toHaveBeenCalled(); + expect(mockS3).toHaveReceivedCommandTimes(GetObjectCommand, 1); + expect(mockS3).not.toHaveReceivedCommand(PutObjectCommand); expect(mockPutMetric).toHaveBeenCalledWith( constants.InvalidPackageJson, 1, @@ -346,38 +348,21 @@ test('throw error when package.json is not valid', async () => { // Helper functions const setupPkgTarS3GetObjectMock = () => { - const spy = jest.fn().mockResolvedValue({ Body: FAKE_TAR_GZ }); - AWSMock.mock( - 'S3', - 'getObject', - (req: AWS.S3.GetObjectRequest, cb: Response) => { - try { - expect(req.Bucket).toBe(MOCK_BUCKET_NAME); - expect(req.Key).toBe(PACKAGE_TGZ); - } catch (e: any) { - return cb(e); - } - return cb(null, spy()); - } - ); - return spy; + mockS3 + .on(GetObjectCommand, { + Bucket: MOCK_BUCKET_NAME, + Key: PACKAGE_TGZ, + }) + .callsFake(() => ({ Body: stringToStream(FAKE_TAR_DATA) })); }; function setupReleaseNotesS3PutObjectMock() { - const spy = jest.fn(); - AWSMock.mock( - 'S3', - 'putObject', - (req: AWS.S3.PutObjectRequest, cb: Response) => { - try { - expect(req.Bucket).toBe(MOCK_BUCKET_NAME); - expect(req.Key).toBe(MOCK_RELEASE_NOTES_KEY); - expect(req.Body).toBe(MOCKED_RELEASE_NOTES); - } catch (e) { - return cb(e as AWSError); - } - return cb(null, spy(req)); - } - ); - return spy; + mockS3 + .on(PutObjectCommand, { + Bucket: MOCK_BUCKET_NAME, + Key: MOCK_RELEASE_NOTES_KEY, + Body: MOCKED_RELEASE_NOTES, + ContentType: 'text/markdown', + }) + .resolves({}); } diff --git a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap index 08c77d194..41d64522d 100644 --- a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap +++ b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap @@ -9675,7 +9675,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "a3260bff5c26ab778f4a65ecb0cb5749a1fcd9ddeee159244a2a1aa4e8eb82c6.zip", + "S3Key": "26b5bec29cce93b76b4d917cea14ddb133f8593766cff0634d13dc097dfaabcc.zip", }, "Description": "ReleaseNotes generator", "Environment": { @@ -9826,7 +9826,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "3963e849d24891020a9ad0f6f17dc5e9c50abf2119b0ba2c27ce84de41e76b13.zip", + "S3Key": "bc5438e29b95b28156e91913408ef4f975c07482b257fa8aab1ddfc41bab0e75.zip", }, "Description": "ReleaseNotes get message from the worker queue", "Environment": { @@ -10097,7 +10097,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "0efd1788fd3f99714e0972250b0b94e1f9d7577c7ee952a54361d0475405c98b.zip", + "S3Key": "74e117df5d2369ea7e1048ffb95f12615afc9fb18b169e2a3fd912b082970813.zip", }, "Description": "backend/release-notes/release-notes-trigger.lambda.ts", "Environment": { diff --git a/src/backend/release-notes/generate-release-notes.lambda.ts b/src/backend/release-notes/generate-release-notes.lambda.ts index 74c388601..970809996 100644 --- a/src/backend/release-notes/generate-release-notes.lambda.ts +++ b/src/backend/release-notes/generate-release-notes.lambda.ts @@ -1,8 +1,9 @@ +import { GetObjectCommand, PutObjectCommand } from '@aws-sdk/client-s3'; import { metricScope, Unit } from 'aws-embedded-metrics'; import type { Context } from 'aws-lambda'; -import * as aws from 'aws-sdk'; import * as constants from './constants'; import { generateReleaseNotes } from './shared/github-changelog-fetcher.lambda-shared'; +import { S3_CLIENT } from '../shared/aws.lambda-shared'; import { requireEnv } from '../shared/env.lambda-shared'; import { extractObjects } from '../shared/tarball.lambda-shared'; @@ -33,20 +34,20 @@ export const handler = metricScope( metrics.putMetric(constants.UnSupportedTarballUrl, 1, Unit.Count); return { error: 'UnSupportedTarballUrl' }; } - const tarball = await new aws.S3() - .getObject({ + const tarball = await S3_CLIENT.send( + new GetObjectCommand({ // Note: we drop anything after the first `.` in the host, as we only care about the bucket name. Bucket: tarballUri.host.split('.')[0], // Note: the pathname part is absolute, so we strip the leading `/`. Key: tarballUri.pathname.replace(/^\//, ''), VersionId: tarballUri.searchParams.get('versionId') ?? undefined, }) - .promise(); + ); let packageJson: Buffer; try { ({ packageJson } = await extractObjects( - Buffer.from(tarball.Body! as any), + Buffer.from(await tarball.Body!.transformToByteArray()), { packageJson: { path: 'package/package.json', required: true }, } @@ -91,8 +92,8 @@ export const handler = metricScope( console.log( `storing release notes to s3://${BUCKET_NAME}${releaseNotesPath}` ); - await new aws.S3() - .putObject({ + await S3_CLIENT.send( + new PutObjectCommand({ Bucket: BUCKET_NAME, Key: releaseNotesPath, Body: releaseNotes, @@ -103,7 +104,7 @@ export const handler = metricScope( 'Lambda-Run-Id': context.awsRequestId, }, }) - .promise(); + ); } else { console.log('No release notes found'); metrics.putMetric(constants.PackageWithChangeLog, 0, Unit.Count); diff --git a/src/backend/release-notes/get-messages-from-worker-queue.lambda.ts b/src/backend/release-notes/get-messages-from-worker-queue.lambda.ts index dbd54f305..d98b4410c 100644 --- a/src/backend/release-notes/get-messages-from-worker-queue.lambda.ts +++ b/src/backend/release-notes/get-messages-from-worker-queue.lambda.ts @@ -1,7 +1,10 @@ +import { ListExecutionsCommand } from '@aws-sdk/client-sfn'; +import type { Message } from '@aws-sdk/client-sqs'; +import { ReceiveMessageCommand } from '@aws-sdk/client-sqs'; import { metricScope, Unit } from 'aws-embedded-metrics'; -import { StepFunctions, SQS } from 'aws-sdk'; import * as constants from './constants'; import { getServiceLimits } from './shared/github-changelog-fetcher.lambda-shared'; +import { SFN_CLIENT, SQS_CLIENT } from '../shared/aws.lambda-shared'; import { requireEnv } from '../shared/env.lambda-shared'; // Each of the release note fetch task can involve making multiple Github @@ -19,7 +22,7 @@ type ServiceLimit = { type ExecutionResult = { error?: string; status?: string; - messages?: SQS.Message[]; + messages?: Message[]; }; /** @@ -86,15 +89,15 @@ export const handler = async (): Promise => { const sfnArn = requireEnv('STEP_FUNCTION_ARN'); // Ensure only one instance of step function is running - const activities = await new StepFunctions() - .listExecutions({ + const activities = await SFN_CLIENT.send( + new ListExecutionsCommand({ stateMachineArn: sfnArn, maxResults: 1, statusFilter: 'RUNNING', }) - .promise(); + ); - if (activities.executions.length > 1) { + if (activities.executions && activities.executions.length > 1) { return { error: 'MaxConcurrentExecutionError' }; } if (serviceLimit.remaining <= MAX_GH_REQUEST_PER_PACKAGE) { @@ -106,15 +109,15 @@ export const handler = async (): Promise => { }; } - const messages = await new SQS() - .receiveMessage({ + const messages = await SQS_CLIENT.send( + new ReceiveMessageCommand({ QueueUrl: process.env.SQS_QUEUE_URL!, MaxNumberOfMessages: Math.min( Math.floor(serviceLimit.remaining / MAX_GH_REQUEST_PER_PACKAGE), 10 ), }) - .promise(); + ); if (messages.Messages?.length) { return { diff --git a/src/backend/release-notes/release-notes-trigger.lambda.ts b/src/backend/release-notes/release-notes-trigger.lambda.ts index 52fdfbeca..c38ebf7ae 100644 --- a/src/backend/release-notes/release-notes-trigger.lambda.ts +++ b/src/backend/release-notes/release-notes-trigger.lambda.ts @@ -1,6 +1,13 @@ +import { + ListExecutionsCommand, + StartExecutionCommand, +} from '@aws-sdk/client-sfn'; +import { + SendMessageBatchCommand, + SendMessageBatchRequestEntry, +} from '@aws-sdk/client-sqs'; import { SQSEvent } from 'aws-lambda'; -import * as aws from 'aws-sdk'; -import { SendMessageBatchRequestEntryList } from 'aws-sdk/clients/sqs'; +import { SFN_CLIENT, SQS_CLIENT } from '../shared/aws.lambda-shared'; import { requireEnv } from '../shared/env.lambda-shared'; const SLEEP_TIME = 1000 * 5; // 5 seconds @@ -23,18 +30,16 @@ export const handler = async (event: SQSEvent) => { const SFN_ARN = requireEnv('SFN_ARN'); const WORKER_QUEUE_URL = requireEnv('WORKER_QUEUE_URL'); - const sqs = new aws.SQS(); - let messages: SendMessageBatchRequestEntryList = []; + let messages: SendMessageBatchRequestEntry[] = []; console.log('attempting to send messages'); const requests = event.Records.map((record, index) => { if (index > 0 && index % 10 === 0) { - return sqs - .sendMessageBatch({ + return SQS_CLIENT.send( + new SendMessageBatchCommand({ Entries: messages, QueueUrl: WORKER_QUEUE_URL, }) - .promise(); - messages = []; + ); } else { console.log('message => ', record.body); messages.push({ Id: record.messageId, MessageBody: record.body }); @@ -45,12 +50,12 @@ export const handler = async (event: SQSEvent) => { // last batch of message if (messages.length) { requests.push( - sqs - .sendMessageBatch({ + SQS_CLIENT.send( + new SendMessageBatchCommand({ Entries: messages, QueueUrl: WORKER_QUEUE_URL, }) - .promise() + ) ); } await Promise.all(requests); @@ -69,25 +74,22 @@ export const handler = async (event: SQSEvent) => { if (await isStepFunctionAlreadyRunning()) return; } - const sfn = new aws.StepFunctions(); - const invocation = await sfn - .startExecution({ + const invocation = await SFN_CLIENT.send( + new StartExecutionCommand({ stateMachineArn: SFN_ARN, }) - .promise(); + ); console.log('invocation done', invocation); return; }; const isStepFunctionAlreadyRunning = async (): Promise => { const SFN_ARN = requireEnv('SFN_ARN'); - const sfn = new aws.StepFunctions(); - - const invocations = await sfn - .listExecutions({ + const invocations = await SFN_CLIENT.send( + new ListExecutionsCommand({ stateMachineArn: SFN_ARN, statusFilter: 'RUNNING', }) - .promise(); - return invocations.executions.length > 0; + ); + return Boolean(invocations.executions && invocations.executions.length > 0); };