Merge branch 'master' of github.com:knihit/aws-solutions-constructs

CONTRIBUTING.md

@@ -218,8 +218,11 @@ Additional scripts that take part in the versioning mechanism:
 ```console
 $ cd <root-of-aws-solutions-constructs-repo>
 $ docker run -u root --rm --net=host -it -v $PWD:$PWD -w $PWD jsii/superchain:1-buster-slim-node14
+# The build-patterns.sh command can take along time, be sure to allocate enough resources in the Docker dashboard
+# (6 CPUs is good)
 docker$ ./deployment/build-patterns.sh
-docker$ exit
+# At this point the container is configured and ready to work on. 
+# To work on a specific construct, execute the Partial Build steps below
 ```
 
 ### Partial Build
@@ -235,7 +238,6 @@ docker$ export PATH=$(npm bin):$PATH
 docker$ cd patterns/@aws-solutions-constructs/my-module
 docker$ npm run build+lint+test
 docker$ ../../../../deployment/align-version.sh revert
-docker$ exit
 ```
 
 ## Code of Conduct

source/patterns/@aws-solutions-constructs/aws-kinesisstreams-gluejob/README.md

@@ -97,53 +97,69 @@ _Parameters_
 | tableProps?         | [`CfnTableProps`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-glue.TableProps.html)                          | User provided AWS Glue Table props to override default props used to create a Glue Table.                        |
 | fieldSchema?        | [`CfnTable.ColumnProperty[]`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-glue.CfnTable.ColumnProperty.html) | User provided schema structure to create an AWS Glue Table.                                                      |
 | outputDataStore?    | [`SinkDataStoreProps`](#sinkdatastoreprops)                                                                                   | User provided properties for S3 bucket that stores Glue Job output. Current datastore types suported is only S3. |
+|createCloudWatchAlarms?|`boolean`|Whether to create recommended CloudWatch alarms for Kinesis Data Stream. Default value is set to `true`|
 
-## SinkDataStoreProps
+### SinkDataStoreProps
 
 | **Name**                | **Type**                                                                                          | **Description**                                                                                                |
 | :---------------------- | :------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------- |
 | existingS3OutputBucket? | [`Bucket`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-s3.Bucket.html)           | Existing instance of S3 bucket where the data should be written. Providing both this and `outputBucketProps` will cause an error. |
 | outputBucketProps       | [`BucketProps`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-s3.BucketProps.html) | User provided bucket properties to create the S3 bucket to store the output from the AWS Glue Job.             |
 | datastoreType           | [`SinkStoreType`](#sinkstoretype)                                                                 | Sink data store type.                                                                                          |
 
-## SinkStoreType
+### SinkStoreType
 
 Enumeration of data store types that could include S3, DynamoDB, DocumentDB, RDS or Redshift. Current construct implementation only supports S3, but potential to add other output types in the future.
 
 | **Name** | **Type** | **Description** |
 | :------- | :------- | --------------- |
 | S3       | `string` | S3 storage type |
 
-# Default settings
+## Pattern Properties
+
+| **Name**     | **Type**        | **Description** |
+|:-------------|:----------------|-----------------|
+|cloudwatchAlarms?|[`cloudwatch.Alarm[]`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-cloudwatch.Alarm.html)|Returns an array of recommended CloudWatch Alarms created by the construct for Kinesis Data stream|
+|glueJob|[`CfnJob`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-glue.CfnJob.html)|Returns an instance of AWS Glue Job created by the construct|
+|glueJobRole|[`iam.Role`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-iam.Role.html)|Returns an instance of the IAM Role created by the construct for the Glue Job|
+|database|[`CfnDatabase`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-glue.CfnDatabase.html)|Returns an instance of AWS Glue Database created by the construct|
+|table|[`CfnTable`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-glue.CfnTable.html)|Returns an instance of the AWS Glue Table created by the construct|
+
+## Default settings
 
 Out of the box implementation of the Construct without any override will set the following defaults:
 
 ### Amazon Kinesis Stream
 
--   Configure least privilege access IAM role for Kinesis Stream
--   Enable server-side encryption for Kinesis Stream using AWS Managed KMS Key
--   Deploy best practices CloudWatch Alarms for the Kinesis Stream
+- Configure least privilege access IAM role for Kinesis Stream
+- Enable server-side encryption for Kinesis Stream using AWS Managed KMS Key
+- Deploy best practices CloudWatch Alarms for the Kinesis Stream
 
 ### Glue Job
 
--   Create a Glue Security Config that configures encryption for CloudWatch, Job Bookmarks, and S3. CloudWatch and Job Bookmarks are encrypted using AWS Managed KMS Key created for AWS Glue Service. The S3 bucket is configured with SSE-S3 encryption mode
--   Configure service role policies that allow AWS Glue to read from Kinesis Data Streams
+- Create a Glue Security Config that configures encryption for CloudWatch, Job Bookmarks, and S3. CloudWatch and Job Bookmarks are encrypted using AWS Managed KMS Key created for AWS Glue Service. The S3 bucket is configured with SSE-S3 encryption mode
+- Configure service role policies that allow AWS Glue to read from Kinesis Data Streams
 
 ### Glue Database
 
--   Create an AWS Glue database. An AWS Glue Table will be added to the database. This table defines the schema for the records buffered in the Amazon Kinesis Data Streams
+- Create an AWS Glue database. An AWS Glue Table will be added to the database. This table defines the schema for the records buffered in the Amazon Kinesis Data Streams
 
 ### Glue Table
 
--   Create an AWS Glue table. The table schema definition is based on the JSON structure of the records buffered in the Amazon Kinesis Data Streams
+- Create an AWS Glue table. The table schema definition is based on the JSON structure of the records buffered in the Amazon Kinesis Data Streams
 
 ### IAM Role
 
--   A job execution role that has privileges to 1) read the ETL script from the S3 bucket location, 2) read records from the Kinesis Stream, and 3) execute the Glue Job
+- A job execution role that has privileges to 1) read the ETL script from the S3 bucket location, 2) read records from the Kinesis Stream, and 3) execute the Glue Job
 
 ### Output S3 Bucket
 
--   An S3 bucket to store the output of the ETL transformation. This bucket will be passed as an argument to the created glue job so that it can be used in the ETL script to write data into it
+- An S3 bucket to store the output of the ETL transformation. This bucket will be passed as an argument to the created glue job so that it can be used in the ETL script to write data into it
+
+### Cloudwatch Alarms
+
+- A CloudWatch Alarm to report when consumer application is reading data slower than expected
+- A CloudWatch Alarm to report when consumer record processing is falling behind (to avoid risk of data loss due to record expiration)
 
 ## Architecture
 
@@ -153,4 +169,4 @@ Out of the box implementation of the Construct without any override will set the
 
 A sample use case which uses this pattern is available under [`use_cases/aws-custom-glue-etl`](https://github.com/awslabs/aws-solutions-constructs/tree/master/source/use_cases/aws-custom-glue-etl).
 
-© Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+© Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.

source/patterns/@aws-solutions-constructs/aws-kinesisstreams-gluejob/lib/index.ts

@@ -16,6 +16,7 @@ import { Effect, IRole, Policy, PolicyStatement } from '@aws-cdk/aws-iam';
 import { Stream, StreamProps } from '@aws-cdk/aws-kinesis';
 import { Bucket } from '@aws-cdk/aws-s3';
 import { Aws, Construct } from '@aws-cdk/core';
+import * as cloudwatch from '@aws-cdk/aws-cloudwatch';
 import * as defaults from '@aws-solutions-constructs/core';
 
 export interface KinesisstreamsToGluejobProps {
@@ -100,6 +101,12 @@ export interface KinesisstreamsToGluejobProps {
    * include only S3, other potential stores may be added in the future.
    */
   readonly outputDataStore?: defaults.SinkDataStoreProps;
+  /**
+   * Whether to create recommended CloudWatch alarms
+   *
+   * @default - Alarms are created
+   */
+  readonly createCloudWatchAlarms?: boolean;
 }
 
 /**
@@ -125,6 +132,7 @@ export class KinesisstreamsToGluejob extends Construct {
    * property is undefined
    */
   public readonly outputBucket?: [Bucket, (Bucket | undefined)?];
+  public readonly cloudwatchAlarms?: cloudwatch.Alarm[];
 
   /**
    * Constructs a new instance of KinesisstreamsToGluejob.Based on the values set in the @props
@@ -165,6 +173,11 @@ export class KinesisstreamsToGluejob extends Construct {
     });
 
     this.glueJobRole = this.buildRolePolicy(scope, id, this.database, this.table, this.glueJob, this.glueJobRole);
+
+    if (props.createCloudWatchAlarms === undefined || props.createCloudWatchAlarms) {
+      // Deploy best practices CW Alarms for Kinesis Stream
+      this.cloudwatchAlarms = defaults.buildKinesisStreamCWAlarms(this);
+    }
   }
 
   /**

source/patterns/@aws-solutions-constructs/aws-kinesisstreams-gluejob/package.json

@@ -61,6 +61,7 @@
     "@aws-cdk/core": "0.0.0",
     "@aws-cdk/aws-logs": "0.0.0",
     "@aws-solutions-constructs/core": "0.0.0",
+    "@aws-cdk/aws-cloudwatch": "0.0.0",
     "constructs": "^3.2.0"
   },
   "devDependencies": {
@@ -90,6 +91,7 @@
     "@aws-cdk/aws-s3-assets": "0.0.0",
     "@aws-cdk/core": "0.0.0",
     "@aws-solutions-constructs/core": "0.0.0",
+    "@aws-cdk/aws-cloudwatch": "0.0.0",
     "constructs": "^3.2.0",
     "@aws-cdk/aws-logs": "0.0.0"
   },

source/patterns/@aws-solutions-constructs/aws-kinesisstreams-gluejob/test/integ.existing-job.expected.json

@@ -83,6 +83,32 @@
         }
       }
     },
+    "testkinesisstreamslambdaKinesisStreamGetRecordsIteratorAgeAlarmFB74C363": {
+      "Type": "AWS::CloudWatch::Alarm",
+      "Properties": {
+        "ComparisonOperator": "GreaterThanOrEqualToThreshold",
+        "EvaluationPeriods": 1,
+        "AlarmDescription": "Consumer Record Processing Falling Behind, there is risk for data loss due to record expiration.",
+        "MetricName": "GetRecords.IteratorAgeMilliseconds",
+        "Namespace": "AWS/Kinesis",
+        "Period": 300,
+        "Statistic": "Maximum",
+        "Threshold": 2592000
+      }
+    },
+    "testkinesisstreamslambdaKinesisStreamReadProvisionedThroughputExceededAlarm5ABF4346": {
+      "Type": "AWS::CloudWatch::Alarm",
+      "Properties": {
+        "ComparisonOperator": "GreaterThanThreshold",
+        "EvaluationPeriods": 1,
+        "AlarmDescription": "Consumer Application is Reading at a Slower Rate Than Expected.",
+        "MetricName": "ReadProvisionedThroughputExceeded",
+        "Namespace": "AWS/Kinesis",
+        "Period": 300,
+        "Statistic": "Average",
+        "Threshold": 0
+      }
+    },
     "GlueDatabase": {
       "Type": "AWS::Glue::Database",
       "Properties": {

source/patterns/@aws-solutions-constructs/aws-kinesisstreams-gluejob/test/integ.no-arguments.expected.json

@@ -410,6 +410,32 @@
         "WorkerType": "G.1X"
       }
     },
+    "testkinesisstreamslambdaKinesisStreamGetRecordsIteratorAgeAlarmFB74C363": {
+      "Type": "AWS::CloudWatch::Alarm",
+      "Properties": {
+        "ComparisonOperator": "GreaterThanOrEqualToThreshold",
+        "EvaluationPeriods": 1,
+        "AlarmDescription": "Consumer Record Processing Falling Behind, there is risk for data loss due to record expiration.",
+        "MetricName": "GetRecords.IteratorAgeMilliseconds",
+        "Namespace": "AWS/Kinesis",
+        "Period": 300,
+        "Statistic": "Maximum",
+        "Threshold": 2592000
+      }
+    },
+    "testkinesisstreamslambdaKinesisStreamReadProvisionedThroughputExceededAlarm5ABF4346": {
+      "Type": "AWS::CloudWatch::Alarm",
+      "Properties": {
+        "ComparisonOperator": "GreaterThanThreshold",
+        "EvaluationPeriods": 1,
+        "AlarmDescription": "Consumer Application is Reading at a Slower Rate Than Expected.",
+        "MetricName": "ReadProvisionedThroughputExceeded",
+        "Namespace": "AWS/Kinesis",
+        "Period": 300,
+        "Statistic": "Average",
+        "Threshold": 0
+      }
+    },
     "GlueDatabase": {
       "Type": "AWS::Glue::Database",
       "Properties": {

source/patterns/@aws-solutions-constructs/aws-kinesisstreams-gluejob/test/test.kinesisstream-gluejob.test.ts

@@ -55,7 +55,7 @@ test('Pattern minimal deployment', () => {
 
   const id = 'test-kinesisstreams-lambda';
 
-  new KinesisstreamsToGluejob(stack, id, props);
+  const construct = new KinesisstreamsToGluejob(stack, id, props);
 
   // check for role creation
   expect(stack).toHaveResourceLike('AWS::IAM::Role', {
@@ -253,6 +253,17 @@ test('Pattern minimal deployment', () => {
       }
     }
   }, ResourcePart.CompleteDefinition);
+
+  // Check for cloudwatch alarm
+  expect(stack).toCountResources('AWS::CloudWatch::Alarm', 2);
+
+  // Check for properties
+  expect(construct.database).toBeDefined();
+  expect(construct.glueJob).toBeDefined();
+  expect(construct.table).toBeDefined();
+  expect(construct.kinesisStream).toBeDefined();
+  expect(construct.glueJobRole).toBeDefined();
+  expect(construct.cloudwatchAlarms).toBeDefined();
 });
 
 // --------------------------------------------------------------
@@ -527,12 +538,13 @@ test('When database and table are provided', () => {
 });
 
 // --------------------------------------------------------------
-// When database and table are not provided
+// When database and table are not provided & cloudwatch alarms set to false
 // --------------------------------------------------------------
-test('When database and table are not provided', () => {
+test('When database and table are not provided & cloudwatch alarms set to false', () => {
   // Initial setup
   const stack = new Stack();
   const props: KinesisstreamsToGluejobProps = {
+    createCloudWatchAlarms: false,
     glueJobProps: {
       command: {
         name: 'glueetl',
@@ -558,7 +570,7 @@ test('When database and table are not provided', () => {
       comment: "Some value associated with the record"
     }]
   };
-  new KinesisstreamsToGluejob(stack, 'test-kinesisstreams-lambda', props);
+  const construct = new KinesisstreamsToGluejob(stack, 'test-kinesisstreams-lambda', props);
   expect(stack).toHaveResourceLike('AWS::Glue::Database', {
     Type: "AWS::Glue::Database",
     Properties: {
@@ -655,4 +667,10 @@ test('When database and table are not provided', () => {
       }
     }
   }, ResourcePart.CompleteDefinition);
+
+  // Cloudwatch alarms is set to false, no CFN def should exist
+  expect(stack).not.toHaveResource('AWS::CloudWatch::Alarm');
+
+  // Since alarms is set to false, cloudwatch alarms property should be undefined
+  expect(construct.cloudwatchAlarms).toBeUndefined();
 });