From 332f984ac0f323179fa9156266ccebfc8705eb8a Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Mon, 11 Jan 2021 10:04:54 -0800 Subject: [PATCH 01/45] Start KAV2 README --- .../@aws-cdk/aws-kinesisanalyticsv2/README.md | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 packages/@aws-cdk/aws-kinesisanalyticsv2/README.md diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/README.md b/packages/@aws-cdk/aws-kinesisanalyticsv2/README.md new file mode 100644 index 0000000000000..4f47943a87f85 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/README.md @@ -0,0 +1,77 @@ +The `@aws-cdk/aws-kinesisanalyticsv2` package provides constructs for +creating Kinesis Data Analytics applications. + +For more information, see the AWS documentation for [Kinesis Data +Analytics](https://aws.amazon.com/kinesis/data-analytics/). + +## Creating a Flink Kinesis Analytics Application + +To create a new Flink application, use the FlinkApplication construct. + +```ts +import * as ka from '@aws-cdk/aws-kinesisanalyticsv2'; + +const flinkApp = new ka.FlinkApplication(this, 'Application', { + code: ka.ApplicationCode.fromBucket(bucket, 'my-app.jar'), + runtime: ka.FlinkRuntime.FLINK_1_11, +}); +``` + +The `code` property can use `fromBucket` as shown above to reference a jar +file in s3 or `fromAsset` to reference a local file. + +```ts +import * as ka from '@aws-cdk/aws-kinesisanalyticsv2'; +import * as path from 'path'; + +const flinkApp = new ka.FlinkApplication(this, 'Application', { + code: ka.ApplicationCode.fromAsset(path.join(__dirname, 'my-app.jar')), + // ... +}); +``` + +The `propertyGroups` property provides a way of passing arbitrary runtime +properties to your Flink application. You can use the +aws-kinesisanalytics-runtime library to [retrieve these +properties](https://docs.aws.amazon.com/kinesisanalytics/latest/java/how-properties.html#how-properties-access). + +```ts +import * as ka from '@aws-cdk/aws-kinesisanalyticsv2'; +import * as path from 'path'; + +const flinkApp = new ka.FlinkApplication(this, 'Application', { + // ... + propertyGroups: [ + new ka.PropertyGroup('FlinkApplicationProperties', { + inputStreamName: 'my-input-kinesis-stream', + outputStreamName: 'my-output-kinesis-stream', + }), + ], +}); +``` + +Flink applications also have specific configuration for passing parameters +when the Flink job starts. These include parameters for checkpointing, +snapshotting, monitoring, and parallelism. + +```ts +import * as ka from '@aws-cdk/aws-kinesisanalyticsv2'; + +const flinkApp = new ka.FlinkApplication(this, 'Application', { + code: ka.ApplicationCode.fromBucket(bucket, 'my-app.jar'), + runtime: ka.FlinkRuntime.FLINK_1_11, + checkpointingEnabled: true, // default is true + checkpointInterval: cdk.Duration.seconds(30), // default is 1 minute + minPausesBetweenCheckpoints: cdk.Duration.seconds(10), // default is 5 seconds + logLevel: ka.LogLevel.ERROR, // default is INFO + metricsLevel: ka.MetricsLevel.PARALLELISM // default is APPLICATION + autoScalingEnabled: false // default is true + parallelism: 32 // default is 1 + parallelismPerKpu: 2 // default is 1 + snapshotsEnabled: false // default is true +}); +``` + +## Creating an SQL Kinesis Analytics Application + +Constructs for SQL applications are not implemented yet. From fedb2cc38357768b6b65860990216507bdba3b18 Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Wed, 20 Jan 2021 09:38:23 -0800 Subject: [PATCH 02/45] Add new construct scaffolding Copied example-resource construct. --- .../aws-kinesisanalyticsv2/.eslintrc.js | 3 + .../aws-kinesisanalyticsv2/.gitignore | 19 ++ .../aws-kinesisanalyticsv2/.npmignore | 26 +++ .../@aws-cdk/aws-kinesisanalyticsv2/LICENSE | 201 +++++++++++++++++ .../@aws-cdk/aws-kinesisanalyticsv2/NOTICE | 2 + .../aws-kinesisanalyticsv2/jest.config.js | 2 + .../lib/flink-application.ts | 193 +++++++++++++++++ .../aws-kinesisanalyticsv2/lib/index.ts | 1 + .../lib/private/example-resource-common.ts | 18 ++ .../aws-kinesisanalyticsv2/package.json | 106 +++++++++ .../test/example-resource.test.ts | 204 ++++++++++++++++++ .../test/integ.example-resource.expected.json | 36 ++++ .../test/integ.example-resource.ts | 25 +++ 13 files changed, 836 insertions(+) create mode 100644 packages/@aws-cdk/aws-kinesisanalyticsv2/.eslintrc.js create mode 100644 packages/@aws-cdk/aws-kinesisanalyticsv2/.gitignore create mode 100644 packages/@aws-cdk/aws-kinesisanalyticsv2/.npmignore create mode 100644 packages/@aws-cdk/aws-kinesisanalyticsv2/LICENSE create mode 100644 packages/@aws-cdk/aws-kinesisanalyticsv2/NOTICE create mode 100644 packages/@aws-cdk/aws-kinesisanalyticsv2/jest.config.js create mode 100644 packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts create mode 100644 packages/@aws-cdk/aws-kinesisanalyticsv2/lib/index.ts create mode 100644 packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/example-resource-common.ts create mode 100644 packages/@aws-cdk/aws-kinesisanalyticsv2/package.json create mode 100644 packages/@aws-cdk/aws-kinesisanalyticsv2/test/example-resource.test.ts create mode 100644 packages/@aws-cdk/aws-kinesisanalyticsv2/test/integ.example-resource.expected.json create mode 100644 packages/@aws-cdk/aws-kinesisanalyticsv2/test/integ.example-resource.ts diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/.eslintrc.js b/packages/@aws-cdk/aws-kinesisanalyticsv2/.eslintrc.js new file mode 100644 index 0000000000000..61dd8dd001f63 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/.eslintrc.js @@ -0,0 +1,3 @@ +const baseConfig = require('cdk-build-tools/config/eslintrc'); +baseConfig.parserOptions.project = __dirname + '/tsconfig.json'; +module.exports = baseConfig; diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/.gitignore b/packages/@aws-cdk/aws-kinesisanalyticsv2/.gitignore new file mode 100644 index 0000000000000..d8a8561d50885 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/.gitignore @@ -0,0 +1,19 @@ +*.js +*.js.map +*.d.ts +tsconfig.json +node_modules +*.generated.ts +dist +.jsii + +.LAST_BUILD +.nyc_output +coverage +nyc.config.js +.LAST_PACKAGE +*.snk +!.eslintrc.js +!jest.config.js + +junit.xml \ No newline at end of file diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/.npmignore b/packages/@aws-cdk/aws-kinesisanalyticsv2/.npmignore new file mode 100644 index 0000000000000..bca0ae2513f1e --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/.npmignore @@ -0,0 +1,26 @@ +# Don't include original .ts files when doing `npm pack` +*.ts +!*.d.ts +coverage +.nyc_output +*.tgz + +dist +.LAST_PACKAGE +.LAST_BUILD +!*.js + +# Include .jsii +!.jsii + +*.snk + +*.tsbuildinfo + +tsconfig.json +.eslintrc.js +jest.config.js + +# exclude cdk artifacts +**/cdk.out +junit.xml \ No newline at end of file diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/LICENSE b/packages/@aws-cdk/aws-kinesisanalyticsv2/LICENSE new file mode 100644 index 0000000000000..b71ec1688783a --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/NOTICE b/packages/@aws-cdk/aws-kinesisanalyticsv2/NOTICE new file mode 100644 index 0000000000000..bfccac9a7f69c --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/NOTICE @@ -0,0 +1,2 @@ +AWS Cloud Development Kit (AWS CDK) +Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/jest.config.js b/packages/@aws-cdk/aws-kinesisanalyticsv2/jest.config.js new file mode 100644 index 0000000000000..54e28beb9798b --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/jest.config.js @@ -0,0 +1,2 @@ +const baseConfig = require('cdk-build-tools/config/jest.config'); +module.exports = baseConfig; diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts new file mode 100644 index 0000000000000..a13a7af03c77d --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts @@ -0,0 +1,193 @@ +import * as iam from '@aws-cdk/aws-iam'; +import * as core from '@aws-cdk/core'; +import { Construct } from 'constructs'; +import { flinkApplicationArnComponents } from './private/example-resource-common'; + +export interface IFlinkApplication extends core.IResource, iam.IGrantable { + /** + * The application ARN. + * + * @attribute + */ + readonly flinkApplicationArn: string; + + /** + * The name of the Flink application. + * + * @attribute + */ + readonly flinkApplicationName: string; + + /** + * The application IAM role. + */ + readonly role?: iam.IRole; + + /** + * Convenience method for adding a policy statement to the application role. + */ + addToRolePolicy(policyStatement: iam.PolicyStatement): boolean; +} + +/** + * Implements the functionality of the Flink applications that is shared + * between CDK created and imported IFlinkApplications. + */ +abstract class FlinkApplicationBase extends core.Resource implements IFlinkApplication { + public abstract readonly flinkApplicationArn: string; + public abstract readonly flinkApplicationName: string; + public abstract readonly role?: iam.IRole; + + // Implement iam.IGrantable interface + public abstract readonly grantPrincipal: iam.IPrincipal; + + /** Implement the convenience {@link IFlinkApplication.addToRolePolicy} method. */ + public addToRolePolicy(policyStatement: iam.PolicyStatement): boolean { + if (this.role) { + this.role.addToPolicy(policyStatement); + return true; + } + + return false; + } +} + +/** + * Props for creating a FlinkApplication construct. + */ +export interface FlinkApplicationProps { + /** + * @default - CloudFormation-generated name + */ + readonly applicationName?: string; + + /** + * A role to use to grant permissions to your application. Omitting this + * property and using the default role is recommended. + * + * @default - a new Role will be created + */ + readonly role?: iam.IRole; + + /** + * Provide a RemovalPolicy to override the default. + * + * @default RemovalPolicy.DESTROY + */ + readonly removalPolicy?: core.RemovalPolicy; +} + +/** + * The L2 construct for Flink Kinesis Data Applications. + * + * @resource AWS::KinesisAnalyticsV2::Application + */ +export class FlinkApplication extends core.Resource { + /** + * Import an existing Flink application, defined outside of the CDK code by name. + */ + public static fromFlinkApplicationName(scope: Construct, id: string, flinkApplicationName: string): IFlinkApplication { + class Import extends FlinkApplicationBase { + // Imported flink applications have no associated role or grantPrincipal + public readonly role = undefined; + public readonly grantPrincipal = new iam.UnknownPrincipal({ resource: this }); + + public readonly flinkApplicationName = flinkApplicationName; + public readonly flinkApplicationArn = core.Stack.of(scope) + .formatArn(flinkApplicationArnComponents(flinkApplicationName)); + } + + return new Import(scope, id); + } + + public readonly flinkApplicationArn: string; + public readonly flinkApplicationName: string; + + // Role must be optional for JSII compatibility + public readonly role?: iam.IRole; + + public readonly grantPrincipal: iam.IPrincipal; + + constructor(scope: Construct, id: string, props: FlinkApplicationProps) { + super(scope, id, { physicalName: props.applicationName }); + + // We often add validations for properties, + // so that customers receive feedback about incorrect properties + // sooner than a CloudFormation deployment. + // However, when validating string (and number!) properties, + // it's important to remember that the value can be a CFN function + // (think a { Ref: ParameterName } expression in CloudFormation), + // and that sort of value would be also encoded as a string; + // so, we need to use the Token.isUnresolved() method from the core library + // to skip validation in that case. + // if (props.waitConditionHandleName !== undefined && + // !core.Token.isUnresolved(props.waitConditionHandleName) && + // !/^[_a-zA-Z]+$/.test(props.waitConditionHandleName)) { + // throw new Error('waitConditionHandleName must be non-empty and contain only letters and underscores, ' + + // `got: '${props.waitConditionHandleName}'`); + // } + + // If this was a real resource, we would use a specific L1 for that resource + // (like a CfnBucket inside the Bucket class), + // but since this is just an example, + // we'll use CloudFormation wait conditions. + + // Remember to always, always, pass 'this' as the first argument + // when creating any constructs inside your L2s! + // This guarantees that they get scoped correctly, + // and the CDK will make sure their locally-unique identifiers + // are globally unique, which makes your L2 compose. + const waitConditionHandle = new core.CfnWaitConditionHandle(this, 'WaitConditionHandle'); + + // The 'main' L1 you create should always have the logical ID 'Resource'. + // This is important, so that the ConstructNode.defaultChild method works correctly. + // The local variable representing the L1 is often called 'resource' as well. + const resource = new core.CfnWaitCondition(this, 'Resource', { + count: 0, + handle: waitConditionHandle.ref, + timeout: '10', + }); + + // The resource's physical name and ARN are set using + // some protected methods from the Resource superclass + // that correctly resolve when your L2 is used in another resource + // that is in a different AWS region or account than this one. + this.flinkApplicationName = this.getResourceNameAttribute( + // A lot of the CloudFormation resources return their physical name + // when the Ref function is used on them. + // If your resource is like that, simply pass 'resource.ref' here. + // However, if Ref for your resource returns something else, + // it's often still possible to use CloudFormation functions to get out the physical name; + // for example, if Ref for your resource returns the ARN, + // and the ARN for your resource is of the form 'arn:aws::::resource/physical-name', + // which is quite common, + // you can use Fn::Select and Fn::Split to take out the part after the '/' from the ARN: + core.Fn.select(1, core.Fn.split('/', resource.ref)), + ); + this.flinkApplicationArn = this.getResourceArnAttribute( + // A lot of the L1 classes have an 'attrArn' property - + // if yours does, use it here. + // However, if it doesn't, + // you can often formulate the ARN yourself, + // using the Stack.formatArn helper function. + // Here, we assume resource.ref returns the physical name of the resource. + core.Stack.of(this).formatArn(flinkApplicationArnComponents(resource.ref)), + // always use the protected physicalName property for this second argument + flinkApplicationArnComponents(this.physicalName)); + + // if a role wasn't passed, create one + const role = props.role || new iam.Role(this, 'Role', { + // of course, fill your correct service principal here + assumedBy: new iam.ServicePrincipal('cloudformation.amazonaws.com'), + }); + this.role = role; + // we need this to correctly implement the iam.IGrantable interface + this.grantPrincipal = role; + + // this is how you apply the removal policy + resource.applyRemovalPolicy(props.removalPolicy, { + // this is the default to apply if props.removalPolicy is undefined + default: core.RemovalPolicy.DESTROY, + }); + } +} diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/index.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/index.ts new file mode 100644 index 0000000000000..3d32418881f9d --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/index.ts @@ -0,0 +1 @@ +export * from './flink-application'; diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/example-resource-common.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/example-resource-common.ts new file mode 100644 index 0000000000000..5b221455096b2 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/example-resource-common.ts @@ -0,0 +1,18 @@ +import * as cdk from '@aws-cdk/core'; + +// This file contains utility functions used in the implementation of ExampleResource +// which we don't want to make part of the public API of this module +// (in fact, we can't, as JSII does not work for standalone functions!). +// So, while the functions are exported from this file, +// this file is not listed in index.ts, +// and so these functions are effectively 'module-private'. +// To make it clear that this file should not be exported, +// we place it in a subdirectory of lib called 'private'. + +export function flinkApplicationArnComponents(resourceName: string): cdk.ArnComponents { + return { + service: 'kinesisanalyticsv2', + resource: 'application', + resourceName, + }; +} diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/package.json b/packages/@aws-cdk/aws-kinesisanalyticsv2/package.json new file mode 100644 index 0000000000000..4bcb212f10d34 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/package.json @@ -0,0 +1,106 @@ +{ + "name": "@aws-cdk/aws-kinesisanalyticsv2", + "private": true, + "version": "0.0.0", + "description": "A CDK Construct Library for Kinesis Analytics V2 constructs", + "main": "lib/index.js", + "types": "lib/index.d.ts", + "jsii": { + "outdir": "dist", + "targets": { + "java": { + "package": "software.amazon.awscdk.services.kinesisanalyticsv2", + "maven": { + "groupId": "software.amazon.awscdk", + "artifactId": "kinesisanalyticsv2" + } + }, + "dotnet": { + "namespace": "Amazon.CDK.AWS.KinesisAnalyticsV2", + "packageId": "Amazon.CDK.AWS.KinesisAnalyticsV2", + "signAssembly": true, + "assemblyOriginatorKeyFile": "../../key.snk", + "iconUrl": "https://raw.githubusercontent.com/aws/aws-cdk/master/logo/default-256-dark.png" + }, + "python": { + "distName": "aws-cdk.aws-kinesisanalyticsv2", + "module": "aws_cdk.aws-kinesisanalyticsv2", + "classifiers": [ + "Framework :: AWS CDK", + "Framework :: AWS CDK :: 1" + ] + } + }, + "projectReferences": true + }, + "repository": { + "type": "git", + "url": "https://github.com/aws/aws-cdk.git", + "directory": "packages/@aws-cdk/aws-kinesisanalyticsv2" + }, + "scripts": { + "build": "cdk-build", + "watch": "cdk-watch", + "lint": "cdk-lint", + "test": "cdk-test", + "integ": "cdk-integ", + "pkglint": "pkglint -f", + "package": "cdk-package", + "awslint": "cdk-awslint", + "build+test": "npm run build && npm test", + "build+test+package": "npm run build+test && npm run package", + "compat": "cdk-compat", + "rosetta:extract": "yarn --silent jsii-rosetta extract" + }, + "keywords": [ + "aws", + "cdk", + "kinesis", + "analytics", + "kinesisanalytcs" + ], + "author": { + "name": "Amazon Web Services", + "url": "https://aws.amazon.com", + "organization": true + }, + "license": "Apache-2.0", + "devDependencies": { + "@aws-cdk/assert": "0.0.0", + "cdk-build-tools": "0.0.0", + "cdk-integ-tools": "0.0.0", + "jest": "^26.6.3", + "pkglint": "0.0.0" + }, + "dependencies": { + "@aws-cdk/core": "0.0.0", + "@aws-cdk/aws-iam": "0.0.0", + "constructs": "^3.2.0" + }, + "homepage": "https://github.com/aws/aws-cdk", + "peerDependencies": { + "@aws-cdk/core": "0.0.0", + "@aws-cdk/aws-iam": "0.0.0", + "constructs": "^3.2.0" + }, + "engines": { + "node": ">= 10.13.0 <13 || >=13.7.0" + }, + "stability": "experimental", + "maturity": "experimental", + "awscdkio": { + "announce": false + }, + "cdk-build": { + "cloudformation": [ + "AWS::KinesisAnalyticsV2" + ], + "jest": true, + "env": { + "AWSLINT_BASE_CONSTRUCT": true + } + }, + "ubergen": { + "exclude": true + } +} diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/example-resource.test.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/example-resource.test.ts new file mode 100644 index 0000000000000..e9f56d52769a6 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/example-resource.test.ts @@ -0,0 +1,204 @@ +/* + * We write unit tests using the Jest framework + * (some modules might still use NodeUnit, + * but it's considered Names, and we want to migrate to Jest). + */ + +// import the various CDK assertion helpers +import { ABSENT, ResourcePart } from '@aws-cdk/assert'; +// always import our Jest-specific helpers +import '@aws-cdk/assert/jest'; + +import * as ec2 from '@aws-cdk/aws-ec2'; +import * as iam from '@aws-cdk/aws-iam'; +import * as core from '@aws-cdk/core'; +// Always import the module you're testing qualified - +// don't import individual classes from it! +// Importing it qualified tests whether everything that needs to be exported +// from the module is. +import * as er from '../lib'; + +/* We allow quotes in the object keys used for CloudFormation template assertions */ +/* eslint-disable quote-props */ + +describe('Example Resource', () => { + let stack: core.Stack; + + beforeEach(() => { + // try to factor out as much boilerplate test setup to before methods - + // makes the tests much more readable + stack = new core.Stack(); + }); + + describe('created with default properties', () => { + let exampleResource: er.IExampleResource; + + beforeEach(() => { + exampleResource = new er.ExampleResource(stack, 'ExampleResource'); + }); + + test('creates a CFN WaitConditionHandle resource', () => { + // you can simply assert that a resource of a given type + // was generated in the resulting template + expect(stack).toHaveResource('AWS::CloudFormation::WaitConditionHandle'); + }); + + describe('creates a CFN WaitCondition resource', () => { + test('with count = 0 and timeout = 10', () => { + // you can also assert the properties of the resulting resource + expect(stack).toHaveResource('AWS::CloudFormation::WaitCondition', { + 'Count': 0, + 'Timeout': '10', + 'Handle': { + // Don't be afraid of using the generated logical IDs in your tests! + // While they look random, they are actually only dependent on the + // path constructs have in the tree. + // Since changing logical IDs as the library changes actually causes + // problems for their customers (their CloudFormation resources will be replaced), + // it's good for the unit tests to verify that the logical IDs are stable. + 'Ref': 'ExampleResourceWaitConditionHandle9C53A8D3', + }, + // this is how you can check a given property is _not_ set + 'RandomProperty': ABSENT, + }); + }); + + test('with retention policy = Retain', () => { + // haveResource asserts _all_ properties of a resource, + // while haveResourceLike only those that you provide + expect(stack).toHaveResourceLike('AWS::CloudFormation::WaitCondition', { + 'DeletionPolicy': 'Retain', + 'UpdateReplacePolicy': 'Retain', + // by default, haveResource and haveResourceLike only assert the properties of a resource - + // here's how you make them look at the entire resource definition + }, ResourcePart.CompleteDefinition); + }); + }); + + test('returns true from addToResourcePolicy', () => { + const result = exampleResource.addToRolePolicy(new iam.PolicyStatement({ + actions: ['kms:*'], + resources: ['*'], + })); + + expect(result).toBe(true); + }); + + test('correctly adds s3:Get* permissions when grantRead() is called', () => { + const role = new iam.Role(stack, 'Role', { + assumedBy: new iam.AnyPrincipal(), + }); + + exampleResource.grantRead(role); + + expect(stack).toHaveResourceLike('AWS::IAM::Policy', { + 'PolicyDocument': { + 'Statement': [ + { + 'Action': 's3:Get*', + 'Resource': { + 'Fn::Join': ['', [ + 'arn:', + { 'Ref': 'AWS::Partition' }, + ':cloudformation:', + { 'Ref': 'AWS::Region' }, + ':', + { 'Ref': 'AWS::AccountId' }, + ':wait-condition/', + { 'Ref': 'ExampleResourceAC53F4AE' }, + ]], + }, + }, + ], + }, + }); + }); + }); + + describe('created with a VPC', () => { + let exampleResource: er.IExampleResource; + let vpc: ec2.IVpc; + + beforeEach(() => { + vpc = new ec2.Vpc(stack, 'Vpc'); + exampleResource = new er.ExampleResource(stack, 'ExampleResource', { + vpc, + }); + }); + + test('allows manipulating its connections object', () => { + exampleResource.connections.allowToAnyIpv4(ec2.Port.allTcp()); + }); + + test('correctly fills out the subnetIds property of the created VPC endpoint', () => { + expect(stack).toHaveResourceLike('AWS::EC2::VPCEndpoint', { + 'SubnetIds': [ + { 'Ref': 'VpcPrivateSubnet1Subnet536B997A' }, + { 'Ref': 'VpcPrivateSubnet2Subnet3788AAA1' }, + ], + }); + }); + }); + + describe('imported by name', () => { + let exampleResource: er.IExampleResource; + + beforeEach(() => { + exampleResource = er.ExampleResource.fromExampleResourceName(stack, 'ExampleResource', + 'my-example-resource-name'); + }); + + test('has the same name as it was imported with', () => { + expect(exampleResource.exampleResourceName).toEqual('my-example-resource-name'); + }); + + test('renders the correct ARN for Example Resource', () => { + // We can't simply compare the value we get from exampleResource.exampleResourceArn, + // as it will contain unresolved late-bound values + // (what we in the CDK call Tokens). + // So, use a utility method on Stack that allows you to resolve those Tokens + // into their correct values. + const arn = stack.resolve(exampleResource.exampleResourceArn); + expect(arn).toEqual({ + 'Fn::Join': ['', [ + 'arn:', + { 'Ref': 'AWS::Partition' }, + ':cloudformation:', + { 'Ref': 'AWS::Region' }, + ':', + { 'Ref': 'AWS::AccountId' }, + ':wait-condition/my-example-resource-name', + ]], + }); + }); + + test('returns false from addToResourcePolicy', () => { + const result = exampleResource.addToRolePolicy(new iam.PolicyStatement({ + actions: ['kms:*'], + resources: ['*'], + })); + + expect(result).toEqual(false); + }); + }); + + test('cannot be created with a physical name containing illegal characters', () => { + // this is how we write tests that expect an exception to be thrown + expect(() => { + new er.ExampleResource(stack, 'ExampleResource', { + waitConditionHandleName: 'a-1234', + }); + // it's not enough to know an exception was thrown - + // we have to verify that its message is what we expected + }).toThrow(/waitConditionHandleName must be non-empty and contain only letters and underscores, got: 'a-1234'/); + }); + + test('does not fail validation if the physical name is a late-bound value', () => { + const parameter = new core.CfnParameter(stack, 'Parameter'); + + // no assertion necessary - the lack of an exception being thrown is all we need in this case + new er.ExampleResource(stack, 'ExampleResource', { + waitConditionHandleName: parameter.valueAsString, + }); + }); +}); diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/integ.example-resource.expected.json b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/integ.example-resource.expected.json new file mode 100644 index 0000000000000..c9e2f4bc4db5a --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/integ.example-resource.expected.json @@ -0,0 +1,36 @@ +{ + "Resources": { + "ExampleResourceWaitConditionHandle9C53A8D3": { + "Type": "AWS::CloudFormation::WaitConditionHandle" + }, + "ExampleResourceAC53F4AE": { + "Type": "AWS::CloudFormation::WaitCondition", + "Properties": { + "Count": 0, + "Handle": { + "Ref": "ExampleResourceWaitConditionHandle9C53A8D3" + }, + "Timeout": "10" + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "ExampleResourceRole0533653E": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "cloudformation.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + } + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/integ.example-resource.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/integ.example-resource.ts new file mode 100644 index 0000000000000..93d082da24989 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/integ.example-resource.ts @@ -0,0 +1,25 @@ +/* + * Our integration tests act as snapshot tests to make sure the rendered template is stable. + * If any changes to the result are required, + * you need to perform an actual CloudFormation deployment of this application, + * and, if it is successful, a new snapshot will be written out. + * + * For more information on CDK integ tests, + * see the main CONTRIBUTING.md file. + */ + +import * as core from '@aws-cdk/core'; +// as in unit tests, we use a qualified import, +// not bring in individual classes +import * as er from '../lib'; + +const app = new core.App(); + +const stack = new core.Stack(app, 'ExampleResourceIntegTestStack'); + +new er.ExampleResource(stack, 'ExampleResource', { + // we don't want to leave trash in the account after running the deployment of this + removalPolicy: core.RemovalPolicy.DESTROY, +}); + +app.synth(); From 86c33236303b76e6a380e588c660d1f5008f1453 Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Mon, 25 Jan 2021 08:53:05 -0800 Subject: [PATCH 03/45] Add initial FlinkApplication construct --- .../@aws-cdk/aws-kinesisanalyticsv2/LICENSE | 2 +- .../@aws-cdk/aws-kinesisanalyticsv2/NOTICE | 2 +- .../@aws-cdk/aws-kinesisanalyticsv2/README.md | 2 + .../lib/application-code.ts | 99 +++++++++ .../lib/flink-application.ts | 108 +++++----- .../aws-kinesisanalyticsv2/lib/index.ts | 2 + .../aws-kinesisanalyticsv2/package.json | 13 +- .../test/code-asset/.gitkeep | 0 .../test/example-resource.test.ts | 204 ------------------ .../test/flink-application.test.ts | 188 ++++++++++++++++ ... => integ.flink-application.expected.json} | 0 ...resource.ts => integ.flink-application.ts} | 7 +- yarn.lock | 23 +- 13 files changed, 383 insertions(+), 267 deletions(-) create mode 100644 packages/@aws-cdk/aws-kinesisanalyticsv2/lib/application-code.ts create mode 100644 packages/@aws-cdk/aws-kinesisanalyticsv2/test/code-asset/.gitkeep delete mode 100644 packages/@aws-cdk/aws-kinesisanalyticsv2/test/example-resource.test.ts create mode 100644 packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts rename packages/@aws-cdk/aws-kinesisanalyticsv2/test/{integ.example-resource.expected.json => integ.flink-application.expected.json} (100%) rename packages/@aws-cdk/aws-kinesisanalyticsv2/test/{integ.example-resource.ts => integ.flink-application.ts} (73%) diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/LICENSE b/packages/@aws-cdk/aws-kinesisanalyticsv2/LICENSE index b71ec1688783a..28e4bdcec77ec 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/LICENSE +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + Copyright 2018-2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/NOTICE b/packages/@aws-cdk/aws-kinesisanalyticsv2/NOTICE index bfccac9a7f69c..5fc3826926b5b 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/NOTICE +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/NOTICE @@ -1,2 +1,2 @@ AWS Cloud Development Kit (AWS CDK) -Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +Copyright 2018-2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/README.md b/packages/@aws-cdk/aws-kinesisanalyticsv2/README.md index 4f47943a87f85..6ad1ae5c8d48e 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/README.md +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/README.md @@ -1,3 +1,5 @@ +# Kinesis Analytics V2 + The `@aws-cdk/aws-kinesisanalyticsv2` package provides constructs for creating Kinesis Data Analytics applications. diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/application-code.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/application-code.ts new file mode 100644 index 0000000000000..64e5cf9da3584 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/application-code.ts @@ -0,0 +1,99 @@ +import * as s3 from '@aws-cdk/aws-s3'; +import * as s3_assets from '@aws-cdk/aws-s3-assets'; +import { Construct } from '@aws-cdk/core'; + +export abstract class ApplicationCode { + public static fromBucket(bucket: s3.IBucket, fileKey: string, objectVersion?: string): BucketApplicationCode { + return new BucketApplicationCode({ + bucket, + fileKey, + objectVersion, + }); + } + + public static fromAsset(path: string, options?: s3_assets.AssetOptions): AssetApplicationCode { + return new AssetApplicationCode(path, options); + } + + public abstract bind(scope: Construct): CodeConfiguration; +} + +interface BucketApplicationCodeProps { + bucket: s3.IBucket; + fileKey: string; + objectVersion?: string; +} + +class BucketApplicationCode extends ApplicationCode { + public readonly bucket: s3.IBucket; + public readonly fileKey: string; + public readonly objectVersion?: string; + + constructor(props: BucketApplicationCodeProps) { + super(); + this.bucket = props.bucket; + this.fileKey = props.fileKey; + this.objectVersion = props.objectVersion; + } + + public bind(_scope: Construct): CodeConfiguration { + return { + codeContent: { + s3ContentLocation: { + bucketArn: this.bucket.bucketArn, + fileKey: this.fileKey, + objectVersion: this.objectVersion, + }, + }, + codeContentType: 'ZIPFILE', + }; + } +} + +class AssetApplicationCode extends ApplicationCode { + private readonly path: string; + private readonly options?: s3_assets.AssetOptions; + private _asset?: s3_assets.Asset; + + constructor(path: string, options?: s3_assets.AssetOptions) { + super(); + this.path = path; + this.options = options; + } + + public bind(scope: Construct): CodeConfiguration { + this._asset = new s3_assets.Asset(scope, 'Code', { + path: this.path, + ...this.options, + }); + + if (!this._asset.isZipArchive) { + throw new Error(`Asset must be a .zip file or a directory (${this.path})`); + } + + return { + codeContent: { + s3ContentLocation: { + bucketArn: this._asset.bucket.bucketArn, + fileKey: this._asset.s3ObjectKey, + }, + }, + codeContentType: 'ZIPFILE', + }; + } + + get asset(): s3_assets.Asset | undefined { + return this._asset; + } +} + +interface CodeConfiguration { + codeContent: { + s3ContentLocation: { + bucketArn: string; + fileKey: string; + objectVersion?: string; + }, + }, + codeContentType: 'ZIPFILE', +} diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts index a13a7af03c77d..7d7464413c95c 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts @@ -1,6 +1,8 @@ import * as iam from '@aws-cdk/aws-iam'; import * as core from '@aws-cdk/core'; import { Construct } from 'constructs'; +import { ApplicationCode } from './application-code'; +import { CfnApplication } from './kinesisanalyticsv2.generated'; import { flinkApplicationArnComponents } from './private/example-resource-common'; export interface IFlinkApplication extends core.IResource, iam.IGrantable { @@ -9,14 +11,14 @@ export interface IFlinkApplication extends core.IResource, iam.IGrantable { * * @attribute */ - readonly flinkApplicationArn: string; + readonly applicationArn: string; /** * The name of the Flink application. * * @attribute */ - readonly flinkApplicationName: string; + readonly applicationName: string; /** * The application IAM role. @@ -26,25 +28,32 @@ export interface IFlinkApplication extends core.IResource, iam.IGrantable { /** * Convenience method for adding a policy statement to the application role. */ - addToRolePolicy(policyStatement: iam.PolicyStatement): boolean; + addToPrincipalPolicy(policyStatement: iam.PolicyStatement): boolean; +} + +export class FlinkRuntime { + public static FLINK_1_6 = new FlinkRuntime('FLINK-1_6'); + public static FLINK_1_8 = new FlinkRuntime('FLINK-1_8'); + public static FLINK_1_11 = new FlinkRuntime('FLINK-1_11'); + public constructor(public readonly value: string) {} } /** - * Implements the functionality of the Flink applications that is shared - * between CDK created and imported IFlinkApplications. + * Implements the functionality shared between CDK created and imported + * IFlinkApplications. */ abstract class FlinkApplicationBase extends core.Resource implements IFlinkApplication { - public abstract readonly flinkApplicationArn: string; - public abstract readonly flinkApplicationName: string; + public abstract readonly applicationArn: string; + public abstract readonly applicationName: string; public abstract readonly role?: iam.IRole; // Implement iam.IGrantable interface public abstract readonly grantPrincipal: iam.IPrincipal; - /** Implement the convenience {@link IFlinkApplication.addToRolePolicy} method. */ - public addToRolePolicy(policyStatement: iam.PolicyStatement): boolean { + /** Implement the convenience {@link IFlinkApplication.addToPrincipalPolicy} method. */ + public addToPrincipalPolicy(policyStatement: iam.PolicyStatement): boolean { if (this.role) { - this.role.addToPolicy(policyStatement); + this.role.addToPrincipalPolicy(policyStatement); return true; } @@ -54,16 +63,29 @@ abstract class FlinkApplicationBase extends core.Resource implements IFlinkAppli /** * Props for creating a FlinkApplication construct. + * @resource AWS::KinesisAnalyticsV2::Application */ export interface FlinkApplicationProps { /** + * A name for your FlinkApplication that is unique to an AWS account. + * * @default - CloudFormation-generated name */ readonly applicationName?: string; /** - * A role to use to grant permissions to your application. Omitting this - * property and using the default role is recommended. + * The Flink version to use for this application. + */ + readonly runtime: FlinkRuntime; + + /** + * The Flink code asset to run. + */ + readonly code: ApplicationCode; + + /** + * A role to use to grant permissions to your application. Prefer omitting + * this property and using the default role. * * @default - a new Role will be created */ @@ -77,31 +99,35 @@ export interface FlinkApplicationProps { readonly removalPolicy?: core.RemovalPolicy; } +interface Attributes { + applicationName: string; + applicationArn: string; +} + /** * The L2 construct for Flink Kinesis Data Applications. - * * @resource AWS::KinesisAnalyticsV2::Application + * @experimental */ -export class FlinkApplication extends core.Resource { +export class FlinkApplication extends FlinkApplicationBase { /** - * Import an existing Flink application, defined outside of the CDK code by name. + * Import an existing Flink application, defined outside of the CDK code. */ - public static fromFlinkApplicationName(scope: Construct, id: string, flinkApplicationName: string): IFlinkApplication { + public static fromAttributes(scope: Construct, id: string, attributes: Attributes): IFlinkApplication { class Import extends FlinkApplicationBase { // Imported flink applications have no associated role or grantPrincipal public readonly role = undefined; public readonly grantPrincipal = new iam.UnknownPrincipal({ resource: this }); - public readonly flinkApplicationName = flinkApplicationName; - public readonly flinkApplicationArn = core.Stack.of(scope) - .formatArn(flinkApplicationArnComponents(flinkApplicationName)); + public readonly applicationName = attributes.applicationName; + public readonly applicationArn = attributes.applicationArn; } return new Import(scope, id); } - public readonly flinkApplicationArn: string; - public readonly flinkApplicationName: string; + public readonly applicationArn: string; + public readonly applicationName: string; // Role must be optional for JSII compatibility public readonly role?: iam.IRole; @@ -132,27 +158,24 @@ export class FlinkApplication extends core.Resource { // but since this is just an example, // we'll use CloudFormation wait conditions. - // Remember to always, always, pass 'this' as the first argument - // when creating any constructs inside your L2s! - // This guarantees that they get scoped correctly, - // and the CDK will make sure their locally-unique identifiers - // are globally unique, which makes your L2 compose. - const waitConditionHandle = new core.CfnWaitConditionHandle(this, 'WaitConditionHandle'); - - // The 'main' L1 you create should always have the logical ID 'Resource'. - // This is important, so that the ConstructNode.defaultChild method works correctly. - // The local variable representing the L1 is often called 'resource' as well. - const resource = new core.CfnWaitCondition(this, 'Resource', { - count: 0, - handle: waitConditionHandle.ref, - timeout: '10', + this.role = props.role ?? new iam.Role(this, 'Role', { + assumedBy: new iam.ServicePrincipal('kinesisanalytics.amazonaws.com'), + }); + this.grantPrincipal = this.role; + + const resource = new CfnApplication(this, 'Resource', { + runtimeEnvironment: props.runtime.value, + serviceExecutionRole: this.role.roleArn, + applicationConfiguration: { + applicationCodeConfiguration: props.code.bind(this), + }, }); // The resource's physical name and ARN are set using // some protected methods from the Resource superclass // that correctly resolve when your L2 is used in another resource // that is in a different AWS region or account than this one. - this.flinkApplicationName = this.getResourceNameAttribute( + this.applicationName = this.getResourceNameAttribute( // A lot of the CloudFormation resources return their physical name // when the Ref function is used on them. // If your resource is like that, simply pass 'resource.ref' here. @@ -164,7 +187,7 @@ export class FlinkApplication extends core.Resource { // you can use Fn::Select and Fn::Split to take out the part after the '/' from the ARN: core.Fn.select(1, core.Fn.split('/', resource.ref)), ); - this.flinkApplicationArn = this.getResourceArnAttribute( + this.applicationArn = this.getResourceArnAttribute( // A lot of the L1 classes have an 'attrArn' property - // if yours does, use it here. // However, if it doesn't, @@ -175,18 +198,7 @@ export class FlinkApplication extends core.Resource { // always use the protected physicalName property for this second argument flinkApplicationArnComponents(this.physicalName)); - // if a role wasn't passed, create one - const role = props.role || new iam.Role(this, 'Role', { - // of course, fill your correct service principal here - assumedBy: new iam.ServicePrincipal('cloudformation.amazonaws.com'), - }); - this.role = role; - // we need this to correctly implement the iam.IGrantable interface - this.grantPrincipal = role; - - // this is how you apply the removal policy resource.applyRemovalPolicy(props.removalPolicy, { - // this is the default to apply if props.removalPolicy is undefined default: core.RemovalPolicy.DESTROY, }); } diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/index.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/index.ts index 3d32418881f9d..b4dd25df3ee32 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/index.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/index.ts @@ -1 +1,3 @@ +export * from './application-code'; export * from './flink-application'; + diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/package.json b/packages/@aws-cdk/aws-kinesisanalyticsv2/package.json index 4bcb212f10d34..3981dfcae5f4f 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/package.json +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/package.json @@ -24,7 +24,7 @@ }, "python": { "distName": "aws-cdk.aws-kinesisanalyticsv2", - "module": "aws_cdk.aws-kinesisanalyticsv2", + "module": "aws_cdk.aws_kinesisanalyticsv2", "classifiers": [ "Framework :: AWS CDK", "Framework :: AWS CDK :: 1" @@ -47,10 +47,11 @@ "pkglint": "pkglint -f", "package": "cdk-package", "awslint": "cdk-awslint", - "build+test": "npm run build && npm test", - "build+test+package": "npm run build+test && npm run package", + "build+test": "yarn build && yarn test", + "build+test+package": "yarn build+test && yarn package", "compat": "cdk-compat", - "rosetta:extract": "yarn --silent jsii-rosetta extract" + "rosetta:extract": "yarn --silent jsii-rosetta extract", + "cfn2ts": "cfn2ts" }, "keywords": [ "aws", @@ -75,12 +76,16 @@ "dependencies": { "@aws-cdk/core": "0.0.0", "@aws-cdk/aws-iam": "0.0.0", + "@aws-cdk/aws-s3": "0.0.0", + "@aws-cdk/aws-s3-assets": "0.0.0", "constructs": "^3.2.0" }, "homepage": "https://github.com/aws/aws-cdk", "peerDependencies": { "@aws-cdk/core": "0.0.0", "@aws-cdk/aws-iam": "0.0.0", + "@aws-cdk/aws-s3": "0.0.0", + "@aws-cdk/aws-s3-assets": "0.0.0", "constructs": "^3.2.0" }, "engines": { diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/code-asset/.gitkeep b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/code-asset/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/example-resource.test.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/example-resource.test.ts deleted file mode 100644 index e9f56d52769a6..0000000000000 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/example-resource.test.ts +++ /dev/null @@ -1,204 +0,0 @@ -/* - * We write unit tests using the Jest framework - * (some modules might still use NodeUnit, - * but it's considered Names, and we want to migrate to Jest). - */ - -// import the various CDK assertion helpers -import { ABSENT, ResourcePart } from '@aws-cdk/assert'; -// always import our Jest-specific helpers -import '@aws-cdk/assert/jest'; - -import * as ec2 from '@aws-cdk/aws-ec2'; -import * as iam from '@aws-cdk/aws-iam'; -import * as core from '@aws-cdk/core'; -// Always import the module you're testing qualified - -// don't import individual classes from it! -// Importing it qualified tests whether everything that needs to be exported -// from the module is. -import * as er from '../lib'; - -/* We allow quotes in the object keys used for CloudFormation template assertions */ -/* eslint-disable quote-props */ - -describe('Example Resource', () => { - let stack: core.Stack; - - beforeEach(() => { - // try to factor out as much boilerplate test setup to before methods - - // makes the tests much more readable - stack = new core.Stack(); - }); - - describe('created with default properties', () => { - let exampleResource: er.IExampleResource; - - beforeEach(() => { - exampleResource = new er.ExampleResource(stack, 'ExampleResource'); - }); - - test('creates a CFN WaitConditionHandle resource', () => { - // you can simply assert that a resource of a given type - // was generated in the resulting template - expect(stack).toHaveResource('AWS::CloudFormation::WaitConditionHandle'); - }); - - describe('creates a CFN WaitCondition resource', () => { - test('with count = 0 and timeout = 10', () => { - // you can also assert the properties of the resulting resource - expect(stack).toHaveResource('AWS::CloudFormation::WaitCondition', { - 'Count': 0, - 'Timeout': '10', - 'Handle': { - // Don't be afraid of using the generated logical IDs in your tests! - // While they look random, they are actually only dependent on the - // path constructs have in the tree. - // Since changing logical IDs as the library changes actually causes - // problems for their customers (their CloudFormation resources will be replaced), - // it's good for the unit tests to verify that the logical IDs are stable. - 'Ref': 'ExampleResourceWaitConditionHandle9C53A8D3', - }, - // this is how you can check a given property is _not_ set - 'RandomProperty': ABSENT, - }); - }); - - test('with retention policy = Retain', () => { - // haveResource asserts _all_ properties of a resource, - // while haveResourceLike only those that you provide - expect(stack).toHaveResourceLike('AWS::CloudFormation::WaitCondition', { - 'DeletionPolicy': 'Retain', - 'UpdateReplacePolicy': 'Retain', - // by default, haveResource and haveResourceLike only assert the properties of a resource - - // here's how you make them look at the entire resource definition - }, ResourcePart.CompleteDefinition); - }); - }); - - test('returns true from addToResourcePolicy', () => { - const result = exampleResource.addToRolePolicy(new iam.PolicyStatement({ - actions: ['kms:*'], - resources: ['*'], - })); - - expect(result).toBe(true); - }); - - test('correctly adds s3:Get* permissions when grantRead() is called', () => { - const role = new iam.Role(stack, 'Role', { - assumedBy: new iam.AnyPrincipal(), - }); - - exampleResource.grantRead(role); - - expect(stack).toHaveResourceLike('AWS::IAM::Policy', { - 'PolicyDocument': { - 'Statement': [ - { - 'Action': 's3:Get*', - 'Resource': { - 'Fn::Join': ['', [ - 'arn:', - { 'Ref': 'AWS::Partition' }, - ':cloudformation:', - { 'Ref': 'AWS::Region' }, - ':', - { 'Ref': 'AWS::AccountId' }, - ':wait-condition/', - { 'Ref': 'ExampleResourceAC53F4AE' }, - ]], - }, - }, - ], - }, - }); - }); - }); - - describe('created with a VPC', () => { - let exampleResource: er.IExampleResource; - let vpc: ec2.IVpc; - - beforeEach(() => { - vpc = new ec2.Vpc(stack, 'Vpc'); - exampleResource = new er.ExampleResource(stack, 'ExampleResource', { - vpc, - }); - }); - - test('allows manipulating its connections object', () => { - exampleResource.connections.allowToAnyIpv4(ec2.Port.allTcp()); - }); - - test('correctly fills out the subnetIds property of the created VPC endpoint', () => { - expect(stack).toHaveResourceLike('AWS::EC2::VPCEndpoint', { - 'SubnetIds': [ - { 'Ref': 'VpcPrivateSubnet1Subnet536B997A' }, - { 'Ref': 'VpcPrivateSubnet2Subnet3788AAA1' }, - ], - }); - }); - }); - - describe('imported by name', () => { - let exampleResource: er.IExampleResource; - - beforeEach(() => { - exampleResource = er.ExampleResource.fromExampleResourceName(stack, 'ExampleResource', - 'my-example-resource-name'); - }); - - test('has the same name as it was imported with', () => { - expect(exampleResource.exampleResourceName).toEqual('my-example-resource-name'); - }); - - test('renders the correct ARN for Example Resource', () => { - // We can't simply compare the value we get from exampleResource.exampleResourceArn, - // as it will contain unresolved late-bound values - // (what we in the CDK call Tokens). - // So, use a utility method on Stack that allows you to resolve those Tokens - // into their correct values. - const arn = stack.resolve(exampleResource.exampleResourceArn); - expect(arn).toEqual({ - 'Fn::Join': ['', [ - 'arn:', - { 'Ref': 'AWS::Partition' }, - ':cloudformation:', - { 'Ref': 'AWS::Region' }, - ':', - { 'Ref': 'AWS::AccountId' }, - ':wait-condition/my-example-resource-name', - ]], - }); - }); - - test('returns false from addToResourcePolicy', () => { - const result = exampleResource.addToRolePolicy(new iam.PolicyStatement({ - actions: ['kms:*'], - resources: ['*'], - })); - - expect(result).toEqual(false); - }); - }); - - test('cannot be created with a physical name containing illegal characters', () => { - // this is how we write tests that expect an exception to be thrown - expect(() => { - new er.ExampleResource(stack, 'ExampleResource', { - waitConditionHandleName: 'a-1234', - }); - // it's not enough to know an exception was thrown - - // we have to verify that its message is what we expected - }).toThrow(/waitConditionHandleName must be non-empty and contain only letters and underscores, got: 'a-1234'/); - }); - - test('does not fail validation if the physical name is a late-bound value', () => { - const parameter = new core.CfnParameter(stack, 'Parameter'); - - // no assertion necessary - the lack of an exception being thrown is all we need in this case - new er.ExampleResource(stack, 'ExampleResource', { - waitConditionHandleName: parameter.valueAsString, - }); - }); -}); diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts new file mode 100644 index 0000000000000..d9e4f521f466d --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts @@ -0,0 +1,188 @@ +import { ResourcePart } from '@aws-cdk/assert'; +import '@aws-cdk/assert/jest'; +import * as iam from '@aws-cdk/aws-iam'; +import * as s3 from '@aws-cdk/aws-s3'; +import * as core from '@aws-cdk/core'; +import * as path from 'path'; +import * as ka from '../lib'; + +function buildStack() { + const stack = new core.Stack(); + const bucket = new s3.Bucket(stack, 'CodeBucket'); + const requiredProps = { + runtime: ka.FlinkRuntime.FLINK_1_11, + code: ka.ApplicationCode.fromBucket(bucket, 'my-app.jar'), + }; + + return { stack, bucket, requiredProps }; +} + +describe('FlinkApplication', () => { + test('default Flink Application', () => { + const { stack, bucket } = buildStack(); + new ka.FlinkApplication(stack, 'FlinkApplication', { + runtime: ka.FlinkRuntime.FLINK_1_11, + code: ka.ApplicationCode.fromBucket(bucket, 'my-app.jar'), + }); + + expect(stack).toHaveResourceLike('AWS::KinesisAnalyticsV2::Application', { + RuntimeEnvironment: 'FLINK-1_11', + ApplicationConfiguration: { + ApplicationCodeConfiguration: { + CodeContent: { + S3ContentLocation: { + BucketARN: stack.resolve(bucket.bucketArn), + FileKey: 'my-app.jar', + }, + }, + CodeContentType: 'ZIPFILE', + }, + }, + }); + + expect(stack).toHaveResourceLike('AWS::KinesisAnalyticsV2::Application', { + DeletionPolicy: 'Delete', + }, ResourcePart.CompleteDefinition); + + expect(stack).toHaveResource('AWS::IAM::Role', { + AssumeRolePolicyDocument: { + Statement: [{ + Action: 'sts:AssumeRole', + Effect: 'Allow', + Principal: { + Service: 'kinesisanalytics.amazonaws.com', + }, + }], + Version: '2012-10-17', + }, + }); + }); + + test('providing a custom role', () => { + const { stack, requiredProps } = buildStack(); + new ka.FlinkApplication(stack, 'FlinkApplication', { + ...requiredProps, + role: new iam.Role(stack, 'CustomRole', { + assumedBy: new iam.ServicePrincipal('custom-principal'), + }), + }); + + expect(stack).toHaveResource('AWS::IAM::Role', { + AssumeRolePolicyDocument: { + Statement: [ + { + Action: 'sts:AssumeRole', + Effect: 'Allow', + Principal: { + Service: 'custom-principal.amazonaws.com', + }, + }, + ], + Version: '2012-10-17', + }, + }); + }); + + test('addToPrincipalPolicy', () => { + const { stack, requiredProps } = buildStack(); + const app = new ka.FlinkApplication(stack, 'FlinkApplication', { + ...requiredProps, + }); + + app.addToPrincipalPolicy(new iam.PolicyStatement({ + actions: ['custom:action'], + resources: ['*'], + })); + + expect(stack).toHaveResource('AWS::IAM::Policy', { + PolicyDocument: { + Version: '2012-10-17', + Statement: [ + { Action: 'custom:action', Effect: 'Allow', Resource: '*' }, + ], + }, + }); + }); + + test('providing a custom runtime', () => { + const { stack, requiredProps } = buildStack(); + new ka.FlinkApplication(stack, 'FlinkApplication', { + ...requiredProps, + runtime: new ka.FlinkRuntime('custom'), + }); + + expect(stack).toHaveResourceLike('AWS::KinesisAnalyticsV2::Application', { + RuntimeEnvironment: 'custom', + }); + }); + + test('providing a custom removal policy', () => { + const { stack, requiredProps } = buildStack(); + new ka.FlinkApplication(stack, 'FlinkApplication', { + ...requiredProps, + removalPolicy: core.RemovalPolicy.RETAIN, + }); + + expect(stack).toHaveResourceLike('AWS::KinesisAnalyticsV2::Application', { + DeletionPolicy: 'Retain', + }, ResourcePart.CompleteDefinition); + }); + + test('granting permissions to resources', () => { + const { stack, requiredProps } = buildStack(); + const app = new ka.FlinkApplication(stack, 'FlinkApplication', { + ...requiredProps, + }); + + const dataBucket = new s3.Bucket(stack, 'DataBucket'); + dataBucket.grantRead(app); + + expect(stack).toHaveResourceLike('AWS::IAM::Policy', { + PolicyDocument: { + Version: '2012-10-17', + Statement: [ + { + Action: ['s3:GetObject*', 's3:GetBucket*', 's3:List*'], + }, + ], + }, + }); + }); + + test('using an asset for code', () => { + const { stack, requiredProps } = buildStack(); + const code = ka.ApplicationCode.fromAsset(path.join(__dirname, 'code-asset')); + new ka.FlinkApplication(stack, 'FlinkApplication', { + ...requiredProps, + code, + }); + + expect(stack).toHaveResourceLike('AWS::KinesisAnalyticsV2::Application', { + RuntimeEnvironment: 'FLINK-1_11', + ApplicationConfiguration: { + ApplicationCodeConfiguration: { + CodeContent: { + S3ContentLocation: { + BucketARN: stack.resolve(code.asset!.bucket.bucketArn), + FileKey: stack.resolve(code.asset!.s3ObjectKey), + }, + }, + CodeContentType: 'ZIPFILE', + }, + }, + }); + }); + + // TODO: Not quite sure what to do with fromAttributes yet. + test('fromAttributes', () => { + const { stack } = buildStack(); + const flinkApp = ka.FlinkApplication.fromAttributes(stack, 'Imported', { + applicationName: 'my-app', + applicationArn: 'my-arn', + }); + + expect(flinkApp.applicationName).toEqual('my-app'); + expect(flinkApp.applicationArn).toEqual('my-arn'); + expect(flinkApp.addToPrincipalPolicy(new iam.PolicyStatement())).toBe(false); + }); +}); diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/integ.example-resource.expected.json b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/integ.flink-application.expected.json similarity index 100% rename from packages/@aws-cdk/aws-kinesisanalyticsv2/test/integ.example-resource.expected.json rename to packages/@aws-cdk/aws-kinesisanalyticsv2/test/integ.flink-application.expected.json diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/integ.example-resource.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/integ.flink-application.ts similarity index 73% rename from packages/@aws-cdk/aws-kinesisanalyticsv2/test/integ.example-resource.ts rename to packages/@aws-cdk/aws-kinesisanalyticsv2/test/integ.flink-application.ts index 93d082da24989..62a42e40bb4b5 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/integ.example-resource.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/integ.flink-application.ts @@ -11,15 +11,12 @@ import * as core from '@aws-cdk/core'; // as in unit tests, we use a qualified import, // not bring in individual classes -import * as er from '../lib'; +import * as ka from '../lib'; const app = new core.App(); const stack = new core.Stack(app, 'ExampleResourceIntegTestStack'); -new er.ExampleResource(stack, 'ExampleResource', { - // we don't want to leave trash in the account after running the deployment of this - removalPolicy: core.RemovalPolicy.DESTROY, -}); +new ka.FlinkApplication(stack, 'ExampleResource', {}); app.synth(); diff --git a/yarn.lock b/yarn.lock index aaee849150073..72da1c73c47d1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2316,9 +2316,9 @@ aws-sdk-mock@^5.1.0: traverse "^0.6.6" aws-sdk@^2.596.0: - version "2.831.0" - resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.831.0.tgz#02607cc911a2136e5aabe624c1282e821830aef2" - integrity sha512-lrOjbGFpjk2xpESyUx2PGsTZgptCy5xycZazPeakNbFO19cOoxjHx3xyxOHsMCYb3pQwns35UvChQT60B4u6cw== + version "2.829.0" + resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.829.0.tgz#da70407a8bddc21b1dc1c9ba61ac3daea39b9364" + integrity sha512-0LV0argbcE1HhOeCeCZWUbpP4rWzwqe+0WmnR+jCJPY0w0n/ntGU/GW8obzhhhUej8pS4AkuAJNgzbwlTnxUmw== dependencies: buffer "4.9.2" events "1.1.1" @@ -2330,7 +2330,22 @@ aws-sdk@^2.596.0: uuid "3.3.2" xml2js "0.4.19" -aws-sdk@^2.637.0, aws-sdk@^2.830.0: +aws-sdk@^2.637.0: + version "2.828.0" + resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.828.0.tgz#6aa599c3582f219568f41fb287eb65753e4a9234" + integrity sha512-JoDujGdncSIF9ka+XFZjop/7G+fNGucwPwYj7OHYMmFIOV5p7YmqomdbVmH/vIzd988YZz8oLOinWc4jM6vvhg== + dependencies: + buffer "4.9.2" + events "1.1.1" + ieee754 "1.1.13" + jmespath "0.15.0" + querystring "0.2.0" + sax "1.2.1" + url "0.10.3" + uuid "3.3.2" + xml2js "0.4.19" + +aws-sdk@^2.830.0: version "2.830.0" resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.830.0.tgz#1d3631d573d18c48373046da7ad92855a7fd1636" integrity sha512-vFatoWkdJmRzpymWbqsuwVsAJdhdAvU2JcM9jKRENTNKJw90ljnLyeP1eKCp4O3/4Lg43PVBwY/KUqPy4wL+OA== From ede24f7d8afa3ca80e8df0409e165d7332edd955 Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Mon, 25 Jan 2021 15:30:12 -0800 Subject: [PATCH 04/45] Add property groups --- .../lib/flink-application.ts | 10 +++++++ .../aws-kinesisanalyticsv2/lib/index.ts | 1 + .../lib/property-group.ts | 20 +++++++++++++ .../test/flink-application.test.ts | 28 +++++++++++++++++++ 4 files changed, 59 insertions(+) create mode 100644 packages/@aws-cdk/aws-kinesisanalyticsv2/lib/property-group.ts diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts index 7d7464413c95c..8f2fcea7f2bcf 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts @@ -4,6 +4,7 @@ import { Construct } from 'constructs'; import { ApplicationCode } from './application-code'; import { CfnApplication } from './kinesisanalyticsv2.generated'; import { flinkApplicationArnComponents } from './private/example-resource-common'; +import { PropertyGroup } from './property-group'; export interface IFlinkApplication extends core.IResource, iam.IGrantable { /** @@ -83,6 +84,12 @@ export interface FlinkApplicationProps { */ readonly code: ApplicationCode; + /** + * Configuration PropertyGroups. You can use these property groups to pass + * arbitrary runtime configuration values to your Flink App. + */ + readonly propertyGroups?: PropertyGroup[]; + /** * A role to use to grant permissions to your application. Prefer omitting * this property and using the default role. @@ -168,6 +175,9 @@ export class FlinkApplication extends FlinkApplicationBase { serviceExecutionRole: this.role.roleArn, applicationConfiguration: { applicationCodeConfiguration: props.code.bind(this), + environmentProperties: { + propertyGroups: props.propertyGroups?.map(pg => pg.toCfn()), + }, }, }); diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/index.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/index.ts index b4dd25df3ee32..1bd447ef6b2d6 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/index.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/index.ts @@ -1,3 +1,4 @@ export * from './application-code'; export * from './flink-application'; +export * from './property-group'; diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/property-group.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/property-group.ts new file mode 100644 index 0000000000000..a7077441905a0 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/property-group.ts @@ -0,0 +1,20 @@ +interface PropertyMap { + [key: string]: string; +} + +export class PropertyGroup { + private readonly propertyGroupId; + private readonly propertyMap; + + constructor(propertyGroupId: string, propertyMap: PropertyMap) { + this.propertyGroupId = propertyGroupId; + this.propertyMap = propertyMap; + } + + public toCfn() { + return { + propertyGroupId: this.propertyGroupId, + propertyMap: this.propertyMap, + }; + } +} diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts index d9e4f521f466d..4da27a6aac3bf 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts @@ -173,6 +173,34 @@ describe('FlinkApplication', () => { }); }); + test('adding property groups', () => { + const { stack, requiredProps } = buildStack(); + new ka.FlinkApplication(stack, 'FlinkApplication', { + ...requiredProps, + propertyGroups: [ + new ka.PropertyGroup('FlinkApplicationProperties', { + SomeProperty: 'SomeValue', + }), + ], + }); + + expect(stack).toHaveResourceLike('AWS::KinesisAnalyticsV2::Application', { + RuntimeEnvironment: 'FLINK-1_11', + ApplicationConfiguration: { + EnvironmentProperties: { + PropertyGroups: [ + { + PropertyGroupId: 'FlinkApplicationProperties', + PropertyMap: { + SomeProperty: 'SomeValue', + }, + }, + ], + }, + }, + }); + }); + // TODO: Not quite sure what to do with fromAttributes yet. test('fromAttributes', () => { const { stack } = buildStack(); From 41dfbd02477c4794872cd99c7a0647cc1dd9fac5 Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Tue, 26 Jan 2021 13:37:00 -0800 Subject: [PATCH 05/45] Add checkpointingEnabled --- .../lib/flink-application.ts | 47 +++++++++++++++++-- .../test/flink-application.test.ts | 28 ++++++++++- 2 files changed, 71 insertions(+), 4 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts index 8f2fcea7f2bcf..3a5600d5bc243 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts @@ -84,6 +84,13 @@ export interface FlinkApplicationProps { */ readonly code: ApplicationCode; + /** + * Whether checkpointing is enabled. + * + * @default true + */ + readonly checkpointingEnabled?: boolean; + /** * Configuration PropertyGroups. You can use these property groups to pass * arbitrary runtime configuration values to your Flink App. @@ -175,9 +182,10 @@ export class FlinkApplication extends FlinkApplicationBase { serviceExecutionRole: this.role.roleArn, applicationConfiguration: { applicationCodeConfiguration: props.code.bind(this), - environmentProperties: { - propertyGroups: props.propertyGroups?.map(pg => pg.toCfn()), - }, + environmentProperties: this.environmentProperties(props.propertyGroups), + flinkApplicationConfiguration: this.flinkApplicationConfiguration({ + checkpointingEnabled: props.checkpointingEnabled, + }), }, }); @@ -212,4 +220,37 @@ export class FlinkApplication extends FlinkApplicationBase { default: core.RemovalPolicy.DESTROY, }); } + + private flinkApplicationConfiguration({ checkpointingEnabled: checkpointingEnabled }: { checkpointingEnabled?: boolean }) { + if (checkpointingEnabled === undefined) { + return; + } + + return { + checkpointConfiguration: this.checkpointConfiguration({ + checkpointingEnabled: checkpointingEnabled, + }), + }; + } + + private environmentProperties(propertyGroups?: PropertyGroup[]) { + if (!propertyGroups || propertyGroups.length === 0) { + return; + } + + return { + propertyGroups: propertyGroups.map(pg => pg.toCfn()), + }; + } + + private checkpointConfiguration({ checkpointingEnabled: checkpointingEnabled }: { checkpointingEnabled?: boolean }) { + if (checkpointingEnabled === undefined) { + return; + } + + return { + checkpointingEnabled, + configurationType: 'CUSTOM', + }; + } } diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts index 4da27a6aac3bf..75c5cabd17a19 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts @@ -25,8 +25,14 @@ describe('FlinkApplication', () => { code: ka.ApplicationCode.fromBucket(bucket, 'my-app.jar'), }); - expect(stack).toHaveResourceLike('AWS::KinesisAnalyticsV2::Application', { + expect(stack).toHaveResource('AWS::KinesisAnalyticsV2::Application', { RuntimeEnvironment: 'FLINK-1_11', + ServiceExecutionRole: { + 'Fn::GetAtt': [ + 'FlinkApplicationRole2F7BCBF6', + 'Arn', + ], + }, ApplicationConfiguration: { ApplicationCodeConfiguration: { CodeContent: { @@ -201,6 +207,26 @@ describe('FlinkApplication', () => { }); }); + test('checkpointEnabled setting', () => { + const { stack, requiredProps } = buildStack(); + new ka.FlinkApplication(stack, 'FlinkApplication', { + ...requiredProps, + checkpointingEnabled: false, + }); + + expect(stack).toHaveResourceLike('AWS::KinesisAnalyticsV2::Application', { + RuntimeEnvironment: 'FLINK-1_11', + ApplicationConfiguration: { + FlinkApplicationConfiguration: { + CheckpointConfiguration: { + ConfigurationType: 'CUSTOM', + CheckpointingEnabled: false, + }, + }, + }, + }); + }); + // TODO: Not quite sure what to do with fromAttributes yet. test('fromAttributes', () => { const { stack } = buildStack(); From 29abb1ab3d0b47a2c9d533f61ffbca48b08a045d Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Tue, 26 Jan 2021 15:07:32 -0800 Subject: [PATCH 06/45] Add minPauseBetweenCheckpoints --- .../lib/flink-application.ts | 30 +++++++++++++++---- .../test/flink-application.test.ts | 20 +++++++++++++ 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts index 3a5600d5bc243..f657066bcd80d 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts @@ -91,6 +91,14 @@ export interface FlinkApplicationProps { */ readonly checkpointingEnabled?: boolean; + /** + * The minumum amount of time in to wait after a checkpoint + * finishes to start a new checkpoint. + * + * @default 5 seconds + */ + readonly minPauseBetweenCheckpoints?: core.Duration; + /** * Configuration PropertyGroups. You can use these property groups to pass * arbitrary runtime configuration values to your Flink App. @@ -185,6 +193,7 @@ export class FlinkApplication extends FlinkApplicationBase { environmentProperties: this.environmentProperties(props.propertyGroups), flinkApplicationConfiguration: this.flinkApplicationConfiguration({ checkpointingEnabled: props.checkpointingEnabled, + minPauseBetweenCheckpoints: props.minPauseBetweenCheckpoints, }), }, }); @@ -221,14 +230,15 @@ export class FlinkApplication extends FlinkApplicationBase { }); } - private flinkApplicationConfiguration({ checkpointingEnabled: checkpointingEnabled }: { checkpointingEnabled?: boolean }) { - if (checkpointingEnabled === undefined) { + private flinkApplicationConfiguration(config: FlinkApplicationConfiguration) { + if (config.checkpointingEnabled === undefined && config.minPauseBetweenCheckpoints === undefined) { return; } return { checkpointConfiguration: this.checkpointConfiguration({ - checkpointingEnabled: checkpointingEnabled, + checkpointingEnabled: config.checkpointingEnabled, + minPauseBetweenCheckpoints: config.minPauseBetweenCheckpoints, }), }; } @@ -243,14 +253,22 @@ export class FlinkApplication extends FlinkApplicationBase { }; } - private checkpointConfiguration({ checkpointingEnabled: checkpointingEnabled }: { checkpointingEnabled?: boolean }) { - if (checkpointingEnabled === undefined) { + private checkpointConfiguration(config: CheckpointConfiguration) { + if (config.checkpointingEnabled === undefined && config.minPauseBetweenCheckpoints === undefined) { return; } return { - checkpointingEnabled, + checkpointingEnabled: config.checkpointingEnabled, + minPauseBetweenCheckpoints: config.minPauseBetweenCheckpoints?.toMilliseconds(), configurationType: 'CUSTOM', }; } } + +interface FlinkApplicationConfiguration extends CheckpointConfiguration {} + +interface CheckpointConfiguration { + checkpointingEnabled?: boolean; + minPauseBetweenCheckpoints?: core.Duration; +} diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts index 75c5cabd17a19..b2e659e7fb8f2 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts @@ -227,6 +227,26 @@ describe('FlinkApplication', () => { }); }); + test('minPauseBetweenCheckpoints setting', () => { + const { stack, requiredProps } = buildStack(); + new ka.FlinkApplication(stack, 'FlinkApplication', { + ...requiredProps, + minPauseBetweenCheckpoints: core.Duration.seconds(10), + }); + + expect(stack).toHaveResourceLike('AWS::KinesisAnalyticsV2::Application', { + RuntimeEnvironment: 'FLINK-1_11', + ApplicationConfiguration: { + FlinkApplicationConfiguration: { + CheckpointConfiguration: { + ConfigurationType: 'CUSTOM', + MinPauseBetweenCheckpoints: 10_000, + }, + }, + }, + }); + }); + // TODO: Not quite sure what to do with fromAttributes yet. test('fromAttributes', () => { const { stack } = buildStack(); From 54df2a73b269f1748f16ea4cfc4995f7194ed16a Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Tue, 26 Jan 2021 15:30:25 -0800 Subject: [PATCH 07/45] Add metricsLevel and logLevel --- .../lib/flink-application.ts | 64 +++++++++++++++++-- .../test/flink-application.test.ts | 20 ++++++ 2 files changed, 78 insertions(+), 6 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts index f657066bcd80d..efbf0d504d4bc 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts @@ -39,6 +39,20 @@ export class FlinkRuntime { public constructor(public readonly value: string) {} } +export enum FlinkLogLevel { + DEBUG = 'DEBUG', + INFO = 'INFO', + WARN = 'WARN', + ERROR = 'ERROR', +} + +export enum FlinkMetricsLevel { + APPLICATION = 'APPLICATION', + TASK = 'TASK', + OPERATOR = 'OPERATOR', + PARALLELISM = 'PARALLELISM', +} + /** * Implements the functionality shared between CDK created and imported * IFlinkApplications. @@ -99,6 +113,19 @@ export interface FlinkApplicationProps { */ readonly minPauseBetweenCheckpoints?: core.Duration; + /** + * The level of verbosity logged from the Flink application to CloudWatch. + */ + readonly logLevel?: FlinkLogLevel; + + /** + * Describes the granularity of the CloudWatch metrics for an application. + * Use caution with the Parallelism level metrics which logs metrics for each + * parallel thread and can quickly become expensive when parallelism is high + * (e.g. > 64). + */ + readonly metricsLevel?: FlinkMetricsLevel; + /** * Configuration PropertyGroups. You can use these property groups to pass * arbitrary runtime configuration values to your Flink App. @@ -194,6 +221,8 @@ export class FlinkApplication extends FlinkApplicationBase { flinkApplicationConfiguration: this.flinkApplicationConfiguration({ checkpointingEnabled: props.checkpointingEnabled, minPauseBetweenCheckpoints: props.minPauseBetweenCheckpoints, + logLevel: props.logLevel, + metricsLevel: props.metricsLevel, }), }, }); @@ -231,15 +260,22 @@ export class FlinkApplication extends FlinkApplicationBase { } private flinkApplicationConfiguration(config: FlinkApplicationConfiguration) { - if (config.checkpointingEnabled === undefined && config.minPauseBetweenCheckpoints === undefined) { + if (config.checkpointingEnabled === undefined && config.minPauseBetweenCheckpoints === undefined && config.logLevel === undefined) { return; } + const checkpointConfiguration = this.checkpointConfiguration({ + checkpointingEnabled: config.checkpointingEnabled, + minPauseBetweenCheckpoints: config.minPauseBetweenCheckpoints, + }); + + const monitoringConfiguration = this.monitoringConfiguration({ + logLevel: config.logLevel, + }); + return { - checkpointConfiguration: this.checkpointConfiguration({ - checkpointingEnabled: config.checkpointingEnabled, - minPauseBetweenCheckpoints: config.minPauseBetweenCheckpoints, - }), + checkpointConfiguration, + monitoringConfiguration, }; } @@ -264,11 +300,27 @@ export class FlinkApplication extends FlinkApplicationBase { configurationType: 'CUSTOM', }; } + + private monitoringConfiguration(config: MonitoringConfiguration) { + if (config.logLevel === undefined && config.metricsLevel === undefined) { + return; + } + + return { + logLevel: config.logLevel, + configurationType: 'CUSTOM', + }; + } } -interface FlinkApplicationConfiguration extends CheckpointConfiguration {} +interface FlinkApplicationConfiguration extends CheckpointConfiguration, MonitoringConfiguration {} interface CheckpointConfiguration { checkpointingEnabled?: boolean; minPauseBetweenCheckpoints?: core.Duration; } + +interface MonitoringConfiguration { + logLevel?: FlinkLogLevel; + metricsLevel?: FlinkMetricsLevel; +} diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts index b2e659e7fb8f2..595f854dfd8fb 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts @@ -247,6 +247,26 @@ describe('FlinkApplication', () => { }); }); + test('logLevel setting', () => { + const { stack, requiredProps } = buildStack(); + new ka.FlinkApplication(stack, 'FlinkApplication', { + ...requiredProps, + logLevel: ka.FlinkLogLevel.DEBUG, + }); + + expect(stack).toHaveResourceLike('AWS::KinesisAnalyticsV2::Application', { + RuntimeEnvironment: 'FLINK-1_11', + ApplicationConfiguration: { + FlinkApplicationConfiguration: { + MonitoringConfiguration: { + ConfigurationType: 'CUSTOM', + LogLevel: 'DEBUG', + }, + }, + }, + }); + }); + // TODO: Not quite sure what to do with fromAttributes yet. test('fromAttributes', () => { const { stack } = buildStack(); From a0bb6693241242e28c7bea0ac1fe9f635c8c8843 Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Wed, 27 Jan 2021 09:10:26 -0800 Subject: [PATCH 08/45] Finish application settings --- .../lib/flink-application.ts | 83 +++++++++++++-- .../test/flink-application.test.ts | 100 +++++++++++++++++- 2 files changed, 172 insertions(+), 11 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts index efbf0d504d4bc..af60ce57063fe 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts @@ -126,6 +126,38 @@ export interface FlinkApplicationProps { */ readonly metricsLevel?: FlinkMetricsLevel; + /** + * Whether the Kinesis Data Analytics service can increase the parallelism of + * the application in response to increased throughput. + * + * @default true + */ + readonly autoScalingEnabled?: boolean; + + /** + * The initial parallelism for the Flink application. Kinesis Data Analytics + * can stop the app, increase the parallelism, and start the app again if + * autoScalingEnabled is true (the default value). + * + * @default 1 + * @minimum 1 + */ + readonly parallelism?: number; + + /** + * The Flink parallelism allowed per Kinesis Processing Unit (KPU). + * + * @default 1 + */ + readonly parallelismPerKpu?: number + + /** + * Determines if Flink snapshots are enabled. + * + * @default true + */ + readonly snapshotsEnabled?: boolean; + /** * Configuration PropertyGroups. You can use these property groups to pass * arbitrary runtime configuration values to your Flink App. @@ -223,7 +255,13 @@ export class FlinkApplication extends FlinkApplicationBase { minPauseBetweenCheckpoints: props.minPauseBetweenCheckpoints, logLevel: props.logLevel, metricsLevel: props.metricsLevel, + autoScalingEnabled: props.autoScalingEnabled, + parallelism: props.parallelism, + parallelismPerKpu: props.parallelismPerKpu, }), + applicationSnapshotConfiguration: { + snapshotsEnabled: props.snapshotsEnabled ?? true, + }, }, }); @@ -260,10 +298,6 @@ export class FlinkApplication extends FlinkApplicationBase { } private flinkApplicationConfiguration(config: FlinkApplicationConfiguration) { - if (config.checkpointingEnabled === undefined && config.minPauseBetweenCheckpoints === undefined && config.logLevel === undefined) { - return; - } - const checkpointConfiguration = this.checkpointConfiguration({ checkpointingEnabled: config.checkpointingEnabled, minPauseBetweenCheckpoints: config.minPauseBetweenCheckpoints, @@ -271,12 +305,26 @@ export class FlinkApplication extends FlinkApplicationBase { const monitoringConfiguration = this.monitoringConfiguration({ logLevel: config.logLevel, + metricsLevel: config.metricsLevel, }); - return { + const parallelismConfiguration = this.parallelismConfiguration({ + autoScalingEnabled: config.autoScalingEnabled, + parallelism: config.parallelism, + parallelismPerKpu: config.parallelismPerKpu, + }); + + const applicationConfiguration = { checkpointConfiguration, monitoringConfiguration, + parallelismConfiguration, }; + + if (Object.values(applicationConfiguration).every(v => v === undefined)) { + return; + } + + return applicationConfiguration; } private environmentProperties(propertyGroups?: PropertyGroup[]) { @@ -308,12 +356,29 @@ export class FlinkApplication extends FlinkApplicationBase { return { logLevel: config.logLevel, + metricsLevel: config.metricsLevel, + configurationType: 'CUSTOM', + }; + } + + private parallelismConfiguration(config: ParallelismConfiguration) { + if (config.autoScalingEnabled === undefined && config.parallelism === undefined && config.parallelismPerKpu === undefined) { + return; + } + + return { + autoScalingEnabled: config.autoScalingEnabled, + parallelism: config.parallelism, + parallelismPerKpu: config.parallelismPerKpu, configurationType: 'CUSTOM', }; } } -interface FlinkApplicationConfiguration extends CheckpointConfiguration, MonitoringConfiguration {} +interface FlinkApplicationConfiguration extends + CheckpointConfiguration, + MonitoringConfiguration, + ParallelismConfiguration {} interface CheckpointConfiguration { checkpointingEnabled?: boolean; @@ -324,3 +389,9 @@ interface MonitoringConfiguration { logLevel?: FlinkLogLevel; metricsLevel?: FlinkMetricsLevel; } + +interface ParallelismConfiguration { + autoScalingEnabled?: boolean; + parallelism?: number; + parallelismPerKpu?: number; +} diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts index 595f854dfd8fb..c9151a17f9921 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts @@ -43,6 +43,9 @@ describe('FlinkApplication', () => { }, CodeContentType: 'ZIPFILE', }, + ApplicationSnapshotConfiguration: { + SnapshotsEnabled: true, + }, }, }); @@ -164,7 +167,6 @@ describe('FlinkApplication', () => { }); expect(stack).toHaveResourceLike('AWS::KinesisAnalyticsV2::Application', { - RuntimeEnvironment: 'FLINK-1_11', ApplicationConfiguration: { ApplicationCodeConfiguration: { CodeContent: { @@ -191,7 +193,6 @@ describe('FlinkApplication', () => { }); expect(stack).toHaveResourceLike('AWS::KinesisAnalyticsV2::Application', { - RuntimeEnvironment: 'FLINK-1_11', ApplicationConfiguration: { EnvironmentProperties: { PropertyGroups: [ @@ -215,7 +216,6 @@ describe('FlinkApplication', () => { }); expect(stack).toHaveResourceLike('AWS::KinesisAnalyticsV2::Application', { - RuntimeEnvironment: 'FLINK-1_11', ApplicationConfiguration: { FlinkApplicationConfiguration: { CheckpointConfiguration: { @@ -235,7 +235,6 @@ describe('FlinkApplication', () => { }); expect(stack).toHaveResourceLike('AWS::KinesisAnalyticsV2::Application', { - RuntimeEnvironment: 'FLINK-1_11', ApplicationConfiguration: { FlinkApplicationConfiguration: { CheckpointConfiguration: { @@ -255,7 +254,6 @@ describe('FlinkApplication', () => { }); expect(stack).toHaveResourceLike('AWS::KinesisAnalyticsV2::Application', { - RuntimeEnvironment: 'FLINK-1_11', ApplicationConfiguration: { FlinkApplicationConfiguration: { MonitoringConfiguration: { @@ -267,6 +265,98 @@ describe('FlinkApplication', () => { }); }); + test('metricsLevel setting', () => { + const { stack, requiredProps } = buildStack(); + new ka.FlinkApplication(stack, 'FlinkApplication', { + ...requiredProps, + metricsLevel: ka.FlinkMetricsLevel.PARALLELISM, + }); + + expect(stack).toHaveResourceLike('AWS::KinesisAnalyticsV2::Application', { + ApplicationConfiguration: { + FlinkApplicationConfiguration: { + MonitoringConfiguration: { + ConfigurationType: 'CUSTOM', + MetricsLevel: 'PARALLELISM', + }, + }, + }, + }); + }); + + test('autoscalingEnabled setting', () => { + const { stack, requiredProps } = buildStack(); + new ka.FlinkApplication(stack, 'FlinkApplication', { + ...requiredProps, + autoScalingEnabled: false, + }); + + expect(stack).toHaveResourceLike('AWS::KinesisAnalyticsV2::Application', { + ApplicationConfiguration: { + FlinkApplicationConfiguration: { + ParallelismConfiguration: { + ConfigurationType: 'CUSTOM', + AutoScalingEnabled: false, + }, + }, + }, + }); + }); + + test('parallelism setting', () => { + const { stack, requiredProps } = buildStack(); + new ka.FlinkApplication(stack, 'FlinkApplication', { + ...requiredProps, + parallelism: 2, + }); + + expect(stack).toHaveResourceLike('AWS::KinesisAnalyticsV2::Application', { + ApplicationConfiguration: { + FlinkApplicationConfiguration: { + ParallelismConfiguration: { + ConfigurationType: 'CUSTOM', + Parallelism: 2, + }, + }, + }, + }); + }); + + test('parallelismPerKpu setting', () => { + const { stack, requiredProps } = buildStack(); + new ka.FlinkApplication(stack, 'FlinkApplication', { + ...requiredProps, + parallelismPerKpu: 2, + }); + + expect(stack).toHaveResourceLike('AWS::KinesisAnalyticsV2::Application', { + ApplicationConfiguration: { + FlinkApplicationConfiguration: { + ParallelismConfiguration: { + ConfigurationType: 'CUSTOM', + ParallelismPerKPU: 2, + }, + }, + }, + }); + }); + + test('snapshotsEnabled setting', () => { + const { stack, requiredProps } = buildStack(); + new ka.FlinkApplication(stack, 'FlinkApplication', { + ...requiredProps, + snapshotsEnabled: false, + }); + + expect(stack).toHaveResourceLike('AWS::KinesisAnalyticsV2::Application', { + ApplicationConfiguration: { + ApplicationSnapshotConfiguration: { + SnapshotsEnabled: false, + }, + }, + }); + }); + // TODO: Not quite sure what to do with fromAttributes yet. test('fromAttributes', () => { const { stack } = buildStack(); From 84c7da6cddfa53607342877a731817e7ec47eb2c Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Wed, 27 Jan 2021 14:51:10 -0800 Subject: [PATCH 09/45] Move flinkAppConfig out of private methods --- .../lib/flink-application.ts | 234 ++++++------------ .../aws-kinesisanalyticsv2/lib/index.ts | 1 + .../lib/private/example-resource-common.ts | 11 +- .../flink-application-configuration.ts | 95 +++++++ .../lib/private/validation.ts | 51 ++++ .../aws-kinesisanalyticsv2/package.json | 2 + .../test/flink-application.test.ts | 149 ++++++++++- 7 files changed, 361 insertions(+), 182 deletions(-) create mode 100644 packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/flink-application-configuration.ts create mode 100644 packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/validation.ts diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts index af60ce57063fe..e235ea9c32904 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts @@ -1,9 +1,12 @@ import * as iam from '@aws-cdk/aws-iam'; +import * as logs from '@aws-cdk/aws-logs'; import * as core from '@aws-cdk/core'; import { Construct } from 'constructs'; import { ApplicationCode } from './application-code'; -import { CfnApplication } from './kinesisanalyticsv2.generated'; +import { CfnApplication, CfnApplicationCloudWatchLoggingOption } from './kinesisanalyticsv2.generated'; import { flinkApplicationArnComponents } from './private/example-resource-common'; +import { flinkApplicationConfiguration, FlinkLogLevel, FlinkMetricsLevel } from './private/flink-application-configuration'; +import { validateFlinkApplicationProps } from './private/validation'; import { PropertyGroup } from './property-group'; export interface IFlinkApplication extends core.IResource, iam.IGrantable { @@ -32,6 +35,13 @@ export interface IFlinkApplication extends core.IResource, iam.IGrantable { addToPrincipalPolicy(policyStatement: iam.PolicyStatement): boolean; } +/** + * Available Flink runtimes for Kinesis Analytics. + * + * @example + * // Creating a new runtime that isn't in CDK yet. + * const runtime = new FlinkRuntime(FLINK-9_99); + */ export class FlinkRuntime { public static FLINK_1_6 = new FlinkRuntime('FLINK-1_6'); public static FLINK_1_8 = new FlinkRuntime('FLINK-1_8'); @@ -39,20 +49,6 @@ export class FlinkRuntime { public constructor(public readonly value: string) {} } -export enum FlinkLogLevel { - DEBUG = 'DEBUG', - INFO = 'INFO', - WARN = 'WARN', - ERROR = 'ERROR', -} - -export enum FlinkMetricsLevel { - APPLICATION = 'APPLICATION', - TASK = 'TASK', - OPERATOR = 'OPERATOR', - PARALLELISM = 'PARALLELISM', -} - /** * Implements the functionality shared between CDK created and imported * IFlinkApplications. @@ -78,7 +74,6 @@ abstract class FlinkApplicationBase extends core.Resource implements IFlinkAppli /** * Props for creating a FlinkApplication construct. - * @resource AWS::KinesisAnalyticsV2::Application */ export interface FlinkApplicationProps { /** @@ -99,36 +94,36 @@ export interface FlinkApplicationProps { readonly code: ApplicationCode; /** - * Whether checkpointing is enabled. + * Whether checkpointing is enabled while your application runs. * * @default true */ readonly checkpointingEnabled?: boolean; /** - * The minumum amount of time in to wait after a checkpoint - * finishes to start a new checkpoint. + * The minimum amount of time in to wait after a checkpoint finishes to start + * a new checkpoint. * * @default 5 seconds */ readonly minPauseBetweenCheckpoints?: core.Duration; /** - * The level of verbosity logged from the Flink application to CloudWatch. + * The level of log verbosity from the Flink application. */ readonly logLevel?: FlinkLogLevel; /** * Describes the granularity of the CloudWatch metrics for an application. - * Use caution with the Parallelism level metrics which logs metrics for each - * parallel thread and can quickly become expensive when parallelism is high - * (e.g. > 64). + * Use caution with Parallelism level metrics. Parallelism granularity logs + * metrics for each parallel thread and can quickly become expensive when + * parallelism is high (e.g. > 64). */ readonly metricsLevel?: FlinkMetricsLevel; /** * Whether the Kinesis Data Analytics service can increase the parallelism of - * the application in response to increased throughput. + * the application in response to resource usage. * * @default true */ @@ -140,7 +135,6 @@ export interface FlinkApplicationProps { * autoScalingEnabled is true (the default value). * * @default 1 - * @minimum 1 */ readonly parallelism?: number; @@ -180,6 +174,9 @@ export interface FlinkApplicationProps { readonly removalPolicy?: core.RemovalPolicy; } +/** + * Attributes passed to the fromAttributes factory method. + */ interface Attributes { applicationName: string; applicationArn: string; @@ -187,12 +184,13 @@ interface Attributes { /** * The L2 construct for Flink Kinesis Data Applications. - * @resource AWS::KinesisAnalyticsV2::Application + * * @experimental + * @resource AWS::KinesisAnalyticsV2::Application */ export class FlinkApplication extends FlinkApplicationBase { /** - * Import an existing Flink application, defined outside of the CDK code. + * Import an existing Flink application, defined outside of CDK code. */ public static fromAttributes(scope: Construct, id: string, attributes: Attributes): IFlinkApplication { class Import extends FlinkApplicationBase { @@ -217,40 +215,26 @@ export class FlinkApplication extends FlinkApplicationBase { constructor(scope: Construct, id: string, props: FlinkApplicationProps) { super(scope, id, { physicalName: props.applicationName }); - - // We often add validations for properties, - // so that customers receive feedback about incorrect properties - // sooner than a CloudFormation deployment. - // However, when validating string (and number!) properties, - // it's important to remember that the value can be a CFN function - // (think a { Ref: ParameterName } expression in CloudFormation), - // and that sort of value would be also encoded as a string; - // so, we need to use the Token.isUnresolved() method from the core library - // to skip validation in that case. - // if (props.waitConditionHandleName !== undefined && - // !core.Token.isUnresolved(props.waitConditionHandleName) && - // !/^[_a-zA-Z]+$/.test(props.waitConditionHandleName)) { - // throw new Error('waitConditionHandleName must be non-empty and contain only letters and underscores, ' + - // `got: '${props.waitConditionHandleName}'`); - // } - - // If this was a real resource, we would use a specific L1 for that resource - // (like a CfnBucket inside the Bucket class), - // but since this is just an example, - // we'll use CloudFormation wait conditions. + validateFlinkApplicationProps(props); this.role = props.role ?? new iam.Role(this, 'Role', { assumedBy: new iam.ServicePrincipal('kinesisanalytics.amazonaws.com'), }); this.grantPrincipal = this.role; + // Permit metric publishing to CloudWatch + this.role.addToPrincipalPolicy(new iam.PolicyStatement({ + actions: ['cloudwatch:PutMetricData'], + resources: ['*'], + })); + const resource = new CfnApplication(this, 'Resource', { runtimeEnvironment: props.runtime.value, serviceExecutionRole: this.role.roleArn, applicationConfiguration: { applicationCodeConfiguration: props.code.bind(this), environmentProperties: this.environmentProperties(props.propertyGroups), - flinkApplicationConfiguration: this.flinkApplicationConfiguration({ + flinkApplicationConfiguration: flinkApplicationConfiguration({ checkpointingEnabled: props.checkpointingEnabled, minPauseBetweenCheckpoints: props.minPauseBetweenCheckpoints, logLevel: props.logLevel, @@ -264,67 +248,53 @@ export class FlinkApplication extends FlinkApplicationBase { }, }, }); + resource.node.addDependency(this.role); - // The resource's physical name and ARN are set using - // some protected methods from the Resource superclass - // that correctly resolve when your L2 is used in another resource - // that is in a different AWS region or account than this one. - this.applicationName = this.getResourceNameAttribute( - // A lot of the CloudFormation resources return their physical name - // when the Ref function is used on them. - // If your resource is like that, simply pass 'resource.ref' here. - // However, if Ref for your resource returns something else, - // it's often still possible to use CloudFormation functions to get out the physical name; - // for example, if Ref for your resource returns the ARN, - // and the ARN for your resource is of the form 'arn:aws::::resource/physical-name', - // which is quite common, - // you can use Fn::Select and Fn::Split to take out the part after the '/' from the ARN: - core.Fn.select(1, core.Fn.split('/', resource.ref)), - ); - this.applicationArn = this.getResourceArnAttribute( - // A lot of the L1 classes have an 'attrArn' property - - // if yours does, use it here. - // However, if it doesn't, - // you can often formulate the ARN yourself, - // using the Stack.formatArn helper function. - // Here, we assume resource.ref returns the physical name of the resource. - core.Stack.of(this).formatArn(flinkApplicationArnComponents(resource.ref)), - // always use the protected physicalName property for this second argument - flinkApplicationArnComponents(this.physicalName)); - - resource.applyRemovalPolicy(props.removalPolicy, { - default: core.RemovalPolicy.DESTROY, - }); - } - - private flinkApplicationConfiguration(config: FlinkApplicationConfiguration) { - const checkpointConfiguration = this.checkpointConfiguration({ - checkpointingEnabled: config.checkpointingEnabled, - minPauseBetweenCheckpoints: config.minPauseBetweenCheckpoints, - }); - - const monitoringConfiguration = this.monitoringConfiguration({ - logLevel: config.logLevel, - metricsLevel: config.metricsLevel, + const logGroup = new logs.LogGroup(this, 'LogGroup', { + retention: logs.RetentionDays.ONE_WEEK, + removalPolicy: core.RemovalPolicy.DESTROY, }); - const parallelismConfiguration = this.parallelismConfiguration({ - autoScalingEnabled: config.autoScalingEnabled, - parallelism: config.parallelism, - parallelismPerKpu: config.parallelismPerKpu, + const logStream = new logs.LogStream(this, 'LogStream', { + logGroup, + removalPolicy: core.RemovalPolicy.DESTROY, }); - const applicationConfiguration = { - checkpointConfiguration, - monitoringConfiguration, - parallelismConfiguration, - }; + const logStreamArn = `${logGroup.logGroupArn}:log-stream:${logStream.logStreamName}`; + + // Permit logging + this.role.addToPrincipalPolicy(new iam.PolicyStatement({ + actions: ['logs:DescribeLogStreams', 'logs:DescribeLogGroups'], + resources: [ + logGroup.logGroupArn, + core.Stack.of(this).formatArn({ + service: 'logs', + resource: 'log-group', + resourceName: '*', + }), + ], + })); + this.role.addToPrincipalPolicy(new iam.PolicyStatement({ + actions: ['logs:PutLogEvents'], + resources: [logStreamArn], + })); + + new CfnApplicationCloudWatchLoggingOption(this, 'LoggingOption', { + applicationName: resource.ref, + cloudWatchLoggingOption: { + logStreamArn, + }, + }).node.addDependency(resource); - if (Object.values(applicationConfiguration).every(v => v === undefined)) { - return; - } + this.applicationName = this.getResourceNameAttribute(resource.ref); + this.applicationArn = this.getResourceArnAttribute( + core.Stack.of(this).formatArn(flinkApplicationArnComponents(resource.ref)), + flinkApplicationArnComponents(this.physicalName), + ); - return applicationConfiguration; + resource.applyRemovalPolicy(props.removalPolicy, { + default: core.RemovalPolicy.DESTROY, + }); } private environmentProperties(propertyGroups?: PropertyGroup[]) { @@ -336,62 +306,4 @@ export class FlinkApplication extends FlinkApplicationBase { propertyGroups: propertyGroups.map(pg => pg.toCfn()), }; } - - private checkpointConfiguration(config: CheckpointConfiguration) { - if (config.checkpointingEnabled === undefined && config.minPauseBetweenCheckpoints === undefined) { - return; - } - - return { - checkpointingEnabled: config.checkpointingEnabled, - minPauseBetweenCheckpoints: config.minPauseBetweenCheckpoints?.toMilliseconds(), - configurationType: 'CUSTOM', - }; - } - - private monitoringConfiguration(config: MonitoringConfiguration) { - if (config.logLevel === undefined && config.metricsLevel === undefined) { - return; - } - - return { - logLevel: config.logLevel, - metricsLevel: config.metricsLevel, - configurationType: 'CUSTOM', - }; - } - - private parallelismConfiguration(config: ParallelismConfiguration) { - if (config.autoScalingEnabled === undefined && config.parallelism === undefined && config.parallelismPerKpu === undefined) { - return; - } - - return { - autoScalingEnabled: config.autoScalingEnabled, - parallelism: config.parallelism, - parallelismPerKpu: config.parallelismPerKpu, - configurationType: 'CUSTOM', - }; - } -} - -interface FlinkApplicationConfiguration extends - CheckpointConfiguration, - MonitoringConfiguration, - ParallelismConfiguration {} - -interface CheckpointConfiguration { - checkpointingEnabled?: boolean; - minPauseBetweenCheckpoints?: core.Duration; -} - -interface MonitoringConfiguration { - logLevel?: FlinkLogLevel; - metricsLevel?: FlinkMetricsLevel; -} - -interface ParallelismConfiguration { - autoScalingEnabled?: boolean; - parallelism?: number; - parallelismPerKpu?: number; } diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/index.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/index.ts index 1bd447ef6b2d6..8bc58fbd2b00e 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/index.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/index.ts @@ -1,4 +1,5 @@ export * from './application-code'; export * from './flink-application'; +export { FlinkLogLevel, FlinkMetricsLevel } from './private/flink-application-configuration'; export * from './property-group'; diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/example-resource-common.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/example-resource-common.ts index 5b221455096b2..1ab626e85bb6d 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/example-resource-common.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/example-resource-common.ts @@ -1,17 +1,8 @@ import * as cdk from '@aws-cdk/core'; -// This file contains utility functions used in the implementation of ExampleResource -// which we don't want to make part of the public API of this module -// (in fact, we can't, as JSII does not work for standalone functions!). -// So, while the functions are exported from this file, -// this file is not listed in index.ts, -// and so these functions are effectively 'module-private'. -// To make it clear that this file should not be exported, -// we place it in a subdirectory of lib called 'private'. - export function flinkApplicationArnComponents(resourceName: string): cdk.ArnComponents { return { - service: 'kinesisanalyticsv2', + service: 'kinesisanalytics', resource: 'application', resourceName, }; diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/flink-application-configuration.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/flink-application-configuration.ts new file mode 100644 index 0000000000000..15ad3c4393884 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/flink-application-configuration.ts @@ -0,0 +1,95 @@ +import * as core from '@aws-cdk/core'; + +/** + * Available log levels for Flink applications. + */ +export enum FlinkLogLevel { + DEBUG = 'DEBUG', + INFO = 'INFO', + WARN = 'WARN', + ERROR = 'ERROR', +} + +/** + * Granularity of metrics sent to CloudWatch. + */ +export enum FlinkMetricsLevel { + APPLICATION = 'APPLICATION', + TASK = 'TASK', + OPERATOR = 'OPERATOR', + PARALLELISM = 'PARALLELISM', +} + +interface FlinkApplicationConfiguration extends + CheckpointConfiguration, + MonitoringConfiguration, + ParallelismConfiguration {} + +interface CheckpointConfiguration { + checkpointingEnabled?: boolean; + minPauseBetweenCheckpoints?: core.Duration; +} + +interface MonitoringConfiguration { + logLevel?: FlinkLogLevel; + metricsLevel?: FlinkMetricsLevel; +} + +interface ParallelismConfiguration { + autoScalingEnabled?: boolean; + parallelism?: number; + parallelismPerKpu?: number; +} + +/** + * Build the nested Cfn FlinkApplicationConfiguration object. This function + * doesn't return empty config objects, returning the minimal config needed to + * express the supplied properties. + * + * This function also handles the quirky configType: 'CUSTOM' setting required + * whenever config in one of the nested groupings. + */ +export function flinkApplicationConfiguration(config: FlinkApplicationConfiguration) { + const checkpointConfiguration = configFor({ + checkpointingEnabled: config.checkpointingEnabled, + minPauseBetweenCheckpoints: config.minPauseBetweenCheckpoints?.toMilliseconds(), + }); + + const monitoringConfiguration = configFor({ + logLevel: config.logLevel, + metricsLevel: config.metricsLevel, + }); + + const parallelismConfiguration = configFor({ + autoScalingEnabled: config.autoScalingEnabled, + parallelism: config.parallelism, + parallelismPerKpu: config.parallelismPerKpu, + }); + + const applicationConfiguration = { + checkpointConfiguration, + monitoringConfiguration, + parallelismConfiguration, + }; + + if (isEmptyObj(applicationConfiguration)) { + return; + } + + return applicationConfiguration; +} + +function configFor(config: {[key: string]: unknown}) { + if (isEmptyObj(config)) { + return; + } + + return { + ...config, + configurationType: 'CUSTOM', + }; +} + +function isEmptyObj(obj: {[key: string]: unknown}) { + return Object.values(obj).every(v => v === undefined); +} diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/validation.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/validation.ts new file mode 100644 index 0000000000000..d229fc19ca58a --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/validation.ts @@ -0,0 +1,51 @@ +import * as core from '@aws-cdk/core'; + +interface ValidatedProps { + applicationName?: string; + parallelism?: number; + parallelismPerKpu?: number; +} + +export function validateFlinkApplicationProps(props: ValidatedProps) { + validateApplicationName(props.applicationName); + validateParallelism(props.parallelism); + validateParallelismPerKpu(props.parallelismPerKpu); +} + +function validateApplicationName(applicationName?: string) { + if (applicationName === undefined || core.Token.isUnresolved(applicationName)) { + return; + } + + if (applicationName.length === 0) { + throw new Error('applicationName cannot be empty. It must contain at least 1 character.'); + } + + if (!/^[a-zA-Z0-9_.-]+$/.test(applicationName)) { + throw new Error(`applicationName may only contain letters, numbers, underscores, hyphens, and periods. Name: ${applicationName}`); + } + + if (applicationName.length > 128) { + throw new Error(`applicationName max length is 128. Name: ${applicationName} is ${applicationName.length} characters.`); + } +} + +function validateParallelism(parallelism?: number) { + if (parallelism === undefined || core.Token.isUnresolved(parallelism)) { + return; + } + + if (parallelism < 1) { + throw new Error('parallelism must be at least 1'); + } +} + +function validateParallelismPerKpu(parallelismPerKpu?: number) { + if (parallelismPerKpu === undefined || core.Token.isUnresolved(parallelismPerKpu)) { + return; + } + + if (parallelismPerKpu < 1) { + throw new Error('parallelismPerKpu must be at least 1'); + } +} diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/package.json b/packages/@aws-cdk/aws-kinesisanalyticsv2/package.json index 3981dfcae5f4f..2031244aac428 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/package.json +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/package.json @@ -76,6 +76,7 @@ "dependencies": { "@aws-cdk/core": "0.0.0", "@aws-cdk/aws-iam": "0.0.0", + "@aws-cdk/aws-logs": "0.0.0", "@aws-cdk/aws-s3": "0.0.0", "@aws-cdk/aws-s3-assets": "0.0.0", "constructs": "^3.2.0" @@ -84,6 +85,7 @@ "peerDependencies": { "@aws-cdk/core": "0.0.0", "@aws-cdk/aws-iam": "0.0.0", + "@aws-cdk/aws-logs": "0.0.0", "@aws-cdk/aws-s3": "0.0.0", "@aws-cdk/aws-s3-assets": "0.0.0", "constructs": "^3.2.0" diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts index c9151a17f9921..c5ca2e84fec1b 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts @@ -1,4 +1,4 @@ -import { ResourcePart } from '@aws-cdk/assert'; +import { arrayWith, objectLike, ResourcePart } from '@aws-cdk/assert'; import '@aws-cdk/assert/jest'; import * as iam from '@aws-cdk/aws-iam'; import * as s3 from '@aws-cdk/aws-s3'; @@ -103,12 +103,11 @@ describe('FlinkApplication', () => { resources: ['*'], })); - expect(stack).toHaveResource('AWS::IAM::Policy', { + expect(stack).toHaveResourceLike('AWS::IAM::Policy', { PolicyDocument: { - Version: '2012-10-17', - Statement: [ - { Action: 'custom:action', Effect: 'Allow', Resource: '*' }, - ], + Statement: arrayWith( + objectLike({ Action: 'custom:action', Effect: 'Allow', Resource: '*' }), + ), }, }); }); @@ -149,11 +148,9 @@ describe('FlinkApplication', () => { expect(stack).toHaveResourceLike('AWS::IAM::Policy', { PolicyDocument: { Version: '2012-10-17', - Statement: [ - { - Action: ['s3:GetObject*', 's3:GetBucket*', 's3:List*'], - }, - ], + Statement: arrayWith( + objectLike({ Action: ['s3:GetObject*', 's3:GetBucket*', 's3:List*'] }), + ), }, }); }); @@ -357,6 +354,135 @@ describe('FlinkApplication', () => { }); }); + // TODO: Put all logging options on FlinkApplication, new ka.Logging or something else? + test('default logging option', () => { + const { stack, requiredProps } = buildStack(); + new ka.FlinkApplication(stack, 'FlinkApplication', { + ...requiredProps, + snapshotsEnabled: false, + }); + + expect(stack).toHaveResource('AWS::KinesisAnalyticsV2::ApplicationCloudWatchLoggingOption', { + ApplicationName: { + Ref: 'FlinkApplicationC5836815', + }, + CloudWatchLoggingOption: { + LogStreamARN: { + 'Fn::Join': [ + '', + [ + { + 'Fn::GetAtt': [ + 'FlinkApplicationLogGroup7739479C', + 'Arn', + ], + }, + ':log-stream:', + { + Ref: 'FlinkApplicationLogStreamB633AF32', + }, + ], + ], + }, + }, + }); + + expect(stack).toHaveResource('AWS::Logs::LogGroup', { + Properties: { + RetentionInDays: 7, + }, + UpdateReplacePolicy: 'Delete', + DeletionPolicy: 'Delete', + }, ResourcePart.CompleteDefinition); + }); + + test('validating applicationName', () => { + const { stack, requiredProps } = buildStack(); + + // Expect no error with valid name + new ka.FlinkApplication(stack, 'ValidString', { + ...requiredProps, + applicationName: 'my-VALID.app_name', + }); + + // Expect no error with ref + new ka.FlinkApplication(stack, 'ValidRef', { + ...requiredProps, + applicationName: new core.CfnParameter(stack, 'Parameter').valueAsString, + }); + + expect(() => { + new ka.FlinkApplication(stack, 'Empty', { + ...requiredProps, + applicationName: '', + }); + }).toThrow(/cannot be empty/); + + expect(() => { + new ka.FlinkApplication(stack, 'InvalidCharacters', { + ...requiredProps, + applicationName: '!!!', + }); + }).toThrow(/may only contain letters, numbers, underscores, hyphens, and periods/); + + expect(() => { + new ka.FlinkApplication(stack, 'TooLong', { + ...requiredProps, + applicationName: 'a'.repeat(129), + }); + }).toThrow(/max length is 128/); + }); + + test('validating parallelism', () => { + const { stack, requiredProps } = buildStack(); + + // Expect no error with valid value + new ka.FlinkApplication(stack, 'ValidNumber', { + ...requiredProps, + parallelism: 32, + }); + + // Expect no error with ref + new ka.FlinkApplication(stack, 'ValidRef', { + ...requiredProps, + parallelism: new core.CfnParameter(stack, 'Parameter', { + type: 'Number', + }).valueAsNumber, + }); + + expect(() => { + new ka.FlinkApplication(stack, 'TooSmall', { + ...requiredProps, + parallelism: 0, + }); + }).toThrow(/must be at least 1/); + }); + + test('validating parallelismPerKpu', () => { + const { stack, requiredProps } = buildStack(); + + // Expect no error with valid value + new ka.FlinkApplication(stack, 'ValidNumber', { + ...requiredProps, + parallelismPerKpu: 10, + }); + + // Expect no error with ref + new ka.FlinkApplication(stack, 'ValidRef', { + ...requiredProps, + parallelismPerKpu: new core.CfnParameter(stack, 'Parameter', { + type: 'Number', + }).valueAsNumber, + }); + + expect(() => { + new ka.FlinkApplication(stack, 'TooSmall', { + ...requiredProps, + parallelismPerKpu: 0, + }); + }).toThrow(/must be at least 1/); + }); + // TODO: Not quite sure what to do with fromAttributes yet. test('fromAttributes', () => { const { stack } = buildStack(); @@ -369,4 +495,5 @@ describe('FlinkApplication', () => { expect(flinkApp.applicationArn).toEqual('my-arn'); expect(flinkApp.addToPrincipalPolicy(new iam.PolicyStatement())).toBe(false); }); + }); From f3ff501cbd8282e27e9b6be04473c7d1f1fbba55 Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Wed, 27 Jan 2021 15:12:28 -0800 Subject: [PATCH 10/45] Move arn components function --- .../lib/flink-application.ts | 21 +++++++++---------- .../lib/private/example-resource-common.ts | 9 -------- .../lib/private/validation.ts | 3 +++ 3 files changed, 13 insertions(+), 20 deletions(-) delete mode 100644 packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/example-resource-common.ts diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts index e235ea9c32904..012eada556cad 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts @@ -4,7 +4,6 @@ import * as core from '@aws-cdk/core'; import { Construct } from 'constructs'; import { ApplicationCode } from './application-code'; import { CfnApplication, CfnApplicationCloudWatchLoggingOption } from './kinesisanalyticsv2.generated'; -import { flinkApplicationArnComponents } from './private/example-resource-common'; import { flinkApplicationConfiguration, FlinkLogLevel, FlinkMetricsLevel } from './private/flink-application-configuration'; import { validateFlinkApplicationProps } from './private/validation'; import { PropertyGroup } from './property-group'; @@ -233,7 +232,9 @@ export class FlinkApplication extends FlinkApplicationBase { serviceExecutionRole: this.role.roleArn, applicationConfiguration: { applicationCodeConfiguration: props.code.bind(this), - environmentProperties: this.environmentProperties(props.propertyGroups), + environmentProperties: props.propertyGroups?.length + ? { propertyGroups: props.propertyGroups.map(pg => pg.toCfn()) } + : undefined, flinkApplicationConfiguration: flinkApplicationConfiguration({ checkpointingEnabled: props.checkpointingEnabled, minPauseBetweenCheckpoints: props.minPauseBetweenCheckpoints, @@ -296,14 +297,12 @@ export class FlinkApplication extends FlinkApplicationBase { default: core.RemovalPolicy.DESTROY, }); } +} - private environmentProperties(propertyGroups?: PropertyGroup[]) { - if (!propertyGroups || propertyGroups.length === 0) { - return; - } - - return { - propertyGroups: propertyGroups.map(pg => pg.toCfn()), - }; - } +function flinkApplicationArnComponents(resourceName: string): core.ArnComponents { + return { + service: 'kinesisanalytics', + resource: 'application', + resourceName, + }; } diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/example-resource-common.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/example-resource-common.ts deleted file mode 100644 index 1ab626e85bb6d..0000000000000 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/example-resource-common.ts +++ /dev/null @@ -1,9 +0,0 @@ -import * as cdk from '@aws-cdk/core'; - -export function flinkApplicationArnComponents(resourceName: string): cdk.ArnComponents { - return { - service: 'kinesisanalytics', - resource: 'application', - resourceName, - }; -} diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/validation.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/validation.ts index d229fc19ca58a..b0f94f56daf77 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/validation.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/validation.ts @@ -6,6 +6,9 @@ interface ValidatedProps { parallelismPerKpu?: number; } +/** + * Early validation for the props used to create FlinkApplications. + */ export function validateFlinkApplicationProps(props: ValidatedProps) { validateApplicationName(props.applicationName); validateParallelism(props.parallelism); From 213c57bf47cb7a4f7a3879ed80e3858a6713cabb Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Wed, 27 Jan 2021 16:08:29 -0800 Subject: [PATCH 11/45] Add first integration test --- .../lib/application-code.ts | 5 + .../lib/flink-application.ts | 11 +- .../test/code-asset/.gitkeep | 0 .../test/code-asset/WordCount.jar | Bin 0 -> 15192 bytes .../integ.flink-application.expected.json | 297 +++++++++++++++++- .../test/integ.flink-application.ts | 21 +- 6 files changed, 300 insertions(+), 34 deletions(-) delete mode 100644 packages/@aws-cdk/aws-kinesisanalyticsv2/test/code-asset/.gitkeep create mode 100644 packages/@aws-cdk/aws-kinesisanalyticsv2/test/code-asset/WordCount.jar diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/application-code.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/application-code.ts index 64e5cf9da3584..ac515c1efd858 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/application-code.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/application-code.ts @@ -15,6 +15,7 @@ export abstract class ApplicationCode { return new AssetApplicationCode(path, options); } + abstract readonly bucket?: s3.IBucket; public abstract bind(scope: Construct): CodeConfiguration; } @@ -85,6 +86,10 @@ class AssetApplicationCode extends ApplicationCode { get asset(): s3_assets.Asset | undefined { return this._asset; } + + get bucket(): s3.IBucket | undefined { + return this._asset?.bucket; + } } interface CodeConfiguration { diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts index 012eada556cad..6b8bf615e20ec 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts @@ -227,11 +227,18 @@ export class FlinkApplication extends FlinkApplicationBase { resources: ['*'], })); + // TODO: Refactor to make this bug impossible + const code = props.code.bind(this); + if (!props.code.bucket) { + throw new Error('BUG: Bound code is missing its bucket.'); + } + props.code.bucket.grantRead(this); + const resource = new CfnApplication(this, 'Resource', { runtimeEnvironment: props.runtime.value, serviceExecutionRole: this.role.roleArn, applicationConfiguration: { - applicationCodeConfiguration: props.code.bind(this), + applicationCodeConfiguration: code, environmentProperties: props.propertyGroups?.length ? { propertyGroups: props.propertyGroups.map(pg => pg.toCfn()) } : undefined, @@ -261,7 +268,7 @@ export class FlinkApplication extends FlinkApplicationBase { removalPolicy: core.RemovalPolicy.DESTROY, }); - const logStreamArn = `${logGroup.logGroupArn}:log-stream:${logStream.logStreamName}`; + const logStreamArn = `arn:${core.Aws.PARTITION}:logs:${core.Aws.REGION}:${core.Aws.ACCOUNT_ID}:log-group:${logGroup.logGroupName}:log-stream:${logStream.logStreamName}`; // Permit logging this.role.addToPrincipalPolicy(new iam.PolicyStatement({ diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/code-asset/.gitkeep b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/code-asset/.gitkeep deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/code-asset/WordCount.jar b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/code-asset/WordCount.jar new file mode 100644 index 0000000000000000000000000000000000000000..9c533e6fea60771f319c0f4923cdbf728bdc72e0 GIT binary patch literal 15192 zcmb_@19)Uxvv$X}Z9AFRw#|ucPc*S@Pdu?EwlT47PBNL;`ZM=n&d2-Rd;ir>_ul<< z)my99?zPsg+Vv_(gMdN!747){A>n5^GvM^=ghmAlE=h5bNP{MHD7%dyfy)`Dju%4m8JKP@_LTr;QQVkKB zlak36(;6H)4>VqLK#KV~zx5R_KI?s@=G+1^f;Z+@*Ht%lOj3c{21^ z`zM$#`3B!x+9oXu(N)GSPmz^CpW&xRmz6_&g=pS27YH9g;q08iJ@ttAjv=QbAFzlR zN6qmJue&UsN6K{iG^lhl@9L@e5%zu=C`CY=m-oVJ*wS}$KzY)jz zXW~xIjwS{+7PkMcL!|#+(Zt=r#@^b*>ED>9{|^z|>>Q1a>|AV}|HOg>n7#eix>jI$ zj6eW@X9xg5{C|w4{)Q_26O~lO&dS8r!o$Ro-pJa($*DqhOc_NI^_5k!$N>{wBtXpV zxhE(vCqJ|pR53DVDXLmkK}-9%5G#a{IZblx3UD+>{R)Gi*Y@3nq3KvnGn@Aj;qJXV z@_Tt;bkcf-@+6f`_NtA+Q*Y zP3!>mb>R+><^@FlVmM5hc;NibLT)fYwfpbg$!!XZeTNOTI=Bf$!A&(DC11mHv z-ZaBwnKMIJEVL&_hSdj{K-L8!dl0&4!OA_R1)Y_$p~AZhS|(g;OB!9>^Eu{L2{)mJ zM60PgymPjYI?24E4DD+XtZ6#y#DOI58yJjlx`&`}c)V0ldGD<1@Cz6cC`r<>y+Cd- zm~0MnRU?8_ar5SiX*L<75lt*TB&4xE5?gRjl{|4RdmtZ6jA!CVo5r7tDTgW2RTL^$ z#ZHjUET3UcoVgU}<+&ZcH}5oazpy)FH-CkISU%G+s$%CW-5ln)V}eEqy!u*^gHSeP z1c%bMG4KU*5;Ewd*I}9)zOsishw6uTd`;>j6a{2aG_c$@s)@G;4O+ch@XbRgzh58t z^d_zur_cW30cH+AN&9!R@PsJo_~Yz?ItUz#nAk>8MQ_Ge{D(;{u9q@<4-j3x@fd?p z>tnr%EiW0_s{xOzj%_Z#VK1sS6G@YZJQObcY}z8>rmn1- z<#0!nj8M6!^#IiIi1p$>qB|!jNU&!h-}0z3$LD!EGiLYk<%tGK_w(uP6l^=Y*eT0{ z?^H-6pP_TW6mot>B%i5s#FUMg8|Iox-RL2H{$nJc!IkgS=SV)ItJtZ7$PWFwoHxR;H*C<*1WyOcav1iDU)^2wU-W`8RUgs(hQ;;oF3HCfm-jpp|1FJS&iT>-zq z%`Wa$q$@N+GjOv~8*fpx+^&LsY31YT}58IKrK6!vC-Z*O6S&g#dpjWUJ5~fHrCjyC~?RZrZ{O`v~!}A zQN;=Y34|6mD{WeX#&dD823Bz_EDaKPnl>VVHkwl^@Gq?WfC)8{i&4i0X|zC_-W96CV>1b?pSe(&fP;maRMb*R9U}-9Iq#ukS?u<{ zI!+6=ZltI+Vw=Ap1^Anl6}P%_o)X)baaw70HBK!s|E@hz>=(R2kON5&rh_Pm zR_TErx@6wu;9Gh+KScZ*l$9wDIT5r+asDwRs)4-HX2`+krOuy=iul*>v!D=+5#C>Sl-1jCLGP!y0oB5zT+Pzl6+I*ylY4HG$!nkcRSb>>y=m zSzX2t|KSEp^nFKPGTo#R1wOp5NP-DvIi5o{Ri9_rg|3pvi#kZQIboE$YOxifiAY#WzsKTm6pkK`b{G80xqcMNM#d)KU zYk7gSR1uvWLV?Z{K*l@~*Cp+iU+B^c$AaIJH8)+(s>dh$B|{b#}zsFt_j{> zn#qNEaDDnj!_IOsJj<+iJT#~_X-=-+=#dRmS@fguqeAfcCDWatuP<--WsX|8Ia#@4e#b!3 z1ZJ!i*a^yqyzhRu7P>anFjQ}(t59KUBp73kV{BSS$nE9y_P8Qu>S*7ojm{jJU<>OB zf-ra39{OcDsPlXexWRsIz|Mbz*NI>8f|^ecG=#{j`lJ}{T?6KiuN5U2esX26}OAVUK&a-O#R-$3Q#p*_2g~? zr5Cb7x_6AWir?k@L5b=IpO#P<&zX zR2(^*Gr(jgt}(V)?8j6<^-Hx^D)}XaGagzw?m-4v-?7v}Sjus{5lY_Sxu6hHg7VKh4-E3|ydd&!Nr7Yn5vwxTOcrp~!Jl=}Kd&scUTH zlf@Th!=E-NpQmQkWZFZRMLEt3AVG~)gHo5;nJ<-`O!pLtWrrpZTLq{T4^_uwiD&a0 zW{hf;MjXhG_ezh%Ye`AldoD>_*A*bSYBEO~6!lhM%`g@3q7UbCzgupQ4X2!~&;`aB zH_J{Gp(ZiWhRSLXUCk=u^@z-~DYMX*J~}@1G#^o&pkS=zOm9Ti8!VwvPT$=YPoNI= z$Y)hrZZ{FuNQtU4w*GJ?3*&=tKS9aSSEpj1DoKrKzKk-HfP7KsPZAkc+LS#X7BgIJH-$Xaq}H!cjPD$1aAK$x zNb;=W5L0fjPMAJ6mE|GRQn~2N4z?dD2#mKIr#^gG8BQNbKaI|ysZlSXFM5r+g_S2p zUmW3TVH)g%{{@v8rG}*>wPQW@Trqv5olCOJkW39p8tqg6GpI^G$WSRqxDj-88Hb-T zHGZEkHD3$^rMpHy$sIA0`Z*a-={A#>W{!1JGIfXl46yV73O6C)f-Db*!-A-kpoDXqmAYoJj~j&u9LAaJIuN0<$C?z8g@nrS~M#EwB0n z%ddV0gVxzsg_-L*kn2P36SYx%N9RLP?E!_{bH&U$u&p9zPn&CJiDC3IebS+# zGo?UoKpPJ`!yMv$1s>A8(3E6LbyZudqEh>=u>OK@mhL~i&|_vmwt zs9F$qL$f-;I&(?FgyZ`nl>?i4YCLQ^ilBwshFw)hp0$3fz}g5?T6MP!Cf>@j#k=u? zVJES(W(^z3Z%ixUIJUB+z^l>MbxU$MICzg*+1MPVJHdvWP29Idc)Wrbix+&nbNOuF zt5--Iq23(U)Jes2zN*3Jlfzw%F#P8#q6bH)DLn90E1qA`C5G z38te0pv)@TKjWgZKfZUe&rz(n<(R*v(}@K~s!V$@ZT#YqZY_61CJva`(gBQsfmY&H z2t}>4Sd6qt=R!=KPI)feK!4}im}BmQ#{dSqzjCKv7HjCKVPp=TNeldrI+ij7lQ_i8 z`Mn{IqRR31460SR+|z3M#Foga)tM~ZxmKY0p1c#CUMe!Ll}^M>drTw4?fveTiCyeT zwc8sBzc1OhE8;ns5^O9R?b_Bjp5tdypM<9-Iwnt^x4Q-=M_UQB6HMVxO`kbCjhfD6 zgZt^(0<4}bO42TRndVCRoo^}N{S;o-x|ekw_~JvSOf2NcMY-xv&-jMXgq^WO*xjr* zOo73&U_xGM`N3nbGXoSIlPyLb(Rx|3pS=9-3n~?&nDR{!rtp?s|)s8U*U^O*V1A06uLd_gO zA*nuEKWG)5o3n=rt&L&4mGMb5&OC#Fc3fyFB$WG{Ks-#Ky=vMG7s$_BgVqcMl)qu_ zA#Zt*(LvBe5gla(Uf;6f6jiDb!viLcmb@O1!>xr)!xeT94}x(A(!YCHQzmXlB^o9= zhd9E*rk;Yy`%zKF)L{u~6i}P`ZDbqly5qikL|4{7Y5gilGmx@a2Rc<0D2X&pd4$dzg| zJ3;OreQc{i@T*aHO$h9pKy9d>?Md(rinGa60^QJvbfOUFmDS=vQ5KZ)K%W$))VdD0 zqvcXFlL4CE^+1`klewucE2XJ}*BM#55u%DK+yp_H* z(X2Xdphm;pAhO?^bF-?MK%IpU0Zux=CcyXsNjTF8o;j3jMC{N?=?gjiuO58!z)k(&gso0SAdBZEnuNyq$w+vui-FC1I0sx>#3jiR#-QfM1 zl$Mbc7L`*LUDMKbS{Xp|xvg4sEK;@rFUopnfpZX6vm0C?Y$4KYOP&xyR!!IlUOhxl z82)_Co6_!AVC?pwCR@ZR`F-ij!9~_-Wl()BwA$LOkI)?CfXFn>S5E)XjRDdHHOZmh-%mKDn?=NG4xRRv@O^*+Of_( ziL#3ArB%bm1j|*2g*t%m=Ob6L?&MOj`OMwN6OGDRwlo-wAhHMl9L3)xDOC|}W z<%_kc$uQ6bq9<5^YX=98S8f4KQ~^zZt<=&rv)O2*-d#IgSMjZl^fhPk*kI7ny|>xj zjP~19OanV2({Sn3R<}3{gNc~9!?w`-3#BjW4S-wCGi2&q?iV*Aa1tpfxxs@r)R!MIXSu2K?#H?+(xqmT8|-|j0v+?oz||es7IDe?%I@^`TC(tt zyPNA%1`BSGeS=;I!JSR;YOV*ss&Ntwvf!?mi+-R|01d@73va^6N46aqoi44Xlq}tS zy^z;Qa)cIa#bmS(UhY6gNOg1YXh7}SC%9ld{a6%0&3mM?^VQl=RFL21@Wr+4R4dAZ z_O7gC(O(0#>x!oB-$`ns;g+}7Vhy9l2IjE@m^+s_xSi#*TvrdCIo1(z6apWItda=o z!SX8vK2=@2)?INo6)IwRty;f;<=U6Fe20I+YcjH?Q*_g}_s&Ak~R{ ziN<|FzP9?{?i@Wh+R*s(#%i3f8rnlbg4zt*!0R<15h8s;@dv0^!&ifv zt2e$mruYGISovZD(k$iz$kh!|11!9eV2$@ zEo0{b(VF@2gc<`PH_UWZC$j=?2XR&EI&3$aGQ zyb7KNwo{N$(E=;C01#~g)F!j{W63fO1x>_c8Gpw+zblnBb=%_N!emZwbUV#d4}IYga->>BEmN?uKivgeOF9uVDj1Y z%ur0Ad&#?!LH)pQn!-FiiaJ#zQ<3oTjY&5sLZ4*6Q!3E!T{Vz=4^a>yh%=ZFBle%O z2D?cno|wD;{w^ry&V|QysB1zgd(}5I=47hK5aO#fY9ES%n5YE2)WKr`sq;}I()Qa_ysCrkDv2`tmB#ZsKh3b~q5iqSqa{;v=`XlRXeyF%SQ3PRzT915q{F?)ASZLm-85L+RX ziDN&hIBmRc1?R5#qs1EW4xB?xlX095!j*&32<6;F1q#G$_>SvX;(70ciLN*9z2i4z zirY)Tn@Du(L3f=bV1ZbZ0$qCrIo0W6otbOy3}Mj1=58u;J(iZ z-2)3Mj$w0&;z0=awFLOwe*{vD#9L%Sg5XiSDUa1B&xDoj;)8De+F@6{_70CDt2}aX zPn8Czq=E6v<6jH!gIx1~>Jec-rf5PHn5GA>_odvfq9aDta2E5l4^M0NnxGjZpzJ zT`gaZ3Ehzv!JQ-@ykF-+g8;P@tq5ta0Ab70oo}Xr-wbL&V$B9Tq=-wD@At}HLoB~y zO1~vfs2Qh4#4G__p+(5dr;eMLNua<_AJKSjt|Ic=~gJD7Uy-; z8qrG@u8NX3#}Cqu&&=32tHz!eo8*cX-}9?K9^^*%Bl_P6x?xhR^0rue1~y?%Q=(|! z(qReLWPTerY51ZFLaKq}?lj+{>;}NbBcNa5uuTQd>w=Y@t2vv}C8Sk@{-7Ke2L(Mv zRG7h!6GJ$D=mcAGmy(K;Zx$=~RW%rrP6dmHJUxMB9KOW7h+1u z%u7>4o?;~CGr>d1{**;l6sPqbvlFRqaAs^Wyawg_#B~PACFnj#HgA;KM8QGhW8+*& zm1Z-SJ3fd@$=JOQym$gox)hn+@y)VmXDS^v{Uu2zn;uJvBe&pY9o?-G_i@RX zUe;I-mG*qp{_b`WI$8ia*B;bCiUQrx0V2L@mgfUKh@)EAVrdOxlO-YpK>zXT&mF@Q z1&bSusUQ4hSY(F!FfA2g@tosjkx_n4JCjxMh&r@kx;h)GYig6m2@LxDU`TZ!E>mot z%b?i|(+m3~tC`u5X-PM(WWrpzpsu7ulZR(2e0QcfRf-g$alGh*q+h~HsU%u-IgZ(| zNFh@+xl!6;39bt;N{E;ASac`UrXFjJ06PW&S8-l+W)Z$YzLL?SH1Zdic#aJY+%H4c zsYy{a*|@r7MKRp?2#%>oHW%M6KhC6oNEfv<1?nYEZBg60iCd@|^^Mp*ftyQ-+A4yb zag{#7OGZ)}i&B8kEJb|}YJhD_e~!b{x(P(3!PFq4efW*?41rI$m_#|Ibd)gLmmRyW zw4%gr5l_W6u7Q<5JeGhA}wSz4>7MxsrqIXwX z$15b)J?4Ih@FaMYa6tjvTIPD)tzM*vy+ zv~X-!S`te%+1sXQVovr)saxw%0VF!?g<^ zm-hVy2z0%HlWln1o2}<Yie0@2Hja=l6#}m04hGCChvXhEx?nRF+SVn4YQ#ifg z$@@i5>sJqh=_gV&K7dVrvL)oz0QZ`ktVVGTl;g&!56qy2%poGca<*JC*MDEwLc49_ z#)P#uFg#(l^Xwl6Jbs1KP9!A$)y4rI>6ES~8KlE+^U(Fl%ZJfJ?kfbmiT?<;EY4#X zpjoZKkvnT^=jcpIz=N~v8v1np%Zix zQN@7v&W>10Ij2}8);-SJ2Q$;Aba`0%>~<6ct*siIUZ7UiqUB@qPk285dDahl+lh#r z-J#(NO6>W2mZTXT-;tYhiQV^Q5!=AJHpfo7acCg?0vtj?pWP7;O!^F3%JJW_Lh~=z za6H62>1awzj4-?pYpVzeV#D!w1FE%GzhceQ6r#|>RVBUwy5V1<$g zyR})Sm0c^AqVB*7;@Dp+RlGv82XpYB;Zfa6PtNAz-ys@(VH0#SV2k=Bgk<+2s5)RY zrAlj)CzYY2a~ep{DBwJQ$%vw_!uTEL5(`S^xVT-Nv#cu)GG}8adM;7rE4JUeZpv@c zG8Aihq$2VOd9<)kGMJv^x-5J~Jx(17Hb>?eQgf4)XVCfsb7@n}b#z3EgbP#QR9WIy zGd5|Ta6!n0)uYF#5fme4n$Pz)hsC%if#_nbv4@}dA_gMa#B@1)tJKvs)F;}(#XA{n z?Y83f_&0K3EVKZj9%oeyyH!87>c^lsMAm*GnG2KHAU=NJW2G3A@XTeVaM=3Yq zxGIoHWVF{1VQxwg?$D&QR($mpdMy@Q7MDKL&+%Q87;LsdMe4fOL&|#vSx6qd)1CIc zo^~tTi2|aKx;=Qia!$-?lWhiwyGPY0A&-D;@y7Db=^Z|a(sHx(VEMRCL~8mozq|3h zy<1A+D-uXhosOqFo=Nw$d$zelqywJ(K4ad!N_2vplYgm5N-swA&oqbYgT<{_iAJ;+ zfXbnD9<;l;w~*XE{&A&qiSY{bx1l+b#VBmZTS(sg_I``de}(39@+xn^xpyQtd=De6 zfZL}Y64)z>B3sBQm^zGV>=}^)heR$=Tlt{AuUG45SNdo4Jk^6M5%Y&1R$ONL>RPJ1 zRJXaC`>9=JJ^5T@7q*SD7~=Nv@>lB>3~@;zgpWu8GV?Z=38_3ZEsT%WH^(>9aQsWs z#sLLquR#!H*pDsrg(_EL)RNNP<$%pU(+U4rZ}m5QwlQ!uvHi2i;2(tv|LWjx!f(}F z^uN_=(f@ZPSbYD1_J3i}{ZUj!_h*6DduDnjrayZI37D>+QrZW;ZzF;Q04yZ|0MY+5 zw!NJVy}OO|iH^423J1DR^hdCTLtz{drRi_Lq6^nu8RQyfIG;n;T=z?I$d*ls74_o7 zjbI-(ZewSIS~%3SFyyMWQHw?64!n5Wq{L62eF%aMqwNKaZaO@-@!gONd~g-mK5jho zaJsF#emr&a_~`Vuj2xkmOZXZ`?)v9+BR`%^aAXgo?Sp0wLcvsn3LWn0d%Zw+lWkF& z7N9tx_A~a!r0qJVOJG96aRwe_#)dLd=>!f0nlI;<4WC`Xj+skv0il^9RqH7>`$DPY zsE*oA!5(fB=7dwhBbhKFT0r8-?WdO60A}{mXYPga-23%E!?b|Pq0|SQe*hQ8EKMi+ zZih!f&H7gD5o(aF-97g1|iHuNkq=v5Y?MmllcTj?*lBm;cU`#Au?Op zH=;Z7T5pLDZ@9Ap^iUy4WlkEapsw1cQ`74}Raur!adEHqdE}CNa}ZEjw!g&Iu4JdI zDSly0aSrx|N_nUh{=sw@7Mg9gro=L%FvIU4uh&Y!%h0NJ%K>Gu;I@mB9MN%n(>4v* zykOSPg3#d7_XQQtIk97SLFR+6;Vy)|tk1Z3&;yz?;vjx@5(RA+HlxD2;~a;)=HcxR z9q8vnMJIYkEu}G=S7{o&YfGRwFdgf%w|eMM*{` zG$JtJ;6#Oo1D7OTPg-tbj~7b@bZO$lrC)tVy}#V}PTqx`K!>K1Zf=$EN@so?uA`R@ z%r>n%qEqR#c7K_CYf0cEoPE?C7SS!ylVGivT&U52ugxeP)RG+X3vJED_R@({ElN!n z5L6e#vo_KfEyBukRUlzxqK`IR3b>gf)AqOEUXv&alyMz~QBqgyB7BVv!08 z%9Lf5OPcODY5;_ohD=a~oh+rgQWB1y#{=imT}(a0IC~v0C90tsE>Rvt8dp-mc#EL| zTn1{%j-e~WtmLwYG^s22j2(_y!I(fTja($TucxrrwGl9Y6j`Y2t)O^aOSnN176Jnng#DvG1;I{%X z-=5yjpx_k5Vr5aR;zogRAmnq0#}R=*@Pdn@n$#5$j$##T+Grj^5~GHM3s`?ZO~7>m zW!%NyRRT_yoQE}qK%}{oVaj}_di~K{`9QP7fw~{NrAa&Hh#@DyTO~xTDwai~@})o8 zDR(?fN|Yrt@QfiDbBtqlGD+)WcwwhN5j4dLSZc77lbY1^oiAn76ZG*St(2qulL$JS zO;o{KKd`S*igIpvuy^xEz=rA(2DM!p9Hz5;MXv1GEbxV!Mk_~6J%QIr}QD%HnxymncJ#0_8=n4J_a%M@x7EdCr(Vx_v4lCjm6dx`()v4949qh z8_&Y`2|tln6j5&QwH!Oev(1d1w-;4{z_1l_S8z0RjZwVdZqjM6?-QSpI*UOy#%bi^ z(b#nRZI53J;G@brgU53zSD{(kO-{`19{UDomy061AKa`cqb#Pa$onItJ4kHd92J zC%Y8RS3c`=aE?~C(9il_&?-2WR>IQ!rB~!I_|jPMUB+JBuNTZx5tpiFitg1xP%7;#j#eeOg);E&U3IJsW$W)4er}e5T#_``OPEmNz99U9}VBJjYj#d&x>uyxEWS7k@bpR9bMC6ph@aMW*QyUjd%Gm{eGW3(fXl!3xwP=~?3#7RFuX*iLXvkZi+cfPxz3)8ZK^p<8VA zd`Gw}P5gDS70x%?-)tAqj@^Z45;F#yo%KF4%$COX1byz_^ER9-=b1|sZ(`;mjJiX& zRV3+!bqPs-o}%QOGO}h5iUes@Z9-w9)-2mI()LNyioT z)pqh;gBYz5@o|H~h31Hs?B!f*FF8wTr$4>19ScxqF6rtGJB#7vDEhom;C9wVc|e^X z|Mvm<$rupg@wShKrW`YE3M<#Xc z)RC}IlpiR!%u$~gzI7GkEY-(}r`FEQ=i=>d&&$*k8rZBa2ax#{!JSse5Ls={WPN_x ziUtE0$Ho=14D;b;%3Ga}ZLGkkYQEj%GLT@H;4*Bs^j4J(iiPo(K$I(D@pD$;5)$&x z4586oWs=@YLS~7g8XAO^g;ILg!;PGSIO5+s0^=3(ey^e_{iz{uz?xNedz2C*DrAfL zhK-1oY`KL1m{%&YWo& ze#_|-Qdq)*U4C{7{Osw*>+`P5wU29BjrE$e$9{-2R1ZG0Rq1 zr+HdI$`=jH9ewM>HQ=pjCWt#KvriO;{DJxEgC>+h{&%T8z&Yr|C(m zN4MBbAshy=z_W^b&Sws1Cs&s5uwcc_7QPLawE4hTXvczWMI3)oGtRKp+_NnVmxm~) zD`7C%DCECBAULj*>?l^j*GuZ3JF0!+5;yXnsrIO)gu6vRXcz7UE!~*Tg4m~-vC7uN zvO(j!SY(2{Qr{(C#Nnc}x=^_n7?4=OP@3MrSAF6jgvo?t54YQf=m{(OMk~=o`ZO30 z0f+{@bqO=ky1M?%W|Q6);rSL=6wsL+4gSSg2_^W zj(MhEFKL|QhRX^@H2;7R`4Q(aO1iz>n)LLC4NPa7K0}Vx1g5s3onB)K-Ryh3y4Bv` zAkQU{sjWqHZ3yu)NO5e8V{ z5l+s- zo0CAms38BE74_!nTSEk#2>v|&`8fKQ+COJW{S^M$3j8SEKL1qv3uo#V!aqizKgmPC zHAKMcs}j&3RHA>~)t_XdUmDK8Yy3$q`qzp-$wj{uv;J1`k2v>VtN$b${Zb$OXX<}O zwr@f0zXtkAKKca&_C_}Q$)5V%f#3S^3+Rui`QIY`BqRMo6#owq|4L2zqqqN@ob(5J zfnUP^82isk)W4@H{ktLh$)xzL{rp#Gga2nk^vj{&c@=-_)!#EK{xDJCSN^{<@TY6P zcox5$`sY==ziigMB{}|+>;B8BKh6DC)B9Ib|E!+mm#HN5zs$?ut4sb~`|nd;zqA?2 z|Dyd{0_>OaKThbMxu@US&r7%aSLHu4QGZ;Pe^e&?L-%bNdVNKDTTuR7PyS_Q>R+S% z%uW5)exk|$2WbCYrs@w{|2$XqhqVH~vj4NK-!oUgMjG Date: Thu, 28 Jan 2021 13:55:40 -0800 Subject: [PATCH 12/45] Fix build errors --- .../lib/application-code.ts | 38 ++++++++++--------- .../lib/flink-application.ts | 15 ++++---- .../lib/property-group.ts | 13 +++++-- 3 files changed, 38 insertions(+), 28 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/application-code.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/application-code.ts index ac515c1efd858..b3dae72cdc066 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/application-code.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/application-code.ts @@ -19,14 +19,14 @@ export abstract class ApplicationCode { public abstract bind(scope: Construct): CodeConfiguration; } -interface BucketApplicationCodeProps { - bucket: s3.IBucket; - fileKey: string; - objectVersion?: string; +export interface BucketApplicationCodeProps { + readonly bucket: s3.IBucket; + readonly fileKey: string; + readonly objectVersion?: string; } -class BucketApplicationCode extends ApplicationCode { - public readonly bucket: s3.IBucket; +export class BucketApplicationCode extends ApplicationCode { + public readonly bucket?: s3.IBucket; public readonly fileKey: string; public readonly objectVersion?: string; @@ -41,7 +41,7 @@ class BucketApplicationCode extends ApplicationCode { return { codeContent: { s3ContentLocation: { - bucketArn: this.bucket.bucketArn, + bucketArn: this.bucket!.bucketArn, fileKey: this.fileKey, objectVersion: this.objectVersion, }, @@ -51,7 +51,7 @@ class BucketApplicationCode extends ApplicationCode { } } -class AssetApplicationCode extends ApplicationCode { +export class AssetApplicationCode extends ApplicationCode { private readonly path: string; private readonly options?: s3_assets.AssetOptions; private _asset?: s3_assets.Asset; @@ -92,13 +92,17 @@ class AssetApplicationCode extends ApplicationCode { } } -interface CodeConfiguration { - codeContent: { - s3ContentLocation: { - bucketArn: string; - fileKey: string; - objectVersion?: string; - }, - }, - codeContentType: 'ZIPFILE', +export interface CodeConfiguration { + readonly codeContent: CodeContent; + readonly codeContentType: 'ZIPFILE', +} + +export interface CodeContent { + readonly s3ContentLocation: S3ContentLocation; +} + +export interface S3ContentLocation { + readonly bucketArn: string; + readonly fileKey: string; + readonly objectVersion?: string; } diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts index 6b8bf615e20ec..19b6f79a52dc6 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts @@ -42,9 +42,9 @@ export interface IFlinkApplication extends core.IResource, iam.IGrantable { * const runtime = new FlinkRuntime(FLINK-9_99); */ export class FlinkRuntime { - public static FLINK_1_6 = new FlinkRuntime('FLINK-1_6'); - public static FLINK_1_8 = new FlinkRuntime('FLINK-1_8'); - public static FLINK_1_11 = new FlinkRuntime('FLINK-1_11'); + public static readonly FLINK_1_6 = new FlinkRuntime('FLINK-1_6'); + public static readonly FLINK_1_8 = new FlinkRuntime('FLINK-1_8'); + public static readonly FLINK_1_11 = new FlinkRuntime('FLINK-1_11'); public constructor(public readonly value: string) {} } @@ -176,16 +176,17 @@ export interface FlinkApplicationProps { /** * Attributes passed to the fromAttributes factory method. */ -interface Attributes { - applicationName: string; - applicationArn: string; +export interface Attributes { + readonly applicationName: string; + readonly applicationArn: string; } /** * The L2 construct for Flink Kinesis Data Applications. * - * @experimental * @resource AWS::KinesisAnalyticsV2::Application + * + * @experimental */ export class FlinkApplication extends FlinkApplicationBase { /** diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/property-group.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/property-group.ts index a7077441905a0..79c9a0af843e9 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/property-group.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/property-group.ts @@ -1,17 +1,22 @@ -interface PropertyMap { +export interface PropertyMap { [key: string]: string; } +export interface CfnPropertyGroup { + readonly propertyGroupId: string; + readonly propertyMap: PropertyMap; +} + export class PropertyGroup { - private readonly propertyGroupId; - private readonly propertyMap; + private readonly propertyGroupId: string; + private readonly propertyMap: PropertyMap; constructor(propertyGroupId: string, propertyMap: PropertyMap) { this.propertyGroupId = propertyGroupId; this.propertyMap = propertyMap; } - public toCfn() { + public toCfn(): CfnPropertyGroup { return { propertyGroupId: this.propertyGroupId, propertyMap: this.propertyMap, From cbee508ef8d22cbbd919fee2bcd7f20ce35ddc7d Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Thu, 28 Jan 2021 14:21:42 -0800 Subject: [PATCH 13/45] Add checkpointInterval setting --- .../lib/flink-application.ts | 8 ++++ .../flink-application-configuration.ts | 2 + .../test/flink-application.test.ts | 37 +++++++++++++++++-- 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts index 19b6f79a52dc6..fcd1e2c26c8c6 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts @@ -99,6 +99,13 @@ export interface FlinkApplicationProps { */ readonly checkpointingEnabled?: boolean; + /** + * The interval between checkpoints. + * + * @default 1 minute + */ + readonly checkpointInterval?: core.Duration; + /** * The minimum amount of time in to wait after a checkpoint finishes to start * a new checkpoint. @@ -245,6 +252,7 @@ export class FlinkApplication extends FlinkApplicationBase { : undefined, flinkApplicationConfiguration: flinkApplicationConfiguration({ checkpointingEnabled: props.checkpointingEnabled, + checkpointInterval: props.checkpointInterval, minPauseBetweenCheckpoints: props.minPauseBetweenCheckpoints, logLevel: props.logLevel, metricsLevel: props.metricsLevel, diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/flink-application-configuration.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/flink-application-configuration.ts index 15ad3c4393884..be0527b36ecf4 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/flink-application-configuration.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/flink-application-configuration.ts @@ -27,6 +27,7 @@ interface FlinkApplicationConfiguration extends interface CheckpointConfiguration { checkpointingEnabled?: boolean; + checkpointInterval?: core.Duration; minPauseBetweenCheckpoints?: core.Duration; } @@ -52,6 +53,7 @@ interface ParallelismConfiguration { export function flinkApplicationConfiguration(config: FlinkApplicationConfiguration) { const checkpointConfiguration = configFor({ checkpointingEnabled: config.checkpointingEnabled, + checkpointInterval: config.checkpointInterval?.toMilliseconds(), minPauseBetweenCheckpoints: config.minPauseBetweenCheckpoints?.toMilliseconds(), }); diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts index c5ca2e84fec1b..17b733a9ca342 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts @@ -224,6 +224,25 @@ describe('FlinkApplication', () => { }); }); + test('checkpointInterval setting', () => { + const { stack, requiredProps } = buildStack(); + new ka.FlinkApplication(stack, 'FlinkApplication', { + ...requiredProps, + checkpointInterval: core.Duration.minutes(5), + }); + + expect(stack).toHaveResourceLike('AWS::KinesisAnalyticsV2::Application', { + ApplicationConfiguration: { + FlinkApplicationConfiguration: { + CheckpointConfiguration: { + ConfigurationType: 'CUSTOM', + CheckpointInterval: 300_000, + }, + }, + }, + }); + }); + test('minPauseBetweenCheckpoints setting', () => { const { stack, requiredProps } = buildStack(); new ka.FlinkApplication(stack, 'FlinkApplication', { @@ -371,11 +390,21 @@ describe('FlinkApplication', () => { 'Fn::Join': [ '', [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':logs:', + { + Ref: 'AWS::Region', + }, + ':', + { + Ref: 'AWS::AccountId', + }, + ':log-group:', { - 'Fn::GetAtt': [ - 'FlinkApplicationLogGroup7739479C', - 'Arn', - ], + Ref: 'FlinkApplicationLogGroup7739479C', }, ':log-stream:', { From 7df3247e225492ebf5b44e7a9eaed16128bf2f90 Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Thu, 28 Jan 2021 14:27:37 -0800 Subject: [PATCH 14/45] Add missing @default annotations --- .../@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts index fcd1e2c26c8c6..23be851b528bf 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts @@ -116,6 +116,8 @@ export interface FlinkApplicationProps { /** * The level of log verbosity from the Flink application. + * + * @default FlinkLogLevel.INFO */ readonly logLevel?: FlinkLogLevel; @@ -124,6 +126,8 @@ export interface FlinkApplicationProps { * Use caution with Parallelism level metrics. Parallelism granularity logs * metrics for each parallel thread and can quickly become expensive when * parallelism is high (e.g. > 64). + * + * @default FlinkMetricsLevel.APPLICATION */ readonly metricsLevel?: FlinkMetricsLevel; From f14556857caa6ed373dca1b03f3716812f0b042a Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Sat, 30 Jan 2021 15:47:20 -0800 Subject: [PATCH 15/45] Fix up package.json file --- .../aws-kinesisanalyticsv2/.npmignore | 3 ++- .../@aws-cdk/aws-kinesisanalyticsv2/README.md | 22 +++++++++++++++++++ .../aws-kinesisanalyticsv2/lib/index.ts | 1 + .../aws-kinesisanalyticsv2/package.json | 11 ++++------ .../test/integ.flink-application.ts | 2 +- 5 files changed, 30 insertions(+), 9 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/.npmignore b/packages/@aws-cdk/aws-kinesisanalyticsv2/.npmignore index bca0ae2513f1e..63ab95621c764 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/.npmignore +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/.npmignore @@ -23,4 +23,5 @@ jest.config.js # exclude cdk artifacts **/cdk.out -junit.xml \ No newline at end of file +junit.xml +test/ \ No newline at end of file diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/README.md b/packages/@aws-cdk/aws-kinesisanalyticsv2/README.md index 6ad1ae5c8d48e..4e5d5f947ca1a 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/README.md +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/README.md @@ -1,4 +1,26 @@ # Kinesis Analytics V2 + + +--- + +![cfn-resources: Stable](https://img.shields.io/badge/cfn--resources-stable-success.svg?style=for-the-badge) + +> All classes with the `Cfn` prefix in this module ([CFN Resources]) are always stable and safe to use. +> +> [CFN Resources]: https://docs.aws.amazon.com/cdk/latest/guide/constructs.html#constructs_lib + +![cdk-constructs: Experimental](https://img.shields.io/badge/cdk--constructs-experimental-important.svg?style=for-the-badge) + +> The APIs of higher level constructs in this module are experimental and under active development. +> They are subject to non-backward compatible changes or removal in any future version. These are +> not subject to the [Semantic Versioning](https://semver.org/) model and breaking changes will be +> announced in the release notes. This means that while you may use them, you may need to update +> your source code when upgrading to a newer version of this package. + +--- + + + The `@aws-cdk/aws-kinesisanalyticsv2` package provides constructs for creating Kinesis Data Analytics applications. diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/index.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/index.ts index 8bc58fbd2b00e..b5bb9c3dcf864 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/index.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/index.ts @@ -1,5 +1,6 @@ export * from './application-code'; export * from './flink-application'; +export * from './kinesisanalyticsv2.generated'; export { FlinkLogLevel, FlinkMetricsLevel } from './private/flink-application-configuration'; export * from './property-group'; diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/package.json b/packages/@aws-cdk/aws-kinesisanalyticsv2/package.json index 2031244aac428..b11ac33147990 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/package.json +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/package.json @@ -1,6 +1,5 @@ { "name": "@aws-cdk/aws-kinesisanalyticsv2", - "private": true, "version": "0.0.0", "description": "A CDK Construct Library for Kinesis Analytics V2 constructs", "main": "lib/index.js", @@ -18,8 +17,6 @@ "dotnet": { "namespace": "Amazon.CDK.AWS.KinesisAnalyticsV2", "packageId": "Amazon.CDK.AWS.KinesisAnalyticsV2", - "signAssembly": true, - "assemblyOriginatorKeyFile": "../../key.snk", "iconUrl": "https://raw.githubusercontent.com/aws/aws-cdk/master/logo/default-256-dark.png" }, "python": { @@ -51,7 +48,8 @@ "build+test+package": "yarn build+test && yarn package", "compat": "cdk-compat", "rosetta:extract": "yarn --silent jsii-rosetta extract", - "cfn2ts": "cfn2ts" + "cfn2ts": "cfn2ts", + "gen": "cfn2ts" }, "keywords": [ "aws", @@ -74,6 +72,7 @@ "pkglint": "0.0.0" }, "dependencies": { + "@aws-cdk/assets": "0.0.0", "@aws-cdk/core": "0.0.0", "@aws-cdk/aws-iam": "0.0.0", "@aws-cdk/aws-logs": "0.0.0", @@ -83,6 +82,7 @@ }, "homepage": "https://github.com/aws/aws-cdk", "peerDependencies": { + "@aws-cdk/assets": "0.0.0", "@aws-cdk/core": "0.0.0", "@aws-cdk/aws-iam": "0.0.0", "@aws-cdk/aws-logs": "0.0.0", @@ -106,8 +106,5 @@ "env": { "AWSLINT_BASE_CONSTRUCT": true } - }, - "ubergen": { - "exclude": true } } diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/integ.flink-application.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/integ.flink-application.ts index 08944b39def6a..1c3b1485b03bb 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/integ.flink-application.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/integ.flink-application.ts @@ -1,5 +1,5 @@ -import * as core from '@aws-cdk/core'; import * as path from 'path'; +import * as core from '@aws-cdk/core'; import * as ka from '../lib'; const app = new core.App(); From 2859c72f4b03f138c561310d1586fa903574ecb4 Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Sat, 30 Jan 2021 16:15:18 -0800 Subject: [PATCH 16/45] Undo yarn.lock changes --- yarn.lock | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/yarn.lock b/yarn.lock index 72da1c73c47d1..aaee849150073 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2316,9 +2316,9 @@ aws-sdk-mock@^5.1.0: traverse "^0.6.6" aws-sdk@^2.596.0: - version "2.829.0" - resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.829.0.tgz#da70407a8bddc21b1dc1c9ba61ac3daea39b9364" - integrity sha512-0LV0argbcE1HhOeCeCZWUbpP4rWzwqe+0WmnR+jCJPY0w0n/ntGU/GW8obzhhhUej8pS4AkuAJNgzbwlTnxUmw== + version "2.831.0" + resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.831.0.tgz#02607cc911a2136e5aabe624c1282e821830aef2" + integrity sha512-lrOjbGFpjk2xpESyUx2PGsTZgptCy5xycZazPeakNbFO19cOoxjHx3xyxOHsMCYb3pQwns35UvChQT60B4u6cw== dependencies: buffer "4.9.2" events "1.1.1" @@ -2330,22 +2330,7 @@ aws-sdk@^2.596.0: uuid "3.3.2" xml2js "0.4.19" -aws-sdk@^2.637.0: - version "2.828.0" - resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.828.0.tgz#6aa599c3582f219568f41fb287eb65753e4a9234" - integrity sha512-JoDujGdncSIF9ka+XFZjop/7G+fNGucwPwYj7OHYMmFIOV5p7YmqomdbVmH/vIzd988YZz8oLOinWc4jM6vvhg== - dependencies: - buffer "4.9.2" - events "1.1.1" - ieee754 "1.1.13" - jmespath "0.15.0" - querystring "0.2.0" - sax "1.2.1" - url "0.10.3" - uuid "3.3.2" - xml2js "0.4.19" - -aws-sdk@^2.830.0: +aws-sdk@^2.637.0, aws-sdk@^2.830.0: version "2.830.0" resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.830.0.tgz#1d3631d573d18c48373046da7ad92855a7fd1636" integrity sha512-vFatoWkdJmRzpymWbqsuwVsAJdhdAvU2JcM9jKRENTNKJw90ljnLyeP1eKCp4O3/4Lg43PVBwY/KUqPy4wL+OA== From 77353f7629d44e92412a58a9936d184523c9e23b Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Sat, 30 Jan 2021 15:55:51 -0800 Subject: [PATCH 17/45] Create types.ts to avoid circular dependencies It looks like there is lots of precedent for this in CDK. Removes public exports from private module. --- .../lib/flink-application.ts | 3 ++- .../aws-kinesisanalyticsv2/lib/index.ts | 2 +- .../flink-application-configuration.ts | 21 +------------------ .../aws-kinesisanalyticsv2/lib/types.ts | 19 +++++++++++++++++ 4 files changed, 23 insertions(+), 22 deletions(-) create mode 100644 packages/@aws-cdk/aws-kinesisanalyticsv2/lib/types.ts diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts index 23be851b528bf..56bab2def9cae 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts @@ -4,9 +4,10 @@ import * as core from '@aws-cdk/core'; import { Construct } from 'constructs'; import { ApplicationCode } from './application-code'; import { CfnApplication, CfnApplicationCloudWatchLoggingOption } from './kinesisanalyticsv2.generated'; -import { flinkApplicationConfiguration, FlinkLogLevel, FlinkMetricsLevel } from './private/flink-application-configuration'; +import { flinkApplicationConfiguration } from './private/flink-application-configuration'; import { validateFlinkApplicationProps } from './private/validation'; import { PropertyGroup } from './property-group'; +import { FlinkLogLevel, FlinkMetricsLevel } from './types'; export interface IFlinkApplication extends core.IResource, iam.IGrantable { /** diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/index.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/index.ts index b5bb9c3dcf864..18e4090d39ad7 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/index.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/index.ts @@ -1,6 +1,6 @@ export * from './application-code'; export * from './flink-application'; export * from './kinesisanalyticsv2.generated'; -export { FlinkLogLevel, FlinkMetricsLevel } from './private/flink-application-configuration'; export * from './property-group'; +export * from './types'; diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/flink-application-configuration.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/flink-application-configuration.ts index be0527b36ecf4..67e0b55d207c0 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/flink-application-configuration.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/flink-application-configuration.ts @@ -1,24 +1,5 @@ import * as core from '@aws-cdk/core'; - -/** - * Available log levels for Flink applications. - */ -export enum FlinkLogLevel { - DEBUG = 'DEBUG', - INFO = 'INFO', - WARN = 'WARN', - ERROR = 'ERROR', -} - -/** - * Granularity of metrics sent to CloudWatch. - */ -export enum FlinkMetricsLevel { - APPLICATION = 'APPLICATION', - TASK = 'TASK', - OPERATOR = 'OPERATOR', - PARALLELISM = 'PARALLELISM', -} +import { FlinkLogLevel, FlinkMetricsLevel } from '../types'; interface FlinkApplicationConfiguration extends CheckpointConfiguration, diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/types.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/types.ts new file mode 100644 index 0000000000000..754efe98dc6f1 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/types.ts @@ -0,0 +1,19 @@ +/** + * Available log levels for Flink applications. + */ +export enum FlinkLogLevel { + DEBUG = 'DEBUG', + INFO = 'INFO', + WARN = 'WARN', + ERROR = 'ERROR', +} + +/** + * Granularity of metrics sent to CloudWatch. + */ +export enum FlinkMetricsLevel { + APPLICATION = 'APPLICATION', + TASK = 'TASK', + OPERATOR = 'OPERATOR', + PARALLELISM = 'PARALLELISM', +} From 3cec557fad94de669332e138bd34b66c85478fbe Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Sat, 30 Jan 2021 16:42:47 -0800 Subject: [PATCH 18/45] Rename addToPrincipalPolicy to addToRolePolicy --- .../@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts | 4 ++-- .../aws-kinesisanalyticsv2/test/flink-application.test.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts index 56bab2def9cae..1f4cc1621b9b7 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts @@ -32,7 +32,7 @@ export interface IFlinkApplication extends core.IResource, iam.IGrantable { /** * Convenience method for adding a policy statement to the application role. */ - addToPrincipalPolicy(policyStatement: iam.PolicyStatement): boolean; + addToRolePolicy(policyStatement: iam.PolicyStatement): boolean; } /** @@ -62,7 +62,7 @@ abstract class FlinkApplicationBase extends core.Resource implements IFlinkAppli public abstract readonly grantPrincipal: iam.IPrincipal; /** Implement the convenience {@link IFlinkApplication.addToPrincipalPolicy} method. */ - public addToPrincipalPolicy(policyStatement: iam.PolicyStatement): boolean { + public addToRolePolicy(policyStatement: iam.PolicyStatement): boolean { if (this.role) { this.role.addToPrincipalPolicy(policyStatement); return true; diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts index 17b733a9ca342..c0ba963fafb71 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts @@ -98,7 +98,7 @@ describe('FlinkApplication', () => { ...requiredProps, }); - app.addToPrincipalPolicy(new iam.PolicyStatement({ + app.addToRolePolicy(new iam.PolicyStatement({ actions: ['custom:action'], resources: ['*'], })); @@ -522,7 +522,7 @@ describe('FlinkApplication', () => { expect(flinkApp.applicationName).toEqual('my-app'); expect(flinkApp.applicationArn).toEqual('my-arn'); - expect(flinkApp.addToPrincipalPolicy(new iam.PolicyStatement())).toBe(false); + expect(flinkApp.addToRolePolicy(new iam.PolicyStatement())).toBe(false); }); }); From fb2c700fed7f2920ac0c20fc478944880dccbf68 Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Sat, 30 Jan 2021 17:00:22 -0800 Subject: [PATCH 19/45] Add fromArn and fromName factory methods --- .../lib/flink-application.ts | 34 ++++++++++++------- .../test/flink-application.test.ts | 29 ++++++++++++---- 2 files changed, 44 insertions(+), 19 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts index 1f4cc1621b9b7..84e524dff0886 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts @@ -185,14 +185,6 @@ export interface FlinkApplicationProps { readonly removalPolicy?: core.RemovalPolicy; } -/** - * Attributes passed to the fromAttributes factory method. - */ -export interface Attributes { - readonly applicationName: string; - readonly applicationArn: string; -} - /** * The L2 construct for Flink Kinesis Data Applications. * @@ -202,16 +194,34 @@ export interface Attributes { */ export class FlinkApplication extends FlinkApplicationBase { /** - * Import an existing Flink application, defined outside of CDK code. + * Import an existing Flink application defined outside of CDK code by + * applicationName. + */ + public static fromFlinkApplicationName(scope: Construct, id: string, flinkApplicationName: string): IFlinkApplication { + class Import extends FlinkApplicationBase { + // Imported flink applications have no associated role or grantPrincipal + public readonly role = undefined; + public readonly grantPrincipal = new iam.UnknownPrincipal({ resource: this }); + + public readonly applicationName = flinkApplicationName; + public readonly applicationArn = core.Stack.of(this).formatArn(flinkApplicationArnComponents(flinkApplicationName)); + } + + return new Import(scope, id); + } + + /** + * Import an existing Flink application defined outside of CDK code by + * applicationArn. */ - public static fromAttributes(scope: Construct, id: string, attributes: Attributes): IFlinkApplication { + public static fromFlinkApplicationArn(scope: Construct, id: string, flinkApplicationArn: string): IFlinkApplication { class Import extends FlinkApplicationBase { // Imported flink applications have no associated role or grantPrincipal public readonly role = undefined; public readonly grantPrincipal = new iam.UnknownPrincipal({ resource: this }); - public readonly applicationName = attributes.applicationName; - public readonly applicationArn = attributes.applicationArn; + public readonly applicationName = flinkApplicationArn.split('/')[1]; + public readonly applicationArn = flinkApplicationArn; } return new Import(scope, id); diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts index c0ba963fafb71..f196d268cdcbb 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts @@ -512,17 +512,32 @@ describe('FlinkApplication', () => { }).toThrow(/must be at least 1/); }); - // TODO: Not quite sure what to do with fromAttributes yet. - test('fromAttributes', () => { + test('fromFlinkApplicationName', () => { const { stack } = buildStack(); - const flinkApp = ka.FlinkApplication.fromAttributes(stack, 'Imported', { - applicationName: 'my-app', - applicationArn: 'my-arn', - }); + const flinkApp = ka.FlinkApplication.fromFlinkApplicationName(stack, 'Imported', 'my-app'); expect(flinkApp.applicationName).toEqual('my-app'); - expect(flinkApp.applicationArn).toEqual('my-arn'); + expect(stack.resolve(flinkApp.applicationArn)).toEqual({ + 'Fn::Join': ['', [ + 'arn:', + { Ref: 'AWS::Partition' }, + ':kinesisanalytics:', + { Ref: 'AWS::Region' }, + ':', + { Ref: 'AWS::AccountId' }, + ':application/my-app', + ]], + }); expect(flinkApp.addToRolePolicy(new iam.PolicyStatement())).toBe(false); }); + test('fromFlinkApplicationArn', () => { + const { stack } = buildStack(); + const arn = 'arn:aws:kinesisanalytics:us-west-2:012345678901:application/my-app'; + const flinkApp = ka.FlinkApplication.fromFlinkApplicationArn(stack, 'Imported', arn); + + expect(flinkApp.applicationName).toEqual('my-app'); + expect(flinkApp.applicationArn).toEqual(arn); + expect(flinkApp.addToRolePolicy(new iam.PolicyStatement())).toBe(false); + }); }); From bbf967a68e7bc292856573a4ad3add8be89e9000 Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Sat, 30 Jan 2021 18:34:19 -0800 Subject: [PATCH 20/45] Add bucket to code return type --- .../lib/application-code.ts | 71 ++++++++++--------- .../lib/flink-application.ts | 8 +-- .../test/flink-application.test.ts | 18 ++++- 3 files changed, 57 insertions(+), 40 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/application-code.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/application-code.ts index b3dae72cdc066..439706c2922dc 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/application-code.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/application-code.ts @@ -1,9 +1,10 @@ import * as s3 from '@aws-cdk/aws-s3'; import * as s3_assets from '@aws-cdk/aws-s3-assets'; import { Construct } from '@aws-cdk/core'; +import { CfnApplication } from './kinesisanalyticsv2.generated'; export abstract class ApplicationCode { - public static fromBucket(bucket: s3.IBucket, fileKey: string, objectVersion?: string): BucketApplicationCode { + public static fromBucket(bucket: s3.IBucket, fileKey: string, objectVersion?: string): ApplicationCode { return new BucketApplicationCode({ bucket, fileKey, @@ -11,21 +12,20 @@ export abstract class ApplicationCode { }); } - public static fromAsset(path: string, options?: s3_assets.AssetOptions): AssetApplicationCode { + public static fromAsset(path: string, options?: s3_assets.AssetOptions): ApplicationCode { return new AssetApplicationCode(path, options); } - abstract readonly bucket?: s3.IBucket; - public abstract bind(scope: Construct): CodeConfiguration; + public abstract bind(scope: Construct): ApplicationCodeConfig; } -export interface BucketApplicationCodeProps { +interface BucketApplicationCodeProps { readonly bucket: s3.IBucket; readonly fileKey: string; readonly objectVersion?: string; } -export class BucketApplicationCode extends ApplicationCode { +class BucketApplicationCode extends ApplicationCode { public readonly bucket?: s3.IBucket; public readonly fileKey: string; public readonly objectVersion?: string; @@ -37,21 +37,26 @@ export class BucketApplicationCode extends ApplicationCode { this.objectVersion = props.objectVersion; } - public bind(_scope: Construct): CodeConfiguration { + public bind(_scope: Construct): ApplicationCodeConfig { return { - codeContent: { - s3ContentLocation: { - bucketArn: this.bucket!.bucketArn, - fileKey: this.fileKey, - objectVersion: this.objectVersion, + applicationCodeConfigurationProperty: { + applicationCodeConfiguration: { + codeContent: { + s3ContentLocation: { + bucketArn: this.bucket!.bucketArn, + fileKey: this.fileKey, + objectVersion: this.objectVersion, + }, + }, + codeContentType: 'ZIPFILE', }, }, - codeContentType: 'ZIPFILE', + bucket: this.bucket!, }; } } -export class AssetApplicationCode extends ApplicationCode { +class AssetApplicationCode extends ApplicationCode { private readonly path: string; private readonly options?: s3_assets.AssetOptions; private _asset?: s3_assets.Asset; @@ -62,7 +67,7 @@ export class AssetApplicationCode extends ApplicationCode { this.options = options; } - public bind(scope: Construct): CodeConfiguration { + public bind(scope: Construct): ApplicationCodeConfig { this._asset = new s3_assets.Asset(scope, 'Code', { path: this.path, ...this.options, @@ -73,13 +78,18 @@ export class AssetApplicationCode extends ApplicationCode { } return { - codeContent: { - s3ContentLocation: { - bucketArn: this._asset.bucket.bucketArn, - fileKey: this._asset.s3ObjectKey, + applicationCodeConfigurationProperty: { + applicationCodeConfiguration: { + codeContent: { + s3ContentLocation: { + bucketArn: this._asset.bucket.bucketArn, + fileKey: this._asset.s3ObjectKey, + }, + }, + codeContentType: 'ZIPFILE', }, }, - codeContentType: 'ZIPFILE', + bucket: this._asset.bucket, }; } @@ -92,17 +102,14 @@ export class AssetApplicationCode extends ApplicationCode { } } -export interface CodeConfiguration { - readonly codeContent: CodeContent; - readonly codeContentType: 'ZIPFILE', -} - -export interface CodeContent { - readonly s3ContentLocation: S3ContentLocation; -} +export interface ApplicationCodeConfig { + /** + * Low-level Cloudformation ApplicationConfigurationProperty + */ + readonly applicationCodeConfigurationProperty: CfnApplication.ApplicationConfigurationProperty; -export interface S3ContentLocation { - readonly bucketArn: string; - readonly fileKey: string; - readonly objectVersion?: string; + /** + * S3 Bucket that stores the Flink application code + */ + readonly bucket: s3.IBucket; } diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts index 84e524dff0886..1f5414470a4de 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts @@ -250,18 +250,14 @@ export class FlinkApplication extends FlinkApplicationBase { resources: ['*'], })); - // TODO: Refactor to make this bug impossible const code = props.code.bind(this); - if (!props.code.bucket) { - throw new Error('BUG: Bound code is missing its bucket.'); - } - props.code.bucket.grantRead(this); + code.bucket.grantRead(this); const resource = new CfnApplication(this, 'Resource', { runtimeEnvironment: props.runtime.value, serviceExecutionRole: this.role.roleArn, applicationConfiguration: { - applicationCodeConfiguration: code, + ...code.applicationCodeConfigurationProperty, environmentProperties: props.propertyGroups?.length ? { propertyGroups: props.propertyGroups.map(pg => pg.toCfn()) } : undefined, diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts index f196d268cdcbb..0662978388e2f 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts @@ -162,14 +162,28 @@ describe('FlinkApplication', () => { ...requiredProps, code, }); + const assetRef = 'AssetParameters8be9e0b5f53d41e9a3b1d51c9572c65f24f8170a7188d0ed57fb7d571de4d577S3BucketEBA17A67'; + const versionKeyRef = 'AssetParameters8be9e0b5f53d41e9a3b1d51c9572c65f24f8170a7188d0ed57fb7d571de4d577S3VersionKey5922697E'; expect(stack).toHaveResourceLike('AWS::KinesisAnalyticsV2::Application', { ApplicationConfiguration: { ApplicationCodeConfiguration: { CodeContent: { S3ContentLocation: { - BucketARN: stack.resolve(code.asset!.bucket.bucketArn), - FileKey: stack.resolve(code.asset!.s3ObjectKey), + BucketARN: { + 'Fn::Join': ['', [ + 'arn:', + { Ref: 'AWS::Partition' }, + ':s3:::', + { Ref: assetRef }, + ]], + }, + FileKey: { + 'Fn::Join': ['', [ + { 'Fn::Select': [0, { 'Fn::Split': ['||', { Ref: versionKeyRef }] }] }, + { 'Fn::Select': [1, { 'Fn::Split': ['||', { Ref: versionKeyRef }] }] }, + ]], + }, }, }, CodeContentType: 'ZIPFILE', From 94638f72a51968e1eb6d9ebf7b7964922ba1d034 Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Sun, 31 Jan 2021 10:46:37 -0800 Subject: [PATCH 21/45] Move const declaration closer to use --- .../@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts index 1f5414470a4de..40673fd4d0fef 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts @@ -288,8 +288,6 @@ export class FlinkApplication extends FlinkApplicationBase { removalPolicy: core.RemovalPolicy.DESTROY, }); - const logStreamArn = `arn:${core.Aws.PARTITION}:logs:${core.Aws.REGION}:${core.Aws.ACCOUNT_ID}:log-group:${logGroup.logGroupName}:log-stream:${logStream.logStreamName}`; - // Permit logging this.role.addToPrincipalPolicy(new iam.PolicyStatement({ actions: ['logs:DescribeLogStreams', 'logs:DescribeLogGroups'], @@ -302,6 +300,8 @@ export class FlinkApplication extends FlinkApplicationBase { }), ], })); + + const logStreamArn = `arn:${core.Aws.PARTITION}:logs:${core.Aws.REGION}:${core.Aws.ACCOUNT_ID}:log-group:${logGroup.logGroupName}:log-stream:${logStream.logStreamName}`; this.role.addToPrincipalPolicy(new iam.PolicyStatement({ actions: ['logs:PutLogEvents'], resources: [logStreamArn], From 8d194e9d95b5b0f386b93908d15571df139a28e7 Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Sun, 31 Jan 2021 10:47:27 -0800 Subject: [PATCH 22/45] Remove redundant resource dependency --- .../@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts index 40673fd4d0fef..161ce72ed3346 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts @@ -312,7 +312,7 @@ export class FlinkApplication extends FlinkApplicationBase { cloudWatchLoggingOption: { logStreamArn, }, - }).node.addDependency(resource); + }); this.applicationName = this.getResourceNameAttribute(resource.ref); this.applicationArn = this.getResourceArnAttribute( From f317827347af0e2f610c55fd7da1673ab9c71aa5 Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Sun, 31 Jan 2021 11:30:56 -0800 Subject: [PATCH 23/45] Use object literals over classes for propertyGroup --- .../lib/flink-application.ts | 10 +++----- .../aws-kinesisanalyticsv2/lib/index.ts | 1 - .../lib/private/environment-properties.ts | 16 ++++++++++++ .../lib/property-group.ts | 25 ------------------- .../aws-kinesisanalyticsv2/lib/types.ts | 8 ++++++ .../test/flink-application.test.ts | 8 +++--- 6 files changed, 32 insertions(+), 36 deletions(-) create mode 100644 packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/environment-properties.ts delete mode 100644 packages/@aws-cdk/aws-kinesisanalyticsv2/lib/property-group.ts diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts index 161ce72ed3346..50bf35000818e 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts @@ -4,10 +4,10 @@ import * as core from '@aws-cdk/core'; import { Construct } from 'constructs'; import { ApplicationCode } from './application-code'; import { CfnApplication, CfnApplicationCloudWatchLoggingOption } from './kinesisanalyticsv2.generated'; +import { environmentProperties } from './private/environment-properties'; import { flinkApplicationConfiguration } from './private/flink-application-configuration'; import { validateFlinkApplicationProps } from './private/validation'; -import { PropertyGroup } from './property-group'; -import { FlinkLogLevel, FlinkMetricsLevel } from './types'; +import { FlinkLogLevel, FlinkMetricsLevel, PropertyGroups } from './types'; export interface IFlinkApplication extends core.IResource, iam.IGrantable { /** @@ -167,7 +167,7 @@ export interface FlinkApplicationProps { * Configuration PropertyGroups. You can use these property groups to pass * arbitrary runtime configuration values to your Flink App. */ - readonly propertyGroups?: PropertyGroup[]; + readonly propertyGroups?: PropertyGroups; /** * A role to use to grant permissions to your application. Prefer omitting @@ -258,9 +258,7 @@ export class FlinkApplication extends FlinkApplicationBase { serviceExecutionRole: this.role.roleArn, applicationConfiguration: { ...code.applicationCodeConfigurationProperty, - environmentProperties: props.propertyGroups?.length - ? { propertyGroups: props.propertyGroups.map(pg => pg.toCfn()) } - : undefined, + environmentProperties: environmentProperties(props.propertyGroups), flinkApplicationConfiguration: flinkApplicationConfiguration({ checkpointingEnabled: props.checkpointingEnabled, checkpointInterval: props.checkpointInterval, diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/index.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/index.ts index 18e4090d39ad7..6f592e4a625f1 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/index.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/index.ts @@ -1,6 +1,5 @@ export * from './application-code'; export * from './flink-application'; export * from './kinesisanalyticsv2.generated'; -export * from './property-group'; export * from './types'; diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/environment-properties.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/environment-properties.ts new file mode 100644 index 0000000000000..ac4123113b20e --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/environment-properties.ts @@ -0,0 +1,16 @@ +import { CfnApplication } from '../kinesisanalyticsv2.generated'; +import { PropertyGroups } from '../types'; + +export function environmentProperties(propertyGroups?: PropertyGroups): CfnApplication.EnvironmentPropertiesProperty | undefined { + const entries = Object.entries(propertyGroups ?? {}); + if (entries.length === 0) { + return; + } + + return { + propertyGroups: entries.map(([id, map]) => ({ + propertyGroupId: id, + propertyMap: map, + })), + }; +} diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/property-group.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/property-group.ts deleted file mode 100644 index 79c9a0af843e9..0000000000000 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/property-group.ts +++ /dev/null @@ -1,25 +0,0 @@ -export interface PropertyMap { - [key: string]: string; -} - -export interface CfnPropertyGroup { - readonly propertyGroupId: string; - readonly propertyMap: PropertyMap; -} - -export class PropertyGroup { - private readonly propertyGroupId: string; - private readonly propertyMap: PropertyMap; - - constructor(propertyGroupId: string, propertyMap: PropertyMap) { - this.propertyGroupId = propertyGroupId; - this.propertyMap = propertyMap; - } - - public toCfn(): CfnPropertyGroup { - return { - propertyGroupId: this.propertyGroupId, - propertyMap: this.propertyMap, - }; - } -} diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/types.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/types.ts index 754efe98dc6f1..c73c2bdde3b93 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/types.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/types.ts @@ -17,3 +17,11 @@ export enum FlinkMetricsLevel { OPERATOR = 'OPERATOR', PARALLELISM = 'PARALLELISM', } + +/** + * Interface for building AWS::KinesisAnalyticsV2::Application PropertyGroup + * configuration. + */ +export interface PropertyGroups { + readonly [propertyId: string]: {[mapKey: string]: string}; +} diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts index 0662978388e2f..8c1c114ff48a5 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts @@ -196,11 +196,11 @@ describe('FlinkApplication', () => { const { stack, requiredProps } = buildStack(); new ka.FlinkApplication(stack, 'FlinkApplication', { ...requiredProps, - propertyGroups: [ - new ka.PropertyGroup('FlinkApplicationProperties', { + propertyGroups: { + FlinkApplicationProperties: { SomeProperty: 'SomeValue', - }), - ], + }, + }, }); expect(stack).toHaveResourceLike('AWS::KinesisAnalyticsV2::Application', { From e855436ce7162f82f389dd3918e87b0687a32bd9 Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Sun, 31 Jan 2021 11:40:48 -0800 Subject: [PATCH 24/45] Use context vars over local vars for fixtures --- .../test/flink-application.test.ts | 49 ++++++------------- 1 file changed, 14 insertions(+), 35 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts index 8c1c114ff48a5..94087dddb26b8 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts @@ -6,20 +6,24 @@ import * as core from '@aws-cdk/core'; import * as path from 'path'; import * as ka from '../lib'; -function buildStack() { - const stack = new core.Stack(); - const bucket = new s3.Bucket(stack, 'CodeBucket'); - const requiredProps = { - runtime: ka.FlinkRuntime.FLINK_1_11, - code: ka.ApplicationCode.fromBucket(bucket, 'my-app.jar'), +describe('FlinkApplication', () => { + let stack: core.Stack; + let bucket: s3.Bucket; + let requiredProps: { + runtime: ka.FlinkRuntime; + code: ka.ApplicationCode; }; - return { stack, bucket, requiredProps }; -} + beforeEach(() => { + stack = new core.Stack(); + bucket = new s3.Bucket(stack, 'CodeBucket'); + requiredProps = { + runtime: ka.FlinkRuntime.FLINK_1_11, + code: ka.ApplicationCode.fromBucket(bucket, 'my-app.jar'), + }; + }); -describe('FlinkApplication', () => { test('default Flink Application', () => { - const { stack, bucket } = buildStack(); new ka.FlinkApplication(stack, 'FlinkApplication', { runtime: ka.FlinkRuntime.FLINK_1_11, code: ka.ApplicationCode.fromBucket(bucket, 'my-app.jar'), @@ -68,7 +72,6 @@ describe('FlinkApplication', () => { }); test('providing a custom role', () => { - const { stack, requiredProps } = buildStack(); new ka.FlinkApplication(stack, 'FlinkApplication', { ...requiredProps, role: new iam.Role(stack, 'CustomRole', { @@ -93,7 +96,6 @@ describe('FlinkApplication', () => { }); test('addToPrincipalPolicy', () => { - const { stack, requiredProps } = buildStack(); const app = new ka.FlinkApplication(stack, 'FlinkApplication', { ...requiredProps, }); @@ -113,7 +115,6 @@ describe('FlinkApplication', () => { }); test('providing a custom runtime', () => { - const { stack, requiredProps } = buildStack(); new ka.FlinkApplication(stack, 'FlinkApplication', { ...requiredProps, runtime: new ka.FlinkRuntime('custom'), @@ -125,7 +126,6 @@ describe('FlinkApplication', () => { }); test('providing a custom removal policy', () => { - const { stack, requiredProps } = buildStack(); new ka.FlinkApplication(stack, 'FlinkApplication', { ...requiredProps, removalPolicy: core.RemovalPolicy.RETAIN, @@ -137,7 +137,6 @@ describe('FlinkApplication', () => { }); test('granting permissions to resources', () => { - const { stack, requiredProps } = buildStack(); const app = new ka.FlinkApplication(stack, 'FlinkApplication', { ...requiredProps, }); @@ -156,7 +155,6 @@ describe('FlinkApplication', () => { }); test('using an asset for code', () => { - const { stack, requiredProps } = buildStack(); const code = ka.ApplicationCode.fromAsset(path.join(__dirname, 'code-asset')); new ka.FlinkApplication(stack, 'FlinkApplication', { ...requiredProps, @@ -193,7 +191,6 @@ describe('FlinkApplication', () => { }); test('adding property groups', () => { - const { stack, requiredProps } = buildStack(); new ka.FlinkApplication(stack, 'FlinkApplication', { ...requiredProps, propertyGroups: { @@ -220,7 +217,6 @@ describe('FlinkApplication', () => { }); test('checkpointEnabled setting', () => { - const { stack, requiredProps } = buildStack(); new ka.FlinkApplication(stack, 'FlinkApplication', { ...requiredProps, checkpointingEnabled: false, @@ -239,7 +235,6 @@ describe('FlinkApplication', () => { }); test('checkpointInterval setting', () => { - const { stack, requiredProps } = buildStack(); new ka.FlinkApplication(stack, 'FlinkApplication', { ...requiredProps, checkpointInterval: core.Duration.minutes(5), @@ -258,7 +253,6 @@ describe('FlinkApplication', () => { }); test('minPauseBetweenCheckpoints setting', () => { - const { stack, requiredProps } = buildStack(); new ka.FlinkApplication(stack, 'FlinkApplication', { ...requiredProps, minPauseBetweenCheckpoints: core.Duration.seconds(10), @@ -277,7 +271,6 @@ describe('FlinkApplication', () => { }); test('logLevel setting', () => { - const { stack, requiredProps } = buildStack(); new ka.FlinkApplication(stack, 'FlinkApplication', { ...requiredProps, logLevel: ka.FlinkLogLevel.DEBUG, @@ -296,7 +289,6 @@ describe('FlinkApplication', () => { }); test('metricsLevel setting', () => { - const { stack, requiredProps } = buildStack(); new ka.FlinkApplication(stack, 'FlinkApplication', { ...requiredProps, metricsLevel: ka.FlinkMetricsLevel.PARALLELISM, @@ -315,7 +307,6 @@ describe('FlinkApplication', () => { }); test('autoscalingEnabled setting', () => { - const { stack, requiredProps } = buildStack(); new ka.FlinkApplication(stack, 'FlinkApplication', { ...requiredProps, autoScalingEnabled: false, @@ -334,7 +325,6 @@ describe('FlinkApplication', () => { }); test('parallelism setting', () => { - const { stack, requiredProps } = buildStack(); new ka.FlinkApplication(stack, 'FlinkApplication', { ...requiredProps, parallelism: 2, @@ -353,7 +343,6 @@ describe('FlinkApplication', () => { }); test('parallelismPerKpu setting', () => { - const { stack, requiredProps } = buildStack(); new ka.FlinkApplication(stack, 'FlinkApplication', { ...requiredProps, parallelismPerKpu: 2, @@ -372,7 +361,6 @@ describe('FlinkApplication', () => { }); test('snapshotsEnabled setting', () => { - const { stack, requiredProps } = buildStack(); new ka.FlinkApplication(stack, 'FlinkApplication', { ...requiredProps, snapshotsEnabled: false, @@ -389,7 +377,6 @@ describe('FlinkApplication', () => { // TODO: Put all logging options on FlinkApplication, new ka.Logging or something else? test('default logging option', () => { - const { stack, requiredProps } = buildStack(); new ka.FlinkApplication(stack, 'FlinkApplication', { ...requiredProps, snapshotsEnabled: false, @@ -440,8 +427,6 @@ describe('FlinkApplication', () => { }); test('validating applicationName', () => { - const { stack, requiredProps } = buildStack(); - // Expect no error with valid name new ka.FlinkApplication(stack, 'ValidString', { ...requiredProps, @@ -477,8 +462,6 @@ describe('FlinkApplication', () => { }); test('validating parallelism', () => { - const { stack, requiredProps } = buildStack(); - // Expect no error with valid value new ka.FlinkApplication(stack, 'ValidNumber', { ...requiredProps, @@ -502,8 +485,6 @@ describe('FlinkApplication', () => { }); test('validating parallelismPerKpu', () => { - const { stack, requiredProps } = buildStack(); - // Expect no error with valid value new ka.FlinkApplication(stack, 'ValidNumber', { ...requiredProps, @@ -527,7 +508,6 @@ describe('FlinkApplication', () => { }); test('fromFlinkApplicationName', () => { - const { stack } = buildStack(); const flinkApp = ka.FlinkApplication.fromFlinkApplicationName(stack, 'Imported', 'my-app'); expect(flinkApp.applicationName).toEqual('my-app'); @@ -546,7 +526,6 @@ describe('FlinkApplication', () => { }); test('fromFlinkApplicationArn', () => { - const { stack } = buildStack(); const arn = 'arn:aws:kinesisanalytics:us-west-2:012345678901:application/my-app'; const flinkApp = ka.FlinkApplication.fromFlinkApplicationArn(stack, 'Imported', arn); From 70fbd3500d02310fbd66ba360815b6a38fc4f2e5 Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Sun, 31 Jan 2021 12:25:58 -0800 Subject: [PATCH 25/45] Add log* props to FlinkApplication --- .../@aws-cdk/aws-kinesisanalyticsv2/README.md | 15 +++-- .../lib/flink-application.ts | 59 ++++++++++++----- .../aws-kinesisanalyticsv2/lib/types.ts | 14 ++++ .../aws-kinesisanalyticsv2/package.json | 2 + .../test/flink-application.test.ts | 66 ++++++++++++++++++- 5 files changed, 131 insertions(+), 25 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/README.md b/packages/@aws-cdk/aws-kinesisanalyticsv2/README.md index 4e5d5f947ca1a..6b7536ad642f8 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/README.md +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/README.md @@ -88,11 +88,16 @@ const flinkApp = new ka.FlinkApplication(this, 'Application', { checkpointInterval: cdk.Duration.seconds(30), // default is 1 minute minPausesBetweenCheckpoints: cdk.Duration.seconds(10), // default is 5 seconds logLevel: ka.LogLevel.ERROR, // default is INFO - metricsLevel: ka.MetricsLevel.PARALLELISM // default is APPLICATION - autoScalingEnabled: false // default is true - parallelism: 32 // default is 1 - parallelismPerKpu: 2 // default is 1 - snapshotsEnabled: false // default is true + metricsLevel: ka.MetricsLevel.PARALLELISM, // default is APPLICATION + autoScalingEnabled: false, // default is true + parallelism: 32, // default is 1 + parallelismPerKpu: 2, // default is 1 + snapshotsEnabled: false, // default is true + logRetention: logs.RetentionDays.ONE_WEEK, // default is TWO_YEARS + logRemovalPolicy: core.RemovalPolicy.DESTROY, // default is RETAIN + logGroupName: 'my-group', // default is automatically generated + logStreamName: 'my-stream', // default is automatically generated + logEncryptionKey: someKmsKey, // default is no encryption }); ``` diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts index 50bf35000818e..6fc67193975d1 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts @@ -1,4 +1,5 @@ import * as iam from '@aws-cdk/aws-iam'; +import * as kms from '@aws-cdk/aws-kms'; import * as logs from '@aws-cdk/aws-logs'; import * as core from '@aws-cdk/core'; import { Construct } from 'constructs'; @@ -7,7 +8,7 @@ import { CfnApplication, CfnApplicationCloudWatchLoggingOption } from './kinesis import { environmentProperties } from './private/environment-properties'; import { flinkApplicationConfiguration } from './private/flink-application-configuration'; import { validateFlinkApplicationProps } from './private/validation'; -import { FlinkLogLevel, FlinkMetricsLevel, PropertyGroups } from './types'; +import { FlinkLogLevel, FlinkMetricsLevel, FlinkRuntime, PropertyGroups } from './types'; export interface IFlinkApplication extends core.IResource, iam.IGrantable { /** @@ -35,20 +36,6 @@ export interface IFlinkApplication extends core.IResource, iam.IGrantable { addToRolePolicy(policyStatement: iam.PolicyStatement): boolean; } -/** - * Available Flink runtimes for Kinesis Analytics. - * - * @example - * // Creating a new runtime that isn't in CDK yet. - * const runtime = new FlinkRuntime(FLINK-9_99); - */ -export class FlinkRuntime { - public static readonly FLINK_1_6 = new FlinkRuntime('FLINK-1_6'); - public static readonly FLINK_1_8 = new FlinkRuntime('FLINK-1_8'); - public static readonly FLINK_1_11 = new FlinkRuntime('FLINK-1_11'); - public constructor(public readonly value: string) {} -} - /** * Implements the functionality shared between CDK created and imported * IFlinkApplications. @@ -183,6 +170,39 @@ export interface FlinkApplicationProps { * @default RemovalPolicy.DESTROY */ readonly removalPolicy?: core.RemovalPolicy; + + /** + * How long to retain logs. + * + * @default two years + */ + readonly logRetention?: core.Duration; + + /** + * Whether to keep or delete logs when removing a FlinkApplication. + * + * @default RETAIN + */ + readonly logRemovalPolicy?: core.RemovalPolicy; + + /** + * The name of the log group for CloudWatch logs. + * + * @default Cloudformation generated + */ + readonly logGroupName?: string; + + /** + * The name of the log stream for CloudWatch logs. + * + * @default Cloudformation generated + */ + readonly logStreamName?: string; + + /** + * The KMS encryption key to use for CloudWatch logs. + */ + readonly logEncryptionKey?: kms.IKey; } /** @@ -277,13 +297,16 @@ export class FlinkApplication extends FlinkApplicationBase { resource.node.addDependency(this.role); const logGroup = new logs.LogGroup(this, 'LogGroup', { - retention: logs.RetentionDays.ONE_WEEK, - removalPolicy: core.RemovalPolicy.DESTROY, + logGroupName: props.logGroupName, + retention: props.logRetention?.toDays(), + removalPolicy: props.logRemovalPolicy, + encryptionKey: props.logEncryptionKey, }); const logStream = new logs.LogStream(this, 'LogStream', { logGroup, - removalPolicy: core.RemovalPolicy.DESTROY, + logStreamName: props.logStreamName, + removalPolicy: props.logRemovalPolicy, }); // Permit logging diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/types.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/types.ts index c73c2bdde3b93..7e14bc3812ead 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/types.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/types.ts @@ -25,3 +25,17 @@ export enum FlinkMetricsLevel { export interface PropertyGroups { readonly [propertyId: string]: {[mapKey: string]: string}; } + +/** + * Available Flink runtimes for Kinesis Analytics. + * + * @example + * // Creating a new runtime that isn't in CDK yet. + * const runtime = new FlinkRuntime(FLINK-9_99); + */ +export class FlinkRuntime { + public static readonly FLINK_1_6 = new FlinkRuntime('FLINK-1_6'); + public static readonly FLINK_1_8 = new FlinkRuntime('FLINK-1_8'); + public static readonly FLINK_1_11 = new FlinkRuntime('FLINK-1_11'); + public constructor(public readonly value: string) {} +} diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/package.json b/packages/@aws-cdk/aws-kinesisanalyticsv2/package.json index b11ac33147990..099d2227d98f7 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/package.json +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/package.json @@ -75,6 +75,7 @@ "@aws-cdk/assets": "0.0.0", "@aws-cdk/core": "0.0.0", "@aws-cdk/aws-iam": "0.0.0", + "@aws-cdk/aws-kms": "0.0.0", "@aws-cdk/aws-logs": "0.0.0", "@aws-cdk/aws-s3": "0.0.0", "@aws-cdk/aws-s3-assets": "0.0.0", @@ -85,6 +86,7 @@ "@aws-cdk/assets": "0.0.0", "@aws-cdk/core": "0.0.0", "@aws-cdk/aws-iam": "0.0.0", + "@aws-cdk/aws-kms": "0.0.0", "@aws-cdk/aws-logs": "0.0.0", "@aws-cdk/aws-s3": "0.0.0", "@aws-cdk/aws-s3-assets": "0.0.0", diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts index 94087dddb26b8..db95408fbda16 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts @@ -1,6 +1,7 @@ import { arrayWith, objectLike, ResourcePart } from '@aws-cdk/assert'; import '@aws-cdk/assert/jest'; import * as iam from '@aws-cdk/aws-iam'; +import * as kms from '@aws-cdk/aws-kms'; import * as s3 from '@aws-cdk/aws-s3'; import * as core from '@aws-cdk/core'; import * as path from 'path'; @@ -375,7 +376,6 @@ describe('FlinkApplication', () => { }); }); - // TODO: Put all logging options on FlinkApplication, new ka.Logging or something else? test('default logging option', () => { new ka.FlinkApplication(stack, 'FlinkApplication', { ...requiredProps, @@ -419,13 +419,75 @@ describe('FlinkApplication', () => { expect(stack).toHaveResource('AWS::Logs::LogGroup', { Properties: { - RetentionInDays: 7, + RetentionInDays: 731, }, + UpdateReplacePolicy: 'Retain', + DeletionPolicy: 'Retain', + }, ResourcePart.CompleteDefinition); + + expect(stack).toHaveResource('AWS::Logs::LogStream', { + UpdateReplacePolicy: 'Retain', + DeletionPolicy: 'Retain', + }, ResourcePart.CompleteDefinition); + }); + + test('logRetentionDays setting', () => { + new ka.FlinkApplication(stack, 'FlinkApplication', { + ...requiredProps, + logRetention: core.Duration.days(5), + }); + + expect(stack).toHaveResource('AWS::Logs::LogGroup', { + RetentionInDays: 5, + }); + }); + + test('logRemovalPolicy setting', () => { + new ka.FlinkApplication(stack, 'FlinkApplication', { + ...requiredProps, + logRemovalPolicy: core.RemovalPolicy.DESTROY, + }); + + expect(stack).toHaveResourceLike('AWS::Logs::LogGroup', { UpdateReplacePolicy: 'Delete', DeletionPolicy: 'Delete', }, ResourcePart.CompleteDefinition); }); + test('logGroupName setting', () => { + new ka.FlinkApplication(stack, 'FlinkApplication', { + ...requiredProps, + logGroupName: 'my-group', + }); + + expect(stack).toHaveResourceLike('AWS::Logs::LogGroup', { + LogGroupName: 'my-group', + }); + }); + + test('logStreamName setting', () => { + new ka.FlinkApplication(stack, 'FlinkApplication', { + ...requiredProps, + logStreamName: 'my-stream', + }); + + expect(stack).toHaveResourceLike('AWS::Logs::LogStream', { + LogStreamName: 'my-stream', + }); + }); + + test('logEncryptionKey setting', () => { + const key = new kms.Key(stack, 'Key'); + new ka.FlinkApplication(stack, 'FlinkApplication', { + ...requiredProps, + logEncryptionKey: key, + }); + + expect(stack).toHaveResourceLike('AWS::Logs::LogGroup', { + KmsKeyId: stack.resolve(key.keyArn), + }); + }); + test('validating applicationName', () => { // Expect no error with valid name new ka.FlinkApplication(stack, 'ValidString', { From 7533b5d63dee534d8af07d9147c87c8350170884 Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Sun, 31 Jan 2021 12:47:07 -0800 Subject: [PATCH 26/45] Show loggings and metric policies in default test --- .../lib/flink-application.ts | 1 + .../test/flink-application.test.ts | 27 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts index 6fc67193975d1..3d4970312cce8 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts @@ -317,6 +317,7 @@ export class FlinkApplication extends FlinkApplicationBase { core.Stack.of(this).formatArn({ service: 'logs', resource: 'log-group', + sep: ':', resourceName: '*', }), ], diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts index db95408fbda16..5cef62df24c87 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts @@ -70,6 +70,33 @@ describe('FlinkApplication', () => { Version: '2012-10-17', }, }); + + expect(stack).toHaveResourceLike('AWS::IAM::Policy', { + PolicyDocument: { + Statement: arrayWith( + { Action: 'cloudwatch:PutMetricData', Effect: 'Allow', Resource: '*' }, + { + Action: ['logs:DescribeLogStreams', 'logs:DescribeLogGroups'], + Effect: 'Allow', + Resource: [{ + // looks like: arn:aws:logs:us-east-1:123456789012:log-group:my-log-group:*, + 'Fn::GetAtt': ['FlinkApplicationLogGroup7739479C', 'Arn'], + }, { + // looks like: arn:aws:logs:us-east-1:123456789012:log-group:*, + 'Fn::Join': ['', [ + 'arn:', + { Ref: 'AWS::Partition' }, + ':logs:', + { Ref: 'AWS::Region' }, + ':', + { Ref: 'AWS::AccountId' }, + ':log-group:*', + ]], + }], + }, + ), + }, + }); }); test('providing a custom role', () => { From f89a8f95f82dd366228dd6f153f9485f6f1f4b1c Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Sun, 31 Jan 2021 14:44:24 -0800 Subject: [PATCH 27/45] Add missing jsdoc comments --- .../lib/application-code.ts | 24 +++++++++++++++++ .../lib/flink-application.ts | 10 ++++++- .../aws-kinesisanalyticsv2/lib/types.ts | 27 ++++++++++++++++++- 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/application-code.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/application-code.ts index 439706c2922dc..ecf4631fb9e79 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/application-code.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/application-code.ts @@ -3,7 +3,17 @@ import * as s3_assets from '@aws-cdk/aws-s3-assets'; import { Construct } from '@aws-cdk/core'; import { CfnApplication } from './kinesisanalyticsv2.generated'; +/** + * Code configuration providing the location to a Flink application JAR file. + */ export abstract class ApplicationCode { + /** + * Reference code from an S3 bucket. + * + * @param bucket - an s3 bucket + * @param fileKey - a key pointing to a Flink JAR file + * @param objectVersion - an optional version string for the provided fileKey + */ public static fromBucket(bucket: s3.IBucket, fileKey: string, objectVersion?: string): ApplicationCode { return new BucketApplicationCode({ bucket, @@ -12,10 +22,19 @@ export abstract class ApplicationCode { }); } + /** + * Reference code from a local directory containing a Flink JAR file. + * + * @param path - a local directory path + * @parm options - standard s3 AssetOptions + */ public static fromAsset(path: string, options?: s3_assets.AssetOptions): ApplicationCode { return new AssetApplicationCode(path, options); } + /** + * A method to lazily bind asset resources to the parent FlinkApplication. + */ public abstract bind(scope: Construct): ApplicationCodeConfig; } @@ -102,6 +121,11 @@ class AssetApplicationCode extends ApplicationCode { } } +/** + * The return type of {@link ApplicationCode.bind}. This represents + * CloudFormation configuration and an s3 bucket holding the Flink application + * JAR file. + */ export interface ApplicationCodeConfig { /** * Low-level Cloudformation ApplicationConfigurationProperty diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts index 3d4970312cce8..d0aa7d5715732 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts @@ -10,6 +10,10 @@ import { flinkApplicationConfiguration } from './private/flink-application-confi import { validateFlinkApplicationProps } from './private/validation'; import { FlinkLogLevel, FlinkMetricsLevel, FlinkRuntime, PropertyGroups } from './types'; +/** + * An interface expressing the public properties on both an imported and + * CDK-created Flink application. + */ export interface IFlinkApplication extends core.IResource, iam.IGrantable { /** * The application ARN. @@ -152,7 +156,9 @@ export interface FlinkApplicationProps { /** * Configuration PropertyGroups. You can use these property groups to pass - * arbitrary runtime configuration values to your Flink App. + * arbitrary runtime configuration values to your Flink app. + * + * @default No property group configuration provided to the Flink app */ readonly propertyGroups?: PropertyGroups; @@ -201,6 +207,8 @@ export interface FlinkApplicationProps { /** * The KMS encryption key to use for CloudWatch logs. + * + * @default No encryption used */ readonly logEncryptionKey?: kms.IKey; } diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/types.ts b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/types.ts index 7e14bc3812ead..c6a275c24fc3e 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/types.ts +++ b/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/types.ts @@ -2,9 +2,16 @@ * Available log levels for Flink applications. */ export enum FlinkLogLevel { + /** Debug level logging */ DEBUG = 'DEBUG', + + /** Info level logging */ INFO = 'INFO', + + /** Warn level logging */ WARN = 'WARN', + + /** Error level logging */ ERROR = 'ERROR', } @@ -12,9 +19,16 @@ export enum FlinkLogLevel { * Granularity of metrics sent to CloudWatch. */ export enum FlinkMetricsLevel { + /** Application sends the least metrics to CloudWatch */ APPLICATION = 'APPLICATION', + + /** Task includes task-level metrics sent to CloudWatch */ TASK = 'TASK', + + /** Operator includes task-level and operator-level metrics sent to CloudWatch */ OPERATOR = 'OPERATOR', + + /** Send all metrics including metrics per task thread */ PARALLELISM = 'PARALLELISM', } @@ -34,8 +48,19 @@ export interface PropertyGroups { * const runtime = new FlinkRuntime(FLINK-9_99); */ export class FlinkRuntime { + /** Flink Version 1.6 */ public static readonly FLINK_1_6 = new FlinkRuntime('FLINK-1_6'); + + /** Flink Version 1.8 */ public static readonly FLINK_1_8 = new FlinkRuntime('FLINK-1_8'); + + /** Flink Version 1.11 */ public static readonly FLINK_1_11 = new FlinkRuntime('FLINK-1_11'); - public constructor(public readonly value: string) {} + + /** The Cfn string that represents a version of Flink */ + public readonly value: string; + + public constructor(value: string) { + this.value = value; + } } From 4b55390a7fed467fb53da35768f3d29bd2f8166b Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Thu, 4 Feb 2021 13:13:00 -0800 Subject: [PATCH 28/45] Move package to aws-kinesis-analytics-flink --- .../.eslintrc.js | 0 .../.gitignore | 0 .../.npmignore | 0 .../LICENSE | 0 .../NOTICE | 0 .../README.md | 2 +- .../jest.config.js | 0 .../lib/application-code.ts | 4 +-- .../lib/flink-application.ts | 8 ++--- .../lib/index.ts | 1 - .../lib/private/environment-properties.ts | 4 +-- .../flink-application-configuration.ts | 0 .../lib/private/validation.ts | 0 .../lib/types.ts | 0 .../package.json | 32 +++++++++++------- .../test/code-asset/WordCount.jar | Bin .../test/flink-application.test.ts | 0 .../integ.flink-application.expected.json | 0 .../test/integ.flink-application.ts | 0 19 files changed, 28 insertions(+), 23 deletions(-) rename packages/@aws-cdk/{aws-kinesisanalyticsv2 => aws-kinesis-analytics-flink}/.eslintrc.js (100%) rename packages/@aws-cdk/{aws-kinesisanalyticsv2 => aws-kinesis-analytics-flink}/.gitignore (100%) rename packages/@aws-cdk/{aws-kinesisanalyticsv2 => aws-kinesis-analytics-flink}/.npmignore (100%) rename packages/@aws-cdk/{aws-kinesisanalyticsv2 => aws-kinesis-analytics-flink}/LICENSE (100%) rename packages/@aws-cdk/{aws-kinesisanalyticsv2 => aws-kinesis-analytics-flink}/NOTICE (100%) rename packages/@aws-cdk/{aws-kinesisanalyticsv2 => aws-kinesis-analytics-flink}/README.md (99%) rename packages/@aws-cdk/{aws-kinesisanalyticsv2 => aws-kinesis-analytics-flink}/jest.config.js (100%) rename packages/@aws-cdk/{aws-kinesisanalyticsv2 => aws-kinesis-analytics-flink}/lib/application-code.ts (95%) rename packages/@aws-cdk/{aws-kinesisanalyticsv2 => aws-kinesis-analytics-flink}/lib/flink-application.ts (97%) rename packages/@aws-cdk/{aws-kinesisanalyticsv2 => aws-kinesis-analytics-flink}/lib/index.ts (67%) rename packages/@aws-cdk/{aws-kinesisanalyticsv2 => aws-kinesis-analytics-flink}/lib/private/environment-properties.ts (71%) rename packages/@aws-cdk/{aws-kinesisanalyticsv2 => aws-kinesis-analytics-flink}/lib/private/flink-application-configuration.ts (100%) rename packages/@aws-cdk/{aws-kinesisanalyticsv2 => aws-kinesis-analytics-flink}/lib/private/validation.ts (100%) rename packages/@aws-cdk/{aws-kinesisanalyticsv2 => aws-kinesis-analytics-flink}/lib/types.ts (100%) rename packages/@aws-cdk/{aws-kinesisanalyticsv2 => aws-kinesis-analytics-flink}/package.json (75%) rename packages/@aws-cdk/{aws-kinesisanalyticsv2 => aws-kinesis-analytics-flink}/test/code-asset/WordCount.jar (100%) rename packages/@aws-cdk/{aws-kinesisanalyticsv2 => aws-kinesis-analytics-flink}/test/flink-application.test.ts (100%) rename packages/@aws-cdk/{aws-kinesisanalyticsv2 => aws-kinesis-analytics-flink}/test/integ.flink-application.expected.json (100%) rename packages/@aws-cdk/{aws-kinesisanalyticsv2 => aws-kinesis-analytics-flink}/test/integ.flink-application.ts (100%) diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/.eslintrc.js b/packages/@aws-cdk/aws-kinesis-analytics-flink/.eslintrc.js similarity index 100% rename from packages/@aws-cdk/aws-kinesisanalyticsv2/.eslintrc.js rename to packages/@aws-cdk/aws-kinesis-analytics-flink/.eslintrc.js diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/.gitignore b/packages/@aws-cdk/aws-kinesis-analytics-flink/.gitignore similarity index 100% rename from packages/@aws-cdk/aws-kinesisanalyticsv2/.gitignore rename to packages/@aws-cdk/aws-kinesis-analytics-flink/.gitignore diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/.npmignore b/packages/@aws-cdk/aws-kinesis-analytics-flink/.npmignore similarity index 100% rename from packages/@aws-cdk/aws-kinesisanalyticsv2/.npmignore rename to packages/@aws-cdk/aws-kinesis-analytics-flink/.npmignore diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/LICENSE b/packages/@aws-cdk/aws-kinesis-analytics-flink/LICENSE similarity index 100% rename from packages/@aws-cdk/aws-kinesisanalyticsv2/LICENSE rename to packages/@aws-cdk/aws-kinesis-analytics-flink/LICENSE diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/NOTICE b/packages/@aws-cdk/aws-kinesis-analytics-flink/NOTICE similarity index 100% rename from packages/@aws-cdk/aws-kinesisanalyticsv2/NOTICE rename to packages/@aws-cdk/aws-kinesis-analytics-flink/NOTICE diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/README.md b/packages/@aws-cdk/aws-kinesis-analytics-flink/README.md similarity index 99% rename from packages/@aws-cdk/aws-kinesisanalyticsv2/README.md rename to packages/@aws-cdk/aws-kinesis-analytics-flink/README.md index 6b7536ad642f8..ed972912d3ea9 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/README.md +++ b/packages/@aws-cdk/aws-kinesis-analytics-flink/README.md @@ -1,4 +1,4 @@ -# Kinesis Analytics V2 +# Kinesis Analytics Flink --- diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/jest.config.js b/packages/@aws-cdk/aws-kinesis-analytics-flink/jest.config.js similarity index 100% rename from packages/@aws-cdk/aws-kinesisanalyticsv2/jest.config.js rename to packages/@aws-cdk/aws-kinesis-analytics-flink/jest.config.js diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/application-code.ts b/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/application-code.ts similarity index 95% rename from packages/@aws-cdk/aws-kinesisanalyticsv2/lib/application-code.ts rename to packages/@aws-cdk/aws-kinesis-analytics-flink/lib/application-code.ts index ecf4631fb9e79..36280678ef553 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/application-code.ts +++ b/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/application-code.ts @@ -1,7 +1,7 @@ +import * as ka from '@aws-cdk/aws-kinesisanalytics'; import * as s3 from '@aws-cdk/aws-s3'; import * as s3_assets from '@aws-cdk/aws-s3-assets'; import { Construct } from '@aws-cdk/core'; -import { CfnApplication } from './kinesisanalyticsv2.generated'; /** * Code configuration providing the location to a Flink application JAR file. @@ -130,7 +130,7 @@ export interface ApplicationCodeConfig { /** * Low-level Cloudformation ApplicationConfigurationProperty */ - readonly applicationCodeConfigurationProperty: CfnApplication.ApplicationConfigurationProperty; + readonly applicationCodeConfigurationProperty: ka.CfnApplicationV2.ApplicationConfigurationProperty; /** * S3 Bucket that stores the Flink application code diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts b/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/flink-application.ts similarity index 97% rename from packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts rename to packages/@aws-cdk/aws-kinesis-analytics-flink/lib/flink-application.ts index d0aa7d5715732..c14be5a185454 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/flink-application.ts +++ b/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/flink-application.ts @@ -1,10 +1,10 @@ import * as iam from '@aws-cdk/aws-iam'; +import * as ka from '@aws-cdk/aws-kinesisanalytics'; import * as kms from '@aws-cdk/aws-kms'; import * as logs from '@aws-cdk/aws-logs'; import * as core from '@aws-cdk/core'; import { Construct } from 'constructs'; import { ApplicationCode } from './application-code'; -import { CfnApplication, CfnApplicationCloudWatchLoggingOption } from './kinesisanalyticsv2.generated'; import { environmentProperties } from './private/environment-properties'; import { flinkApplicationConfiguration } from './private/flink-application-configuration'; import { validateFlinkApplicationProps } from './private/validation'; @@ -18,7 +18,7 @@ export interface IFlinkApplication extends core.IResource, iam.IGrantable { /** * The application ARN. * - * @attribute + * @attribute ApplicationV2Name */ readonly applicationArn: string; @@ -281,7 +281,7 @@ export class FlinkApplication extends FlinkApplicationBase { const code = props.code.bind(this); code.bucket.grantRead(this); - const resource = new CfnApplication(this, 'Resource', { + const resource = new ka.CfnApplicationV2(this, 'Resource', { runtimeEnvironment: props.runtime.value, serviceExecutionRole: this.role.roleArn, applicationConfiguration: { @@ -337,7 +337,7 @@ export class FlinkApplication extends FlinkApplicationBase { resources: [logStreamArn], })); - new CfnApplicationCloudWatchLoggingOption(this, 'LoggingOption', { + new ka.CfnApplicationCloudWatchLoggingOptionV2(this, 'LoggingOption', { applicationName: resource.ref, cloudWatchLoggingOption: { logStreamArn, diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/index.ts b/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/index.ts similarity index 67% rename from packages/@aws-cdk/aws-kinesisanalyticsv2/lib/index.ts rename to packages/@aws-cdk/aws-kinesis-analytics-flink/lib/index.ts index 6f592e4a625f1..180618667f51a 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/index.ts +++ b/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/index.ts @@ -1,5 +1,4 @@ export * from './application-code'; export * from './flink-application'; -export * from './kinesisanalyticsv2.generated'; export * from './types'; diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/environment-properties.ts b/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/private/environment-properties.ts similarity index 71% rename from packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/environment-properties.ts rename to packages/@aws-cdk/aws-kinesis-analytics-flink/lib/private/environment-properties.ts index ac4123113b20e..d13a0e870e23e 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/environment-properties.ts +++ b/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/private/environment-properties.ts @@ -1,7 +1,7 @@ -import { CfnApplication } from '../kinesisanalyticsv2.generated'; +import * as ka from '@aws-cdk/aws-kinesisanalytics'; import { PropertyGroups } from '../types'; -export function environmentProperties(propertyGroups?: PropertyGroups): CfnApplication.EnvironmentPropertiesProperty | undefined { +export function environmentProperties(propertyGroups?: PropertyGroups): ka.CfnApplicationV2.EnvironmentPropertiesProperty | undefined { const entries = Object.entries(propertyGroups ?? {}); if (entries.length === 0) { return; diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/flink-application-configuration.ts b/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/private/flink-application-configuration.ts similarity index 100% rename from packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/flink-application-configuration.ts rename to packages/@aws-cdk/aws-kinesis-analytics-flink/lib/private/flink-application-configuration.ts diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/validation.ts b/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/private/validation.ts similarity index 100% rename from packages/@aws-cdk/aws-kinesisanalyticsv2/lib/private/validation.ts rename to packages/@aws-cdk/aws-kinesis-analytics-flink/lib/private/validation.ts diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/lib/types.ts b/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/types.ts similarity index 100% rename from packages/@aws-cdk/aws-kinesisanalyticsv2/lib/types.ts rename to packages/@aws-cdk/aws-kinesis-analytics-flink/lib/types.ts diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/package.json b/packages/@aws-cdk/aws-kinesis-analytics-flink/package.json similarity index 75% rename from packages/@aws-cdk/aws-kinesisanalyticsv2/package.json rename to packages/@aws-cdk/aws-kinesis-analytics-flink/package.json index 099d2227d98f7..31eafd9674f14 100644 --- a/packages/@aws-cdk/aws-kinesisanalyticsv2/package.json +++ b/packages/@aws-cdk/aws-kinesis-analytics-flink/package.json @@ -1,27 +1,27 @@ { - "name": "@aws-cdk/aws-kinesisanalyticsv2", + "name": "@aws-cdk/aws-kinesis-analytics-flink", "version": "0.0.0", - "description": "A CDK Construct Library for Kinesis Analytics V2 constructs", + "description": "A CDK Construct Library for Kinesis Analytics Flink applications", "main": "lib/index.js", "types": "lib/index.d.ts", "jsii": { "outdir": "dist", "targets": { "java": { - "package": "software.amazon.awscdk.services.kinesisanalyticsv2", + "package": "software.amazon.awscdk.services.kinesis.analytics.flink", "maven": { "groupId": "software.amazon.awscdk", - "artifactId": "kinesisanalyticsv2" + "artifactId": "kinesis-analytics-flink" } }, "dotnet": { - "namespace": "Amazon.CDK.AWS.KinesisAnalyticsV2", - "packageId": "Amazon.CDK.AWS.KinesisAnalyticsV2", + "namespace": "Amazon.CDK.AWS.KinesisAnalyticsFlink", + "packageId": "Amazon.CDK.AWS.KinesisAnalyticsFlink", "iconUrl": "https://raw.githubusercontent.com/aws/aws-cdk/master/logo/default-256-dark.png" }, "python": { - "distName": "aws-cdk.aws-kinesisanalyticsv2", - "module": "aws_cdk.aws_kinesisanalyticsv2", + "distName": "aws-cdk.aws-kinesis-analytics-flink", + "module": "aws_cdk.aws_kinesis_analytics_flink", "classifiers": [ "Framework :: AWS CDK", "Framework :: AWS CDK :: 1" @@ -33,7 +33,7 @@ "repository": { "type": "git", "url": "https://github.com/aws/aws-cdk.git", - "directory": "packages/@aws-cdk/aws-kinesisanalyticsv2" + "directory": "packages/@aws-cdk/aws-kinesis-analytics-flink" }, "scripts": { "build": "cdk-build", @@ -56,7 +56,8 @@ "cdk", "kinesis", "analytics", - "kinesisanalytcs" + "kinesisanalytcs", + "flink" ], "author": { "name": "Amazon Web Services", @@ -75,6 +76,7 @@ "@aws-cdk/assets": "0.0.0", "@aws-cdk/core": "0.0.0", "@aws-cdk/aws-iam": "0.0.0", + "@aws-cdk/aws-kinesisanalytics": "0.0.0", "@aws-cdk/aws-kms": "0.0.0", "@aws-cdk/aws-logs": "0.0.0", "@aws-cdk/aws-s3": "0.0.0", @@ -86,6 +88,7 @@ "@aws-cdk/assets": "0.0.0", "@aws-cdk/core": "0.0.0", "@aws-cdk/aws-iam": "0.0.0", + "@aws-cdk/aws-kinesisanalytics": "0.0.0", "@aws-cdk/aws-kms": "0.0.0", "@aws-cdk/aws-logs": "0.0.0", "@aws-cdk/aws-s3": "0.0.0", @@ -95,16 +98,19 @@ "engines": { "node": ">= 10.13.0 <13 || >=13.7.0" }, + "awslint": { + "exclude": [ + "props-physical-name:@aws-cdk/aws-kinesis-analytics-flink.FlinkApplicationProps" + ] + }, "stability": "experimental", "maturity": "experimental", "awscdkio": { "announce": false }, "cdk-build": { - "cloudformation": [ - "AWS::KinesisAnalyticsV2" - ], "jest": true, + "cloudformation": [], "env": { "AWSLINT_BASE_CONSTRUCT": true } diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/code-asset/WordCount.jar b/packages/@aws-cdk/aws-kinesis-analytics-flink/test/code-asset/WordCount.jar similarity index 100% rename from packages/@aws-cdk/aws-kinesisanalyticsv2/test/code-asset/WordCount.jar rename to packages/@aws-cdk/aws-kinesis-analytics-flink/test/code-asset/WordCount.jar diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts b/packages/@aws-cdk/aws-kinesis-analytics-flink/test/flink-application.test.ts similarity index 100% rename from packages/@aws-cdk/aws-kinesisanalyticsv2/test/flink-application.test.ts rename to packages/@aws-cdk/aws-kinesis-analytics-flink/test/flink-application.test.ts diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/integ.flink-application.expected.json b/packages/@aws-cdk/aws-kinesis-analytics-flink/test/integ.flink-application.expected.json similarity index 100% rename from packages/@aws-cdk/aws-kinesisanalyticsv2/test/integ.flink-application.expected.json rename to packages/@aws-cdk/aws-kinesis-analytics-flink/test/integ.flink-application.expected.json diff --git a/packages/@aws-cdk/aws-kinesisanalyticsv2/test/integ.flink-application.ts b/packages/@aws-cdk/aws-kinesis-analytics-flink/test/integ.flink-application.ts similarity index 100% rename from packages/@aws-cdk/aws-kinesisanalyticsv2/test/integ.flink-application.ts rename to packages/@aws-cdk/aws-kinesis-analytics-flink/test/integ.flink-application.ts From 6c7cde3655fd271b4f4f4a46005ff8ff0a2a36a9 Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Thu, 4 Feb 2021 13:35:23 -0800 Subject: [PATCH 29/45] Mass rename to align with package name --- .../{flink-application.ts => application.ts} | 68 ++++++------- .../aws-kinesis-analytics-flink/lib/index.ts | 2 +- .../flink-application-configuration.ts | 6 +- .../aws-kinesis-analytics-flink/lib/types.ts | 12 +-- ...pplication.test.ts => application.test.ts} | 96 +++++++++---------- ...d.json => integ.application.expected.json} | 76 +++++++-------- ...nk-application.ts => integ.application.ts} | 6 +- 7 files changed, 131 insertions(+), 135 deletions(-) rename packages/@aws-cdk/aws-kinesis-analytics-flink/lib/{flink-application.ts => application.ts} (80%) rename packages/@aws-cdk/aws-kinesis-analytics-flink/test/{flink-application.test.ts => application.test.ts} (85%) rename packages/@aws-cdk/aws-kinesis-analytics-flink/test/{integ.flink-application.expected.json => integ.application.expected.json} (87%) rename packages/@aws-cdk/aws-kinesis-analytics-flink/test/{integ.flink-application.ts => integ.application.ts} (77%) diff --git a/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/flink-application.ts b/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/application.ts similarity index 80% rename from packages/@aws-cdk/aws-kinesis-analytics-flink/lib/flink-application.ts rename to packages/@aws-cdk/aws-kinesis-analytics-flink/lib/application.ts index c14be5a185454..dd31ae2683035 100644 --- a/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/flink-application.ts +++ b/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/application.ts @@ -7,25 +7,25 @@ import { Construct } from 'constructs'; import { ApplicationCode } from './application-code'; import { environmentProperties } from './private/environment-properties'; import { flinkApplicationConfiguration } from './private/flink-application-configuration'; -import { validateFlinkApplicationProps } from './private/validation'; -import { FlinkLogLevel, FlinkMetricsLevel, FlinkRuntime, PropertyGroups } from './types'; +import { validateFlinkApplicationProps as validateApplicationProps } from './private/validation'; +import { LogLevel as LogLevel, MetricsLevel as MetricsLevel, PropertyGroups, Runtime as Runtime } from './types'; /** * An interface expressing the public properties on both an imported and * CDK-created Flink application. */ -export interface IFlinkApplication extends core.IResource, iam.IGrantable { +export interface IApplication extends core.IResource, iam.IGrantable { /** * The application ARN. * - * @attribute ApplicationV2Name + * @attribute */ readonly applicationArn: string; /** * The name of the Flink application. * - * @attribute + * @attribute ApplicationV2Name */ readonly applicationName: string; @@ -42,9 +42,9 @@ export interface IFlinkApplication extends core.IResource, iam.IGrantable { /** * Implements the functionality shared between CDK created and imported - * IFlinkApplications. + * IApplications. */ -abstract class FlinkApplicationBase extends core.Resource implements IFlinkApplication { +abstract class ApplicationBase extends core.Resource implements IApplication { public abstract readonly applicationArn: string; public abstract readonly applicationName: string; public abstract readonly role?: iam.IRole; @@ -52,7 +52,7 @@ abstract class FlinkApplicationBase extends core.Resource implements IFlinkAppli // Implement iam.IGrantable interface public abstract readonly grantPrincipal: iam.IPrincipal; - /** Implement the convenience {@link IFlinkApplication.addToPrincipalPolicy} method. */ + /** Implement the convenience {@link IApplication.addToPrincipalPolicy} method. */ public addToRolePolicy(policyStatement: iam.PolicyStatement): boolean { if (this.role) { this.role.addToPrincipalPolicy(policyStatement); @@ -64,11 +64,11 @@ abstract class FlinkApplicationBase extends core.Resource implements IFlinkAppli } /** - * Props for creating a FlinkApplication construct. + * Props for creating an Application construct. */ -export interface FlinkApplicationProps { +export interface ApplicationProps { /** - * A name for your FlinkApplication that is unique to an AWS account. + * A name for your Application that is unique to an AWS account. * * @default - CloudFormation-generated name */ @@ -77,7 +77,7 @@ export interface FlinkApplicationProps { /** * The Flink version to use for this application. */ - readonly runtime: FlinkRuntime; + readonly runtime: Runtime; /** * The Flink code asset to run. @@ -111,7 +111,7 @@ export interface FlinkApplicationProps { * * @default FlinkLogLevel.INFO */ - readonly logLevel?: FlinkLogLevel; + readonly logLevel?: LogLevel; /** * Describes the granularity of the CloudWatch metrics for an application. @@ -119,9 +119,9 @@ export interface FlinkApplicationProps { * metrics for each parallel thread and can quickly become expensive when * parallelism is high (e.g. > 64). * - * @default FlinkMetricsLevel.APPLICATION + * @default MetricsLevel.APPLICATION */ - readonly metricsLevel?: FlinkMetricsLevel; + readonly metricsLevel?: MetricsLevel; /** * Whether the Kinesis Data Analytics service can increase the parallelism of @@ -132,8 +132,8 @@ export interface FlinkApplicationProps { readonly autoScalingEnabled?: boolean; /** - * The initial parallelism for the Flink application. Kinesis Data Analytics - * can stop the app, increase the parallelism, and start the app again if + * The initial parallelism for the application. Kinesis Data Analytics can + * stop the app, increase the parallelism, and start the app again if * autoScalingEnabled is true (the default value). * * @default 1 @@ -220,36 +220,36 @@ export interface FlinkApplicationProps { * * @experimental */ -export class FlinkApplication extends FlinkApplicationBase { +export class Application extends ApplicationBase { /** * Import an existing Flink application defined outside of CDK code by * applicationName. */ - public static fromFlinkApplicationName(scope: Construct, id: string, flinkApplicationName: string): IFlinkApplication { - class Import extends FlinkApplicationBase { - // Imported flink applications have no associated role or grantPrincipal + public static fromApplicationName(scope: Construct, id: string, applicationName: string): IApplication { + class Import extends ApplicationBase { + // Imported applications have no associated role or grantPrincipal public readonly role = undefined; public readonly grantPrincipal = new iam.UnknownPrincipal({ resource: this }); - public readonly applicationName = flinkApplicationName; - public readonly applicationArn = core.Stack.of(this).formatArn(flinkApplicationArnComponents(flinkApplicationName)); + public readonly applicationName = applicationName; + public readonly applicationArn = core.Stack.of(this).formatArn(applicationArnComponents(applicationName)); } return new Import(scope, id); } /** - * Import an existing Flink application defined outside of CDK code by + * Import an existing application defined outside of CDK code by * applicationArn. */ - public static fromFlinkApplicationArn(scope: Construct, id: string, flinkApplicationArn: string): IFlinkApplication { - class Import extends FlinkApplicationBase { - // Imported flink applications have no associated role or grantPrincipal + public static fromApplicationArn(scope: Construct, id: string, applicationArn: string): IApplication { + class Import extends ApplicationBase { + // Imported applications have no associated role or grantPrincipal public readonly role = undefined; public readonly grantPrincipal = new iam.UnknownPrincipal({ resource: this }); - public readonly applicationName = flinkApplicationArn.split('/')[1]; - public readonly applicationArn = flinkApplicationArn; + public readonly applicationName = applicationArn.split('/')[1]; + public readonly applicationArn = applicationArn; } return new Import(scope, id); @@ -263,9 +263,9 @@ export class FlinkApplication extends FlinkApplicationBase { public readonly grantPrincipal: iam.IPrincipal; - constructor(scope: Construct, id: string, props: FlinkApplicationProps) { + constructor(scope: Construct, id: string, props: ApplicationProps) { super(scope, id, { physicalName: props.applicationName }); - validateFlinkApplicationProps(props); + validateApplicationProps(props); this.role = props.role ?? new iam.Role(this, 'Role', { assumedBy: new iam.ServicePrincipal('kinesisanalytics.amazonaws.com'), @@ -346,8 +346,8 @@ export class FlinkApplication extends FlinkApplicationBase { this.applicationName = this.getResourceNameAttribute(resource.ref); this.applicationArn = this.getResourceArnAttribute( - core.Stack.of(this).formatArn(flinkApplicationArnComponents(resource.ref)), - flinkApplicationArnComponents(this.physicalName), + core.Stack.of(this).formatArn(applicationArnComponents(resource.ref)), + applicationArnComponents(this.physicalName), ); resource.applyRemovalPolicy(props.removalPolicy, { @@ -356,7 +356,7 @@ export class FlinkApplication extends FlinkApplicationBase { } } -function flinkApplicationArnComponents(resourceName: string): core.ArnComponents { +function applicationArnComponents(resourceName: string): core.ArnComponents { return { service: 'kinesisanalytics', resource: 'application', diff --git a/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/index.ts b/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/index.ts index 180618667f51a..6e2bea43a0ebf 100644 --- a/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/index.ts +++ b/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/index.ts @@ -1,4 +1,4 @@ +export * from './application'; export * from './application-code'; -export * from './flink-application'; export * from './types'; diff --git a/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/private/flink-application-configuration.ts b/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/private/flink-application-configuration.ts index 67e0b55d207c0..c0a93edac20d6 100644 --- a/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/private/flink-application-configuration.ts +++ b/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/private/flink-application-configuration.ts @@ -1,5 +1,5 @@ import * as core from '@aws-cdk/core'; -import { FlinkLogLevel, FlinkMetricsLevel } from '../types'; +import { LogLevel, MetricsLevel } from '../types'; interface FlinkApplicationConfiguration extends CheckpointConfiguration, @@ -13,8 +13,8 @@ interface CheckpointConfiguration { } interface MonitoringConfiguration { - logLevel?: FlinkLogLevel; - metricsLevel?: FlinkMetricsLevel; + logLevel?: LogLevel; + metricsLevel?: MetricsLevel; } interface ParallelismConfiguration { diff --git a/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/types.ts b/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/types.ts index c6a275c24fc3e..38347fc500963 100644 --- a/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/types.ts +++ b/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/types.ts @@ -1,7 +1,7 @@ /** * Available log levels for Flink applications. */ -export enum FlinkLogLevel { +export enum LogLevel { /** Debug level logging */ DEBUG = 'DEBUG', @@ -18,7 +18,7 @@ export enum FlinkLogLevel { /** * Granularity of metrics sent to CloudWatch. */ -export enum FlinkMetricsLevel { +export enum MetricsLevel { /** Application sends the least metrics to CloudWatch */ APPLICATION = 'APPLICATION', @@ -47,15 +47,15 @@ export interface PropertyGroups { * // Creating a new runtime that isn't in CDK yet. * const runtime = new FlinkRuntime(FLINK-9_99); */ -export class FlinkRuntime { +export class Runtime { /** Flink Version 1.6 */ - public static readonly FLINK_1_6 = new FlinkRuntime('FLINK-1_6'); + public static readonly FLINK_1_6 = new Runtime('FLINK-1_6'); /** Flink Version 1.8 */ - public static readonly FLINK_1_8 = new FlinkRuntime('FLINK-1_8'); + public static readonly FLINK_1_8 = new Runtime('FLINK-1_8'); /** Flink Version 1.11 */ - public static readonly FLINK_1_11 = new FlinkRuntime('FLINK-1_11'); + public static readonly FLINK_1_11 = new Runtime('FLINK-1_11'); /** The Cfn string that represents a version of Flink */ public readonly value: string; diff --git a/packages/@aws-cdk/aws-kinesis-analytics-flink/test/flink-application.test.ts b/packages/@aws-cdk/aws-kinesis-analytics-flink/test/application.test.ts similarity index 85% rename from packages/@aws-cdk/aws-kinesis-analytics-flink/test/flink-application.test.ts rename to packages/@aws-cdk/aws-kinesis-analytics-flink/test/application.test.ts index 5cef62df24c87..97a341bf8213d 100644 --- a/packages/@aws-cdk/aws-kinesis-analytics-flink/test/flink-application.test.ts +++ b/packages/@aws-cdk/aws-kinesis-analytics-flink/test/application.test.ts @@ -5,29 +5,29 @@ import * as kms from '@aws-cdk/aws-kms'; import * as s3 from '@aws-cdk/aws-s3'; import * as core from '@aws-cdk/core'; import * as path from 'path'; -import * as ka from '../lib'; +import * as flink from '../lib'; -describe('FlinkApplication', () => { +describe('Application', () => { let stack: core.Stack; let bucket: s3.Bucket; let requiredProps: { - runtime: ka.FlinkRuntime; - code: ka.ApplicationCode; + runtime: flink.Runtime; + code: flink.ApplicationCode; }; beforeEach(() => { stack = new core.Stack(); bucket = new s3.Bucket(stack, 'CodeBucket'); requiredProps = { - runtime: ka.FlinkRuntime.FLINK_1_11, - code: ka.ApplicationCode.fromBucket(bucket, 'my-app.jar'), + runtime: flink.Runtime.FLINK_1_11, + code: flink.ApplicationCode.fromBucket(bucket, 'my-app.jar'), }; }); test('default Flink Application', () => { - new ka.FlinkApplication(stack, 'FlinkApplication', { - runtime: ka.FlinkRuntime.FLINK_1_11, - code: ka.ApplicationCode.fromBucket(bucket, 'my-app.jar'), + new flink.Application(stack, 'FlinkApplication', { + runtime: flink.Runtime.FLINK_1_11, + code: flink.ApplicationCode.fromBucket(bucket, 'my-app.jar'), }); expect(stack).toHaveResource('AWS::KinesisAnalyticsV2::Application', { @@ -100,7 +100,7 @@ describe('FlinkApplication', () => { }); test('providing a custom role', () => { - new ka.FlinkApplication(stack, 'FlinkApplication', { + new flink.Application(stack, 'FlinkApplication', { ...requiredProps, role: new iam.Role(stack, 'CustomRole', { assumedBy: new iam.ServicePrincipal('custom-principal'), @@ -124,7 +124,7 @@ describe('FlinkApplication', () => { }); test('addToPrincipalPolicy', () => { - const app = new ka.FlinkApplication(stack, 'FlinkApplication', { + const app = new flink.Application(stack, 'FlinkApplication', { ...requiredProps, }); @@ -143,9 +143,9 @@ describe('FlinkApplication', () => { }); test('providing a custom runtime', () => { - new ka.FlinkApplication(stack, 'FlinkApplication', { + new flink.Application(stack, 'FlinkApplication', { ...requiredProps, - runtime: new ka.FlinkRuntime('custom'), + runtime: new flink.Runtime('custom'), }); expect(stack).toHaveResourceLike('AWS::KinesisAnalyticsV2::Application', { @@ -154,7 +154,7 @@ describe('FlinkApplication', () => { }); test('providing a custom removal policy', () => { - new ka.FlinkApplication(stack, 'FlinkApplication', { + new flink.Application(stack, 'FlinkApplication', { ...requiredProps, removalPolicy: core.RemovalPolicy.RETAIN, }); @@ -165,7 +165,7 @@ describe('FlinkApplication', () => { }); test('granting permissions to resources', () => { - const app = new ka.FlinkApplication(stack, 'FlinkApplication', { + const app = new flink.Application(stack, 'FlinkApplication', { ...requiredProps, }); @@ -183,8 +183,8 @@ describe('FlinkApplication', () => { }); test('using an asset for code', () => { - const code = ka.ApplicationCode.fromAsset(path.join(__dirname, 'code-asset')); - new ka.FlinkApplication(stack, 'FlinkApplication', { + const code = flink.ApplicationCode.fromAsset(path.join(__dirname, 'code-asset')); + new flink.Application(stack, 'FlinkApplication', { ...requiredProps, code, }); @@ -219,7 +219,7 @@ describe('FlinkApplication', () => { }); test('adding property groups', () => { - new ka.FlinkApplication(stack, 'FlinkApplication', { + new flink.Application(stack, 'FlinkApplication', { ...requiredProps, propertyGroups: { FlinkApplicationProperties: { @@ -245,7 +245,7 @@ describe('FlinkApplication', () => { }); test('checkpointEnabled setting', () => { - new ka.FlinkApplication(stack, 'FlinkApplication', { + new flink.Application(stack, 'FlinkApplication', { ...requiredProps, checkpointingEnabled: false, }); @@ -263,7 +263,7 @@ describe('FlinkApplication', () => { }); test('checkpointInterval setting', () => { - new ka.FlinkApplication(stack, 'FlinkApplication', { + new flink.Application(stack, 'FlinkApplication', { ...requiredProps, checkpointInterval: core.Duration.minutes(5), }); @@ -281,7 +281,7 @@ describe('FlinkApplication', () => { }); test('minPauseBetweenCheckpoints setting', () => { - new ka.FlinkApplication(stack, 'FlinkApplication', { + new flink.Application(stack, 'FlinkApplication', { ...requiredProps, minPauseBetweenCheckpoints: core.Duration.seconds(10), }); @@ -299,9 +299,9 @@ describe('FlinkApplication', () => { }); test('logLevel setting', () => { - new ka.FlinkApplication(stack, 'FlinkApplication', { + new flink.Application(stack, 'FlinkApplication', { ...requiredProps, - logLevel: ka.FlinkLogLevel.DEBUG, + logLevel: flink.LogLevel.DEBUG, }); expect(stack).toHaveResourceLike('AWS::KinesisAnalyticsV2::Application', { @@ -317,9 +317,9 @@ describe('FlinkApplication', () => { }); test('metricsLevel setting', () => { - new ka.FlinkApplication(stack, 'FlinkApplication', { + new flink.Application(stack, 'FlinkApplication', { ...requiredProps, - metricsLevel: ka.FlinkMetricsLevel.PARALLELISM, + metricsLevel: flink.MetricsLevel.PARALLELISM, }); expect(stack).toHaveResourceLike('AWS::KinesisAnalyticsV2::Application', { @@ -335,7 +335,7 @@ describe('FlinkApplication', () => { }); test('autoscalingEnabled setting', () => { - new ka.FlinkApplication(stack, 'FlinkApplication', { + new flink.Application(stack, 'FlinkApplication', { ...requiredProps, autoScalingEnabled: false, }); @@ -353,7 +353,7 @@ describe('FlinkApplication', () => { }); test('parallelism setting', () => { - new ka.FlinkApplication(stack, 'FlinkApplication', { + new flink.Application(stack, 'FlinkApplication', { ...requiredProps, parallelism: 2, }); @@ -371,7 +371,7 @@ describe('FlinkApplication', () => { }); test('parallelismPerKpu setting', () => { - new ka.FlinkApplication(stack, 'FlinkApplication', { + new flink.Application(stack, 'FlinkApplication', { ...requiredProps, parallelismPerKpu: 2, }); @@ -389,7 +389,7 @@ describe('FlinkApplication', () => { }); test('snapshotsEnabled setting', () => { - new ka.FlinkApplication(stack, 'FlinkApplication', { + new flink.Application(stack, 'FlinkApplication', { ...requiredProps, snapshotsEnabled: false, }); @@ -404,7 +404,7 @@ describe('FlinkApplication', () => { }); test('default logging option', () => { - new ka.FlinkApplication(stack, 'FlinkApplication', { + new flink.Application(stack, 'FlinkApplication', { ...requiredProps, snapshotsEnabled: false, }); @@ -459,7 +459,7 @@ describe('FlinkApplication', () => { }); test('logRetentionDays setting', () => { - new ka.FlinkApplication(stack, 'FlinkApplication', { + new flink.Application(stack, 'FlinkApplication', { ...requiredProps, logRetention: core.Duration.days(5), }); @@ -470,7 +470,7 @@ describe('FlinkApplication', () => { }); test('logRemovalPolicy setting', () => { - new ka.FlinkApplication(stack, 'FlinkApplication', { + new flink.Application(stack, 'FlinkApplication', { ...requiredProps, logRemovalPolicy: core.RemovalPolicy.DESTROY, }); @@ -482,7 +482,7 @@ describe('FlinkApplication', () => { }); test('logGroupName setting', () => { - new ka.FlinkApplication(stack, 'FlinkApplication', { + new flink.Application(stack, 'FlinkApplication', { ...requiredProps, logGroupName: 'my-group', }); @@ -493,7 +493,7 @@ describe('FlinkApplication', () => { }); test('logStreamName setting', () => { - new ka.FlinkApplication(stack, 'FlinkApplication', { + new flink.Application(stack, 'FlinkApplication', { ...requiredProps, logStreamName: 'my-stream', }); @@ -505,7 +505,7 @@ describe('FlinkApplication', () => { test('logEncryptionKey setting', () => { const key = new kms.Key(stack, 'Key'); - new ka.FlinkApplication(stack, 'FlinkApplication', { + new flink.Application(stack, 'FlinkApplication', { ...requiredProps, logEncryptionKey: key, }); @@ -517,33 +517,33 @@ describe('FlinkApplication', () => { test('validating applicationName', () => { // Expect no error with valid name - new ka.FlinkApplication(stack, 'ValidString', { + new flink.Application(stack, 'ValidString', { ...requiredProps, applicationName: 'my-VALID.app_name', }); // Expect no error with ref - new ka.FlinkApplication(stack, 'ValidRef', { + new flink.Application(stack, 'ValidRef', { ...requiredProps, applicationName: new core.CfnParameter(stack, 'Parameter').valueAsString, }); expect(() => { - new ka.FlinkApplication(stack, 'Empty', { + new flink.Application(stack, 'Empty', { ...requiredProps, applicationName: '', }); }).toThrow(/cannot be empty/); expect(() => { - new ka.FlinkApplication(stack, 'InvalidCharacters', { + new flink.Application(stack, 'InvalidCharacters', { ...requiredProps, applicationName: '!!!', }); }).toThrow(/may only contain letters, numbers, underscores, hyphens, and periods/); expect(() => { - new ka.FlinkApplication(stack, 'TooLong', { + new flink.Application(stack, 'TooLong', { ...requiredProps, applicationName: 'a'.repeat(129), }); @@ -552,13 +552,13 @@ describe('FlinkApplication', () => { test('validating parallelism', () => { // Expect no error with valid value - new ka.FlinkApplication(stack, 'ValidNumber', { + new flink.Application(stack, 'ValidNumber', { ...requiredProps, parallelism: 32, }); // Expect no error with ref - new ka.FlinkApplication(stack, 'ValidRef', { + new flink.Application(stack, 'ValidRef', { ...requiredProps, parallelism: new core.CfnParameter(stack, 'Parameter', { type: 'Number', @@ -566,7 +566,7 @@ describe('FlinkApplication', () => { }); expect(() => { - new ka.FlinkApplication(stack, 'TooSmall', { + new flink.Application(stack, 'TooSmall', { ...requiredProps, parallelism: 0, }); @@ -575,13 +575,13 @@ describe('FlinkApplication', () => { test('validating parallelismPerKpu', () => { // Expect no error with valid value - new ka.FlinkApplication(stack, 'ValidNumber', { + new flink.Application(stack, 'ValidNumber', { ...requiredProps, parallelismPerKpu: 10, }); // Expect no error with ref - new ka.FlinkApplication(stack, 'ValidRef', { + new flink.Application(stack, 'ValidRef', { ...requiredProps, parallelismPerKpu: new core.CfnParameter(stack, 'Parameter', { type: 'Number', @@ -589,7 +589,7 @@ describe('FlinkApplication', () => { }); expect(() => { - new ka.FlinkApplication(stack, 'TooSmall', { + new flink.Application(stack, 'TooSmall', { ...requiredProps, parallelismPerKpu: 0, }); @@ -597,7 +597,7 @@ describe('FlinkApplication', () => { }); test('fromFlinkApplicationName', () => { - const flinkApp = ka.FlinkApplication.fromFlinkApplicationName(stack, 'Imported', 'my-app'); + const flinkApp = flink.Application.fromApplicationName(stack, 'Imported', 'my-app'); expect(flinkApp.applicationName).toEqual('my-app'); expect(stack.resolve(flinkApp.applicationArn)).toEqual({ @@ -616,7 +616,7 @@ describe('FlinkApplication', () => { test('fromFlinkApplicationArn', () => { const arn = 'arn:aws:kinesisanalytics:us-west-2:012345678901:application/my-app'; - const flinkApp = ka.FlinkApplication.fromFlinkApplicationArn(stack, 'Imported', arn); + const flinkApp = flink.Application.fromApplicationArn(stack, 'Imported', arn); expect(flinkApp.applicationName).toEqual('my-app'); expect(flinkApp.applicationArn).toEqual(arn); diff --git a/packages/@aws-cdk/aws-kinesis-analytics-flink/test/integ.flink-application.expected.json b/packages/@aws-cdk/aws-kinesis-analytics-flink/test/integ.application.expected.json similarity index 87% rename from packages/@aws-cdk/aws-kinesis-analytics-flink/test/integ.flink-application.expected.json rename to packages/@aws-cdk/aws-kinesis-analytics-flink/test/integ.application.expected.json index 023c53864d8ee..3b4f7ecf64f7e 100644 --- a/packages/@aws-cdk/aws-kinesis-analytics-flink/test/integ.flink-application.expected.json +++ b/packages/@aws-cdk/aws-kinesis-analytics-flink/test/integ.application.expected.json @@ -69,39 +69,38 @@ ] }, { - "Action": [ - "logs:DescribeLogStreams", - "logs:DescribeLogGroups" - ], + "Action": "logs:DescribeLogGroups", "Effect": "Allow", - "Resource": [ - { - "Fn::GetAtt": [ - "AppLogGroupC72EEC8C", - "Arn" - ] - }, - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":logs:", - { - "Ref": "AWS::Region" - }, - ":", - { - "Ref": "AWS::AccountId" - }, - ":log-group/*" - ] + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":logs:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":log-group:*" ] - } - ] + ] + } + }, + { + "Action": "logs:DescribeLogStreams", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "AppLogGroupC72EEC8C", + "Arn" + ] + } }, { "Action": "logs:PutLogEvents", @@ -226,10 +225,10 @@ "AppLogGroupC72EEC8C": { "Type": "AWS::Logs::LogGroup", "Properties": { - "RetentionInDays": 7 + "RetentionInDays": 731 }, - "UpdateReplacePolicy": "Delete", - "DeletionPolicy": "Delete" + "UpdateReplacePolicy": "Retain", + "DeletionPolicy": "Retain" }, "AppLogStream3CAF66A7": { "Type": "AWS::Logs::LogStream", @@ -238,8 +237,8 @@ "Ref": "AppLogGroupC72EEC8C" } }, - "UpdateReplacePolicy": "Delete", - "DeletionPolicy": "Delete" + "UpdateReplacePolicy": "Retain", + "DeletionPolicy": "Retain" }, "AppLoggingOption75BE995E": { "Type": "AWS::KinesisAnalyticsV2::ApplicationCloudWatchLoggingOption", @@ -276,10 +275,7 @@ ] } } - }, - "DependsOn": [ - "AppF1B96344" - ] + } } }, "Parameters": { diff --git a/packages/@aws-cdk/aws-kinesis-analytics-flink/test/integ.flink-application.ts b/packages/@aws-cdk/aws-kinesis-analytics-flink/test/integ.application.ts similarity index 77% rename from packages/@aws-cdk/aws-kinesis-analytics-flink/test/integ.flink-application.ts rename to packages/@aws-cdk/aws-kinesis-analytics-flink/test/integ.application.ts index 1c3b1485b03bb..0ac3ec5bea02d 100644 --- a/packages/@aws-cdk/aws-kinesis-analytics-flink/test/integ.flink-application.ts +++ b/packages/@aws-cdk/aws-kinesis-analytics-flink/test/integ.application.ts @@ -1,13 +1,13 @@ -import * as path from 'path'; import * as core from '@aws-cdk/core'; +import * as path from 'path'; import * as ka from '../lib'; const app = new core.App(); const stack = new core.Stack(app, 'FlinkAppTest'); -new ka.FlinkApplication(stack, 'App', { +new ka.Application(stack, 'App', { code: ka.ApplicationCode.fromAsset(path.join(__dirname, 'code-asset')), - runtime: ka.FlinkRuntime.FLINK_1_11, + runtime: ka.Runtime.FLINK_1_11, }); app.synth(); From 03ade5e4db341d1a1126b0bbdd7356c6e6e507d1 Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Thu, 4 Feb 2021 13:49:23 -0800 Subject: [PATCH 30/45] Split logging into more ganular policy statements --- .../lib/application.ts | 11 ++++-- .../test/application.test.ts | 35 +++++++++++++++---- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/application.ts b/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/application.ts index dd31ae2683035..3f22235598724 100644 --- a/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/application.ts +++ b/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/application.ts @@ -317,11 +317,11 @@ export class Application extends ApplicationBase { removalPolicy: props.logRemovalPolicy, }); - // Permit logging + /* Permit logging */ + this.role.addToPrincipalPolicy(new iam.PolicyStatement({ - actions: ['logs:DescribeLogStreams', 'logs:DescribeLogGroups'], + actions: ['logs:DescribeLogGroups'], resources: [ - logGroup.logGroupArn, core.Stack.of(this).formatArn({ service: 'logs', resource: 'log-group', @@ -331,6 +331,11 @@ export class Application extends ApplicationBase { ], })); + this.role.addToPrincipalPolicy(new iam.PolicyStatement({ + actions: ['logs:DescribeLogStreams'], + resources: [logGroup.logGroupArn], + })); + const logStreamArn = `arn:${core.Aws.PARTITION}:logs:${core.Aws.REGION}:${core.Aws.ACCOUNT_ID}:log-group:${logGroup.logGroupName}:log-stream:${logStream.logStreamName}`; this.role.addToPrincipalPolicy(new iam.PolicyStatement({ actions: ['logs:PutLogEvents'], diff --git a/packages/@aws-cdk/aws-kinesis-analytics-flink/test/application.test.ts b/packages/@aws-cdk/aws-kinesis-analytics-flink/test/application.test.ts index 97a341bf8213d..ae74488863868 100644 --- a/packages/@aws-cdk/aws-kinesis-analytics-flink/test/application.test.ts +++ b/packages/@aws-cdk/aws-kinesis-analytics-flink/test/application.test.ts @@ -76,13 +76,33 @@ describe('Application', () => { Statement: arrayWith( { Action: 'cloudwatch:PutMetricData', Effect: 'Allow', Resource: '*' }, { - Action: ['logs:DescribeLogStreams', 'logs:DescribeLogGroups'], + Action: 'logs:DescribeLogGroups', Effect: 'Allow', - Resource: [{ + Resource: { + // looks like arn:aws:logs:us-east-1:123456789012:log-group:*, + 'Fn::Join': ['', [ + 'arn:', + { Ref: 'AWS::Partition' }, + ':logs:', + { Ref: 'AWS::Region' }, + ':', + { Ref: 'AWS::AccountId' }, + ':log-group:*', + ]], + }, + }, + { + Action: 'logs:DescribeLogStreams', + Effect: 'Allow', + Resource: { // looks like: arn:aws:logs:us-east-1:123456789012:log-group:my-log-group:*, 'Fn::GetAtt': ['FlinkApplicationLogGroup7739479C', 'Arn'], - }, { - // looks like: arn:aws:logs:us-east-1:123456789012:log-group:*, + }, + }, + { + Action: 'logs:PutLogEvents', + Effect: 'Allow', + Resource: { 'Fn::Join': ['', [ 'arn:', { Ref: 'AWS::Partition' }, @@ -90,9 +110,12 @@ describe('Application', () => { { Ref: 'AWS::Region' }, ':', { Ref: 'AWS::AccountId' }, - ':log-group:*', + ':log-group:', + { Ref: 'FlinkApplicationLogGroup7739479C' }, + ':log-stream:', + { Ref: 'FlinkApplicationLogStreamB633AF32' }, ]], - }], + }, }, ), }, From a18e6bf0c6f1fed3f9924cdf35c84efb5ad441f8 Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Thu, 4 Feb 2021 14:06:57 -0800 Subject: [PATCH 31/45] Reference flink pkg from main kinesisanalytics pkg --- packages/@aws-cdk/aws-kinesisanalytics/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/@aws-cdk/aws-kinesisanalytics/README.md b/packages/@aws-cdk/aws-kinesisanalytics/README.md index e65b812cefc97..9426a7cfd3db3 100644 --- a/packages/@aws-cdk/aws-kinesisanalytics/README.md +++ b/packages/@aws-cdk/aws-kinesisanalytics/README.md @@ -14,3 +14,9 @@ This module is part of the [AWS Cloud Development Kit](https://github.com/aws/aws-cdk) project. + +## Kinesis Analytics Flink + +The `aws-kinesisanalytics-flink` package provides constructs for building Flink applications. + * [Github](https://github.com/aws/aws-cdk/tree/master/packages/%40aws-cdk/aws-kinesisanalytics-flink) + * [CDK Docs](https://docs.aws.amazon.com/cdk/api/latest/docs/aws-kinesisanalytics-flink.html) From ffc071c201bc3879e96170bf3b9026768e1b1509 Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Thu, 4 Feb 2021 14:07:38 -0800 Subject: [PATCH 32/45] Remove hyphen from kinesis-analytics --- .../.eslintrc.js | 0 .../.gitignore | 0 .../.npmignore | 0 .../LICENSE | 0 .../NOTICE | 0 .../README.md | 0 .../jest.config.js | 0 .../lib/application-code.ts | 0 .../lib/application.ts | 0 .../lib/index.ts | 0 .../lib/private/environment-properties.ts | 0 .../lib/private/flink-application-configuration.ts | 0 .../lib/private/validation.ts | 0 .../lib/types.ts | 0 .../package.json | 10 +++++----- .../test/application.test.ts | 0 .../test/code-asset/WordCount.jar | Bin .../test/integ.application.expected.json | 0 .../test/integ.application.ts | 0 19 files changed, 5 insertions(+), 5 deletions(-) rename packages/@aws-cdk/{aws-kinesis-analytics-flink => aws-kinesisanalytics-flink}/.eslintrc.js (100%) rename packages/@aws-cdk/{aws-kinesis-analytics-flink => aws-kinesisanalytics-flink}/.gitignore (100%) rename packages/@aws-cdk/{aws-kinesis-analytics-flink => aws-kinesisanalytics-flink}/.npmignore (100%) rename packages/@aws-cdk/{aws-kinesis-analytics-flink => aws-kinesisanalytics-flink}/LICENSE (100%) rename packages/@aws-cdk/{aws-kinesis-analytics-flink => aws-kinesisanalytics-flink}/NOTICE (100%) rename packages/@aws-cdk/{aws-kinesis-analytics-flink => aws-kinesisanalytics-flink}/README.md (100%) rename packages/@aws-cdk/{aws-kinesis-analytics-flink => aws-kinesisanalytics-flink}/jest.config.js (100%) rename packages/@aws-cdk/{aws-kinesis-analytics-flink => aws-kinesisanalytics-flink}/lib/application-code.ts (100%) rename packages/@aws-cdk/{aws-kinesis-analytics-flink => aws-kinesisanalytics-flink}/lib/application.ts (100%) rename packages/@aws-cdk/{aws-kinesis-analytics-flink => aws-kinesisanalytics-flink}/lib/index.ts (100%) rename packages/@aws-cdk/{aws-kinesis-analytics-flink => aws-kinesisanalytics-flink}/lib/private/environment-properties.ts (100%) rename packages/@aws-cdk/{aws-kinesis-analytics-flink => aws-kinesisanalytics-flink}/lib/private/flink-application-configuration.ts (100%) rename packages/@aws-cdk/{aws-kinesis-analytics-flink => aws-kinesisanalytics-flink}/lib/private/validation.ts (100%) rename packages/@aws-cdk/{aws-kinesis-analytics-flink => aws-kinesisanalytics-flink}/lib/types.ts (100%) rename packages/@aws-cdk/{aws-kinesis-analytics-flink => aws-kinesisanalytics-flink}/package.json (90%) rename packages/@aws-cdk/{aws-kinesis-analytics-flink => aws-kinesisanalytics-flink}/test/application.test.ts (100%) rename packages/@aws-cdk/{aws-kinesis-analytics-flink => aws-kinesisanalytics-flink}/test/code-asset/WordCount.jar (100%) rename packages/@aws-cdk/{aws-kinesis-analytics-flink => aws-kinesisanalytics-flink}/test/integ.application.expected.json (100%) rename packages/@aws-cdk/{aws-kinesis-analytics-flink => aws-kinesisanalytics-flink}/test/integ.application.ts (100%) diff --git a/packages/@aws-cdk/aws-kinesis-analytics-flink/.eslintrc.js b/packages/@aws-cdk/aws-kinesisanalytics-flink/.eslintrc.js similarity index 100% rename from packages/@aws-cdk/aws-kinesis-analytics-flink/.eslintrc.js rename to packages/@aws-cdk/aws-kinesisanalytics-flink/.eslintrc.js diff --git a/packages/@aws-cdk/aws-kinesis-analytics-flink/.gitignore b/packages/@aws-cdk/aws-kinesisanalytics-flink/.gitignore similarity index 100% rename from packages/@aws-cdk/aws-kinesis-analytics-flink/.gitignore rename to packages/@aws-cdk/aws-kinesisanalytics-flink/.gitignore diff --git a/packages/@aws-cdk/aws-kinesis-analytics-flink/.npmignore b/packages/@aws-cdk/aws-kinesisanalytics-flink/.npmignore similarity index 100% rename from packages/@aws-cdk/aws-kinesis-analytics-flink/.npmignore rename to packages/@aws-cdk/aws-kinesisanalytics-flink/.npmignore diff --git a/packages/@aws-cdk/aws-kinesis-analytics-flink/LICENSE b/packages/@aws-cdk/aws-kinesisanalytics-flink/LICENSE similarity index 100% rename from packages/@aws-cdk/aws-kinesis-analytics-flink/LICENSE rename to packages/@aws-cdk/aws-kinesisanalytics-flink/LICENSE diff --git a/packages/@aws-cdk/aws-kinesis-analytics-flink/NOTICE b/packages/@aws-cdk/aws-kinesisanalytics-flink/NOTICE similarity index 100% rename from packages/@aws-cdk/aws-kinesis-analytics-flink/NOTICE rename to packages/@aws-cdk/aws-kinesisanalytics-flink/NOTICE diff --git a/packages/@aws-cdk/aws-kinesis-analytics-flink/README.md b/packages/@aws-cdk/aws-kinesisanalytics-flink/README.md similarity index 100% rename from packages/@aws-cdk/aws-kinesis-analytics-flink/README.md rename to packages/@aws-cdk/aws-kinesisanalytics-flink/README.md diff --git a/packages/@aws-cdk/aws-kinesis-analytics-flink/jest.config.js b/packages/@aws-cdk/aws-kinesisanalytics-flink/jest.config.js similarity index 100% rename from packages/@aws-cdk/aws-kinesis-analytics-flink/jest.config.js rename to packages/@aws-cdk/aws-kinesisanalytics-flink/jest.config.js diff --git a/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/application-code.ts b/packages/@aws-cdk/aws-kinesisanalytics-flink/lib/application-code.ts similarity index 100% rename from packages/@aws-cdk/aws-kinesis-analytics-flink/lib/application-code.ts rename to packages/@aws-cdk/aws-kinesisanalytics-flink/lib/application-code.ts diff --git a/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/application.ts b/packages/@aws-cdk/aws-kinesisanalytics-flink/lib/application.ts similarity index 100% rename from packages/@aws-cdk/aws-kinesis-analytics-flink/lib/application.ts rename to packages/@aws-cdk/aws-kinesisanalytics-flink/lib/application.ts diff --git a/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/index.ts b/packages/@aws-cdk/aws-kinesisanalytics-flink/lib/index.ts similarity index 100% rename from packages/@aws-cdk/aws-kinesis-analytics-flink/lib/index.ts rename to packages/@aws-cdk/aws-kinesisanalytics-flink/lib/index.ts diff --git a/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/private/environment-properties.ts b/packages/@aws-cdk/aws-kinesisanalytics-flink/lib/private/environment-properties.ts similarity index 100% rename from packages/@aws-cdk/aws-kinesis-analytics-flink/lib/private/environment-properties.ts rename to packages/@aws-cdk/aws-kinesisanalytics-flink/lib/private/environment-properties.ts diff --git a/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/private/flink-application-configuration.ts b/packages/@aws-cdk/aws-kinesisanalytics-flink/lib/private/flink-application-configuration.ts similarity index 100% rename from packages/@aws-cdk/aws-kinesis-analytics-flink/lib/private/flink-application-configuration.ts rename to packages/@aws-cdk/aws-kinesisanalytics-flink/lib/private/flink-application-configuration.ts diff --git a/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/private/validation.ts b/packages/@aws-cdk/aws-kinesisanalytics-flink/lib/private/validation.ts similarity index 100% rename from packages/@aws-cdk/aws-kinesis-analytics-flink/lib/private/validation.ts rename to packages/@aws-cdk/aws-kinesisanalytics-flink/lib/private/validation.ts diff --git a/packages/@aws-cdk/aws-kinesis-analytics-flink/lib/types.ts b/packages/@aws-cdk/aws-kinesisanalytics-flink/lib/types.ts similarity index 100% rename from packages/@aws-cdk/aws-kinesis-analytics-flink/lib/types.ts rename to packages/@aws-cdk/aws-kinesisanalytics-flink/lib/types.ts diff --git a/packages/@aws-cdk/aws-kinesis-analytics-flink/package.json b/packages/@aws-cdk/aws-kinesisanalytics-flink/package.json similarity index 90% rename from packages/@aws-cdk/aws-kinesis-analytics-flink/package.json rename to packages/@aws-cdk/aws-kinesisanalytics-flink/package.json index 31eafd9674f14..d7a1c7bd8409d 100644 --- a/packages/@aws-cdk/aws-kinesis-analytics-flink/package.json +++ b/packages/@aws-cdk/aws-kinesisanalytics-flink/package.json @@ -1,5 +1,5 @@ { - "name": "@aws-cdk/aws-kinesis-analytics-flink", + "name": "@aws-cdk/aws-kinesisanalytics-flink", "version": "0.0.0", "description": "A CDK Construct Library for Kinesis Analytics Flink applications", "main": "lib/index.js", @@ -11,7 +11,7 @@ "package": "software.amazon.awscdk.services.kinesis.analytics.flink", "maven": { "groupId": "software.amazon.awscdk", - "artifactId": "kinesis-analytics-flink" + "artifactId": "kinesisanalytics-flink" } }, "dotnet": { @@ -20,7 +20,7 @@ "iconUrl": "https://raw.githubusercontent.com/aws/aws-cdk/master/logo/default-256-dark.png" }, "python": { - "distName": "aws-cdk.aws-kinesis-analytics-flink", + "distName": "aws-cdk.aws-kinesisanalytics-flink", "module": "aws_cdk.aws_kinesis_analytics_flink", "classifiers": [ "Framework :: AWS CDK", @@ -33,7 +33,7 @@ "repository": { "type": "git", "url": "https://github.com/aws/aws-cdk.git", - "directory": "packages/@aws-cdk/aws-kinesis-analytics-flink" + "directory": "packages/@aws-cdk/aws-kinesisanalytics-flink" }, "scripts": { "build": "cdk-build", @@ -100,7 +100,7 @@ }, "awslint": { "exclude": [ - "props-physical-name:@aws-cdk/aws-kinesis-analytics-flink.FlinkApplicationProps" + "props-physical-name:@aws-cdk/aws-kinesisanalytics-flink.FlinkApplicationProps" ] }, "stability": "experimental", diff --git a/packages/@aws-cdk/aws-kinesis-analytics-flink/test/application.test.ts b/packages/@aws-cdk/aws-kinesisanalytics-flink/test/application.test.ts similarity index 100% rename from packages/@aws-cdk/aws-kinesis-analytics-flink/test/application.test.ts rename to packages/@aws-cdk/aws-kinesisanalytics-flink/test/application.test.ts diff --git a/packages/@aws-cdk/aws-kinesis-analytics-flink/test/code-asset/WordCount.jar b/packages/@aws-cdk/aws-kinesisanalytics-flink/test/code-asset/WordCount.jar similarity index 100% rename from packages/@aws-cdk/aws-kinesis-analytics-flink/test/code-asset/WordCount.jar rename to packages/@aws-cdk/aws-kinesisanalytics-flink/test/code-asset/WordCount.jar diff --git a/packages/@aws-cdk/aws-kinesis-analytics-flink/test/integ.application.expected.json b/packages/@aws-cdk/aws-kinesisanalytics-flink/test/integ.application.expected.json similarity index 100% rename from packages/@aws-cdk/aws-kinesis-analytics-flink/test/integ.application.expected.json rename to packages/@aws-cdk/aws-kinesisanalytics-flink/test/integ.application.expected.json diff --git a/packages/@aws-cdk/aws-kinesis-analytics-flink/test/integ.application.ts b/packages/@aws-cdk/aws-kinesisanalytics-flink/test/integ.application.ts similarity index 100% rename from packages/@aws-cdk/aws-kinesis-analytics-flink/test/integ.application.ts rename to packages/@aws-cdk/aws-kinesisanalytics-flink/test/integ.application.ts From 550a7e42f40caef009770e021ad625793ea25c35 Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Thu, 4 Feb 2021 15:43:11 -0800 Subject: [PATCH 33/45] Add fromBucket integration test --- .../aws-kinesisanalytics-flink/README.md | 50 +-- .../lib/application.ts | 2 +- .../aws-kinesisanalytics-flink/package.json | 4 +- ...ication-code-from-bucket.lit.expected.json | 295 ++++++++++++++++++ .../integ.application-code-from-bucket.lit.ts | 22 ++ .../test/integ.application.lit.ts | 15 + .../test/integ.application.ts | 13 - 7 files changed, 351 insertions(+), 50 deletions(-) create mode 100644 packages/@aws-cdk/aws-kinesisanalytics-flink/test/integ.application-code-from-bucket.lit.expected.json create mode 100644 packages/@aws-cdk/aws-kinesisanalytics-flink/test/integ.application-code-from-bucket.lit.ts create mode 100644 packages/@aws-cdk/aws-kinesisanalytics-flink/test/integ.application.lit.ts delete mode 100644 packages/@aws-cdk/aws-kinesisanalytics-flink/test/integ.application.ts diff --git a/packages/@aws-cdk/aws-kinesisanalytics-flink/README.md b/packages/@aws-cdk/aws-kinesisanalytics-flink/README.md index ed972912d3ea9..86e4c1aa4562c 100644 --- a/packages/@aws-cdk/aws-kinesisanalytics-flink/README.md +++ b/packages/@aws-cdk/aws-kinesisanalytics-flink/README.md @@ -32,27 +32,12 @@ Analytics](https://aws.amazon.com/kinesis/data-analytics/). To create a new Flink application, use the FlinkApplication construct. -```ts -import * as ka from '@aws-cdk/aws-kinesisanalyticsv2'; - -const flinkApp = new ka.FlinkApplication(this, 'Application', { - code: ka.ApplicationCode.fromBucket(bucket, 'my-app.jar'), - runtime: ka.FlinkRuntime.FLINK_1_11, -}); -``` +[simple flink application](test/integ.application.lit.ts) -The `code` property can use `fromBucket` as shown above to reference a jar -file in s3 or `fromAsset` to reference a local file. +The `code` property can use `fromAsset` as shown above to reference a local jar +file in s3 or `fromBucket` to reference a file in s3. -```ts -import * as ka from '@aws-cdk/aws-kinesisanalyticsv2'; -import * as path from 'path'; - -const flinkApp = new ka.FlinkApplication(this, 'Application', { - code: ka.ApplicationCode.fromAsset(path.join(__dirname, 'my-app.jar')), - // ... -}); -``` +[flink application using code from bucket](test/integ.application-code-from-bucket.lit.ts) The `propertyGroups` property provides a way of passing arbitrary runtime properties to your Flink application. You can use the @@ -60,16 +45,15 @@ aws-kinesisanalytics-runtime library to [retrieve these properties](https://docs.aws.amazon.com/kinesisanalytics/latest/java/how-properties.html#how-properties-access). ```ts -import * as ka from '@aws-cdk/aws-kinesisanalyticsv2'; -import * as path from 'path'; - -const flinkApp = new ka.FlinkApplication(this, 'Application', { +const flinkApp = new flink.Application(this, 'Application', { // ... propertyGroups: [ - new ka.PropertyGroup('FlinkApplicationProperties', { - inputStreamName: 'my-input-kinesis-stream', - outputStreamName: 'my-output-kinesis-stream', - }), + { + FlinkApplicationProperties: { + inputStreamName: 'my-input-kinesis-stream', + outputStreamName: 'my-output-kinesis-stream', + }, + }, ], }); ``` @@ -79,16 +63,14 @@ when the Flink job starts. These include parameters for checkpointing, snapshotting, monitoring, and parallelism. ```ts -import * as ka from '@aws-cdk/aws-kinesisanalyticsv2'; - -const flinkApp = new ka.FlinkApplication(this, 'Application', { - code: ka.ApplicationCode.fromBucket(bucket, 'my-app.jar'), - runtime: ka.FlinkRuntime.FLINK_1_11, +const flinkApp = new flink.Application(this, 'Application', { + code: flink.ApplicationCode.fromBucket(bucket, 'my-app.jar'), + runtime: file.Runtime.FLINK_1_11, checkpointingEnabled: true, // default is true checkpointInterval: cdk.Duration.seconds(30), // default is 1 minute minPausesBetweenCheckpoints: cdk.Duration.seconds(10), // default is 5 seconds - logLevel: ka.LogLevel.ERROR, // default is INFO - metricsLevel: ka.MetricsLevel.PARALLELISM, // default is APPLICATION + logLevel: flink.LogLevel.ERROR, // default is INFO + metricsLevel: flink.MetricsLevel.PARALLELISM, // default is APPLICATION autoScalingEnabled: false, // default is true parallelism: 32, // default is 1 parallelismPerKpu: 2, // default is 1 diff --git a/packages/@aws-cdk/aws-kinesisanalytics-flink/lib/application.ts b/packages/@aws-cdk/aws-kinesisanalytics-flink/lib/application.ts index 3f22235598724..7a65b7261423d 100644 --- a/packages/@aws-cdk/aws-kinesisanalytics-flink/lib/application.ts +++ b/packages/@aws-cdk/aws-kinesisanalytics-flink/lib/application.ts @@ -25,7 +25,7 @@ export interface IApplication extends core.IResource, iam.IGrantable { /** * The name of the Flink application. * - * @attribute ApplicationV2Name + * @attribute */ readonly applicationName: string; diff --git a/packages/@aws-cdk/aws-kinesisanalytics-flink/package.json b/packages/@aws-cdk/aws-kinesisanalytics-flink/package.json index d7a1c7bd8409d..3d1da8603c649 100644 --- a/packages/@aws-cdk/aws-kinesisanalytics-flink/package.json +++ b/packages/@aws-cdk/aws-kinesisanalytics-flink/package.json @@ -21,7 +21,7 @@ }, "python": { "distName": "aws-cdk.aws-kinesisanalytics-flink", - "module": "aws_cdk.aws_kinesis_analytics_flink", + "module": "aws_cdk.aws_kinesisanalytics_flink", "classifiers": [ "Framework :: AWS CDK", "Framework :: AWS CDK :: 1" @@ -100,7 +100,7 @@ }, "awslint": { "exclude": [ - "props-physical-name:@aws-cdk/aws-kinesisanalytics-flink.FlinkApplicationProps" + "props-physical-name:@aws-cdk/aws-kinesisanalytics-flink.ApplicationProps" ] }, "stability": "experimental", diff --git a/packages/@aws-cdk/aws-kinesisanalytics-flink/test/integ.application-code-from-bucket.lit.expected.json b/packages/@aws-cdk/aws-kinesisanalytics-flink/test/integ.application-code-from-bucket.lit.expected.json new file mode 100644 index 0000000000000..6ab6a5f40bcab --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisanalytics-flink/test/integ.application-code-from-bucket.lit.expected.json @@ -0,0 +1,295 @@ +{ + "Parameters": { + "AssetParameters8be9e0b5f53d41e9a3b1d51c9572c65f24f8170a7188d0ed57fb7d571de4d577S3BucketEBA17A67": { + "Type": "String", + "Description": "S3 bucket for asset \"8be9e0b5f53d41e9a3b1d51c9572c65f24f8170a7188d0ed57fb7d571de4d577\"" + }, + "AssetParameters8be9e0b5f53d41e9a3b1d51c9572c65f24f8170a7188d0ed57fb7d571de4d577S3VersionKey5922697E": { + "Type": "String", + "Description": "S3 key for asset version \"8be9e0b5f53d41e9a3b1d51c9572c65f24f8170a7188d0ed57fb7d571de4d577\"" + }, + "AssetParameters8be9e0b5f53d41e9a3b1d51c9572c65f24f8170a7188d0ed57fb7d571de4d577ArtifactHash211A4F2F": { + "Type": "String", + "Description": "Artifact hash for asset \"8be9e0b5f53d41e9a3b1d51c9572c65f24f8170a7188d0ed57fb7d571de4d577\"" + } + }, + "Resources": { + "AppRole1AF9B530": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "kinesisanalytics.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + } + }, + "AppRoleDefaultPolicy9CADBAA1": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "cloudwatch:PutMetricData", + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Ref": "AssetParameters8be9e0b5f53d41e9a3b1d51c9572c65f24f8170a7188d0ed57fb7d571de4d577S3BucketEBA17A67" + } + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Ref": "AssetParameters8be9e0b5f53d41e9a3b1d51c9572c65f24f8170a7188d0ed57fb7d571de4d577S3BucketEBA17A67" + }, + "/*" + ] + ] + } + ] + }, + { + "Action": "logs:DescribeLogGroups", + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":logs:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":log-group:*" + ] + ] + } + }, + { + "Action": "logs:DescribeLogStreams", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "AppLogGroupC72EEC8C", + "Arn" + ] + } + }, + { + "Action": "logs:PutLogEvents", + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":logs:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":log-group:", + { + "Ref": "AppLogGroupC72EEC8C" + }, + ":log-stream:", + { + "Ref": "AppLogStream3CAF66A7" + } + ] + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "AppRoleDefaultPolicy9CADBAA1", + "Roles": [ + { + "Ref": "AppRole1AF9B530" + } + ] + } + }, + "AppF1B96344": { + "Type": "AWS::KinesisAnalyticsV2::Application", + "Properties": { + "RuntimeEnvironment": "FLINK-1_11", + "ServiceExecutionRole": { + "Fn::GetAtt": [ + "AppRole1AF9B530", + "Arn" + ] + }, + "ApplicationConfiguration": { + "ApplicationCodeConfiguration": { + "CodeContent": { + "S3ContentLocation": { + "BucketARN": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Ref": "AssetParameters8be9e0b5f53d41e9a3b1d51c9572c65f24f8170a7188d0ed57fb7d571de4d577S3BucketEBA17A67" + } + ] + ] + }, + "FileKey": { + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameters8be9e0b5f53d41e9a3b1d51c9572c65f24f8170a7188d0ed57fb7d571de4d577S3VersionKey5922697E" + } + ] + } + ] + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameters8be9e0b5f53d41e9a3b1d51c9572c65f24f8170a7188d0ed57fb7d571de4d577S3VersionKey5922697E" + } + ] + } + ] + } + ] + ] + } + } + }, + "CodeContentType": "ZIPFILE" + }, + "ApplicationSnapshotConfiguration": { + "SnapshotsEnabled": true + } + } + }, + "DependsOn": [ + "AppRoleDefaultPolicy9CADBAA1", + "AppRole1AF9B530" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "AppLogGroupC72EEC8C": { + "Type": "AWS::Logs::LogGroup", + "Properties": { + "RetentionInDays": 731 + }, + "UpdateReplacePolicy": "Retain", + "DeletionPolicy": "Retain" + }, + "AppLogStream3CAF66A7": { + "Type": "AWS::Logs::LogStream", + "Properties": { + "LogGroupName": { + "Ref": "AppLogGroupC72EEC8C" + } + }, + "UpdateReplacePolicy": "Retain", + "DeletionPolicy": "Retain" + }, + "AppLoggingOption75BE995E": { + "Type": "AWS::KinesisAnalyticsV2::ApplicationCloudWatchLoggingOption", + "Properties": { + "ApplicationName": { + "Ref": "AppF1B96344" + }, + "CloudWatchLoggingOption": { + "LogStreamARN": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":logs:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":log-group:", + { + "Ref": "AppLogGroupC72EEC8C" + }, + ":log-stream:", + { + "Ref": "AppLogStream3CAF66A7" + } + ] + ] + } + } + } + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-kinesisanalytics-flink/test/integ.application-code-from-bucket.lit.ts b/packages/@aws-cdk/aws-kinesisanalytics-flink/test/integ.application-code-from-bucket.lit.ts new file mode 100644 index 0000000000000..eb8c5ce3e9d03 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisanalytics-flink/test/integ.application-code-from-bucket.lit.ts @@ -0,0 +1,22 @@ +import * as path from 'path'; +import * as assets from '@aws-cdk/aws-s3-assets'; +import * as core from '@aws-cdk/core'; +import * as flink from '../lib'; + +const app = new core.App(); +const stack = new core.Stack(app, 'FlinkAppCodeFromBucketTest'); + +const asset = new assets.Asset(stack, 'CodeAsset', { + path: path.join(__dirname, 'code-asset'), +}); +const bucket = asset.bucket; +const fileKey = asset.s3ObjectKey; + +///! show +new flink.Application(stack, 'App', { + code: flink.ApplicationCode.fromBucket(bucket, fileKey), + runtime: flink.Runtime.FLINK_1_11, +}); +///! hide + +app.synth(); diff --git a/packages/@aws-cdk/aws-kinesisanalytics-flink/test/integ.application.lit.ts b/packages/@aws-cdk/aws-kinesisanalytics-flink/test/integ.application.lit.ts new file mode 100644 index 0000000000000..82e0682e2adc8 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisanalytics-flink/test/integ.application.lit.ts @@ -0,0 +1,15 @@ +import * as path from 'path'; +import * as core from '@aws-cdk/core'; +import * as flink from '../lib'; + +const app = new core.App(); +const stack = new core.Stack(app, 'FlinkAppTest'); + +///! show +new flink.Application(stack, 'App', { + code: flink.ApplicationCode.fromAsset(path.join(__dirname, 'code-asset')), + runtime: flink.Runtime.FLINK_1_11, +}); +///! hide + +app.synth(); diff --git a/packages/@aws-cdk/aws-kinesisanalytics-flink/test/integ.application.ts b/packages/@aws-cdk/aws-kinesisanalytics-flink/test/integ.application.ts deleted file mode 100644 index 0ac3ec5bea02d..0000000000000 --- a/packages/@aws-cdk/aws-kinesisanalytics-flink/test/integ.application.ts +++ /dev/null @@ -1,13 +0,0 @@ -import * as core from '@aws-cdk/core'; -import * as path from 'path'; -import * as ka from '../lib'; - -const app = new core.App(); -const stack = new core.Stack(app, 'FlinkAppTest'); - -new ka.Application(stack, 'App', { - code: ka.ApplicationCode.fromAsset(path.join(__dirname, 'code-asset')), - runtime: ka.Runtime.FLINK_1_11, -}); - -app.synth(); From 671847c7243a9aab0a8cb1394f04dd00a5400f86 Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Thu, 4 Feb 2021 16:02:30 -0800 Subject: [PATCH 34/45] Fix reference to FlinkRuntime in comment --- packages/@aws-cdk/aws-kinesisanalytics-flink/lib/types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-kinesisanalytics-flink/lib/types.ts b/packages/@aws-cdk/aws-kinesisanalytics-flink/lib/types.ts index 38347fc500963..3149cdfb17996 100644 --- a/packages/@aws-cdk/aws-kinesisanalytics-flink/lib/types.ts +++ b/packages/@aws-cdk/aws-kinesisanalytics-flink/lib/types.ts @@ -45,7 +45,7 @@ export interface PropertyGroups { * * @example * // Creating a new runtime that isn't in CDK yet. - * const runtime = new FlinkRuntime(FLINK-9_99); + * const runtime = new Runtime(FLINK-9_99); */ export class Runtime { /** Flink Version 1.6 */ From b12d0a537c1bdab8907a475f0951c5d4fe7a713e Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Thu, 4 Feb 2021 16:17:38 -0800 Subject: [PATCH 35/45] Fix markdown lint error --- packages/@aws-cdk/aws-kinesisanalytics/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/@aws-cdk/aws-kinesisanalytics/README.md b/packages/@aws-cdk/aws-kinesisanalytics/README.md index 9426a7cfd3db3..33dfe482719b0 100644 --- a/packages/@aws-cdk/aws-kinesisanalytics/README.md +++ b/packages/@aws-cdk/aws-kinesisanalytics/README.md @@ -18,5 +18,6 @@ This module is part of the [AWS Cloud Development Kit](https://github.com/aws/aw ## Kinesis Analytics Flink The `aws-kinesisanalytics-flink` package provides constructs for building Flink applications. + * [Github](https://github.com/aws/aws-cdk/tree/master/packages/%40aws-cdk/aws-kinesisanalytics-flink) * [CDK Docs](https://docs.aws.amazon.com/cdk/api/latest/docs/aws-kinesisanalytics-flink.html) From 004337f8323c5d78d6c7ce189708251083851d86 Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Thu, 4 Feb 2021 18:22:03 -0800 Subject: [PATCH 36/45] Update integ snapshots --- ...lication.expected.json => integ.application.lit.expected.json} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/@aws-cdk/aws-kinesisanalytics-flink/test/{integ.application.expected.json => integ.application.lit.expected.json} (100%) diff --git a/packages/@aws-cdk/aws-kinesisanalytics-flink/test/integ.application.expected.json b/packages/@aws-cdk/aws-kinesisanalytics-flink/test/integ.application.lit.expected.json similarity index 100% rename from packages/@aws-cdk/aws-kinesisanalytics-flink/test/integ.application.expected.json rename to packages/@aws-cdk/aws-kinesisanalytics-flink/test/integ.application.lit.expected.json From 9148324c3b6b60cf9b56d48277207febed51d8dc Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Thu, 4 Feb 2021 19:11:25 -0800 Subject: [PATCH 37/45] Add to cloudformation-include --- packages/@aws-cdk/cloudformation-include/package.json | 2 ++ packages/aws-cdk-lib/package.json | 1 + packages/decdk/package.json | 1 + packages/monocdk/package.json | 1 + 4 files changed, 5 insertions(+) diff --git a/packages/@aws-cdk/cloudformation-include/package.json b/packages/@aws-cdk/cloudformation-include/package.json index 6a3dd4c17346d..c5d95f3000ecc 100644 --- a/packages/@aws-cdk/cloudformation-include/package.json +++ b/packages/@aws-cdk/cloudformation-include/package.json @@ -153,6 +153,7 @@ "@aws-cdk/aws-kendra": "0.0.0", "@aws-cdk/aws-kinesis": "0.0.0", "@aws-cdk/aws-kinesisanalytics": "0.0.0", + "@aws-cdk/aws-kinesisanalytics-flink": "0.0.0", "@aws-cdk/aws-kinesisfirehose": "0.0.0", "@aws-cdk/aws-kms": "0.0.0", "@aws-cdk/aws-lakeformation": "0.0.0", @@ -300,6 +301,7 @@ "@aws-cdk/aws-kendra": "0.0.0", "@aws-cdk/aws-kinesis": "0.0.0", "@aws-cdk/aws-kinesisanalytics": "0.0.0", + "@aws-cdk/aws-kinesisanalytics-flink": "0.0.0", "@aws-cdk/aws-kinesisfirehose": "0.0.0", "@aws-cdk/aws-kms": "0.0.0", "@aws-cdk/aws-lakeformation": "0.0.0", diff --git a/packages/aws-cdk-lib/package.json b/packages/aws-cdk-lib/package.json index 7c938b86bd7e0..163ad88412c82 100644 --- a/packages/aws-cdk-lib/package.json +++ b/packages/aws-cdk-lib/package.json @@ -202,6 +202,7 @@ "@aws-cdk/aws-kendra": "0.0.0", "@aws-cdk/aws-kinesis": "0.0.0", "@aws-cdk/aws-kinesisanalytics": "0.0.0", + "@aws-cdk/aws-kinesisanalytics-flink": "0.0.0", "@aws-cdk/aws-kinesisfirehose": "0.0.0", "@aws-cdk/aws-kms": "0.0.0", "@aws-cdk/aws-lakeformation": "0.0.0", diff --git a/packages/decdk/package.json b/packages/decdk/package.json index 7f8ace298e383..9b6e67228a98a 100644 --- a/packages/decdk/package.json +++ b/packages/decdk/package.json @@ -128,6 +128,7 @@ "@aws-cdk/aws-kendra": "0.0.0", "@aws-cdk/aws-kinesis": "0.0.0", "@aws-cdk/aws-kinesisanalytics": "0.0.0", + "@aws-cdk/aws-kinesisanalytics-flink": "0.0.0", "@aws-cdk/aws-kinesisfirehose": "0.0.0", "@aws-cdk/aws-kms": "0.0.0", "@aws-cdk/aws-lakeformation": "0.0.0", diff --git a/packages/monocdk/package.json b/packages/monocdk/package.json index dddf5eb0b77fd..0f6ef93d562f2 100644 --- a/packages/monocdk/package.json +++ b/packages/monocdk/package.json @@ -207,6 +207,7 @@ "@aws-cdk/aws-kendra": "0.0.0", "@aws-cdk/aws-kinesis": "0.0.0", "@aws-cdk/aws-kinesisanalytics": "0.0.0", + "@aws-cdk/aws-kinesisanalytics-flink": "0.0.0", "@aws-cdk/aws-kinesisfirehose": "0.0.0", "@aws-cdk/aws-kms": "0.0.0", "@aws-cdk/aws-lakeformation": "0.0.0", From 88c24ffc3458faf9a4efd4b8fcf8b1c61fb914ec Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Sun, 7 Feb 2021 12:38:19 -0800 Subject: [PATCH 38/45] Update package.json to indicate no L1 constructs --- packages/@aws-cdk/aws-kinesisanalytics-flink/README.md | 7 ------- packages/@aws-cdk/aws-kinesisanalytics-flink/package.json | 5 +---- packages/@aws-cdk/cloudformation-include/package.json | 1 - 3 files changed, 1 insertion(+), 12 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisanalytics-flink/README.md b/packages/@aws-cdk/aws-kinesisanalytics-flink/README.md index 86e4c1aa4562c..0dfd9197a457f 100644 --- a/packages/@aws-cdk/aws-kinesisanalytics-flink/README.md +++ b/packages/@aws-cdk/aws-kinesisanalytics-flink/README.md @@ -3,12 +3,6 @@ --- -![cfn-resources: Stable](https://img.shields.io/badge/cfn--resources-stable-success.svg?style=for-the-badge) - -> All classes with the `Cfn` prefix in this module ([CFN Resources]) are always stable and safe to use. -> -> [CFN Resources]: https://docs.aws.amazon.com/cdk/latest/guide/constructs.html#constructs_lib - ![cdk-constructs: Experimental](https://img.shields.io/badge/cdk--constructs-experimental-important.svg?style=for-the-badge) > The APIs of higher level constructs in this module are experimental and under active development. @@ -21,7 +15,6 @@ - The `@aws-cdk/aws-kinesisanalyticsv2` package provides constructs for creating Kinesis Data Analytics applications. diff --git a/packages/@aws-cdk/aws-kinesisanalytics-flink/package.json b/packages/@aws-cdk/aws-kinesisanalytics-flink/package.json index 3d1da8603c649..80dbdfd4aeb76 100644 --- a/packages/@aws-cdk/aws-kinesisanalytics-flink/package.json +++ b/packages/@aws-cdk/aws-kinesisanalytics-flink/package.json @@ -47,9 +47,7 @@ "build+test": "yarn build && yarn test", "build+test+package": "yarn build+test && yarn package", "compat": "cdk-compat", - "rosetta:extract": "yarn --silent jsii-rosetta extract", - "cfn2ts": "cfn2ts", - "gen": "cfn2ts" + "rosetta:extract": "yarn --silent jsii-rosetta extract" }, "keywords": [ "aws", @@ -110,7 +108,6 @@ }, "cdk-build": { "jest": true, - "cloudformation": [], "env": { "AWSLINT_BASE_CONSTRUCT": true } diff --git a/packages/@aws-cdk/cloudformation-include/package.json b/packages/@aws-cdk/cloudformation-include/package.json index c5d95f3000ecc..8ee3ae0b1aa19 100644 --- a/packages/@aws-cdk/cloudformation-include/package.json +++ b/packages/@aws-cdk/cloudformation-include/package.json @@ -153,7 +153,6 @@ "@aws-cdk/aws-kendra": "0.0.0", "@aws-cdk/aws-kinesis": "0.0.0", "@aws-cdk/aws-kinesisanalytics": "0.0.0", - "@aws-cdk/aws-kinesisanalytics-flink": "0.0.0", "@aws-cdk/aws-kinesisfirehose": "0.0.0", "@aws-cdk/aws-kms": "0.0.0", "@aws-cdk/aws-lakeformation": "0.0.0", From 2fd57730769575dc60f3283c5166f2c2f662520d Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Sun, 7 Feb 2021 12:48:55 -0800 Subject: [PATCH 39/45] Update README for package focus change --- .../aws-kinesisanalytics-flink/README.md | 29 +++++++------------ .../test/integ.application.lit.ts | 2 +- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisanalytics-flink/README.md b/packages/@aws-cdk/aws-kinesisanalytics-flink/README.md index 0dfd9197a457f..057b4e0dd85dd 100644 --- a/packages/@aws-cdk/aws-kinesisanalytics-flink/README.md +++ b/packages/@aws-cdk/aws-kinesisanalytics-flink/README.md @@ -15,15 +15,14 @@ -The `@aws-cdk/aws-kinesisanalyticsv2` package provides constructs for -creating Kinesis Data Analytics applications. +This package provides constructs for creating Kinesis Analytics Flink +applications. To learn more about using using managed Flink applications, see +the [AWS developer +guide](https://docs.aws.amazon.com/kinesisanalytics/latest/java/what-is.html). -For more information, see the AWS documentation for [Kinesis Data -Analytics](https://aws.amazon.com/kinesis/data-analytics/). +## Creating Flink Applications -## Creating a Flink Kinesis Analytics Application - -To create a new Flink application, use the FlinkApplication construct. +To create a new Flink application, use the `Application` construct: [simple flink application](test/integ.application.lit.ts) @@ -40,14 +39,12 @@ properties](https://docs.aws.amazon.com/kinesisanalytics/latest/java/how-propert ```ts const flinkApp = new flink.Application(this, 'Application', { // ... - propertyGroups: [ - { - FlinkApplicationProperties: { - inputStreamName: 'my-input-kinesis-stream', - outputStreamName: 'my-output-kinesis-stream', - }, + propertyGroups: { + FlinkApplicationProperties: { + inputStreamName: 'my-input-kinesis-stream', + outputStreamName: 'my-output-kinesis-stream', }, - ], + }, }); ``` @@ -75,7 +72,3 @@ const flinkApp = new flink.Application(this, 'Application', { logEncryptionKey: someKmsKey, // default is no encryption }); ``` - -## Creating an SQL Kinesis Analytics Application - -Constructs for SQL applications are not implemented yet. diff --git a/packages/@aws-cdk/aws-kinesisanalytics-flink/test/integ.application.lit.ts b/packages/@aws-cdk/aws-kinesisanalytics-flink/test/integ.application.lit.ts index 82e0682e2adc8..02a6a9949dcfa 100644 --- a/packages/@aws-cdk/aws-kinesisanalytics-flink/test/integ.application.lit.ts +++ b/packages/@aws-cdk/aws-kinesisanalytics-flink/test/integ.application.lit.ts @@ -1,3 +1,4 @@ +///! show import * as path from 'path'; import * as core from '@aws-cdk/core'; import * as flink from '../lib'; @@ -5,7 +6,6 @@ import * as flink from '../lib'; const app = new core.App(); const stack = new core.Stack(app, 'FlinkAppTest'); -///! show new flink.Application(stack, 'App', { code: flink.ApplicationCode.fromAsset(path.join(__dirname, 'code-asset')), runtime: flink.Runtime.FLINK_1_11, From 49dbe4ed164668091ce2a8110ef283a15e28e7f1 Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Sun, 7 Feb 2021 14:00:36 -0800 Subject: [PATCH 40/45] Declare interface before use --- .../lib/application-code.ts | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisanalytics-flink/lib/application-code.ts b/packages/@aws-cdk/aws-kinesisanalytics-flink/lib/application-code.ts index 36280678ef553..c7d7b49180086 100644 --- a/packages/@aws-cdk/aws-kinesisanalytics-flink/lib/application-code.ts +++ b/packages/@aws-cdk/aws-kinesisanalytics-flink/lib/application-code.ts @@ -3,6 +3,23 @@ import * as s3 from '@aws-cdk/aws-s3'; import * as s3_assets from '@aws-cdk/aws-s3-assets'; import { Construct } from '@aws-cdk/core'; +/** + * The return type of {@link ApplicationCode.bind}. This represents + * CloudFormation configuration and an s3 bucket holding the Flink application + * JAR file. + */ +export interface ApplicationCodeConfig { + /** + * Low-level Cloudformation ApplicationConfigurationProperty + */ + readonly applicationCodeConfigurationProperty: ka.CfnApplicationV2.ApplicationConfigurationProperty; + + /** + * S3 Bucket that stores the Flink application code + */ + readonly bucket: s3.IBucket; +} + /** * Code configuration providing the location to a Flink application JAR file. */ @@ -120,20 +137,3 @@ class AssetApplicationCode extends ApplicationCode { return this._asset?.bucket; } } - -/** - * The return type of {@link ApplicationCode.bind}. This represents - * CloudFormation configuration and an s3 bucket holding the Flink application - * JAR file. - */ -export interface ApplicationCodeConfig { - /** - * Low-level Cloudformation ApplicationConfigurationProperty - */ - readonly applicationCodeConfigurationProperty: ka.CfnApplicationV2.ApplicationConfigurationProperty; - - /** - * S3 Bucket that stores the Flink application code - */ - readonly bucket: s3.IBucket; -} From 0b828fa21c5800d40cb5417453a03926199fa479 Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Sun, 7 Feb 2021 14:01:04 -0800 Subject: [PATCH 41/45] Use factory over constructor for FlinkRuntime --- .../aws-kinesisanalytics-flink/lib/types.ts | 17 +++++++++-------- .../test/application.test.ts | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisanalytics-flink/lib/types.ts b/packages/@aws-cdk/aws-kinesisanalytics-flink/lib/types.ts index 3149cdfb17996..b92d55cb48518 100644 --- a/packages/@aws-cdk/aws-kinesisanalytics-flink/lib/types.ts +++ b/packages/@aws-cdk/aws-kinesisanalytics-flink/lib/types.ts @@ -42,25 +42,26 @@ export interface PropertyGroups { /** * Available Flink runtimes for Kinesis Analytics. - * - * @example - * // Creating a new runtime that isn't in CDK yet. - * const runtime = new Runtime(FLINK-9_99); */ export class Runtime { /** Flink Version 1.6 */ - public static readonly FLINK_1_6 = new Runtime('FLINK-1_6'); + public static readonly FLINK_1_6 = Runtime.of('FLINK-1_6'); /** Flink Version 1.8 */ - public static readonly FLINK_1_8 = new Runtime('FLINK-1_8'); + public static readonly FLINK_1_8 = Runtime.of('FLINK-1_8'); /** Flink Version 1.11 */ - public static readonly FLINK_1_11 = new Runtime('FLINK-1_11'); + public static readonly FLINK_1_11 = Runtime.of('FLINK-1_11'); + + /** Create a new Runtime with with an arbitrary Flink version string */ + public static of(value: string) { + return new Runtime(value); + } /** The Cfn string that represents a version of Flink */ public readonly value: string; - public constructor(value: string) { + private constructor(value: string) { this.value = value; } } diff --git a/packages/@aws-cdk/aws-kinesisanalytics-flink/test/application.test.ts b/packages/@aws-cdk/aws-kinesisanalytics-flink/test/application.test.ts index ae74488863868..83e3643a5e576 100644 --- a/packages/@aws-cdk/aws-kinesisanalytics-flink/test/application.test.ts +++ b/packages/@aws-cdk/aws-kinesisanalytics-flink/test/application.test.ts @@ -168,7 +168,7 @@ describe('Application', () => { test('providing a custom runtime', () => { new flink.Application(stack, 'FlinkApplication', { ...requiredProps, - runtime: new flink.Runtime('custom'), + runtime: flink.Runtime.of('custom'), }); expect(stack).toHaveResourceLike('AWS::KinesisAnalyticsV2::Application', { From 36f9eb9cc35724200f6e856e20064cb0d6b0dc9d Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Sun, 7 Feb 2021 14:01:44 -0800 Subject: [PATCH 42/45] Declare one Import class in module scope --- .../lib/application.ts | 44 ++++++++++++------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisanalytics-flink/lib/application.ts b/packages/@aws-cdk/aws-kinesisanalytics-flink/lib/application.ts index 7a65b7261423d..7bd534f866ea0 100644 --- a/packages/@aws-cdk/aws-kinesisanalytics-flink/lib/application.ts +++ b/packages/@aws-cdk/aws-kinesisanalytics-flink/lib/application.ts @@ -213,6 +213,27 @@ export interface ApplicationProps { readonly logEncryptionKey?: kms.IKey; } +/** + * An imported Flink application. + */ +class Import extends ApplicationBase { + public readonly grantPrincipal: iam.IPrincipal; + public readonly role?: iam.IRole; + public readonly applicationName: string; + public readonly applicationArn: string; + + constructor(scope: Construct, id: string, attrs: { applicationArn: string, applicationName: string }) { + super(scope, id); + + // Imported applications have no associated role or grantPrincipal + this.grantPrincipal = new iam.UnknownPrincipal({ resource: this }); + this.role = undefined; + + this.applicationArn = attrs.applicationArn; + this.applicationName = attrs.applicationName; + } +} + /** * The L2 construct for Flink Kinesis Data Applications. * @@ -226,16 +247,9 @@ export class Application extends ApplicationBase { * applicationName. */ public static fromApplicationName(scope: Construct, id: string, applicationName: string): IApplication { - class Import extends ApplicationBase { - // Imported applications have no associated role or grantPrincipal - public readonly role = undefined; - public readonly grantPrincipal = new iam.UnknownPrincipal({ resource: this }); + const applicationArn = core.Stack.of(scope).formatArn(applicationArnComponents(applicationName)); - public readonly applicationName = applicationName; - public readonly applicationArn = core.Stack.of(this).formatArn(applicationArnComponents(applicationName)); - } - - return new Import(scope, id); + return new Import(scope, id, { applicationArn, applicationName }); } /** @@ -243,16 +257,12 @@ export class Application extends ApplicationBase { * applicationArn. */ public static fromApplicationArn(scope: Construct, id: string, applicationArn: string): IApplication { - class Import extends ApplicationBase { - // Imported applications have no associated role or grantPrincipal - public readonly role = undefined; - public readonly grantPrincipal = new iam.UnknownPrincipal({ resource: this }); - - public readonly applicationName = applicationArn.split('/')[1]; - public readonly applicationArn = applicationArn; + const applicationName = core.Stack.of(scope).parseArn(applicationArn).resourceName; + if (!applicationName) { + throw new Error(`applicationArn for fromApplicationArn (${applicationArn}) must include resource name`); } - return new Import(scope, id); + return new Import(scope, id, { applicationArn, applicationName }); } public readonly applicationArn: string; From b951b975eb1212c453723eaa5945dc07372a8ec3 Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Sun, 7 Feb 2021 14:36:29 -0800 Subject: [PATCH 43/45] Pass logGroup rather than all log settings The trade off here is that we won't allow users to control the stream properties. --- .../lib/application.ts | 51 ++--------------- .../test/application.test.ts | 56 ++----------------- 2 files changed, 12 insertions(+), 95 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisanalytics-flink/lib/application.ts b/packages/@aws-cdk/aws-kinesisanalytics-flink/lib/application.ts index 7bd534f866ea0..1bcfa0fc4cbbf 100644 --- a/packages/@aws-cdk/aws-kinesisanalytics-flink/lib/application.ts +++ b/packages/@aws-cdk/aws-kinesisanalytics-flink/lib/application.ts @@ -1,6 +1,5 @@ import * as iam from '@aws-cdk/aws-iam'; import * as ka from '@aws-cdk/aws-kinesisanalytics'; -import * as kms from '@aws-cdk/aws-kms'; import * as logs from '@aws-cdk/aws-logs'; import * as core from '@aws-cdk/core'; import { Construct } from 'constructs'; @@ -8,7 +7,7 @@ import { ApplicationCode } from './application-code'; import { environmentProperties } from './private/environment-properties'; import { flinkApplicationConfiguration } from './private/flink-application-configuration'; import { validateFlinkApplicationProps as validateApplicationProps } from './private/validation'; -import { LogLevel as LogLevel, MetricsLevel as MetricsLevel, PropertyGroups, Runtime as Runtime } from './types'; +import { LogLevel, MetricsLevel, PropertyGroups, Runtime } from './types'; /** * An interface expressing the public properties on both an imported and @@ -178,39 +177,11 @@ export interface ApplicationProps { readonly removalPolicy?: core.RemovalPolicy; /** - * How long to retain logs. + * The log group to send log entries to. * - * @default two years + * @default CDK's default LogGroup */ - readonly logRetention?: core.Duration; - - /** - * Whether to keep or delete logs when removing a FlinkApplication. - * - * @default RETAIN - */ - readonly logRemovalPolicy?: core.RemovalPolicy; - - /** - * The name of the log group for CloudWatch logs. - * - * @default Cloudformation generated - */ - readonly logGroupName?: string; - - /** - * The name of the log stream for CloudWatch logs. - * - * @default Cloudformation generated - */ - readonly logStreamName?: string; - - /** - * The KMS encryption key to use for CloudWatch logs. - * - * @default No encryption used - */ - readonly logEncryptionKey?: kms.IKey; + readonly logGroup?: logs.ILogGroup; } /** @@ -314,18 +285,8 @@ export class Application extends ApplicationBase { }); resource.node.addDependency(this.role); - const logGroup = new logs.LogGroup(this, 'LogGroup', { - logGroupName: props.logGroupName, - retention: props.logRetention?.toDays(), - removalPolicy: props.logRemovalPolicy, - encryptionKey: props.logEncryptionKey, - }); - - const logStream = new logs.LogStream(this, 'LogStream', { - logGroup, - logStreamName: props.logStreamName, - removalPolicy: props.logRemovalPolicy, - }); + const logGroup = props.logGroup ?? new logs.LogGroup(this, 'LogGroup'); + const logStream = new logs.LogStream(this, 'LogStream', { logGroup }); /* Permit logging */ diff --git a/packages/@aws-cdk/aws-kinesisanalytics-flink/test/application.test.ts b/packages/@aws-cdk/aws-kinesisanalytics-flink/test/application.test.ts index 83e3643a5e576..6dc35fadd44b6 100644 --- a/packages/@aws-cdk/aws-kinesisanalytics-flink/test/application.test.ts +++ b/packages/@aws-cdk/aws-kinesisanalytics-flink/test/application.test.ts @@ -1,7 +1,7 @@ import { arrayWith, objectLike, ResourcePart } from '@aws-cdk/assert'; import '@aws-cdk/assert/jest'; import * as iam from '@aws-cdk/aws-iam'; -import * as kms from '@aws-cdk/aws-kms'; +import * as logs from '@aws-cdk/aws-logs'; import * as s3 from '@aws-cdk/aws-s3'; import * as core from '@aws-cdk/core'; import * as path from 'path'; @@ -481,60 +481,16 @@ describe('Application', () => { }, ResourcePart.CompleteDefinition); }); - test('logRetentionDays setting', () => { + test('logGroup setting', () => { new flink.Application(stack, 'FlinkApplication', { ...requiredProps, - logRetention: core.Duration.days(5), + logGroup: new logs.LogGroup(stack, 'LogGroup', { + logGroupName: 'custom', + }), }); expect(stack).toHaveResource('AWS::Logs::LogGroup', { - RetentionInDays: 5, - }); - }); - - test('logRemovalPolicy setting', () => { - new flink.Application(stack, 'FlinkApplication', { - ...requiredProps, - logRemovalPolicy: core.RemovalPolicy.DESTROY, - }); - - expect(stack).toHaveResourceLike('AWS::Logs::LogGroup', { - UpdateReplacePolicy: 'Delete', - DeletionPolicy: 'Delete', - }, ResourcePart.CompleteDefinition); - }); - - test('logGroupName setting', () => { - new flink.Application(stack, 'FlinkApplication', { - ...requiredProps, - logGroupName: 'my-group', - }); - - expect(stack).toHaveResourceLike('AWS::Logs::LogGroup', { - LogGroupName: 'my-group', - }); - }); - - test('logStreamName setting', () => { - new flink.Application(stack, 'FlinkApplication', { - ...requiredProps, - logStreamName: 'my-stream', - }); - - expect(stack).toHaveResourceLike('AWS::Logs::LogStream', { - LogStreamName: 'my-stream', - }); - }); - - test('logEncryptionKey setting', () => { - const key = new kms.Key(stack, 'Key'); - new flink.Application(stack, 'FlinkApplication', { - ...requiredProps, - logEncryptionKey: key, - }); - - expect(stack).toHaveResourceLike('AWS::Logs::LogGroup', { - KmsKeyId: stack.resolve(key.keyArn), + LogGroupName: 'custom', }); }); From 4fc6d79524225b839685ff19e37205fb8f1f6f1f Mon Sep 17 00:00:00 2001 From: Adam Ruka Date: Mon, 8 Feb 2021 12:30:37 -0800 Subject: [PATCH 44/45] Remove dependency from `cloudformation-include`. --- packages/@aws-cdk/cloudformation-include/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/@aws-cdk/cloudformation-include/package.json b/packages/@aws-cdk/cloudformation-include/package.json index 14cd41d7f3eb2..1732065bd3a96 100644 --- a/packages/@aws-cdk/cloudformation-include/package.json +++ b/packages/@aws-cdk/cloudformation-include/package.json @@ -301,7 +301,6 @@ "@aws-cdk/aws-kendra": "0.0.0", "@aws-cdk/aws-kinesis": "0.0.0", "@aws-cdk/aws-kinesisanalytics": "0.0.0", - "@aws-cdk/aws-kinesisanalytics-flink": "0.0.0", "@aws-cdk/aws-kinesisfirehose": "0.0.0", "@aws-cdk/aws-kms": "0.0.0", "@aws-cdk/aws-lakeformation": "0.0.0", From 2571ab276d41fad920f0b5f5e1112dc9f3d42046 Mon Sep 17 00:00:00 2001 From: Adam Ruka Date: Mon, 8 Feb 2021 12:37:57 -0800 Subject: [PATCH 45/45] Update ReadMe.md with the latest log properties --- packages/@aws-cdk/aws-kinesisanalytics-flink/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisanalytics-flink/README.md b/packages/@aws-cdk/aws-kinesisanalytics-flink/README.md index 057b4e0dd85dd..7dc9b1a088b16 100644 --- a/packages/@aws-cdk/aws-kinesisanalytics-flink/README.md +++ b/packages/@aws-cdk/aws-kinesisanalytics-flink/README.md @@ -37,6 +37,8 @@ aws-kinesisanalytics-runtime library to [retrieve these properties](https://docs.aws.amazon.com/kinesisanalytics/latest/java/how-properties.html#how-properties-access). ```ts +import * as flink from '@aws-cdk/aws-kinesisanalytics-flink'; + const flinkApp = new flink.Application(this, 'Application', { // ... propertyGroups: { @@ -53,6 +55,8 @@ when the Flink job starts. These include parameters for checkpointing, snapshotting, monitoring, and parallelism. ```ts +import * as logs from '@aws-cdk/aws-logs'; + const flinkApp = new flink.Application(this, 'Application', { code: flink.ApplicationCode.fromBucket(bucket, 'my-app.jar'), runtime: file.Runtime.FLINK_1_11, @@ -65,10 +69,6 @@ const flinkApp = new flink.Application(this, 'Application', { parallelism: 32, // default is 1 parallelismPerKpu: 2, // default is 1 snapshotsEnabled: false, // default is true - logRetention: logs.RetentionDays.ONE_WEEK, // default is TWO_YEARS - logRemovalPolicy: core.RemovalPolicy.DESTROY, // default is RETAIN - logGroupName: 'my-group', // default is automatically generated - logStreamName: 'my-stream', // default is automatically generated - logEncryptionKey: someKmsKey, // default is no encryption + logGroup: new logs.LogGroup(this, 'LogGroup'), // by default, a new LogGroup will be created }); ```