From 5b2de2b2642f4c03b455fbb7aa5c2b2009840735 Mon Sep 17 00:00:00 2001 From: Elad Ben-Israel Date: Tue, 29 Jan 2019 17:23:56 +0200 Subject: [PATCH] chore(toolkit): integ tests moved to release pipeline (#1632) Integration tests are now executed automatically as part of our release pipeline. Fixes #294 --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- packages/aws-cdk/integ-tests/README.md | 4 - packages/aws-cdk/integ-tests/app/.gitignore | 1 - packages/aws-cdk/integ-tests/app/app.js | 61 ----------- packages/aws-cdk/integ-tests/app/cdk.json | 4 - packages/aws-cdk/integ-tests/common.bash | 103 ------------------ .../aws-cdk/integ-tests/test-cdk-context.sh | 30 ----- .../integ-tests/test-cdk-deploy-all.sh | 23 ---- .../integ-tests/test-cdk-deploy-no-tty.sh | 16 --- .../integ-tests/test-cdk-deploy-with-role.sh | 63 ----------- .../aws-cdk/integ-tests/test-cdk-deploy.sh | 26 ----- packages/aws-cdk/integ-tests/test-cdk-diff.sh | 16 --- .../aws-cdk/integ-tests/test-cdk-iam-diff.sh | 27 ----- packages/aws-cdk/integ-tests/test-cdk-ls.sh | 17 --- .../test-cdk-multiple-toolkit-stacks.sh | 24 ---- .../aws-cdk/integ-tests/test-cdk-order.sh | 15 --- .../aws-cdk/integ-tests/test-cdk-synth.sh | 31 ------ packages/aws-cdk/integ-tests/test.sh | 11 -- 18 files changed, 1 insertion(+), 473 deletions(-) delete mode 100644 packages/aws-cdk/integ-tests/README.md delete mode 100644 packages/aws-cdk/integ-tests/app/.gitignore delete mode 100644 packages/aws-cdk/integ-tests/app/app.js delete mode 100644 packages/aws-cdk/integ-tests/app/cdk.json delete mode 100644 packages/aws-cdk/integ-tests/common.bash delete mode 100755 packages/aws-cdk/integ-tests/test-cdk-context.sh delete mode 100755 packages/aws-cdk/integ-tests/test-cdk-deploy-all.sh delete mode 100755 packages/aws-cdk/integ-tests/test-cdk-deploy-no-tty.sh delete mode 100755 packages/aws-cdk/integ-tests/test-cdk-deploy-with-role.sh delete mode 100755 packages/aws-cdk/integ-tests/test-cdk-deploy.sh delete mode 100755 packages/aws-cdk/integ-tests/test-cdk-diff.sh delete mode 100755 packages/aws-cdk/integ-tests/test-cdk-iam-diff.sh delete mode 100755 packages/aws-cdk/integ-tests/test-cdk-ls.sh delete mode 100755 packages/aws-cdk/integ-tests/test-cdk-multiple-toolkit-stacks.sh delete mode 100755 packages/aws-cdk/integ-tests/test-cdk-order.sh delete mode 100755 packages/aws-cdk/integ-tests/test-cdk-synth.sh delete mode 100755 packages/aws-cdk/integ-tests/test.sh diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index c95565be055d2..540040d4acf54 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -5,7 +5,7 @@ * [ ] Testing - Unit test added - - __CLI change?:__ manually run [integration tests](https://github.com/awslabs/aws-cdk/blob/master/packages/aws-cdk/integ-tests/test.sh) and paste output as a PR comment + - __CLI change?:__ coordinate update of integration tests with team - __cdk-init template change?:__ coordinated update of integration tests with team * [ ] Docs - __jsdocs__: All public APIs documented diff --git a/packages/aws-cdk/integ-tests/README.md b/packages/aws-cdk/integ-tests/README.md deleted file mode 100644 index 425d53b1bfe84..0000000000000 --- a/packages/aws-cdk/integ-tests/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# CDK toolkit integreation tests - -To run, just execute `./test.sh`. The test uses the default AWS credentials. - diff --git a/packages/aws-cdk/integ-tests/app/.gitignore b/packages/aws-cdk/integ-tests/app/.gitignore deleted file mode 100644 index d4aa116a26c73..0000000000000 --- a/packages/aws-cdk/integ-tests/app/.gitignore +++ /dev/null @@ -1 +0,0 @@ -!*.js diff --git a/packages/aws-cdk/integ-tests/app/app.js b/packages/aws-cdk/integ-tests/app/app.js deleted file mode 100644 index 28c231e442326..0000000000000 --- a/packages/aws-cdk/integ-tests/app/app.js +++ /dev/null @@ -1,61 +0,0 @@ -const cdk = require('@aws-cdk/cdk'); -const iam = require('@aws-cdk/aws-iam'); -const sns = require('@aws-cdk/aws-sns'); - -class MyStack extends cdk.Stack { - constructor(parent, id) { - super(parent, id); - new sns.Topic(this, 'topic'); - - new cdk.AvailabilityZoneProvider(this).availabilityZones; - new cdk.SSMParameterProvider(this, { parameterName: '/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2' }).parameterValue(''); - } -} - -class YourStack extends cdk.Stack { - constructor(parent, id) { - super(parent, id); - new sns.Topic(this, 'topic1'); - new sns.Topic(this, 'topic2'); - } -} - -class IamStack extends cdk.Stack { - constructor(parent, id) { - super(parent, id); - - new iam.Role(this, 'SomeRole', { - assumedBy: new iam.ServicePrincipal('ec2.amazon.aws.com') - }); - } -} - -class ProvidingStack extends cdk.Stack { - constructor(parent, id) { - super(parent, id); - - new sns.Topic(this, 'BogusTopic'); // Some filler - } -} - -class ConsumingStack extends cdk.Stack { - constructor(parent, id, providingStack) { - super(parent, id); - - - new sns.Topic(this, 'BogusTopic'); // Some filler - new cdk.Output(this, 'IConsumedSomething', { value: providingStack.stackName }); - } -} - -const app = new cdk.App(); - -// Deploy all does a wildcard cdk-toolkit-integration-test-* -new MyStack(app, 'cdk-toolkit-integration-test-1'); -new YourStack(app, 'cdk-toolkit-integration-test-2'); -// Not included in wildcard -new IamStack(app, 'cdk-toolkit-integration-iam-test'); -const providing = new ProvidingStack(app, 'cdk-toolkit-integration-order-providing'); -new ConsumingStack(app, 'cdk-toolkit-integration-order-consuming', providing); - -app.run(); diff --git a/packages/aws-cdk/integ-tests/app/cdk.json b/packages/aws-cdk/integ-tests/app/cdk.json deleted file mode 100644 index f0075b1c9e33b..0000000000000 --- a/packages/aws-cdk/integ-tests/app/cdk.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "app": "node app.js", - "versionReporting": false -} diff --git a/packages/aws-cdk/integ-tests/common.bash b/packages/aws-cdk/integ-tests/common.bash deleted file mode 100644 index 6af9d8bec095c..0000000000000 --- a/packages/aws-cdk/integ-tests/common.bash +++ /dev/null @@ -1,103 +0,0 @@ -scriptdir=$(cd $(dirname $0) && pwd) - -toolkit_bin="${scriptdir}/../bin" - -if [ ! -x ${toolkit_bin}/cdk ]; then - echo "Unable to find 'cdk' under ${toolkit_bin}" - exit 1 -fi - -# make sure "this" toolkit is in the path -export PATH=${toolkit_bin}:$PATH - -function cleanup_stack() { - local stack_arn=$1 - echo "| ensuring ${stack_arn} is cleaned up" - if aws cloudformation describe-stacks --stack-name ${stack_arn} 2> /dev/null; then - aws cloudformation delete-stack --stack-name ${stack_arn} - fi -} - -function cleanup() { - cleanup_stack cdk-toolkit-integration-test-1 - cleanup_stack cdk-toolkit-integration-test-2 - cleanup_stack cdk-toolkit-integration-iam-test -} - -function setup() { - cleanup - rm -rf /tmp/cdk-integ-test - mkdir -p /tmp/cdk-integ-test - cp -R app/* /tmp/cdk-integ-test - cd /tmp/cdk-integ-test - - # "install" symlinks to the cdk core and SNS modules - # we don't use "npm install" here so that the modules will - # be from the same version as the toolkit we are testing - mkdir -p node_modules/@aws-cdk - ( - cd node_modules/@aws-cdk - ln -s ${scriptdir}/../../@aws-cdk/aws-sns - ln -s ${scriptdir}/../../@aws-cdk/aws-iam - ln -s ${scriptdir}/../../@aws-cdk/cdk - ) -} - -function fail() { - echo "❌ $@" - exit 1 -} - -function assert_diff() { - local test=$1 - local actual=$2 - local expected=$3 - - diff -w ${actual} ${expected} || { - echo - echo "+-----------" - echo "| expected:" - cat ${expected} - echo "|--" - echo - echo "+-----------" - echo "| actual:" - cat ${actual} - echo "|--" - echo - fail "assertion failed. ${test}" - } -} - -function assert() { - local command="$1" - - local expected=$(mktemp) - local actual=$(mktemp) - - echo "| running ${command}" - - eval "$command" > ${actual} || { - fail "command ${command} non-zero exit code" - } - - cat > ${expected} - - assert_diff "command: ${command}" "${actual}" "${expected}" -} - -function assert_lines() { - local data="$1" - local expected="$2" - echo "| assert that last command returned ${expected} line(s)" - - local lines="$(echo "${data}" | wc -l)" - if [ "${lines}" -ne "${expected}" ]; then - echo "${data}" - fail "response has ${lines} lines and we expected ${expected} lines to be returned" - fi -} - -function strip_color_codes() { - perl -pe 's/\e\[?.*?[\@-~]//g' -} diff --git a/packages/aws-cdk/integ-tests/test-cdk-context.sh b/packages/aws-cdk/integ-tests/test-cdk-context.sh deleted file mode 100755 index f543428d4e6a1..0000000000000 --- a/packages/aws-cdk/integ-tests/test-cdk-context.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -set -euo pipefail -scriptdir=$(cd $(dirname $0) && pwd) -source ${scriptdir}/common.bash -# ---------------------------------------------------------- - -rm -rf /tmp/cdk-integ-test -mkdir -p /tmp/cdk-integ-test -cd /tmp/cdk-integ-test - -cat > cdk.json <&1 | grep "this is the context value" > /dev/null - -# Test that deleting the contextkey works -cdk context --reset contextkey -cdk context 2>&1 | grep "this is the context value" > /dev/null && { echo "Should not contain key"; exit 1; } || true - -# Test that forced delete of the context key does not error -cdk context -f --reset contextkey - -echo "✅ success" diff --git a/packages/aws-cdk/integ-tests/test-cdk-deploy-all.sh b/packages/aws-cdk/integ-tests/test-cdk-deploy-all.sh deleted file mode 100755 index dc712453b5594..0000000000000 --- a/packages/aws-cdk/integ-tests/test-cdk-deploy-all.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -set -euo pipefail -scriptdir=$(cd $(dirname $0) && pwd) -source ${scriptdir}/common.bash -# ---------------------------------------------------------- - -setup - -stack_arns=$(cdk deploy cdk-toolkit-integration-test-\*) -echo "Stack deployed successfully" - -# verify that we only deployed a single stack (there's a single ARN in the output) -lines="$(echo "${stack_arns}" | wc -l)" -if [ "${lines}" -ne 2 ]; then - echo "-- output -----------" - echo "${stack_arns}" - echo "---------------------" - fail "cdk deploy returned ${lines} arns and we expected 2" -fi - -cdk destroy -f cdk-toolkit-integration-test-\* - -echo "✅ success" diff --git a/packages/aws-cdk/integ-tests/test-cdk-deploy-no-tty.sh b/packages/aws-cdk/integ-tests/test-cdk-deploy-no-tty.sh deleted file mode 100755 index 8c3ae3dfaa27c..0000000000000 --- a/packages/aws-cdk/integ-tests/test-cdk-deploy-no-tty.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -set -euo pipefail -scriptdir=$(cd $(dirname $0) && pwd) -source ${scriptdir}/common.bash -# ---------------------------------------------------------- - -setup - -# redirect /dev/null to stdin, which means there will not be tty attached -# since this stack includes security-related changes, the deployment should -# immediately fail because we can't confirm the changes -if cdk deploy cdk-toolkit-integration-iam-test < /dev/null; then - fail "test failed. we expect 'cdk deploy' to fail if there are security-related changes and no tty" -fi - -echo "✅ success" diff --git a/packages/aws-cdk/integ-tests/test-cdk-deploy-with-role.sh b/packages/aws-cdk/integ-tests/test-cdk-deploy-with-role.sh deleted file mode 100755 index 2b6cc8c2d6d44..0000000000000 --- a/packages/aws-cdk/integ-tests/test-cdk-deploy-with-role.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash -set -euo pipefail -scriptdir=$(cd $(dirname $0) && pwd) -source ${scriptdir}/common.bash -# ---------------------------------------------------------- - -role_name=cdk-integ-test-role -delete_role() { - for policy_name in $(aws iam list-role-policies --role-name $role_name --output text --query PolicyNames); do - aws iam delete-role-policy --role-name $role_name --policy-name $policy_name - done - aws iam delete-role --role-name $role_name -} - -delete_role || echo 'Role does not exist yet' - -role_arn=$(aws iam create-role \ - --output text --query Role.Arn \ - --role-name $role_name \ - --assume-role-policy-document file://<(echo '{ - "Version": "2012-10-17", - "Statement": [{ - "Action": "sts:AssumeRole", - "Principal": { "Service": "cloudformation.amazonaws.com" }, - "Effect": "Allow" - }] - }')) -trap delete_role EXIT -aws iam put-role-policy \ - --role-name $role_name \ - --policy-name DefaultPolicy \ - --policy-document file://<(echo '{ - "Version": "2012-10-17", - "Statement": [{ - "Action": "*", - "Resource": "*", - "Effect": "Allow" - }] - }') - -echo "Sleeping a bit to improve chances of the role having propagated" -sleep 5 - -setup - -stack_arn=$(cdk --role-arn $role_arn deploy cdk-toolkit-integration-test-2) -echo "Stack deployed successfully" - -# verify that we only deployed a single stack (there's a single ARN in the output) -assert_lines "${stack_arn}" 1 - -# verify the number of resources in the stack -response_json=$(mktemp).json -aws cloudformation describe-stack-resources --stack-name ${stack_arn} > ${response_json} -resource_count=$(node -e "console.log(require('${response_json}').StackResources.length)") -if [ "${resource_count}" -ne 2 ]; then - fail "stack has ${resource_count} resources, and we expected two" -fi - -# destroy -cdk destroy --role-arn $role_arn -f cdk-toolkit-integration-test-2 - -echo "✅ success" diff --git a/packages/aws-cdk/integ-tests/test-cdk-deploy.sh b/packages/aws-cdk/integ-tests/test-cdk-deploy.sh deleted file mode 100755 index a44253ec9c375..0000000000000 --- a/packages/aws-cdk/integ-tests/test-cdk-deploy.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -set -euo pipefail -scriptdir=$(cd $(dirname $0) && pwd) -source ${scriptdir}/common.bash -# ---------------------------------------------------------- - -setup - -stack_arn=$(cdk deploy cdk-toolkit-integration-test-2) -echo "Stack deployed successfully" - -# verify that we only deployed a single stack (there's a single ARN in the output) -assert_lines "${stack_arn}" 1 - -# verify the number of resources in the stack -response_json=$(mktemp).json -aws cloudformation describe-stack-resources --stack-name ${stack_arn} > ${response_json} -resource_count=$(node -e "console.log(require('${response_json}').StackResources.length)") -if [ "${resource_count}" -ne 2 ]; then - fail "stack has ${resource_count} resources, and we expected two" -fi - -# destroy -cdk destroy -f cdk-toolkit-integration-test-2 - -echo "✅ success" diff --git a/packages/aws-cdk/integ-tests/test-cdk-diff.sh b/packages/aws-cdk/integ-tests/test-cdk-diff.sh deleted file mode 100755 index c6a0ac87e0c40..0000000000000 --- a/packages/aws-cdk/integ-tests/test-cdk-diff.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -set -euo pipefail -scriptdir=$(cd $(dirname $0) && pwd) -source ${scriptdir}/common.bash -# ---------------------------------------------------------- - -setup - -function cdk_diff() { - cdk diff $1 2>&1 || true -} - -assert_lines "$(cdk_diff cdk-toolkit-integration-test-1)" 2 -assert_lines "$(cdk_diff cdk-toolkit-integration-test-2)" 3 - -echo "✅ success" diff --git a/packages/aws-cdk/integ-tests/test-cdk-iam-diff.sh b/packages/aws-cdk/integ-tests/test-cdk-iam-diff.sh deleted file mode 100755 index acc7f9d01320c..0000000000000 --- a/packages/aws-cdk/integ-tests/test-cdk-iam-diff.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -set -euo pipefail -scriptdir=$(cd $(dirname $0) && pwd) -source ${scriptdir}/common.bash -# ---------------------------------------------------------- - -setup - -function nonfailing_diff() { - ( cdk diff $1 2>&1 || true ) | strip_color_codes -} - -assert "nonfailing_diff cdk-toolkit-integration-iam-test" <