Skip to content

Commit

Permalink
Merged branch 'main' into 'github' for 1.0.10 release
Browse files Browse the repository at this point in the history
  • Loading branch information
jonemo committed Mar 28, 2022
2 parents ce18410 + f6b0f36 commit a85f232
Show file tree
Hide file tree
Showing 333 changed files with 17,018 additions and 3,576 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -121,3 +121,4 @@ env
ggv2_core_device_install_script.txt
*.yaml.build
*.yml.build

227 changes: 227 additions & 0 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,227 @@
# Official framework image. Look for the different tagged releases at: https://hub.docker.com/r/library/node/tags/

variables:
CONFIG_PACKAGE_NAME: cdf-infrastructure-demo
RUSH_PARALLELISM: 8
APP_CONFIG_DIR: common/build/configdir
BRANCH: main

.staging-variables: &staging-variables
ENVIRONMENT: development-staging
AWS_REGION: us-west-2

.live-variables: &live-variables
ENVIRONMENT: development
AWS_REGION: us-west-2

.default-run-condition: &default-run-condition
- if: ($CI_COMMIT_BRANCH == "main" && $CI_COMMIT_MESSAGE != /.*\[skip ci\].*/)

.branch-run-condition: &branch-run-condition
- if: ($CI_COMMIT_BRANCH != "main")

.common-build-steps: &common-build-steps # install dependencies
- cd ${CI_PROJECT_DIR}/source
- rush update
# build
- rush build

.common-install-deployment-dependencies: &common-install-deployment-dependencies
- env | sort
- yum update -y
# install tool dependencies (required to test some services)
- yum install -y git tar openssl zip unzip
# install specific version of pnpm needed by rush
- curl -f https://get.pnpm.io/v5.js | node - add --global pnpm@5
- npm install -g @microsoft/rush
# install aws cli
- curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && unzip -qq awscliv2.zip
- ./aws/install && rm -rf ./aws && rm awscliv2.zip
# install jq
- curl -s -qL -o /usr/bin/jq https://stedolan.github.io/jq/download/linux64/jq
- chmod +x /usr/bin/jq

.common-install-node-dependencies: &common-install-node-dependencies # install dependencies
- curl -f https://get.pnpm.io/v5.js | node - add --global pnpm@5
- npm install -g @microsoft/rush

.common-git-steps: &common-git-steps # install dependencies
- git config --global user.name "gitlab-ci-token"
- git config --global user.email "gitlab-ci-token"
- git remote set-url origin https://gitlab-ci-token:$PROJECT_ACCESS_TOKEN@$CI_SERVER_HOST/$CI_PROJECT_PATH.git

.common-artifacts: &common-artifacts
artifacts:
paths:
- source/common/
- source/infrastructure/**/build.zip
- source/packages/**/.rush/
- source/packages/**/dist/
- source/packages/**/tsconfig.tsbuildinfo
- source/packages/**/bundle.zip

stages:
- build
- test
- deployStage
- integrationTest
- deployLive
- upload
- triggerCpp

### not working at the moment, therefore using artifacts as a temp workaround
# cache:
# # enable per-branch caching
# key: "$CI_COMMIT_REF_SLUG"
# paths:
# - *common-artifacts

default:
image: public.ecr.aws/lambda/nodejs:14
before_script:
- *common-install-deployment-dependencies
- *common-git-steps

downloadInfastructureConfig:
rules:
- *default-run-condition
stage: build
before_script:
- env | sort
- echo 'download jq'
- curl -s -qL -o /usr/bin/jq https://stedolan.github.io/jq/download/linux64/jq
- chmod +x /usr/bin/jq
- echo 'download wget'
- yum install -y wget unzip
image:
name: public.ecr.aws/amazonlinux/amazonlinux:2
entrypoint: [""]
script:
- 'CONFIG_PACKAGE_VERSION=$(curl -H "PRIVATE-TOKEN: $CDF_INFRASTRUCTURE_GIT_TOKEN" "${CI_API_V4_URL}/projects/${CONFIG_REPO_PROJECT_ID}/packages?sort=desc" | jq -r ".[0].version")'
- 'wget --header="PRIVATE-TOKEN: $CDF_INFRASTRUCTURE_GIT_TOKEN" ${CI_API_V4_URL}/projects/${CONFIG_REPO_PROJECT_ID}/packages/generic/${CONFIG_PACKAGE_NAME}/${CONFIG_PACKAGE_VERSION}/config.zip'
- mkdir -p common/build
- cp config.zip common/build/config.zip
- unzip -d $APP_CONFIG_DIR common/build/config.zip
artifacts:
paths:
- $APP_CONFIG_DIR/

buildLambaLayersZip:
rules:
- *default-run-condition
stage: build
before_script:
- yum update -y && yum install -y openssl zip
image:
name: public.ecr.aws/amazonlinux/amazonlinux:2
entrypoint: [""]
script:
- cd ${CI_PROJECT_DIR}/source && mkdir -p common/build
- cd common/build && mkdir -p bin lib && cp /bin/openssl bin/ && cp /usr/lib64/libbz2.so.1 lib/ && zip -r build.zip * && rm -rf bin lib
- cd ${CI_PROJECT_DIR}/source && mkdir -p infrastructure/lambdaLayers/openssl/build && cp common/build/build.zip infrastructure/lambdaLayers/openssl/build/build.zip
artifacts:
paths:
- source/infrastructure/lambdaLayers/openssl/build/build.zip


branchTest:
rules:
- *default-run-condition
stage: test
script:
- *common-build-steps
# build
- rush build
- export RUSH_PARALLELISM=1
- rush test
# Bundling the node project when it past
- export RUSH_PARALLELISM=8
- cd ${CI_PROJECT_DIR}
- bash -c source/cicd/buildproject_postbuild.bash
<<: *common-artifacts

test:
rules:
- *branch-run-condition
stage: test
script:
- *common-build-steps
- rush build
- export RUSH_PARALLELISM=1
- rush test

deployStage:
rules:
- *default-run-condition
stage: deployStage
variables:
<<: *staging-variables
script:
- pushd source
# rush install
- rush update --purge
- popd
# export the location of the config
- export CODEBUILD_SRC_DIR_source_infrastructure="${CI_PROJECT_DIR}/${APP_CONFIG_DIR}"
# deploy to staging environment
- bash -c source/cicd/deployproject_build.bash
after_script:
- bash -c source/cicd/deployproject_postbuild.bash

integrationTest:
rules:
- *default-run-condition
stage: integrationTest
before_script:
- *common-install-deployment-dependencies
variables:
<<: *staging-variables
script:
- pushd source
# rush install
- rush update --purge
- popd
# export the location of the config
- export CODEBUILD_SRC_DIR_source_infrastructure="${CI_PROJECT_DIR}/${APP_CONFIG_DIR}"
# run in the integration tests
- bash -c source/cicd/integrationtestsproject_build.bash

deployLive:
rules:
- *default-run-condition
stage: deployLive
variables:
<<: *live-variables
script:
# deploy to live environment
- pushd source
# rush install
- rush update --purge
- popd
- export CODEBUILD_SRC_DIR_source_infrastructure="${CI_PROJECT_DIR}/${APP_CONFIG_DIR}"
- bash -c source/cicd/deployproject_build.bash
after_script:
- bash -c source/cicd/deployproject_postbuild.bash

upload:
before_script:
- yum update -y
- yum install -y git
image:
name: public.ecr.aws/amazonlinux/amazonlinux:2
entrypoint: [""]
stage: upload
rules:
- if: ($CI_COMMIT_BRANCH == "feat_command_and_control")
script:
# Create archive based on latest commit
- git archive --format=zip --output=cdf.zip $CI_COMMIT_SHA
- 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file cdf.zip "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/aws-connected-device-framework/${CI_JOB_ID}/cdf.zip"'

triggerCpp:
stage: triggerCpp
rules:
- if: ($CI_COMMIT_BRANCH == "feat_command_and_control")
trigger:
project: robotics-and-engineering-gsp/cdf/aws-connected-product-platform
branch: main
12 changes: 3 additions & 9 deletions NOTICE.txt
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ json-schema, 0.2.3," ""AFLv2.1"," BSD""",,
@cdf/request-queue, 3.3.1," ""Apache 2.0", ISC, MIT," New BSD"""
@cdf/simulation-launcher, 0.0.1," ""Apache 2.0", ISC, MIT," New BSD"""
@cdf/simulation-manager, 0.0.1," ""Apache 2.0", ISC, MIT," New BSD"""
@cucumber/cucumber, 7.3.2, MIT,,,
cdf," """""," ""Apache 2.0", ISC, MIT," New BSD"""
amazon-dax-client, 1.2.2," ""Apache 2.0""",,,
aws-iot-device-sdk, 2.2.6," ""Apache 2.0""",,,
Expand Down Expand Up @@ -163,7 +164,6 @@ wrappy, 1.0.2, ISC,,,
@types/config, 0.0.36, MIT,,,
@types/connect, 3.4.33, MIT,,,
@types/cors, 2.8.6, MIT,,,
@types/cucumber, 6.0.0, MIT,,,
@types/dot, 1.1.3, MIT,,,
@types/express, 4.17.2, MIT,,,
@types/express, 4.17.8, MIT,,,
Expand All @@ -177,8 +177,6 @@ wrappy, 1.0.2, ISC,,,
@types/jsonpath, 0.2.0, MIT,,,
@types/jsonwebtoken, 8.3.5, MIT,,,
@types/jsonwebtoken, 8.5.0, MIT,,,
@types/jszip, 3.1.6, MIT,,,
@types/jszip, 3.4.1, MIT,,,
@types/jwk-to-pem, 2.0.0, MIT,,,
@types/mime, 2.0.3, MIT,,,
@types/morgan, 1.7.37, MIT,,,
Expand All @@ -193,9 +191,7 @@ wrappy, 1.0.2, ISC,,,
@types/serve-static, 1.13.6, MIT,,,
@types/shortid, 0.0.29, MIT,,,
@types/sinon, 7.5.1, MIT,,,
@types/superagent, 4.1.10, MIT,,,
@types/superagent, 4.1.4, MIT,,,
@types/superagent, 4.1.7, MIT,,,
@types/superagent, 4.1.15, MIT,,,
@types/uuid, 3.4.6, MIT,,,
accepts, 1.3.7, MIT,,,
ajv, 6.10.2, MIT,,,
Expand Down Expand Up @@ -270,7 +266,6 @@ core-util-is, 1.0.2, MIT,,,
cors, 2.8.5, MIT,,,
cpy-cli, 3.0.0, MIT,,,
crypto-js, 3.1.6, MIT,,,
cucumber, 6.0.5, MIT,,,
cucumber-expressions, 8.3.0, MIT,,,
cucumber-tag-expressions, 2.0.3, MIT,,,
curriable, 1.3.0, MIT,,,
Expand Down Expand Up @@ -505,8 +500,7 @@ string-argv, 0.3.1, MIT,,,
string-width, 2.1.1, MIT,,,
string_decoder, 1.1.1, MIT,,,
strip-ansi, 4.0.0, MIT,,,
superagent, 5.1.2, MIT,,,
superagent, 5.3.1, MIT,,,
superagent, 7.2.0, MIT,,,
superagent-retry-delay, 2.4.2, MIT,,,
supports-color, 5.5.0, MIT,,,
text-hex, 1.0.0, MIT,,,
Expand Down
10 changes: 10 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -153,12 +153,22 @@ See [overview](source/packages/services/device-monitoring/README.md).

### Commands

> Note: this module is deprecated and has been replaced with the _Command & Control_ module.
Utilizes AWS IoT Jobs to issue commands to a device or set of devices, and optionally inspect their execution status. It augments AWS IoT jobs by providing the ability to create Job templates (job document, parameters, and files), and enforcing that each requested command adheres to a template before executing.

Also allows for sending jobs to thousands of devices, by automatically managing temporary groups to overcome any limitations with the no. of allowed targets. Can optionally use Asset Library devices, groups, and search queries as Job targets.

See [overview](source/packages/services/commands/README.md).

### Command & Control

This module provides a simple zero-code approach to implement command and control functionality using AWS IoT Shadows, AWS IoT Jobs, and/or MQTT topics as desired. In addition, it supports sending to a variety of different targets regardless of the delivery method configured: a thing or list of things, a thing group or list of thing groups, a dynamic group or list of dynamic groups, an Asset Library device or list of devices, an Asset Library group or list of groups, an Asset Library search query, or any combination of.

Finally, any messages sent back from the device related to the command message are correlated to the original message where the entire conversation can be easily retrieved.

See [overview](source/packages/services/command-and-control/README.md).

### Asset Library History

Tracks and stores all changes made to the Asset Library (devices, groups, policies and/or templates) for auditing purposes.
Expand Down
1 change: 1 addition & 0 deletions source/.cz-config.js
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ module.exports = {
{ name: 'certificateactivator' },
{ name: 'certificatevendor' },
{ name: 'commands' },
{ name: 'command-and-control' },
{ name: 'device-patcher' },
{ name: 'events-alerts' },
{ name: 'events-processor' },
Expand Down
9 changes: 5 additions & 4 deletions source/.eslintrc.js
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,12 @@ module.exports = {
],
ignorePatterns: [ "**/dist", "**/build", "**/deploy", "**/node_modules", "**/*.spec.ts" ] ,
rules: {
"no-underscore-dangle": "off",
"no-mixed-spaces-and-tabs": ["error", "smart-tabs"],
"no-unused-vars": "off",
"@typescript-eslint/no-unused-vars-experimental": "error",
"@typescript-eslint/no-unused-vars": "off",
"@typescript-eslint/no-unused-vars-experimental": "error"
"no-mixed-spaces-and-tabs": ["error", "smart-tabs"],
"no-prototype-builtins": "off",
"no-underscore-dangle": "off",
"no-unused-vars": "off"
}
};

39 changes: 17 additions & 22 deletions source/cicd/buildproject_postbuild.bash
Original file line number Diff line number Diff line change
Expand Up @@ -16,29 +16,24 @@ set -e

echo buildproject_postbuild started on `date`

if [ "$CODEBUILD_BUILD_SUCCEEDING" -eq 1 ]; then
cd source

cd source
echo "enforce need for changelog"
rush change -v

echo Adding workaround for unsupported characters in the auto-generated codecommit passwords
### pnpm requires GIT_CREDENTIALS configuring for when it pushes version changes to codecommit
export GIT_CREDENTIALS=$(echo "$CDF_CODECOMMIT_USERNAME:$CDF_CODECOMMIT_PASSWORD" | sed -r 's:/:%2F:g' )

echo Applying version updates
rush publish -a

echo Committing updated files
set +e
git add -A
git update-index --refresh
git diff-index --quiet HEAD -- ; differences=$?
set -e
if [ $differences -eq 1 ]; then
git commit -am 'ci(@cdf) buildproject_prebuild [skip ci]'
git push origin HEAD:$BRANCH
fi

echo Bundling...
infrastructure/bundle-core.bash
echo "Applying version updates"
rush publish -a

echo Committing updated files
set +e
git add -A
git update-index --refresh
git diff-index --quiet HEAD -- ; differences=$?
set -e
if [ $differences -eq 1 ]; then
git commit -am 'ci(@cdf) buildproject_prebuild [skip ci]'
git push origin HEAD:$BRANCH
fi

echo Bundling...
infrastructure/bundle-core.bash -z
2 changes: 0 additions & 2 deletions source/cicd/buildproject_prebuild.bash
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,4 @@ cd source
echo Installing...
rush update

echo Enforcing the need for change log
rush change -v

Loading

0 comments on commit a85f232

Please sign in to comment.