From 949e630dfbe8d9b996595f1984e7d47f8fc7a69f Mon Sep 17 00:00:00 2001 From: Michel Daviot Date: Fri, 16 Aug 2024 17:27:56 +0200 Subject: [PATCH 1/3] Fix path in CFT so retry has access to the proper s3 prefix --- aws/logs_monitoring/template.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/logs_monitoring/template.yaml b/aws/logs_monitoring/template.yaml index 4a752966..bf2889ff 100644 --- a/aws/logs_monitoring/template.yaml +++ b/aws/logs_monitoring/template.yaml @@ -583,7 +583,7 @@ Resources: Condition: StringLike: s3:prefix: - - "retry/*" + - "failed_events/*" - "log-group-cache/*" Effect: Allow - !Ref AWS::NoValue From d76f25f060f9be6ffe307fe656dd98648b52b58e Mon Sep 17 00:00:00 2001 From: Michel Daviot Date: Fri, 16 Aug 2024 18:24:14 +0200 Subject: [PATCH 2/3] start to setup taskcat --- aws/logs_monitoring/tools/taskcat/.gitignore | 2 + .../tools/taskcat/.taskcat.yml | 14 +++++++ .../tools/taskcat/run-taskcat.sh | 38 +++++++++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 aws/logs_monitoring/tools/taskcat/.gitignore create mode 100644 aws/logs_monitoring/tools/taskcat/.taskcat.yml create mode 100755 aws/logs_monitoring/tools/taskcat/run-taskcat.sh diff --git a/aws/logs_monitoring/tools/taskcat/.gitignore b/aws/logs_monitoring/tools/taskcat/.gitignore new file mode 100644 index 00000000..26410b8c --- /dev/null +++ b/aws/logs_monitoring/tools/taskcat/.gitignore @@ -0,0 +1,2 @@ +tmp +taskcat_outputs diff --git a/aws/logs_monitoring/tools/taskcat/.taskcat.yml b/aws/logs_monitoring/tools/taskcat/.taskcat.yml new file mode 100644 index 00000000..35059dce --- /dev/null +++ b/aws/logs_monitoring/tools/taskcat/.taskcat.yml @@ -0,0 +1,14 @@ +general: + auth: + default: "" + s3_bucket: datadog-cloudformation-templates-aws-taskcat-test + +project: + name: aws-quickstart + regions: + - us-east-2 +tests: + default: + template: ./template.yaml + parameters: + DdApiKey: "" diff --git a/aws/logs_monitoring/tools/taskcat/run-taskcat.sh b/aws/logs_monitoring/tools/taskcat/run-taskcat.sh new file mode 100755 index 00000000..a3260e9a --- /dev/null +++ b/aws/logs_monitoring/tools/taskcat/run-taskcat.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +set -ex + +if [ -z "$AWS_SSO_PROFILE_NAME" ]; then + echo "Missing AWS_SSO_PROFILE_NAME - Must specify an AWS profile name" + exit 1 +fi + +# aws sso login --profile ${AWS_SSO_PROFILE_NAME} + +TASKCAT_S3_BUCKET="lambdaforwarder-taskcat-test" +TASKCAT_PROJECT="aws-lambda-forwarder-taskcat-tests" +# +#if [ -z "$DD_API_KEY" ]; then +# echo "Missing DD_API_KEY - Must specify a Datadog API key" +# exit 1 +#fi +# +#if [ -z "$DD_APP_KEY" ]; then +# echo "Missing DD_APP_KEY - Must specify a Datadog APP key" +# exit 1 +#fi +# +mkdir -p ./tmp + +for f in ../../template.yaml; do + sed "s|.s3.amazonaws.com/aws/|${TASKCAT_S3_BUCKET}.s3.amazonaws.com/${TASKCAT_PROJECT}|g" $f > ./tmp/$(basename $f) +done + +sed "s||${DD_API_KEY}|g ; s||${DD_APP_KEY}|g ; s||${AWS_SSO_PROFILE_NAME}|g" ./.taskcat.yml > ./tmp/.taskcat.yml + +taskcat upload -b ${TASKCAT_S3_BUCKET} -k ${TASKCAT_PROJECT} -p tmp + +taskcat test run --skip-upload --project-root tmp --no-delete + +echo "To delete test stacks, run:" +echo " taskcat test clean ${TASKCAT_PROJECT} -a ${AWS_SSO_PROFILE_NAME}" From ce2eaf9bb72728e4bf6a668b5ce0c00a3fb3d01d Mon Sep 17 00:00:00 2001 From: Michel Daviot Date: Fri, 16 Aug 2024 18:25:35 +0200 Subject: [PATCH 3/3] properly pass default values as strings --- aws/logs_monitoring/template.yaml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/aws/logs_monitoring/template.yaml b/aws/logs_monitoring/template.yaml index bf2889ff..ab7758a9 100644 --- a/aws/logs_monitoring/template.yaml +++ b/aws/logs_monitoring/template.yaml @@ -57,7 +57,7 @@ Parameters: Description: DO NOT CHANGE unless you know what you are doing. Override the default location of the function source code. InstallAsLayer: Type: String - Default: true + Default: "true" Description: Whether to use the layer-based installation flow. Set to false to use our legacy installation flow, which installs a second function that copies the forwarder code from Github to an S3 bucket. Defaults to true. AllowedValues: - true @@ -72,35 +72,35 @@ Parameters: Description: Add custom tags to forwarded logs, comma-delimited string, no trailing comma, e.g., env:prod,stack:classic DdFetchLambdaTags: Type: String - Default: true + Default: "true" AllowedValues: - true - false Description: Let the forwarder fetch Lambda tags using GetResources API calls and apply them to logs, metrics and traces. If set to true, permission tag:GetResources will be automatically added to the Lambda execution IAM role. The tags are cached in memory and S3 so that they'll only be fetched when the function cold starts or when the TTL (1 hour) expires. The forwarder increments the aws.lambda.enhanced.get_resources_api_calls metric for each API call made. DdFetchLogGroupTags: Type: String - Default: true + Default: "true" AllowedValues: - true - false Description: Let the forwarder fetch Log Group tags using ListTagsLogGroup and apply them to logs, metrics and traces. If set to true, permission logs:ListTagsLogGroup will be automatically added to the Lambda execution IAM role. The tags are cached in memory and S3 so that they'll only be fetched when the function cold starts or when the TTL (1 hour) expires. The forwarder increments the aws.lambda.enhanced.list_tags_log_group_api_call metric for each API call made. DdFetchStepFunctionsTags: Type: String - Default: true + Default: "true" AllowedValues: - true - false Description: Let the forwarder fetch Step Functions tags using GetResources API calls and apply them to logs, metrics and traces. If set to true, permission tag:GetResources will be automatically added to the Lambda execution IAM role. The tags are cached in memory and S3 so that they'll only be fetched when the function cold starts or when the TTL (1 hour) expires. The forwarder increments the aws.lambda.enhanced.get_resources_api_calls metric for each API call made. DdUseTcp: Type: String - Default: false + Default: "false" AllowedValues: - true - false Description: By default, the forwarder sends logs using HTTPS through the port 443. To send logs over an SSL encrypted TCP connection, set this parameter to true. DdNoSsl: Type: String - Default: false + Default: "false" AllowedValues: - true - false @@ -115,21 +115,21 @@ Parameters: Description: The endpoint port to forward the logs to, useful for forwarding logs through a proxy DdSkipSslValidation: Type: String - Default: false + Default: "false" AllowedValues: - true - false Description: Send logs over HTTPS, while NOT validating the certificate provided by the endpoint. This will still encrypt the traffic between the forwarder and the log intake endpoint, but will not verify if the destination SSL certificate is valid. RedactIp: Type: String - Default: false + Default: "false" AllowedValues: - true - false Description: Replace text matching \d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3} with xxx.xxx.xxx.xxx RedactEmail: Type: String - Default: false + Default: "false" AllowedValues: - true - false @@ -156,35 +156,35 @@ Parameters: Description: Use the supplied regular expression to detect for a new log line for multiline logs from S3, e.g., use expression "\d{2}\/\d{2}\/\d{4}" for multiline logs beginning with pattern "11/10/2014". DdForwardLog: Type: String - Default: true + Default: "true" AllowedValues: - true - false Description: Set to false to disable log forwarding, while continuing to forward other observability data, such as metrics and traces from Lambda functions. DdStepFunctionsTraceEnabled: Type: String - Default: false + Default: "false" AllowedValues: - true - false Description: Set to true to enable tracing for all Step Functions. DdUseCompression: Type: String - Default: true + Default: "true" AllowedValues: - true - false Description: Set to false to disable log compression. Only valid when sending logs over HTTP. DdUsePrivateLink: Type: String - Default: false + Default: "false" AllowedValues: - true - false Description: DEPRECATED, DO NOT CHANGE. See README.md for details. Set to true to deploy the Forwarder to a VPC and send logs, metrics, and traces via AWS PrivateLink. When set to true, must also set VPCSecurityGroupIds and VPCSubnetIds. DdUseVPC: Type: String - Default: false + Default: "false" AllowedValues: - true - false @@ -240,7 +240,7 @@ Parameters: Description: (Optional) The name of the S3 bucket to store access logs. Leave empty if access logging is not needed. DdStoreFailedEvents: Type: String - Default: false + Default: "false" AllowedValues: - true - false