From 9843f9296d86895181c970a48df97e981d305e7c Mon Sep 17 00:00:00 2001 From: Bryan Robitaille Date: Tue, 10 Sep 2024 11:36:16 -0400 Subject: [PATCH 01/11] add second sqs queue and rename app audit log queue --- aws/api/ecs.tf | 16 ++++++++++- aws/api/inputs.tf | 7 ++++- aws/app/ecs.tf | 2 +- aws/app/ecs_iam.tf | 2 +- aws/app/inputs.tf | 4 +-- aws/lambdas/audit_log.tf | 2 +- aws/lambdas/inputs.tf | 2 +- aws/sqs/outputs.tf | 18 +++++++++--- aws/sqs/sqs.tf | 49 ++++++++++++++++++++++++++++---- env/cloud/app/terragrunt.hcl | 8 +++--- env/cloud/lambdas/terragrunt.hcl | 4 +-- 11 files changed, 90 insertions(+), 24 deletions(-) diff --git a/aws/api/ecs.tf b/aws/api/ecs.tf index 49382eea4..dfd89063c 100644 --- a/aws/api/ecs.tf +++ b/aws/api/ecs.tf @@ -58,7 +58,8 @@ module "api_ecs" { data.aws_iam_policy_document.api_ecs_kms_vault.json, data.aws_iam_policy_document.api_ecs_dynamodb_vault.json, data.aws_iam_policy_document.api_ecs_s3_vault.json, - data.aws_iam_policy_document.api_ecs_secrets_manager_runtime.json + data.aws_iam_policy_document.api_ecs_secrets_manager_runtime.json, + data.aws_iam_policy_document.api_sqs.json ] task_exec_role_policy_documents = [ @@ -158,3 +159,16 @@ data "aws_iam_policy_document" "api_ecs_secrets_manager" { ] } } +data "aws_iam_policy_document" "api_sqs" { + statement { + effect = "Allow" + actions = [ + "sqs:GetQueueUrl", + "sqs:SendMessage" + ] + + resources = [ + var.sqs_api_audit_log_queue_arn + ] + } +} diff --git a/aws/api/inputs.tf b/aws/api/inputs.tf index 612d73b58..7c1768ae1 100644 --- a/aws/api/inputs.tf +++ b/aws/api/inputs.tf @@ -74,4 +74,9 @@ variable "rds_connection_url_secret_arn" { description = "The RDS connection URL secret used by the ECS task" type = string sensitive = true -} \ No newline at end of file +} +variable "sqs_api_audit_log_queue_arn" { + description = "SQS audit log queue ARN" + type = string +} + diff --git a/aws/app/ecs.tf b/aws/app/ecs.tf index ba4143b5f..ed164fb1a 100644 --- a/aws/app/ecs.tf +++ b/aws/app/ecs.tf @@ -42,7 +42,7 @@ data "template_file" "form_viewer_task" { email_address_contact_us = var.email_address_contact_us email_address_support = var.email_address_support reprocess_submission_queue = var.sqs_reprocess_submission_queue_id - audit_log_queue_url = var.sqs_audit_log_queue_id + audit_log_queue_url = var.sqs_app_audit_log_queue_id zitadel_provider = var.zitadel_provider zitadel_administration_key = var.zitadel_administration_key_secret_arn sentry_api_key = var.sentry_api_key_secret_arn diff --git a/aws/app/ecs_iam.tf b/aws/app/ecs_iam.tf index 20e14a47b..eb613f651 100644 --- a/aws/app/ecs_iam.tf +++ b/aws/app/ecs_iam.tf @@ -142,7 +142,7 @@ data "aws_iam_policy_document" "forms_sqs" { resources = [ var.sqs_reprocess_submission_queue_arn, - var.sqs_audit_log_queue_arn + var.sqs_app_audit_log_queue_arn ] } } diff --git a/aws/app/inputs.tf b/aws/app/inputs.tf index 23016269f..628c437c3 100644 --- a/aws/app/inputs.tf +++ b/aws/app/inputs.tf @@ -173,12 +173,12 @@ variable "sqs_reprocess_submission_queue_id" { type = string } -variable "sqs_audit_log_queue_arn" { +variable "sqs_app_audit_log_queue_arn" { description = "SQS audit log queue ARN" type = string } -variable "sqs_audit_log_queue_id" { +variable "sqs_app_audit_log_queue_id" { description = "SQS audit log queue URL" type = string } diff --git a/aws/lambdas/audit_log.tf b/aws/lambdas/audit_log.tf index 80406d81b..6aebd808a 100644 --- a/aws/lambdas/audit_log.tf +++ b/aws/lambdas/audit_log.tf @@ -32,7 +32,7 @@ resource "aws_lambda_function" "audit_logs" { } resource "aws_lambda_event_source_mapping" "audit_logs" { - event_source_arn = var.sqs_audit_log_queue_arn + event_source_arn = var.sqs_app_audit_log_queue_arn function_name = aws_lambda_function.audit_logs.arn function_response_types = ["ReportBatchItemFailures"] batch_size = 10 diff --git a/aws/lambdas/inputs.tf b/aws/lambdas/inputs.tf index 36daac322..fad066c80 100644 --- a/aws/lambdas/inputs.tf +++ b/aws/lambdas/inputs.tf @@ -44,7 +44,7 @@ variable "sqs_reprocess_submission_queue_arn" { type = string } -variable "sqs_audit_log_queue_arn" { +variable "sqs_app_audit_log_queue_arn" { description = "SQS audit log queue ARN" type = string } diff --git a/aws/sqs/outputs.tf b/aws/sqs/outputs.tf index 2a0a7da82..449346b6c 100644 --- a/aws/sqs/outputs.tf +++ b/aws/sqs/outputs.tf @@ -28,14 +28,24 @@ output "sqs_reliability_deadletter_queue_arn" { value = aws_sqs_queue.reliability_deadletter_queue.name } -output "sqs_audit_log_queue_arn" { +output "sqs_app_audit_log_queue_arn" { description = "SQS audit log queue ARN" - value = aws_sqs_queue.audit_log_queue.arn + value = aws_sqs_queue.app_audit_log_queue.arn } -output "sqs_audit_log_queue_id" { +output "sqs_app_audit_log_queue_id" { description = "SQS audit log queue URL" - value = aws_sqs_queue.audit_log_queue.id + value = aws_sqs_queue.app_audit_log_queue.id +} + +output "sqs_api_audit_log_queue_arn" { + description = "SQS audit log queue ARN" + value = aws_sqs_queue.api_audit_log_queue.arn +} + +output "sqs_api_audit_log_queue_id" { + description = "SQS audit log queue URL" + value = aws_sqs_queue.api_audit_log_queue.id } output "sqs_audit_log_deadletter_queue_arn" { diff --git a/aws/sqs/sqs.tf b/aws/sqs/sqs.tf index 79cf84892..c277b1f32 100644 --- a/aws/sqs/sqs.tf +++ b/aws/sqs/sqs.tf @@ -59,9 +59,46 @@ resource "aws_sqs_queue" "reprocess_submission_queue" { }) } -# Audit Log Queue +# App Audit Log Queue -resource "aws_sqs_queue" "audit_log_queue" { +resource "aws_sqs_queue" "app_audit_log_queue" { + name = "app_audit_log_queue" + delay_seconds = 0 + max_message_size = 262144 + message_retention_seconds = 172800 // 2 days + visibility_timeout_seconds = 1960 + # https://aws.amazon.com/premiumsupport/knowledge-center/lambda-function-process-sqs-messages/ + # The SQS visibility timeout must be at least six times the total of the function timeout and the batch window timeout. + # Lambda function timeout is 300. + + kms_master_key_id = "alias/aws/sqs" + kms_data_key_reuse_period_seconds = 300 + + redrive_policy = jsonencode({ + deadLetterTargetArn = aws_sqs_queue.app_audit_log_deadletter_queue.arn + maxReceiveCount = 5 + }) + + redrive_allow_policy = jsonencode({ + redrivePermission = "byQueue", + sourceQueueArns = [aws_sqs_queue.app_audit_log_deadletter_queue.arn] + }) +} + +resource "aws_sqs_queue" "app_audit_log_deadletter_queue" { + name = "app_audit_log_deadletter_queue" + delay_seconds = 60 + max_message_size = 262144 + message_retention_seconds = 1209600 + receive_wait_time_seconds = 5 + + kms_master_key_id = "alias/aws/sqs" + kms_data_key_reuse_period_seconds = 300 +} + +# API Audit Log Queue + +resource "aws_sqs_queue" "api_audit_log_queue" { name = "audit_log_queue" delay_seconds = 0 max_message_size = 262144 @@ -75,18 +112,18 @@ resource "aws_sqs_queue" "audit_log_queue" { kms_data_key_reuse_period_seconds = 300 redrive_policy = jsonencode({ - deadLetterTargetArn = aws_sqs_queue.audit_log_deadletter_queue.arn + deadLetterTargetArn = aws_sqs_queue.api_audit_log_deadletter_queue.arn maxReceiveCount = 5 }) redrive_allow_policy = jsonencode({ redrivePermission = "byQueue", - sourceQueueArns = [aws_sqs_queue.audit_log_deadletter_queue.arn] + sourceQueueArns = [aws_sqs_queue.api_audit_log_deadletter_queue.arn] }) } -resource "aws_sqs_queue" "audit_log_deadletter_queue" { - name = "audit_log_deadletter_queue" +resource "aws_sqs_queue" "api_audit_log_deadletter_queue" { + name = "api_audit_log_deadletter_queue" delay_seconds = 60 max_message_size = 262144 message_retention_seconds = 1209600 diff --git a/env/cloud/app/terragrunt.hcl b/env/cloud/app/terragrunt.hcl index 9cbc40a03..4571615d3 100644 --- a/env/cloud/app/terragrunt.hcl +++ b/env/cloud/app/terragrunt.hcl @@ -82,8 +82,8 @@ dependency "sqs" { mock_outputs_allowed_terraform_commands = ["init", "fmt", "validate", "plan", "show"] mock_outputs = { sqs_reprocess_submission_queue_arn = null - sqs_audit_log_queue_arn = null - sqs_audit_log_queue_id = null + sqs_app_audit_log_queue_arn = null + sqs_app_audit_log_queue_id = null sqs_reprocess_submission_queue_id = null } } @@ -168,8 +168,8 @@ inputs = { database_url_secret_arn = dependency.rds.outputs.database_url_secret_arn sqs_reprocess_submission_queue_arn = dependency.sqs.outputs.sqs_reprocess_submission_queue_arn - sqs_audit_log_queue_arn = dependency.sqs.outputs.sqs_audit_log_queue_arn - sqs_audit_log_queue_id = dependency.sqs.outputs.sqs_audit_log_queue_id + sqs_app_audit_log_queue_arn = dependency.sqs.outputs.sqs_app_audit_log_queue_arn + sqs_app_audit_log_queue_id = dependency.sqs.outputs.sqs_app_audit_log_queue_id sqs_reprocess_submission_queue_id = dependency.sqs.outputs.sqs_reprocess_submission_queue_id cognito_endpoint_url = dependency.cognito.outputs.cognito_endpoint_url diff --git a/env/cloud/lambdas/terragrunt.hcl b/env/cloud/lambdas/terragrunt.hcl index b99cf1ac1..83c9bc4cc 100644 --- a/env/cloud/lambdas/terragrunt.hcl +++ b/env/cloud/lambdas/terragrunt.hcl @@ -65,7 +65,7 @@ dependency "sqs" { sqs_reliability_queue_id = null sqs_reprocess_submission_queue_arn = null sqs_reliability_dead_letter_queue_id = null - sqs_audit_log_queue_arn = null + sqs_app_audit_log_queue_arn = null } } @@ -174,7 +174,7 @@ inputs = { sqs_reliability_queue_id = dependency.sqs.outputs.sqs_reliability_queue_id sqs_reprocess_submission_queue_arn = dependency.sqs.outputs.sqs_reprocess_submission_queue_arn sqs_reliability_dead_letter_queue_id = dependency.sqs.outputs.sqs_reliability_dead_letter_queue_id - sqs_audit_log_queue_arn = dependency.sqs.outputs.sqs_audit_log_queue_arn + sqs_app_audit_log_queue_arn = dependency.sqs.outputs.sqs_app_audit_log_queue_arn sns_topic_alert_critical_arn = dependency.sns.outputs.sns_topic_alert_critical_arn From 19a7ad957dea374490298578b61f5ae778ac3057 Mon Sep 17 00:00:00 2001 From: Bryan Robitaille Date: Tue, 10 Sep 2024 12:29:38 -0400 Subject: [PATCH 02/11] only change the sqs app audit log output names --- aws/alarms/cloudwatch_app.tf | 4 ++-- aws/alarms/inputs.tf | 2 +- aws/sqs/outputs.tf | 10 +++++++--- aws/sqs/sqs.tf | 14 +++++++------- env/cloud/alarms/terragrunt.hcl | 4 ++-- 5 files changed, 19 insertions(+), 15 deletions(-) diff --git a/aws/alarms/cloudwatch_app.tf b/aws/alarms/cloudwatch_app.tf index d8ee8dd57..13e7454a0 100644 --- a/aws/alarms/cloudwatch_app.tf +++ b/aws/alarms/cloudwatch_app.tf @@ -174,7 +174,7 @@ resource "aws_cloudwatch_metric_alarm" "audit_log_dead_letter_queue_warn" { unit = "Count" dimensions = { - QueueName = var.sqs_audit_log_deadletter_queue_arn + QueueName = var.sqs_app_audit_log_deadletter_queue_arn } } } @@ -190,7 +190,7 @@ resource "aws_cloudwatch_metric_alarm" "audit_log_dead_letter_queue_warn" { unit = "Count" dimensions = { - QueueName = var.sqs_audit_log_deadletter_queue_arn + QueueName = var.sqs_app_audit_log_deadletter_queue_arn } } } diff --git a/aws/alarms/inputs.tf b/aws/alarms/inputs.tf index 8c8eb52bd..53bf300d3 100644 --- a/aws/alarms/inputs.tf +++ b/aws/alarms/inputs.tf @@ -201,7 +201,7 @@ variable "sqs_reliability_deadletter_queue_arn" { type = string } -variable "sqs_audit_log_deadletter_queue_arn" { +variable "sqs_app_audit_log_deadletter_queue_arn" { description = "ARN of the Audit Log queue's SQS Dead Letter Queue" type = string } diff --git a/aws/sqs/outputs.tf b/aws/sqs/outputs.tf index 449346b6c..4bb0ee227 100644 --- a/aws/sqs/outputs.tf +++ b/aws/sqs/outputs.tf @@ -30,12 +30,12 @@ output "sqs_reliability_deadletter_queue_arn" { output "sqs_app_audit_log_queue_arn" { description = "SQS audit log queue ARN" - value = aws_sqs_queue.app_audit_log_queue.arn + value = aws_sqs_queue.audit_log_queue.arn } output "sqs_app_audit_log_queue_id" { description = "SQS audit log queue URL" - value = aws_sqs_queue.app_audit_log_queue.id + value = aws_sqs_queue.audit_log_queue.id } output "sqs_api_audit_log_queue_arn" { @@ -48,8 +48,12 @@ output "sqs_api_audit_log_queue_id" { value = aws_sqs_queue.api_audit_log_queue.id } -output "sqs_audit_log_deadletter_queue_arn" { +output "sqs_app_audit_log_deadletter_queue_arn" { description = "Audit Log queues dead-letter queue ARN" value = aws_sqs_queue.audit_log_deadletter_queue.arn } +output "sqs_api_audit_log_deadletter_queue_arn" { + description = "Audit Log queues dead-letter queue ARN" + value = aws_sqs_queue.api_audit_log_deadletter_queue.arn +} diff --git a/aws/sqs/sqs.tf b/aws/sqs/sqs.tf index c277b1f32..694d59ab9 100644 --- a/aws/sqs/sqs.tf +++ b/aws/sqs/sqs.tf @@ -61,8 +61,8 @@ resource "aws_sqs_queue" "reprocess_submission_queue" { # App Audit Log Queue -resource "aws_sqs_queue" "app_audit_log_queue" { - name = "app_audit_log_queue" +resource "aws_sqs_queue" "audit_log_queue" { + name = "audit_log_queue" delay_seconds = 0 max_message_size = 262144 message_retention_seconds = 172800 // 2 days @@ -75,18 +75,18 @@ resource "aws_sqs_queue" "app_audit_log_queue" { kms_data_key_reuse_period_seconds = 300 redrive_policy = jsonencode({ - deadLetterTargetArn = aws_sqs_queue.app_audit_log_deadletter_queue.arn + deadLetterTargetArn = aws_sqs_queue.audit_log_deadletter_queue.arn maxReceiveCount = 5 }) redrive_allow_policy = jsonencode({ redrivePermission = "byQueue", - sourceQueueArns = [aws_sqs_queue.app_audit_log_deadletter_queue.arn] + sourceQueueArns = [aws_sqs_queue.audit_log_deadletter_queue.arn] }) } -resource "aws_sqs_queue" "app_audit_log_deadletter_queue" { - name = "app_audit_log_deadletter_queue" +resource "aws_sqs_queue" "audit_log_deadletter_queue" { + name = "audit_log_deadletter_queue" delay_seconds = 60 max_message_size = 262144 message_retention_seconds = 1209600 @@ -99,7 +99,7 @@ resource "aws_sqs_queue" "app_audit_log_deadletter_queue" { # API Audit Log Queue resource "aws_sqs_queue" "api_audit_log_queue" { - name = "audit_log_queue" + name = "api_audit_log_queue" delay_seconds = 0 max_message_size = 262144 message_retention_seconds = 172800 // 2 days diff --git a/env/cloud/alarms/terragrunt.hcl b/env/cloud/alarms/terragrunt.hcl index d7113532a..d1f432db3 100644 --- a/env/cloud/alarms/terragrunt.hcl +++ b/env/cloud/alarms/terragrunt.hcl @@ -66,7 +66,7 @@ dependency "sqs" { mock_outputs_merge_strategy_with_state = "shallow" mock_outputs = { sqs_reliability_deadletter_queue_arn = null - sqs_audit_log_deadletter_queue_arn = null + sqs_app_audit_log_deadletter_queue_arn = null } } @@ -195,7 +195,7 @@ inputs = { lb_api_target_group_arn_suffix = dependency.load_balancer.outputs.lb_target_group_api_arn_suffix sqs_reliability_deadletter_queue_arn = dependency.sqs.outputs.sqs_reliability_deadletter_queue_arn - sqs_audit_log_deadletter_queue_arn = dependency.sqs.outputs.sqs_audit_log_deadletter_queue_arn + sqs_app_audit_log_deadletter_queue_arn = dependency.sqs.outputs.sqs_app_audit_log_deadletter_queue_arn ecs_cloudwatch_log_group_name = dependency.app.outputs.ecs_cloudwatch_log_group_name ecs_cluster_name = dependency.app.outputs.ecs_cluster_name From 41acd37b69c5dd71bbfccd7da8875b4d67a813a3 Mon Sep 17 00:00:00 2001 From: Bryan Robitaille Date: Thu, 12 Sep 2024 10:04:43 -0400 Subject: [PATCH 03/11] Add infra and lambda changes for api audit logs --- aws/alarms/athena.tf | 2 +- aws/alarms/inputs.tf | 2 +- aws/dynamodb/dynamo.tf | 53 ++++++ aws/dynamodb/outputs.tf | 14 +- aws/lambdas/audit_log.tf | 13 +- aws/lambdas/audit_logs_archiver.tf | 2 +- aws/lambdas/iam.tf | 6 +- aws/lambdas/inputs.tf | 19 ++- env/cloud/alarms/terragrunt.hcl | 4 +- env/cloud/lambdas/terragrunt.hcl | 14 +- lambda-code/audit-logs/main.ts | 255 +++++++++++++++++++---------- 11 files changed, 283 insertions(+), 101 deletions(-) diff --git a/aws/alarms/athena.tf b/aws/alarms/athena.tf index 13301266e..14c55490d 100644 --- a/aws/alarms/athena.tf +++ b/aws/alarms/athena.tf @@ -140,7 +140,7 @@ resource "aws_iam_role_policy" "athena_dynamodb_policy" { "dynamodb:Scan", "dynamodb:PartiQLSelect" ], - "Resource" : ["${var.dynamodb_audit_logs_arn}", "${lower(var.dynamodb_audit_logs_arn)}"] + "Resource" : ["${var.dynamodb_app_audit_logs_arn}", "${lower(var.dynamodb_app_audit_logs_arn)}"] "Effect" : "Allow" }, { diff --git a/aws/alarms/inputs.tf b/aws/alarms/inputs.tf index 53bf300d3..664da1ad5 100644 --- a/aws/alarms/inputs.tf +++ b/aws/alarms/inputs.tf @@ -251,7 +251,7 @@ variable "ecr_repository_url_notify_slack_lambda" { type = string } -variable "dynamodb_audit_logs_arn" { +variable "dynamodb_app_audit_logs_arn" { description = "Audit Logs table ARN" type = string } diff --git a/aws/dynamodb/dynamo.tf b/aws/dynamodb/dynamo.tf index d141f7545..3703fb56a 100644 --- a/aws/dynamodb/dynamo.tf +++ b/aws/dynamodb/dynamo.tf @@ -129,3 +129,56 @@ resource "aws_dynamodb_table" "audit_logs" { enabled = var.env == "local" ? false : true } } + +resource "aws_dynamodb_table" "api_audit_logs" { + name = "ApiAuditLogs" + billing_mode = "PAY_PER_REQUEST" + hash_key = "UserID" + range_key = "Event#SubjectID#TimeStamp" + deletion_protection_enabled = true + stream_enabled = false # Can be removed in the future when this gets applied to production + + attribute { + name = "UserID" + type = "S" + } + + attribute { + name = "Event#SubjectID#TimeStamp" + type = "S" + } + + attribute { + name = "TimeStamp" + type = "N" + } + + attribute { + name = "Status" + type = "S" + } + + global_secondary_index { + name = "UserByTime" + hash_key = "UserID" + range_key = "TimeStamp" + projection_type = "KEYS_ONLY" + } + + global_secondary_index { + name = "StatusByTimestamp" + hash_key = "Status" + range_key = "TimeStamp" + projection_type = "ALL" + } + + server_side_encryption { + enabled = true + kms_key_arn = var.kms_key_dynamodb_arn + } + + point_in_time_recovery { + enabled = var.env == "local" ? false : true + } +} + diff --git a/aws/dynamodb/outputs.tf b/aws/dynamodb/outputs.tf index c02132930..cb9375a06 100644 --- a/aws/dynamodb/outputs.tf +++ b/aws/dynamodb/outputs.tf @@ -18,12 +18,22 @@ output "dynamodb_vault_stream_arn" { value = aws_dynamodb_table.vault.stream_arn } -output "dynamodb_audit_logs_arn" { +output "dynamodb_app_audit_logs_arn" { description = "Audit Logs table ARN" value = aws_dynamodb_table.audit_logs.arn } -output "dynamodb_audit_logs_table_name" { +output "dynamodb_app_audit_logs_table_name" { description = "Audit Logs table name" value = aws_dynamodb_table.audit_logs.name +} + +output "dynamodb_api_audit_logs_arn" { + description = "Audit Logs table ARN" + value = aws_dynamodb_table.api_audit_logs.arn +} + +output "dynamodb_api_audit_logs_table_name" { + description = "Audit Logs table name" + value = aws_dynamodb_table.api_audit_logs.name } \ No newline at end of file diff --git a/aws/lambdas/audit_log.tf b/aws/lambdas/audit_log.tf index 6aebd808a..7cda14658 100644 --- a/aws/lambdas/audit_log.tf +++ b/aws/lambdas/audit_log.tf @@ -18,6 +18,8 @@ resource "aws_lambda_function" "audit_logs" { variables = { REGION = var.region LOCALSTACK = var.localstack_hosted + APP_AUDIT_LOGS_SQS_ARN = var.sqs_app_audit_log_queue_arn + API_AUDIT_LOGS_SQS_ARN = var.sqs_api_audit_log_queue_arn } } @@ -31,7 +33,7 @@ resource "aws_lambda_function" "audit_logs" { } } -resource "aws_lambda_event_source_mapping" "audit_logs" { +resource "aws_lambda_event_source_mapping" "app_audit_logs" { event_source_arn = var.sqs_app_audit_log_queue_arn function_name = aws_lambda_function.audit_logs.arn function_response_types = ["ReportBatchItemFailures"] @@ -40,6 +42,15 @@ resource "aws_lambda_event_source_mapping" "audit_logs" { enabled = true } +resource "aws_lambda_event_source_mapping" "api_audit_logs" { + event_source_arn = var.sqs_api_audit_log_queue_arn + function_name = aws_lambda_function.audit_logs.arn + function_response_types = ["ReportBatchItemFailures"] + batch_size = 10 + maximum_batching_window_in_seconds = 30 + enabled = true +} + /* * When implementing containerized Lambda we had to rename some of the functions. * In order to keep existing log groups we decided to hardcode the group name and make the Lambda write to that legacy group. diff --git a/aws/lambdas/audit_logs_archiver.tf b/aws/lambdas/audit_logs_archiver.tf index 856704aa5..b3224b6de 100644 --- a/aws/lambdas/audit_logs_archiver.tf +++ b/aws/lambdas/audit_logs_archiver.tf @@ -13,7 +13,7 @@ resource "aws_lambda_function" "audit_logs_archiver" { variables = { REGION = var.region LOCALSTACK = var.localstack_hosted - AUDIT_LOGS_DYNAMODB_TABLE_NAME = var.dynamodb_audit_logs_table_name + AUDIT_LOGS_DYNAMODB_TABLE_NAME = var.dynamodb_app_audit_logs_table_name AUDIT_LOGS_ARCHIVE_STORAGE_S3_BUCKET = var.audit_logs_archive_storage_id } } diff --git a/aws/lambdas/iam.tf b/aws/lambdas/iam.tf index 306329410..a92bb3607 100644 --- a/aws/lambdas/iam.tf +++ b/aws/lambdas/iam.tf @@ -135,8 +135,10 @@ data "aws_iam_policy_document" "lambda_dynamodb" { var.dynamodb_vault_arn, "${var.dynamodb_vault_arn}/index/*", var.dynamodb_vault_stream_arn, - var.dynamodb_audit_logs_arn, - "${var.dynamodb_audit_logs_arn}/index/*" + var.dynamodb_app_audit_logs_arn, + "${var.dynamodb_app_audit_logs_arn}/index/*", + var.dynamodb_api_audit_logs_arn, + "${var.dynamodb_api_audit_logs_arn}/index/*" ] } } diff --git a/aws/lambdas/inputs.tf b/aws/lambdas/inputs.tf index fad066c80..a2caf2a49 100644 --- a/aws/lambdas/inputs.tf +++ b/aws/lambdas/inputs.tf @@ -49,6 +49,11 @@ variable "sqs_app_audit_log_queue_arn" { type = string } +variable "sqs_api_audit_log_queue_arn" { + description = "SQS Api audit log queue ARN" + type = string +} + variable "kms_key_cloudwatch_arn" { description = "CloudWatch KMS key ARN, used by the ECS task's CloudWatch log group" type = string @@ -79,16 +84,26 @@ variable "dynamodb_relability_queue_arn" { type = string } -variable "dynamodb_audit_logs_arn" { +variable "dynamodb_app_audit_logs_arn" { description = "Audit Logs table ARN" type = string } -variable "dynamodb_audit_logs_table_name" { +variable "dynamodb_app_audit_logs_table_name" { description = "Audit Logs DynamodDB table name" type = string } +variable "dynamodb_api_audit_logs_arn" { + description = "API Audit Logs table ARN" + type = string +} + +variable "dynamodb_api_audit_logs_table_name" { + description = "API Audit Logs DynamodDB table name" + type = string +} + variable "sns_topic_alert_critical_arn" { description = "SNS topic ARN that critical alerts are sent to" type = string diff --git a/env/cloud/alarms/terragrunt.hcl b/env/cloud/alarms/terragrunt.hcl index d1f432db3..cac30895a 100644 --- a/env/cloud/alarms/terragrunt.hcl +++ b/env/cloud/alarms/terragrunt.hcl @@ -163,7 +163,7 @@ dependency "dynamodb" { mock_outputs_allowed_terraform_commands = ["init", "fmt", "validate", "plan", "show"] mock_outputs_merge_strategy_with_state = "shallow" mock_outputs = { - dynamodb_audit_logs_arn = "arn:aws:dynamodb:ca-central-1:123456789012:table/AuditLogs" + dynamodb_app_audit_logs_arn = "arn:aws:dynamodb:ca-central-1:123456789012:table/AuditLogs" } } @@ -239,7 +239,7 @@ inputs = { rds_idp_cluster_identifier = local.feature_flag_idp == "true" ? dependency.idp.outputs.rds_idp_cluster_identifier : "" rds_idp_cpu_maxiumum = 80 - dynamodb_audit_logs_arn = dependency.dynamodb.outputs.dynamodb_audit_logs_arn + dynamodb_app_audit_logs_arn = dependency.dynamodb.outputs.dynamodb_app_audit_logs_arn kms_key_dynamodb_arn = dependency.kms.outputs.kms_key_dynamodb_arn private_subnet_ids = dependency.network.outputs.private_subnet_ids diff --git a/env/cloud/lambdas/terragrunt.hcl b/env/cloud/lambdas/terragrunt.hcl index 83c9bc4cc..3d2283df1 100644 --- a/env/cloud/lambdas/terragrunt.hcl +++ b/env/cloud/lambdas/terragrunt.hcl @@ -66,6 +66,7 @@ dependency "sqs" { sqs_reprocess_submission_queue_arn = null sqs_reliability_dead_letter_queue_id = null sqs_app_audit_log_queue_arn = null + sqs_api_audit_log_queue_arn = null } } @@ -99,8 +100,10 @@ dependency "dynamodb" { dynamodb_vault_arn = "arn:aws:dynamodb:ca-central-1:123456789012:table/Vault" dynamodb_vault_table_name = "Vault" dynamodb_vault_stream_arn = "arn:aws:dynamodb:ca-central-1:123456789012:table/Vault/stream/2023-03-14T15:54:31.086" - dynamodb_audit_logs_table_name = "AuditLogs" - dynamodb_audit_logs_arn = "arn:aws:dynamodb:ca-central-1:123456789012:table/AuditLogs" + dynamodb_app_audit_logs_table_name = "AuditLogs" + dynamodb_app_audit_logs_arn = "arn:aws:dynamodb:ca-central-1:123456789012:table/AuditLogs" + dynamodb_api_audit_logs_table_name = "ApiAuditLogs" + dynamodb_api_audit_logs_arn = "arn:aws:dynamodb:ca-central-1:123456789012:table/ApiAuditLogs" } } @@ -157,8 +160,10 @@ inputs = { dynamodb_vault_arn = dependency.dynamodb.outputs.dynamodb_vault_arn dynamodb_vault_table_name = dependency.dynamodb.outputs.dynamodb_vault_table_name dynamodb_vault_stream_arn = dependency.dynamodb.outputs.dynamodb_vault_stream_arn - dynamodb_audit_logs_table_name = dependency.dynamodb.outputs.dynamodb_audit_logs_table_name - dynamodb_audit_logs_arn = dependency.dynamodb.outputs.dynamodb_audit_logs_arn + dynamodb_app_audit_logs_table_name = dependency.dynamodb.outputs.dynamodb_app_audit_logs_table_name + dynamodb_app_audit_logs_arn = dependency.dynamodb.outputs.dynamodb_app_audit_logs_arn + dynamodb_api_audit_logs_table_name = dependency.dynamodb.outputs.dynamodb_api_audit_logs_table_name + dynamodb_api_audit_logs_arn = dependency.dynamodb.outputs.dynamodb_api_audit_logs_arn kms_key_cloudwatch_arn = dependency.kms.outputs.kms_key_cloudwatch_arn kms_key_dynamodb_arn = dependency.kms.outputs.kms_key_dynamodb_arn @@ -175,6 +180,7 @@ inputs = { sqs_reprocess_submission_queue_arn = dependency.sqs.outputs.sqs_reprocess_submission_queue_arn sqs_reliability_dead_letter_queue_id = dependency.sqs.outputs.sqs_reliability_dead_letter_queue_id sqs_app_audit_log_queue_arn = dependency.sqs.outputs.sqs_app_audit_log_queue_arn + sqs_api_audit_log_queue_arn = dependency.sqs.outputs.sqs_api_audit_log_queue_arn sns_topic_alert_critical_arn = dependency.sns.outputs.sns_topic_alert_critical_arn diff --git a/lambda-code/audit-logs/main.ts b/lambda-code/audit-logs/main.ts index 7bfa3f798..dccedffbe 100644 --- a/lambda-code/audit-logs/main.ts +++ b/lambda-code/audit-logs/main.ts @@ -1,6 +1,7 @@ import { DynamoDBClient } from "@aws-sdk/client-dynamodb"; import { DynamoDBDocumentClient, BatchWriteCommand } from "@aws-sdk/lib-dynamodb"; import { Handler, SQSEvent } from "aws-lambda"; +import type { BatchWriteCommandOutput } from "@aws-sdk/lib-dynamodb"; type LogEvent = { userID: string; @@ -10,6 +11,20 @@ type LogEvent = { description: string; }; +type TransactionRequest = { + PutRequest: { + Item: { + UserID: string; + "Event#SubjectID#TimeStamp": string; + Event: string; + Subject: string; + TimeStamp: string; + description?: string; + Status: string; + }; + }; +}; + const awsProperties = { region: process.env.REGION ?? "ca-central-1", ...(process.env.LOCALSTACK === "true" && { @@ -17,6 +32,9 @@ const awsProperties = { }), }; +const AppAuditLogArn = process.env.APP_AUDIT_LOGS_SQS_ARN; +const ApiAuditLogArn = process.env.API_AUDIT_LOGS_SQS_ARN; + const warnOnEvents = [ // Form Events "GrantFormAccess", @@ -49,89 +67,62 @@ const notifyOnEvent = async (logEvents: Array) => { ); }; -export const handler: Handler = async (event: SQSEvent) => { - try { - const logEvents = event.Records.map((record) => ({ - messageId: record.messageId, - logEvent: JSON.parse(record.body) as LogEvent, - })); +const detectAndRemoveDuplicateEvents = (transactionItems: TransactionRequest[]) => { + const uniqueTransactionItems = [ + ...new Map( + transactionItems.map((v) => [ + JSON.stringify([v.PutRequest.Item.UserID, v.PutRequest.Item["Event#SubjectID#TimeStamp"]]), + v, + ]) + ).values(), + ]; - // Warn on events that should be notified - await notifyOnEvent(logEvents.map((event) => event.logEvent)); - - const putTransactionItems = logEvents.map(({ logEvent }) => ({ - PutRequest: { - Item: { - UserID: logEvent.userID, - "Event#SubjectID#TimeStamp": `${logEvent.event}#${ - logEvent.subject.id ?? logEvent.subject.type - }#${logEvent.timestamp}`, - Event: logEvent.event, - Subject: `${logEvent.subject.type}${ - logEvent.subject.id ? `#${logEvent.subject.id}` : "" - }`, - TimeStamp: logEvent.timestamp, - ...(logEvent.description && { Description: logEvent.description }), - Status: "Archivable", - }, - }, - })); + if (transactionItems.length !== uniqueTransactionItems.length) { + // Find duplicated items that were removed - const uniquePutTransactionItems = [ - ...new Map( - putTransactionItems.map((v) => [ - JSON.stringify([ - v.PutRequest.Item.UserID, - v.PutRequest.Item["Event#SubjectID#TimeStamp"], - ]), - v, - ]) - ).values(), - ]; - - if (putTransactionItems.length !== uniquePutTransactionItems.length) { - // Find duplicated items that were removed - - // clone array so the original is not modified - const clonedPutTransactionItems = [...putTransactionItems]; - - uniquePutTransactionItems.forEach((u) => { - const itemIndex = clonedPutTransactionItems.findIndex( - (o) => - o.PutRequest.Item.UserID === u.PutRequest.Item.UserID && - o.PutRequest.Item["Event#SubjectID#TimeStamp"] === - u.PutRequest.Item["Event#SubjectID#TimeStamp"] - ); - // If it exists, remove it from the cloned array so we are only left with duplicates - if (itemIndex >= 0) { - clonedPutTransactionItems.splice(itemIndex, 1); - } - }); + // clone array so the original is not modified + const clonedPutTransactionItems = [...transactionItems]; - console.warn( - JSON.stringify({ - level: "warn", - severity: 3, - msg: `Duplicate log events were detected and removed. List of duplicate events: ${JSON.stringify( - clonedPutTransactionItems - )}`, - }) + uniqueTransactionItems.forEach((u) => { + const itemIndex = clonedPutTransactionItems.findIndex( + (o) => + o.PutRequest.Item.UserID === u.PutRequest.Item.UserID && + o.PutRequest.Item["Event#SubjectID#TimeStamp"] === + u.PutRequest.Item["Event#SubjectID#TimeStamp"] ); - } - - const dynamoDb = DynamoDBDocumentClient.from(new DynamoDBClient(awsProperties)); + // If it exists, remove it from the cloned array so we are only left with duplicates + if (itemIndex >= 0) { + clonedPutTransactionItems.splice(itemIndex, 1); + } + }); - const { UnprocessedItems: { AuditLogs = [], ...UnprocessedItems } = {} } = await dynamoDb.send( - new BatchWriteCommand({ - RequestItems: { - AuditLogs: uniquePutTransactionItems, - }, + console.warn( + JSON.stringify({ + level: "warn", + severity: 3, + msg: `Duplicate log events were detected and removed. List of duplicate events: ${JSON.stringify( + clonedPutTransactionItems + )}`, }) ); + } + + return uniqueTransactionItems; +}; + +const detectUnprocessedItems = ( + unprocessedItems: BatchWriteCommandOutput["UnprocessedItems"], + logEvents: Array<{ logEvent: LogEvent; messageId: string }> +) => { + let batchItemFailures: { itemIdentifier: string }[] = []; - if (AuditLogs.length > 0) { - console.log(AuditLogs); - const unprocessedIDs = AuditLogs.map(({ PutRequest: { Item } = {} }, index) => { + // If there are no unprocessed items, return empty array + if (!unprocessedItems) return batchItemFailures; + + for (const [tableName, items] of Object.entries(unprocessedItems)) { + if (items.length > 0) { + console.log(items); + const unprocessedIDs = items.map(({ PutRequest: { Item } = {} }, index) => { // Find the original LogEvent item that has the messageID const unprocessedItem = logEvents.filter( ({ logEvent }) => @@ -142,8 +133,8 @@ export const handler: Handler = async (event: SQSEvent) => { if (!unprocessedItem) throw new Error( - `Unprocessed LogEvent could not be found. ${JSON.stringify( - AuditLogs[index] + `Unprocessed ${tableName} Event could not be found. ${JSON.stringify( + items[index] )} not found.` ); @@ -155,19 +146,113 @@ export const handler: Handler = async (event: SQSEvent) => { level: "error", severity: 2, msg: `Failed to process ${ - unprocessedIDs.length - } log events. List of unprocessed IDs: ${unprocessedIDs.join(",")}.`, + items.length + } ${tableName} events. List of unprocessed items: ${JSON.stringify(items)}`, }) ); + batchItemFailures.push(...unprocessedIDs.map((id) => ({ itemIdentifier: id }))); + } + } + return { batchItemFailures }; +}; - return { - batchItemFailures: unprocessedIDs.map((id) => ({ itemIdentifier: id })), +const buildTransactionItems = ( + logEvents: Array<{ + logEvent: LogEvent; + messageId: string; + eventSourceARN: string; + }> +) => { + const { apiAuditLogTransactions, appAuditLogTransactions } = logEvents.reduce( + ( + acc: { + appAuditLogTransactions: TransactionRequest[]; + apiAuditLogTransactions: TransactionRequest[]; + }, + { logEvent, eventSourceARN } + ) => { + const item = { + PutRequest: { + Item: { + UserID: logEvent.userID, + "Event#SubjectID#TimeStamp": `${logEvent.event}#${ + logEvent.subject.id ?? logEvent.subject.type + }#${logEvent.timestamp}`, + Event: logEvent.event, + Subject: `${logEvent.subject.type}${ + logEvent.subject.id ? `#${logEvent.subject.id}` : "" + }`, + TimeStamp: logEvent.timestamp, + ...(logEvent.description && { Description: logEvent.description }), + Status: "Archivable", + }, + }, }; - } + switch (eventSourceARN) { + case AppAuditLogArn: + acc.appAuditLogTransactions.push(item); + return acc; + case ApiAuditLogArn: + acc.apiAuditLogTransactions.push(item); + return acc; + default: + throw new Error(`Unknown event source ARN: ${eventSourceARN}`); + } + }, + { appAuditLogTransactions: [], apiAuditLogTransactions: [] } + ); + return { + apiAuditLogTransactions: + apiAuditLogTransactions.length > 0 ? apiAuditLogTransactions : undefined, + appAuditLogTransactions: + appAuditLogTransactions.length > 0 ? appAuditLogTransactions : undefined, + }; +}; - return { - batchItemFailures: [], - }; +export const handler: Handler = async (event: SQSEvent) => { + try { + const logEvents = event.Records.map((record) => ({ + messageId: record.messageId, + eventSourceARN: record.eventSourceARN, + logEvent: JSON.parse(record.body) as LogEvent, + })); + + logEvents.forEach((log) => { + console.info( + `Event ARN ${log.eventSourceARN === ApiAuditLogArn ? "API" : "APP"} / Event: ${ + log.logEvent.event + }` + ); + }); + + // Warn on events that should be notified + await notifyOnEvent( + logEvents + .filter((event) => event.eventSourceARN === AppAuditLogArn) + .map((event) => event.logEvent) + ); + + const { apiAuditLogTransactions, appAuditLogTransactions } = buildTransactionItems(logEvents); + + const dynamoDb = DynamoDBDocumentClient.from(new DynamoDBClient(awsProperties)); + + // Handle Unprocessed Items + // Start here tomorrow + + const { UnprocessedItems } = await dynamoDb.send( + new BatchWriteCommand({ + RequestItems: { + ...(appAuditLogTransactions && { + AuditLogs: detectAndRemoveDuplicateEvents(appAuditLogTransactions), + }), + ...(apiAuditLogTransactions && { + ApiAuditLogs: detectAndRemoveDuplicateEvents(apiAuditLogTransactions), + }), + }, + }) + ); + + return detectUnprocessedItems(UnprocessedItems, logEvents); } catch (error) { // Catastrophic Error - Fail whole batch -- Error Message will be sent to slack console.error( From fcc928bd6d2a835d74861249f990d99eea359ca9 Mon Sep 17 00:00:00 2001 From: Bryan Robitaille Date: Fri, 13 Sep 2024 09:12:00 -0400 Subject: [PATCH 04/11] use consistent naming convention for userId --- lambda-code/audit-logs/main.ts | 46 +++++++++++++--------------------- 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/lambda-code/audit-logs/main.ts b/lambda-code/audit-logs/main.ts index dccedffbe..5e33c2e55 100644 --- a/lambda-code/audit-logs/main.ts +++ b/lambda-code/audit-logs/main.ts @@ -4,7 +4,7 @@ import { Handler, SQSEvent } from "aws-lambda"; import type { BatchWriteCommandOutput } from "@aws-sdk/lib-dynamodb"; type LogEvent = { - userID: string; + userId: string; event: string; timestamp: string; subject: { type: string; id?: string }; @@ -59,7 +59,7 @@ const notifyOnEvent = async (logEvents: Array) => { console.warn( JSON.stringify({ level: "warn", - msg: `User ${logEvent.userID} performed ${logEvent.event} on ${logEvent.subject?.type} ${ + msg: `User ${logEvent.userId} performed ${logEvent.event} on ${logEvent.subject?.type} ${ logEvent.subject.id ?? `with id ${logEvent.subject.id}.` }${logEvent.description ? "\n".concat(logEvent.description) : ""}`, }) @@ -126,7 +126,7 @@ const detectUnprocessedItems = ( // Find the original LogEvent item that has the messageID const unprocessedItem = logEvents.filter( ({ logEvent }) => - logEvent.userID === Item?.UserID && + logEvent.userId === Item?.UserID && logEvent.event === Item?.Event && logEvent.timestamp === Item?.TimeStamp )[0]; @@ -163,7 +163,7 @@ const buildTransactionItems = ( eventSourceARN: string; }> ) => { - const { apiAuditLogTransactions, appAuditLogTransactions } = logEvents.reduce( + return logEvents.reduce( ( acc: { appAuditLogTransactions: TransactionRequest[]; @@ -174,7 +174,7 @@ const buildTransactionItems = ( const item = { PutRequest: { Item: { - UserID: logEvent.userID, + UserID: logEvent.userId, "Event#SubjectID#TimeStamp": `${logEvent.event}#${ logEvent.subject.id ?? logEvent.subject.type }#${logEvent.timestamp}`, @@ -201,31 +201,24 @@ const buildTransactionItems = ( }, { appAuditLogTransactions: [], apiAuditLogTransactions: [] } ); - return { - apiAuditLogTransactions: - apiAuditLogTransactions.length > 0 ? apiAuditLogTransactions : undefined, - appAuditLogTransactions: - appAuditLogTransactions.length > 0 ? appAuditLogTransactions : undefined, - }; }; export const handler: Handler = async (event: SQSEvent) => { try { - const logEvents = event.Records.map((record) => ({ - messageId: record.messageId, - eventSourceARN: record.eventSourceARN, - logEvent: JSON.parse(record.body) as LogEvent, - })); + const logEvents = event.Records.map((record) => { + const logEvent = JSON.parse(record.body); + // App currently does not use userId, but userID + // Opening an issue to correct + logEvent.userId = logEvent.userId ?? logEvent.userID; - logEvents.forEach((log) => { - console.info( - `Event ARN ${log.eventSourceARN === ApiAuditLogArn ? "API" : "APP"} / Event: ${ - log.logEvent.event - }` - ); + return { + messageId: record.messageId, + eventSourceARN: record.eventSourceARN, + logEvent: logEvent as LogEvent, + }; }); - // Warn on events that should be notified + // Warn on App events that should be notified await notifyOnEvent( logEvents .filter((event) => event.eventSourceARN === AppAuditLogArn) @@ -236,16 +229,13 @@ export const handler: Handler = async (event: SQSEvent) => { const dynamoDb = DynamoDBDocumentClient.from(new DynamoDBClient(awsProperties)); - // Handle Unprocessed Items - // Start here tomorrow - const { UnprocessedItems } = await dynamoDb.send( new BatchWriteCommand({ RequestItems: { - ...(appAuditLogTransactions && { + ...(appAuditLogTransactions.length && { AuditLogs: detectAndRemoveDuplicateEvents(appAuditLogTransactions), }), - ...(apiAuditLogTransactions && { + ...(apiAuditLogTransactions.length && { ApiAuditLogs: detectAndRemoveDuplicateEvents(apiAuditLogTransactions), }), }, From fd649ed7b112de60242b3430e240327d8c7aab05 Mon Sep 17 00:00:00 2001 From: Bryan Robitaille Date: Fri, 13 Sep 2024 10:07:02 -0400 Subject: [PATCH 05/11] update mock outputs --- env/cloud/app/terragrunt.hcl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/env/cloud/app/terragrunt.hcl b/env/cloud/app/terragrunt.hcl index 4571615d3..da056d455 100644 --- a/env/cloud/app/terragrunt.hcl +++ b/env/cloud/app/terragrunt.hcl @@ -81,10 +81,10 @@ dependency "sqs" { mock_outputs_merge_strategy_with_state = "shallow" mock_outputs_allowed_terraform_commands = ["init", "fmt", "validate", "plan", "show"] mock_outputs = { - sqs_reprocess_submission_queue_arn = null - sqs_app_audit_log_queue_arn = null - sqs_app_audit_log_queue_id = null - sqs_reprocess_submission_queue_id = null + sqs_reprocess_submission_queue_arn = "arn:aws:sqs:ca-central-1:000000000000:reprocess_submission_queue.fifo" + sqs_app_audit_log_queue_arn = "arn:aws:sqs:ca-central-1:000000000000:audit_log_queue" + sqs_app_audit_log_queue_id = "http://sqs.ca-central-1.localhost.localstack.cloud:4566/000000000000/audit_log_queue" + sqs_reprocess_submission_queue_id = "https://localhost.localstack.cloud:4566/000000000000/reprocess_submission_queue.fifo" } } From d2731aeecb1ef42c1d10e8a6b7703c5ed5fc7108 Mon Sep 17 00:00:00 2001 From: Bryan Robitaille Date: Fri, 13 Sep 2024 10:16:26 -0400 Subject: [PATCH 06/11] add missing input on api module --- env/cloud/api/terragrunt.hcl | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/env/cloud/api/terragrunt.hcl b/env/cloud/api/terragrunt.hcl index 80c5cdab4..196668ea6 100644 --- a/env/cloud/api/terragrunt.hcl +++ b/env/cloud/api/terragrunt.hcl @@ -3,7 +3,7 @@ terraform { } dependencies { - paths = ["../kms", "../network", "../dynamodb", "../load_balancer", "../ecr", "../redis", "../s3", "../app", "../secrets", "../rds"] + paths = ["../kms", "../network", "../dynamodb", "../load_balancer", "../ecr", "../redis", "../s3", "../app", "../secrets", "../rds", "../sqs"] } dependency "app" { @@ -102,6 +102,15 @@ dependency "rds" { } } +dependency "sqs" { + config_path = "../sqs" + mock_outputs_allowed_terraform_commands = ["init", "fmt", "validate", "plan", "show"] + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs = { + sqs_api_audit_log_queue_arn = ""arn:aws:sqs:ca-central-1:000000000000:api_audit_log_queue"" + } +} + locals { zitadel_domain = get_env("ZITADEL_PROVIDER", "https://localhost") } @@ -127,6 +136,8 @@ inputs = { freshdesk_api_key_secret_arn = dependency.secrets.outputs.freshdesk_api_key_secret_arn rds_connection_url_secret_arn = dependency.rds.outputs.database_url_secret_arn + + sqs_api_audit_log_queue_arn = dependency.sqs.outputs.sqs_api_audit_log_queue_arn } include { From 976ac74e014db5024b275c8a2156308486160482 Mon Sep 17 00:00:00 2001 From: Bryan Robitaille Date: Fri, 13 Sep 2024 10:28:28 -0400 Subject: [PATCH 07/11] formatting --- aws/lambdas/audit_log.tf | 4 ++-- env/cloud/alarms/terragrunt.hcl | 10 +++++----- env/cloud/api/terragrunt.hcl | 2 +- env/cloud/lambdas/terragrunt.hcl | 16 ++++++++-------- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/aws/lambdas/audit_log.tf b/aws/lambdas/audit_log.tf index 7cda14658..e267d327e 100644 --- a/aws/lambdas/audit_log.tf +++ b/aws/lambdas/audit_log.tf @@ -16,8 +16,8 @@ resource "aws_lambda_function" "audit_logs" { environment { variables = { - REGION = var.region - LOCALSTACK = var.localstack_hosted + REGION = var.region + LOCALSTACK = var.localstack_hosted APP_AUDIT_LOGS_SQS_ARN = var.sqs_app_audit_log_queue_arn API_AUDIT_LOGS_SQS_ARN = var.sqs_api_audit_log_queue_arn } diff --git a/env/cloud/alarms/terragrunt.hcl b/env/cloud/alarms/terragrunt.hcl index cac30895a..a4147463e 100644 --- a/env/cloud/alarms/terragrunt.hcl +++ b/env/cloud/alarms/terragrunt.hcl @@ -65,8 +65,8 @@ dependency "sqs" { mock_outputs_allowed_terraform_commands = ["init", "fmt", "validate", "plan", "show"] mock_outputs_merge_strategy_with_state = "shallow" mock_outputs = { - sqs_reliability_deadletter_queue_arn = null - sqs_app_audit_log_deadletter_queue_arn = null + sqs_reliability_deadletter_queue_arn = null + sqs_app_audit_log_deadletter_queue_arn = null } } @@ -194,8 +194,8 @@ inputs = { lb_target_group_2_arn_suffix = dependency.load_balancer.outputs.lb_target_group_2_arn_suffix lb_api_target_group_arn_suffix = dependency.load_balancer.outputs.lb_target_group_api_arn_suffix - sqs_reliability_deadletter_queue_arn = dependency.sqs.outputs.sqs_reliability_deadletter_queue_arn - sqs_app_audit_log_deadletter_queue_arn = dependency.sqs.outputs.sqs_app_audit_log_deadletter_queue_arn + sqs_reliability_deadletter_queue_arn = dependency.sqs.outputs.sqs_reliability_deadletter_queue_arn + sqs_app_audit_log_deadletter_queue_arn = dependency.sqs.outputs.sqs_app_audit_log_deadletter_queue_arn ecs_cloudwatch_log_group_name = dependency.app.outputs.ecs_cloudwatch_log_group_name ecs_cluster_name = dependency.app.outputs.ecs_cluster_name @@ -240,7 +240,7 @@ inputs = { rds_idp_cpu_maxiumum = 80 dynamodb_app_audit_logs_arn = dependency.dynamodb.outputs.dynamodb_app_audit_logs_arn - kms_key_dynamodb_arn = dependency.kms.outputs.kms_key_dynamodb_arn + kms_key_dynamodb_arn = dependency.kms.outputs.kms_key_dynamodb_arn private_subnet_ids = dependency.network.outputs.private_subnet_ids connector_security_group_id = dependency.network.outputs.connector_security_group_id diff --git a/env/cloud/api/terragrunt.hcl b/env/cloud/api/terragrunt.hcl index 196668ea6..1f0a3ec76 100644 --- a/env/cloud/api/terragrunt.hcl +++ b/env/cloud/api/terragrunt.hcl @@ -107,7 +107,7 @@ dependency "sqs" { mock_outputs_allowed_terraform_commands = ["init", "fmt", "validate", "plan", "show"] mock_outputs_merge_strategy_with_state = "shallow" mock_outputs = { - sqs_api_audit_log_queue_arn = ""arn:aws:sqs:ca-central-1:000000000000:api_audit_log_queue"" + sqs_api_audit_log_queue_arn = "arn:aws:sqs:ca-central-1:000000000000:api_audit_log_queue" } } diff --git a/env/cloud/lambdas/terragrunt.hcl b/env/cloud/lambdas/terragrunt.hcl index 3d2283df1..d8414c9fa 100644 --- a/env/cloud/lambdas/terragrunt.hcl +++ b/env/cloud/lambdas/terragrunt.hcl @@ -96,10 +96,10 @@ dependency "dynamodb" { mock_outputs_allowed_terraform_commands = ["init", "fmt", "validate", "plan", "show"] mock_outputs_merge_strategy_with_state = "shallow" mock_outputs = { - dynamodb_relability_queue_arn = "arn:aws:dynamodb:ca-central-1:123456789012:table/ReliabilityQueue" - dynamodb_vault_arn = "arn:aws:dynamodb:ca-central-1:123456789012:table/Vault" - dynamodb_vault_table_name = "Vault" - dynamodb_vault_stream_arn = "arn:aws:dynamodb:ca-central-1:123456789012:table/Vault/stream/2023-03-14T15:54:31.086" + dynamodb_relability_queue_arn = "arn:aws:dynamodb:ca-central-1:123456789012:table/ReliabilityQueue" + dynamodb_vault_arn = "arn:aws:dynamodb:ca-central-1:123456789012:table/Vault" + dynamodb_vault_table_name = "Vault" + dynamodb_vault_stream_arn = "arn:aws:dynamodb:ca-central-1:123456789012:table/Vault/stream/2023-03-14T15:54:31.086" dynamodb_app_audit_logs_table_name = "AuditLogs" dynamodb_app_audit_logs_arn = "arn:aws:dynamodb:ca-central-1:123456789012:table/AuditLogs" dynamodb_api_audit_logs_table_name = "ApiAuditLogs" @@ -156,10 +156,10 @@ inputs = { lambda_nagware_security_group_id = dependency.network.outputs.lambda_nagware_security_group_id private_subnet_ids = dependency.network.outputs.private_subnet_ids - dynamodb_relability_queue_arn = dependency.dynamodb.outputs.dynamodb_relability_queue_arn - dynamodb_vault_arn = dependency.dynamodb.outputs.dynamodb_vault_arn - dynamodb_vault_table_name = dependency.dynamodb.outputs.dynamodb_vault_table_name - dynamodb_vault_stream_arn = dependency.dynamodb.outputs.dynamodb_vault_stream_arn + dynamodb_relability_queue_arn = dependency.dynamodb.outputs.dynamodb_relability_queue_arn + dynamodb_vault_arn = dependency.dynamodb.outputs.dynamodb_vault_arn + dynamodb_vault_table_name = dependency.dynamodb.outputs.dynamodb_vault_table_name + dynamodb_vault_stream_arn = dependency.dynamodb.outputs.dynamodb_vault_stream_arn dynamodb_app_audit_logs_table_name = dependency.dynamodb.outputs.dynamodb_app_audit_logs_table_name dynamodb_app_audit_logs_arn = dependency.dynamodb.outputs.dynamodb_app_audit_logs_arn dynamodb_api_audit_logs_table_name = dependency.dynamodb.outputs.dynamodb_api_audit_logs_table_name From c0ef7d367f626b35a4d22afc54e901b8f47fe7e2 Mon Sep 17 00:00:00 2001 From: Bryan Robitaille Date: Fri, 13 Sep 2024 10:41:49 -0400 Subject: [PATCH 08/11] add non null defaults to lambda inputs --- env/cloud/lambdas/terragrunt.hcl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/env/cloud/lambdas/terragrunt.hcl b/env/cloud/lambdas/terragrunt.hcl index d8414c9fa..ea93d3fd1 100644 --- a/env/cloud/lambdas/terragrunt.hcl +++ b/env/cloud/lambdas/terragrunt.hcl @@ -61,12 +61,12 @@ dependency "sqs" { mock_outputs_merge_strategy_with_state = "shallow" mock_outputs_allowed_terraform_commands = ["init", "fmt", "validate", "plan", "show"] mock_outputs = { - sqs_reliability_queue_arn = null - sqs_reliability_queue_id = null - sqs_reprocess_submission_queue_arn = null - sqs_reliability_dead_letter_queue_id = null - sqs_app_audit_log_queue_arn = null - sqs_api_audit_log_queue_arn = null + sqs_reliability_queue_arn = "arn:aws:sqs:ca-central-1:000000000000:reliability_queue" + sqs_reliability_queue_id = "https://localhost.localstack.cloud:4566/000000000000/submission_processing.fifo" + sqs_reprocess_submission_queue_arn = "arn:aws:sqs:ca-central-1:000000000000:reprocess_submission_queue.fifo" + sqs_reliability_dead_letter_queue_id = "https://localhost.localstack.cloud:4566/000000000000/reliability_deadletter_queue.fifo" + sqs_app_audit_log_queue_arn = "arn:aws:sqs:ca-central-1:000000000000:audit_log_queue" + sqs_api_audit_log_queue_arn = "arn:aws:sqs:ca-central-1:000000000000:api_audit_log_queue" } } From 8eb466f1c2820ed0ec4df21f35d7f885ee9ef4c4 Mon Sep 17 00:00:00 2001 From: Bryan Robitaille Date: Mon, 16 Sep 2024 09:53:25 -0400 Subject: [PATCH 09/11] update descriptions for dynomodb outputs --- aws/dynamodb/outputs.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/aws/dynamodb/outputs.tf b/aws/dynamodb/outputs.tf index cb9375a06..954b2fed1 100644 --- a/aws/dynamodb/outputs.tf +++ b/aws/dynamodb/outputs.tf @@ -19,21 +19,21 @@ output "dynamodb_vault_stream_arn" { } output "dynamodb_app_audit_logs_arn" { - description = "Audit Logs table ARN" + description = "App Audit Logs table ARN" value = aws_dynamodb_table.audit_logs.arn } output "dynamodb_app_audit_logs_table_name" { - description = "Audit Logs table name" + description = "App Audit Logs table name" value = aws_dynamodb_table.audit_logs.name } output "dynamodb_api_audit_logs_arn" { - description = "Audit Logs table ARN" + description = "API Audit Logs table ARN" value = aws_dynamodb_table.api_audit_logs.arn } output "dynamodb_api_audit_logs_table_name" { - description = "Audit Logs table name" + description = "API Audit Logs table name" value = aws_dynamodb_table.api_audit_logs.name } \ No newline at end of file From 9ab47e14482e3e53ebdae43a8ce0d6ade60baa5f Mon Sep 17 00:00:00 2001 From: Bryan Robitaille Date: Mon, 16 Sep 2024 10:01:45 -0400 Subject: [PATCH 10/11] Add alarm on api audit log deadletter queue --- aws/alarms/cloudwatch_api.tf | 51 +++++++++++++++++++++++++++++++++ aws/alarms/inputs.tf | 5 ++++ aws/sqs/outputs.tf | 6 ++-- env/cloud/alarms/terragrunt.hcl | 6 ++-- 4 files changed, 63 insertions(+), 5 deletions(-) diff --git a/aws/alarms/cloudwatch_api.tf b/aws/alarms/cloudwatch_api.tf index 85ac8e013..c784c8205 100644 --- a/aws/alarms/cloudwatch_api.tf +++ b/aws/alarms/cloudwatch_api.tf @@ -110,3 +110,54 @@ resource "aws_cloudwatch_metric_alarm" "api_response_time_warn" { } } } + +# +# Audit Log Dead Letter Queue +# +resource "aws_cloudwatch_metric_alarm" "api_audit_log_dead_letter_queue_warn" { + alarm_name = "ApiAuditLogDeadLetterQueueWarn" + comparison_operator = "GreaterThanThreshold" + evaluation_periods = "1" + threshold = "0" + alarm_description = "Detect when a message is sent to the API Audit Log Dead Letter Queue" + alarm_actions = [var.sns_topic_alert_warning_arn] + + metric_query { + id = "e1" + expression = "RATE(m2+m1)" + label = "Error Rate" + return_data = "true" + } + + metric_query { + id = "m1" + + metric { + metric_name = "ApproximateNumberOfMessagesVisible" + namespace = "AWS/SQS" + period = "60" + stat = "Sum" + unit = "Count" + + dimensions = { + QueueName = var.sqs_api_audit_log_deadletter_queue_arn + } + } + } + + metric_query { + id = "m2" + + metric { + metric_name = "ApproximateNumberOfMessagesNotVisible" + namespace = "AWS/SQS" + period = "60" + stat = "Sum" + unit = "Count" + + dimensions = { + QueueName = var.sqs_api_audit_log_deadletter_queue_arn + } + } + } +} \ No newline at end of file diff --git a/aws/alarms/inputs.tf b/aws/alarms/inputs.tf index 664da1ad5..31dab89d0 100644 --- a/aws/alarms/inputs.tf +++ b/aws/alarms/inputs.tf @@ -206,6 +206,11 @@ variable "sqs_app_audit_log_deadletter_queue_arn" { type = string } +variable "sqs_api_audit_log_deadletter_queue_arn" { + description = "ARN of the API Audit Log queue's SQS Dead Letter Queue" + type = string +} + variable "threshold_ecs_cpu_utilization_high" { description = "ECS cluster CPU average use threshold, above which an alarm is triggered (4 minute period)" type = string diff --git a/aws/sqs/outputs.tf b/aws/sqs/outputs.tf index 4bb0ee227..299c5ed2d 100644 --- a/aws/sqs/outputs.tf +++ b/aws/sqs/outputs.tf @@ -39,12 +39,12 @@ output "sqs_app_audit_log_queue_id" { } output "sqs_api_audit_log_queue_arn" { - description = "SQS audit log queue ARN" + description = "SQS API audit log queue ARN" value = aws_sqs_queue.api_audit_log_queue.arn } output "sqs_api_audit_log_queue_id" { - description = "SQS audit log queue URL" + description = "SQS API audit log queue URL" value = aws_sqs_queue.api_audit_log_queue.id } @@ -54,6 +54,6 @@ output "sqs_app_audit_log_deadletter_queue_arn" { } output "sqs_api_audit_log_deadletter_queue_arn" { - description = "Audit Log queues dead-letter queue ARN" + description = "API Audit Log queues dead-letter queue ARN" value = aws_sqs_queue.api_audit_log_deadletter_queue.arn } diff --git a/env/cloud/alarms/terragrunt.hcl b/env/cloud/alarms/terragrunt.hcl index a4147463e..643e5f853 100644 --- a/env/cloud/alarms/terragrunt.hcl +++ b/env/cloud/alarms/terragrunt.hcl @@ -65,8 +65,9 @@ dependency "sqs" { mock_outputs_allowed_terraform_commands = ["init", "fmt", "validate", "plan", "show"] mock_outputs_merge_strategy_with_state = "shallow" mock_outputs = { - sqs_reliability_deadletter_queue_arn = null - sqs_app_audit_log_deadletter_queue_arn = null + sqs_reliability_deadletter_queue_arn = "arn:aws:sqs:ca-central-1:000000000000:reliability_deadletter_queue.fifo" + sqs_app_audit_log_deadletter_queue_arn ="arn:aws:sqs:ca-central-1:000000000000:audit_log_deadletter_queue" + sqs_api_audit_log_deadletter_queue_arn = "arn:aws:sqs:ca-central-1:000000000000:api_audit_log_deadletter_queue" } } @@ -196,6 +197,7 @@ inputs = { sqs_reliability_deadletter_queue_arn = dependency.sqs.outputs.sqs_reliability_deadletter_queue_arn sqs_app_audit_log_deadletter_queue_arn = dependency.sqs.outputs.sqs_app_audit_log_deadletter_queue_arn + sqs_api_audit_log_deadletter_queue_arn = dependency.sqs.outputs.sqs_api_audit_log_deadletter_queue_arn ecs_cloudwatch_log_group_name = dependency.app.outputs.ecs_cloudwatch_log_group_name ecs_cluster_name = dependency.app.outputs.ecs_cluster_name From a79f5e3e857980b098693caa5964aed35bdff435 Mon Sep 17 00:00:00 2001 From: Bryan Robitaille Date: Mon, 16 Sep 2024 10:10:35 -0400 Subject: [PATCH 11/11] formatting --- aws/alarms/inputs.tf | 2 +- env/cloud/alarms/terragrunt.hcl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aws/alarms/inputs.tf b/aws/alarms/inputs.tf index 31dab89d0..2276afc6a 100644 --- a/aws/alarms/inputs.tf +++ b/aws/alarms/inputs.tf @@ -208,7 +208,7 @@ variable "sqs_app_audit_log_deadletter_queue_arn" { variable "sqs_api_audit_log_deadletter_queue_arn" { description = "ARN of the API Audit Log queue's SQS Dead Letter Queue" - type = string + type = string } variable "threshold_ecs_cpu_utilization_high" { diff --git a/env/cloud/alarms/terragrunt.hcl b/env/cloud/alarms/terragrunt.hcl index 643e5f853..8f3069594 100644 --- a/env/cloud/alarms/terragrunt.hcl +++ b/env/cloud/alarms/terragrunt.hcl @@ -66,7 +66,7 @@ dependency "sqs" { mock_outputs_merge_strategy_with_state = "shallow" mock_outputs = { sqs_reliability_deadletter_queue_arn = "arn:aws:sqs:ca-central-1:000000000000:reliability_deadletter_queue.fifo" - sqs_app_audit_log_deadletter_queue_arn ="arn:aws:sqs:ca-central-1:000000000000:audit_log_deadletter_queue" + sqs_app_audit_log_deadletter_queue_arn = "arn:aws:sqs:ca-central-1:000000000000:audit_log_deadletter_queue" sqs_api_audit_log_deadletter_queue_arn = "arn:aws:sqs:ca-central-1:000000000000:api_audit_log_deadletter_queue" } }