Merge pull request #104 from Sage-Bionetworks/etl-610

[ETL-610] Add SQS queue for input to raw buckets processing
Sage-Bionetworks · Feb 9, 2024 · 33ea17e · 33ea17e
2 parents 68a0b99 + cef6b08
commit 33ea17e
Show file tree

Hide file tree

Showing 10 changed files with 41 additions and 15 deletions.
diff --git a/config/develop/namespaced/s3-event-config-lambda.yaml b/config/develop/namespaced/s3-event-config-lambda.yaml
@@ -6,12 +6,12 @@ template:
 dependencies:
   - develop/namespaced/s3-event-config-lambda-role.yaml
   - develop/namespaced/s3-to-glue-lambda.yaml
-  - develop/namespaced/sqs-queue.yaml
+  - develop/namespaced/sqs-input-to-intermediate.yaml
 stack_name: '{{ stack_group_config.namespace }}-lambda-S3EventConfig'
 stack_tags: {{ stack_group_config.default_stack_tags }}
 parameters:
   Namespace: {{ stack_group_config.namespace }}
-  S3ToGlueDestinationArn: !stack_output_external "{{ stack_group_config.namespace }}-sqs-S3ToLambda::PrimaryQueueArn"
+  S3ToGlueDestinationArn: !stack_output_external "{{ stack_group_config.namespace }}-sqs-input-to-intermediate::PrimaryQueueArn"
   S3ToGlueDestinationType: "Queue"
   S3EventConfigRoleArn: !stack_output_external "{{ stack_group_config.namespace }}-s3-event-config-lambda-role::RoleArn"
   S3SourceBucketName: {{ stack_group_config.input_bucket_name }}
diff --git a/config/develop/namespaced/s3-to-glue-lambda-role.yaml b/config/develop/namespaced/s3-to-glue-lambda-role.yaml
@@ -2,8 +2,8 @@ template:
   path: s3-to-glue-lambda-role.yaml
 stack_name: '{{ stack_group_config.namespace }}-s3-to-glue-lambda-role'
 dependencies:
-  - develop/namespaced/sqs-queue.yaml
+  - develop/namespaced/sqs-input-to-intermediate.yaml
 parameters:
-  SQSQueueArn: !stack_output_external "{{ stack_group_config.namespace }}-sqs-S3ToLambda::PrimaryQueueArn"
+  SQSQueueArn: !stack_output_external "{{ stack_group_config.namespace }}-sqs-input-to-intermediate::PrimaryQueueArn"
 stack_tags:
   {{ stack_group_config.default_stack_tags }}
diff --git a/config/develop/namespaced/s3-to-glue-lambda.yaml b/config/develop/namespaced/s3-to-glue-lambda.yaml
@@ -6,10 +6,11 @@ template:
 dependencies:
   - develop/namespaced/s3-to-glue-lambda-role.yaml
   - develop/namespaced/glue-workflow.yaml
+  - develop/namespaced/sqs-input-to-intermediate.yaml
 stack_name: '{{ stack_group_config.namespace }}-lambda-S3ToGlue'
 stack_tags: {{ stack_group_config.default_stack_tags }}
 parameters:
-  SQSQueueArn: !stack_output_external "{{ stack_group_config.namespace }}-sqs-S3ToLambda::PrimaryQueueArn"
+  SQSQueueArn: !stack_output_external "{{ stack_group_config.namespace }}-sqs-input-to-intermediate::PrimaryQueueArn"
   S3ToGlueRoleArn: !stack_output_external "{{ stack_group_config.namespace }}-s3-to-glue-lambda-role::RoleArn"
   S3ToJsonWorkflowName: !stack_output_external "{{ stack_group_config.namespace }}-glue-workflow::S3ToJsonWorkflowName"
   LambdaBatchSize: '10'

diff --git a/config/develop/namespaced/sqs-queue.yaml → ...namespaced/sqs-input-to-intermediate.yaml b/config/develop/namespaced/sqs-queue.yaml → ...namespaced/sqs-input-to-intermediate.yaml
@@ -1,12 +1,12 @@
 template:
   path: sqs-queue.yaml
 parameters:
-  MessageRetentionPeriod: '86400'
-  ReceiveMessageWaitTimeSeconds: '20'
-  VisibilityTimeout: '120'
+  MessageRetentionPeriod: "86400"
+  ReceiveMessageWaitTimeSeconds: "20"
+  VisibilityTimeout: "120"
   S3SourceBucketArn: !stack_output_external recover-dev-input-bucket::BucketArn
 dependencies:
   - develop/s3-input-bucket.yaml
-stack_name: '{{ stack_group_config.namespace }}-sqs-S3ToLambda'
+stack_name: '{{ stack_group_config.namespace }}-sqs-input-to-intermediate'
 stack_tags:
   {{ stack_group_config.default_stack_tags }}
diff --git a/config/develop/namespaced/sqs-input-to-raw.yaml b/config/develop/namespaced/sqs-input-to-raw.yaml
@@ -0,0 +1,12 @@
+template:
+  path: sqs-queue.yaml
+parameters:
+  MessageRetentionPeriod: "1209600"
+  ReceiveMessageWaitTimeSeconds: "20"
+  VisibilityTimeout: "120"
+  S3SourceBucketArn: !stack_output_external recover-dev-raw-bucket::BucketArn
+dependencies:
+  - develop/s3-raw-bucket.yaml
+stack_name: '{{ stack_group_config.namespace }}-sqs-input-to-raw'
+stack_tags:
+  {{ stack_group_config.default_stack_tags }}
diff --git a/config/prod/namespaced/s3-event-config-lambda.yaml b/config/prod/namespaced/s3-event-config-lambda.yaml
@@ -6,12 +6,12 @@ template:
 dependencies:
   - prod/namespaced/s3-event-config-lambda-role.yaml
   - prod/namespaced/s3-to-glue-lambda.yaml
-  - prod/namespaced/sqs-queue.yaml
+  - prod/namespaced/sqs-input-to-intermediate.yaml
 stack_name: '{{ stack_group_config.namespace }}-lambda-S3EventConfig'
 stack_tags: {{ stack_group_config.default_stack_tags }}
 parameters:
   Namespace: {{ stack_group_config.namespace }}
-  S3ToGlueDestinationArn: !stack_output_external "{{ stack_group_config.namespace }}-sqs-S3ToLambda::PrimaryQueueArn"
+  S3ToGlueDestinationArn: !stack_output_external "{{ stack_group_config.namespace }}-sqs-input-to-intermediate::PrimaryQueueArn"
   S3ToGlueDestinationType: "Queue"
   S3EventConfigRoleArn: !stack_output_external "{{ stack_group_config.namespace }}-s3-event-config-lambda-role::RoleArn"
   S3SourceBucketName: {{ stack_group_config.input_bucket_name }}
diff --git a/config/prod/namespaced/s3-to-glue-lambda-role.yaml b/config/prod/namespaced/s3-to-glue-lambda-role.yaml
@@ -2,8 +2,8 @@ template:
   path: s3-to-glue-lambda-role.yaml
 stack_name: '{{ stack_group_config.namespace }}-s3-to-glue-lambda-role'
 dependencies:
-  - prod/namespaced/sqs-queue.yaml
+  - prod/namespaced/sqs-input-to-intermediate.yaml
 parameters:
-  SQSQueueArn: !stack_output_external "{{ stack_group_config.namespace }}-sqs-S3ToLambda::PrimaryQueueArn"
+  SQSQueueArn: !stack_output_external "{{ stack_group_config.namespace }}-sqs-input-to-intermediate::PrimaryQueueArn"
 stack_tags:
   {{ stack_group_config.default_stack_tags }}
diff --git a/config/prod/namespaced/s3-to-glue-lambda.yaml b/config/prod/namespaced/s3-to-glue-lambda.yaml
@@ -6,10 +6,11 @@ template:
 dependencies:
   - prod/namespaced/s3-to-glue-lambda-role.yaml
   - prod/namespaced/glue-workflow.yaml
+  - prod/namespaced/sqs-input-to-intermediate.yaml
 stack_name: '{{ stack_group_config.namespace }}-lambda-S3ToGlue'
 stack_tags: {{ stack_group_config.default_stack_tags }}
 parameters:
-  SQSQueueArn: !stack_output_external "{{ stack_group_config.namespace }}-sqs-S3ToLambda::PrimaryQueueArn"
+  SQSQueueArn: !stack_output_external "{{ stack_group_config.namespace }}-sqs-input-to-intermediate::PrimaryQueueArn"
   S3ToGlueRoleArn: !stack_output_external "{{ stack_group_config.namespace }}-s3-to-glue-lambda-role::RoleArn"
   S3ToJsonWorkflowName: !stack_output_external "{{ stack_group_config.namespace }}-glue-workflow::S3ToJsonWorkflowName"
   LambdaBatchSize: '10'

diff --git a/config/prod/namespaced/sqs-queue.yaml → ...namespaced/sqs-input-to-intermediate.yaml b/config/prod/namespaced/sqs-queue.yaml → ...namespaced/sqs-input-to-intermediate.yaml
@@ -7,6 +7,6 @@ parameters:
   S3SourceBucketArn: !stack_output_external recover-input-bucket::BucketArn
 dependencies:
   - prod/s3-input-bucket.yaml
-stack_name: '{{ stack_group_config.namespace }}-sqs-S3ToLambda'
+stack_name: '{{ stack_group_config.namespace }}-sqs-input-to-intermediate'
 stack_tags:
   {{ stack_group_config.default_stack_tags }}
diff --git a/config/prod/namespaced/sqs-input-to-raw.yaml b/config/prod/namespaced/sqs-input-to-raw.yaml
@@ -0,0 +1,12 @@
+template:
+  path: sqs-queue.yaml
+parameters:
+  MessageRetentionPeriod: "1209600"
+  ReceiveMessageWaitTimeSeconds: "20"
+  VisibilityTimeout: "120"
+  S3SourceBucketArn: !stack_output_external recover-raw-bucket::BucketArn
+dependencies:
+  - prod/s3-raw-bucket.yaml
+stack_name: '{{ stack_group_config.namespace }}-sqs-input-to-raw'
+stack_tags:
+  {{ stack_group_config.default_stack_tags }}