From 1594f37e36c45609f3296f7ff928da64b0bab71e Mon Sep 17 00:00:00 2001
From: Rixing Xu <rxu@w197.local>
Date: Wed, 10 May 2023 15:55:37 -0700
Subject: [PATCH 1/7] initial design for new s3-event-config lambda and dep

---
 .../s3-event-config-lambda-role.yaml          |  7 ++
 .../namespaced/s3-event-config-lambda.yaml    | 14 ++++
 .../develop/namespaced/s3-to-glue-lambda.yaml |  3 +-
 src/lambda_function/s3_event_config/app.py    | 67 +++++++++++++++++++
 .../events/test-trigger-event.json            |  3 +
 .../s3_event_config/template.yaml             | 48 +++++++++++++
 .../{ => s3_to_glue}/README.md                |  0
 .../events/generate_test_event.py             |  0
 .../events/test-trigger-event.json            |  0
 .../{ => s3_to_glue}/template.yaml            | 16 ++++-
 templates/s3-event-config-lambda-role.yaml    | 57 ++++++++++++++++
 11 files changed, 213 insertions(+), 2 deletions(-)
 create mode 100644 config/develop/namespaced/s3-event-config-lambda-role.yaml
 create mode 100644 config/develop/namespaced/s3-event-config-lambda.yaml
 create mode 100644 src/lambda_function/s3_event_config/app.py
 create mode 100644 src/lambda_function/s3_event_config/events/test-trigger-event.json
 create mode 100644 src/lambda_function/s3_event_config/template.yaml
 rename src/lambda_function/{ => s3_to_glue}/README.md (100%)
 rename src/lambda_function/{ => s3_to_glue}/events/generate_test_event.py (100%)
 rename src/lambda_function/{ => s3_to_glue}/events/test-trigger-event.json (100%)
 rename src/lambda_function/{ => s3_to_glue}/template.yaml (71%)
 create mode 100644 templates/s3-event-config-lambda-role.yaml

diff --git a/config/develop/namespaced/s3-event-config-lambda-role.yaml b/config/develop/namespaced/s3-event-config-lambda-role.yaml
new file mode 100644
index 00000000..8b611a1b
--- /dev/null
+++ b/config/develop/namespaced/s3-event-config-lambda-role.yaml
@@ -0,0 +1,7 @@
+template:
+  path: s3-event-config-lambda-role.yaml
+stack_name: "{{ stack_group_config.namespace }}-s3-event-config-lambda-role"
+parameters:
+  S3SourceBucketName: {{ stack_group_config.input_bucket_name }}
+stack_tags:
+  {{ stack_group_config.default_stack_tags }}
diff --git a/config/develop/namespaced/s3-event-config-lambda.yaml b/config/develop/namespaced/s3-event-config-lambda.yaml
new file mode 100644
index 00000000..8c800561
--- /dev/null
+++ b/config/develop/namespaced/s3-event-config-lambda.yaml
@@ -0,0 +1,14 @@
+template:
+  type: sam
+  path: src/lambda_function/s3_event_config/template.yaml
+  artifact_bucket_name: {{ stack_group_config.cloudformation_artifact_bucket_name }}
+  artifact_prefix: '{{ stack_group_config.namespace }}/src/lambda'
+dependencies:
+  - develop/namespaced/s3-event-config-lambda-role.yaml
+  - develop/namespaced/s3-to-glue-lambda.yaml
+stack_name: '{{ stack_group_config.namespace }}-lambda-S3EventConfig'
+stack_tags: {{ stack_group_config.default_stack_tags }}
+parameters:
+  S3ToGlueFunctionArn: !stack_output_external "{{ stack_group_config.namespace }}-lambda-S3ToGlue::S3ToGlueFunctionArn"
+  S3EventConfigRoleArn: !stack_output_external "{{ stack_group_config.namespace }}-s3-event-config-lambda-role::RoleArn"
+  S3SourceBucketName: {{ stack_group_config.input_bucket_name }}
diff --git a/config/develop/namespaced/s3-to-glue-lambda.yaml b/config/develop/namespaced/s3-to-glue-lambda.yaml
index 6382cf42..32cafb4e 100644
--- a/config/develop/namespaced/s3-to-glue-lambda.yaml
+++ b/config/develop/namespaced/s3-to-glue-lambda.yaml
@@ -1,6 +1,6 @@
 template:
   type: sam
-  path: src/lambda_function/template.yaml
+  path: src/lambda_function/s3_to_glue/template.yaml
   artifact_bucket_name: {{ stack_group_config.cloudformation_artifact_bucket_name }}
   artifact_prefix: '{{ stack_group_config.namespace }}/src/lambda'
 dependencies:
@@ -9,5 +9,6 @@ dependencies:
 stack_name: '{{ stack_group_config.namespace }}-lambda-S3ToGlue'
 stack_tags: {{ stack_group_config.default_stack_tags }}
 parameters:
+  S3SourceBucketName: {{ stack_group_config.input_bucket_name }}
   S3ToGlueRoleArn: !stack_output_external s3-to-glue-lambda-role::RoleArn
   PrimaryWorkflowName: !stack_output_external "{{ stack_group_config.namespace }}-glue-workflow::WorkflowName"
diff --git a/src/lambda_function/s3_event_config/app.py b/src/lambda_function/s3_event_config/app.py
new file mode 100644
index 00000000..d1a1b08c
--- /dev/null
+++ b/src/lambda_function/s3_event_config/app.py
@@ -0,0 +1,67 @@
+"""
+This Lambda app responds to an S3 event notification and starts a Glue workflow.
+The Glue workflow name is set by the environment variable `PRIMARY_WORKFLOW_NAME`.
+Subsequently, the S3 objects which were contained in the event are written as a
+JSON string to the `messages` workflow run property.
+"""
+import os
+import json
+import logging
+import boto3
+
+logger = logging.getLogger()
+logger.setLevel(logging.INFO)
+
+SUCCESS = "SUCCESS"
+FAILED = "FAILED"
+
+logger.info("Loading function")
+s3 = boto3.resource("s3")
+
+
+def lambda_handler(event, context):
+    logger.info(f"Received event: {json.dumps(event, indent=2)}")
+    try:
+        if event["RequestType"] == "Delete":
+            logger.info(f'Request Type:{event["RequestType"]}')
+            bucket = os.environ["S3_SOURCE_BUCKET_NAME"]
+            delete_notification(bucket)
+            logger.info("Sending response to custom resource after Delete")
+
+        elif event["RequestType"] == "Create" or event["RequestType"] == "Update":
+            logger.info(f'Request Type: {event["RequestType"]}')
+            lambda_arn = os.environ["S3_TO_GLUE_FUNCTION_ARN"]
+            bucket = os.environ["S3_SOURCE_BUCKET_NAME"]
+            add_notification(lambda_arn, bucket)
+            logger.info("Sending response to custom resource")
+    except Exception as e:
+        logger.info(f"Failed to process:{e}")
+
+
+def add_notification(lambda_arn: str, bucket: str) -> None:
+    """Adds the S3 notification configuration to an existing bucket
+
+    Args:
+        lambda_arn (str): Arn of the lambda s3 event config function
+        bucket (str): bucket name of the s3 bucket to add the config to
+    """
+    bucket_notification = s3.BucketNotification(bucket)
+    response = bucket_notification.put(
+        NotificationConfiguration={
+            "LambdaFunctionConfigurations": [
+                {"LambdaFunctionArn": lambda_arn, "Events": ["s3:ObjectCreated:*"]}
+            ]
+        }
+    )
+    logger.info("Put request completed....")
+
+
+def delete_notification(bucket: str) -> None:
+    """Deletes the S3 notification configuration from an existing bucket
+
+    Args:
+        bucket (str): bucket name of the s3 bucket to delete the config in
+    """
+    bucket_notification = s3.BucketNotification(bucket)
+    response = bucket_notification.put(NotificationConfiguration={})
+    logger.info("Delete request completed....")
diff --git a/src/lambda_function/s3_event_config/events/test-trigger-event.json b/src/lambda_function/s3_event_config/events/test-trigger-event.json
new file mode 100644
index 00000000..a6f92aea
--- /dev/null
+++ b/src/lambda_function/s3_event_config/events/test-trigger-event.json
@@ -0,0 +1,3 @@
+{
+    "RequestType":"Create"
+}
diff --git a/src/lambda_function/s3_event_config/template.yaml b/src/lambda_function/s3_event_config/template.yaml
new file mode 100644
index 00000000..0ecfd3fe
--- /dev/null
+++ b/src/lambda_function/s3_event_config/template.yaml
@@ -0,0 +1,48 @@
+AWSTemplateFormatVersion: '2010-09-09'
+
+Transform: AWS::Serverless-2016-10-31
+
+Description: >
+  SAM Template for s3-event-config lambda function
+
+Parameters:
+
+  S3ToGlueFunctionArn:
+    Type: String
+    Description: Arn for the S3 Event Config Lambda Function
+
+  S3EventConfigRoleArn:
+    Type: String
+    Description: Arn for the S3 Event Config Lambda Role
+
+  S3SourceBucketName:
+    Type: String
+    Description: Name of the S3 bucket where source data are stored.
+
+  LambdaPythonVersion:
+    Type: String
+    Description: Python version to use for this lambda function
+    Default: "3.9"
+
+
+Resources:
+  S3EventConfigFunction:
+    Type: AWS::Serverless::Function
+    Properties:
+      PackageType: Zip
+      CodeUri: ./
+      Handler: app.lambda_handler
+      Runtime: !Sub "python${LambdaPythonVersion}"
+      Role: !Ref S3EventConfigRoleArn
+      Timeout: 30
+      Environment:
+        Variables:
+          S3_SOURCE_BUCKET_NAME: !Ref S3SourceBucketName
+          S3_TO_GLUE_FUNCTION_ARN: !Ref S3ToGlueFunctionArn
+
+Outputs:
+  S3EventConfigFunctionArn:
+    Description: Arn of the S3EventConfigFunction function
+    Value: !GetAtt S3EventConfigFunction.Arn
+    Export:
+      Name: !Sub "${AWS::Region}-${AWS::StackName}-S3EventConfigFunctionArn"
diff --git a/src/lambda_function/README.md b/src/lambda_function/s3_to_glue/README.md
similarity index 100%
rename from src/lambda_function/README.md
rename to src/lambda_function/s3_to_glue/README.md
diff --git a/src/lambda_function/events/generate_test_event.py b/src/lambda_function/s3_to_glue/events/generate_test_event.py
similarity index 100%
rename from src/lambda_function/events/generate_test_event.py
rename to src/lambda_function/s3_to_glue/events/generate_test_event.py
diff --git a/src/lambda_function/events/test-trigger-event.json b/src/lambda_function/s3_to_glue/events/test-trigger-event.json
similarity index 100%
rename from src/lambda_function/events/test-trigger-event.json
rename to src/lambda_function/s3_to_glue/events/test-trigger-event.json
diff --git a/src/lambda_function/template.yaml b/src/lambda_function/s3_to_glue/template.yaml
similarity index 71%
rename from src/lambda_function/template.yaml
rename to src/lambda_function/s3_to_glue/template.yaml
index 46238b8f..4acd096e 100644
--- a/src/lambda_function/template.yaml
+++ b/src/lambda_function/s3_to_glue/template.yaml
@@ -7,6 +7,10 @@ Description: >
 
 Parameters:
 
+  S3SourceBucketName:
+    Type: String
+    Description: Name of the S3 bucket where source data are stored.
+
   S3ToGlueRoleArn:
     Type: String
     Description: Arn for the S3 to Glue Lambda Role
@@ -27,7 +31,7 @@ Resources:
     Type: AWS::Serverless::Function
     Properties:
       PackageType: Zip
-      CodeUri: ./s3_to_glue
+      CodeUri: ./
       Handler: app.lambda_handler
       Runtime: !Sub "python${LambdaPythonVersion}"
       Role: !Ref S3ToGlueRoleArn
@@ -36,6 +40,16 @@ Resources:
         Variables:
           PRIMARY_WORKFLOW_NAME: !Ref PrimaryWorkflowName
 
+  LambdaInvokePermission:
+    Type: AWS::Lambda::Permission
+    Properties:
+      FunctionName: !GetAtt S3ToGlueFunction.Arn
+      Action: lambda:InvokeFunction
+      Principal: s3.amazonaws.com
+      SourceAccount: !Ref 'AWS::AccountId'
+      SourceArn: !Sub 'arn:aws:s3:::${S3SourceBucketName}'
+
+
 Outputs:
   S3ToGlueFunctionArn:
     Description: Arn of the S3ToGlueFunction function
diff --git a/templates/s3-event-config-lambda-role.yaml b/templates/s3-event-config-lambda-role.yaml
new file mode 100644
index 00000000..99bb616f
--- /dev/null
+++ b/templates/s3-event-config-lambda-role.yaml
@@ -0,0 +1,57 @@
+AWSTemplateFormatVersion: '2010-09-09'
+
+Transform: AWS::Serverless-2016-10-31
+
+Description: >
+  An IAM Role for the S3 Event Config lambda allowing one to put
+  s3 event notification configuration in the source bucket
+
+Parameters:
+  S3SourceBucketName:
+    Type: String
+    Description: Name of the S3 bucket where source data are stored.
+
+
+Resources:
+  S3EventConfigRole:
+      Type: AWS::IAM::Role
+      Properties:
+        AssumeRolePolicyDocument:
+          Version: '2012-10-17'
+          Statement:
+          - Effect: Allow
+            Principal:
+              Service:
+              - lambda.amazonaws.com
+            Action:
+            - sts:AssumeRole
+        ManagedPolicyArns:
+          - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
+        Policies:
+        - PolicyName: PutS3NotificationConfiguration
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+            - Effect: Allow
+              Action:
+                - 's3:GetBucketNotification'
+                - 's3:PutBucketNotification'
+              Resource:
+              - !Sub arn:aws:s3:::${S3SourceBucketName}
+            - Effect: Allow
+              Action:
+                - 'logs:CreateLogGroup'
+                - 'logs:CreateLogStream'
+                - 'logs:PutLogEvents'
+              Resource: 'arn:aws:logs:*:*:*'
+
+Outputs:
+  RoleName:
+    Value: !Ref S3EventConfigRole
+    Export:
+      Name: !Sub '${AWS::Region}-${AWS::StackName}-RoleName'
+
+  RoleArn:
+    Value: !GetAtt S3EventConfigRole.Arn
+    Export:
+      Name: !Sub '${AWS::Region}-${AWS::StackName}-RoleArn'

From d49ec81cc1ed3633796b9374b9a203861b6c1f22 Mon Sep 17 00:00:00 2001
From: Rixing Xu <rxu@w197.local>
Date: Wed, 10 May 2023 18:20:03 -0700
Subject: [PATCH 2/7] add prod stacks

---
 .../namespaced/s3-event-config-lambda-role.yaml    |  7 +++++++
 config/prod/namespaced/s3-event-config-lambda.yaml | 14 ++++++++++++++
 2 files changed, 21 insertions(+)
 create mode 100644 config/prod/namespaced/s3-event-config-lambda-role.yaml
 create mode 100644 config/prod/namespaced/s3-event-config-lambda.yaml

diff --git a/config/prod/namespaced/s3-event-config-lambda-role.yaml b/config/prod/namespaced/s3-event-config-lambda-role.yaml
new file mode 100644
index 00000000..8b611a1b
--- /dev/null
+++ b/config/prod/namespaced/s3-event-config-lambda-role.yaml
@@ -0,0 +1,7 @@
+template:
+  path: s3-event-config-lambda-role.yaml
+stack_name: "{{ stack_group_config.namespace }}-s3-event-config-lambda-role"
+parameters:
+  S3SourceBucketName: {{ stack_group_config.input_bucket_name }}
+stack_tags:
+  {{ stack_group_config.default_stack_tags }}
diff --git a/config/prod/namespaced/s3-event-config-lambda.yaml b/config/prod/namespaced/s3-event-config-lambda.yaml
new file mode 100644
index 00000000..52fcd07f
--- /dev/null
+++ b/config/prod/namespaced/s3-event-config-lambda.yaml
@@ -0,0 +1,14 @@
+template:
+  type: sam
+  path: src/lambda_function/s3_event_config/template.yaml
+  artifact_bucket_name: {{ stack_group_config.cloudformation_artifact_bucket_name }}
+  artifact_prefix: '{{ stack_group_config.namespace }}/src/lambda'
+dependencies:
+  - prod/namespaced/s3-event-config-lambda-role.yaml
+  - prod/namespaced/s3-to-glue-lambda.yaml
+stack_name: '{{ stack_group_config.namespace }}-lambda-S3EventConfig'
+stack_tags: {{ stack_group_config.default_stack_tags }}
+parameters:
+  S3ToGlueFunctionArn: !stack_output_external "{{ stack_group_config.namespace }}-lambda-S3ToGlue::S3ToGlueFunctionArn"
+  S3EventConfigRoleArn: !stack_output_external "{{ stack_group_config.namespace }}-s3-event-config-lambda-role::RoleArn"
+  S3SourceBucketName: {{ stack_group_config.input_bucket_name }}

From 8032add765c68718ac80286b0e6964b7fe7dc178 Mon Sep 17 00:00:00 2001
From: Rixing Xu <rxu@w197.local>
Date: Wed, 10 May 2023 18:20:16 -0700
Subject: [PATCH 3/7] add prod stacks

---
 config/prod/namespaced/s3-to-glue-lambda.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/config/prod/namespaced/s3-to-glue-lambda.yaml b/config/prod/namespaced/s3-to-glue-lambda.yaml
index 231530fb..79e3c384 100644
--- a/config/prod/namespaced/s3-to-glue-lambda.yaml
+++ b/config/prod/namespaced/s3-to-glue-lambda.yaml
@@ -1,6 +1,6 @@
 template:
   type: sam
-  path: src/lambda_function/template.yaml
+  path: src/lambda_function/s3_to_glue/template.yaml
   artifact_bucket_name: {{ stack_group_config.cloudformation_artifact_bucket_name }}
   artifact_prefix: '{{ stack_group_config.namespace }}/src/lambda'
 dependencies:
@@ -9,5 +9,6 @@ dependencies:
 stack_name: '{{ stack_group_config.namespace }}-lambda-S3ToGlue'
 stack_tags: {{ stack_group_config.default_stack_tags }}
 parameters:
+  S3SourceBucketName: {{ stack_group_config.input_bucket_name }}
   S3ToGlueRoleArn: !stack_output_external s3-to-glue-lambda-role::RoleArn
   PrimaryWorkflowName: !stack_output_external "{{ stack_group_config.namespace }}-glue-workflow::WorkflowName"

From a152f6a10f2f4a47b24cb8679ff085be9f88af3d Mon Sep 17 00:00:00 2001
From: Rixing Xu <rxu@w197.local>
Date: Thu, 11 May 2023 11:29:26 -0700
Subject: [PATCH 4/7] add error catching, update docs due to lambda structure
 changes

---
 src/lambda_function/s3_event_config/README.md | 42 +++++++++++++
 .../s3_event_config/__init__.py               |  0
 src/lambda_function/s3_event_config/app.py    | 59 ++++++++++---------
 .../events/test-trigger-event.json            |  3 -
 src/lambda_function/s3_to_glue/README.md      |  8 +--
 .../{ => s3_to_glue}/test-env-vars.json       |  0
 6 files changed, 77 insertions(+), 35 deletions(-)
 create mode 100644 src/lambda_function/s3_event_config/README.md
 create mode 100644 src/lambda_function/s3_event_config/__init__.py
 delete mode 100644 src/lambda_function/s3_event_config/events/test-trigger-event.json
 rename src/lambda_function/{ => s3_to_glue}/test-env-vars.json (100%)

diff --git a/src/lambda_function/s3_event_config/README.md b/src/lambda_function/s3_event_config/README.md
new file mode 100644
index 00000000..9d15d82f
--- /dev/null
+++ b/src/lambda_function/s3_event_config/README.md
@@ -0,0 +1,42 @@
+# s3_event_config lambda
+
+The s3_event_config lambda is triggered by a github action during deployment or manually through the AWS console.
+
+It will then put a S3 event notification configuration into
+the input data bucket which allows the input data bucket to
+trigger the S3 to JSON lambda with S3 new object notifications whenever new objects are added
+to it and eventually lead to the start of the S3-to-JSON workflow.
+
+## Event format
+
+The events that will trigger the s3-event-config-lambda
+should be in the form of something like:
+
+```
+{
+    "RequestType": "Create"
+}
+```
+
+Where the allowed RequestType values are:
+- "Create"
+- "Update"
+- "Delete"
+
+## Launching Lambda stack in AWS
+
+There are two main stacks involved in the s3_event_config lambda. They are the
+`s3_event_config lambda role` stack and the `s3_event_config lambda` stack.
+
+Note that they depend on the `s3 to json` lambda stacks.
+
+### Sceptre
+
+#### Launching in development
+
+Run the following command to create the lambda stack in your AWS account. Note this will
+also create the lambda s3 to glue IAM role stack as well:
+
+```shell script
+sceptre --var namespace='test-namespace' launch develop/namespaced/s3-event-config-lambda.yaml
+```
diff --git a/src/lambda_function/s3_event_config/__init__.py b/src/lambda_function/s3_event_config/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src/lambda_function/s3_event_config/app.py b/src/lambda_function/s3_event_config/app.py
index d1a1b08c..15e45af9 100644
--- a/src/lambda_function/s3_event_config/app.py
+++ b/src/lambda_function/s3_event_config/app.py
@@ -1,51 +1,53 @@
 """
-This Lambda app responds to an S3 event notification and starts a Glue workflow.
-The Glue workflow name is set by the environment variable `PRIMARY_WORKFLOW_NAME`.
-Subsequently, the S3 objects which were contained in the event are written as a
-JSON string to the `messages` workflow run property.
+This Lambda app responds to an external trigger (usually github action or aws console) and
+puts a s3 event notification configuration for the S3 to Glue lambda in the
+input data S3 bucket set by the environment variable `S3_SOURCE_BUCKET_NAME`.
+
+This Lambda app also has the option of deleting the notification configuration
+from an S3 bucket
 """
 import os
 import json
 import logging
+
 import boto3
 
 logger = logging.getLogger()
 logger.setLevel(logging.INFO)
 
-SUCCESS = "SUCCESS"
-FAILED = "FAILED"
-
-logger.info("Loading function")
-s3 = boto3.resource("s3")
+REQUEST_TYPE_VALS = ["Delete", "Create", "Update"]
 
 
 def lambda_handler(event, context):
+    s3 = boto3.resource("s3")
     logger.info(f"Received event: {json.dumps(event, indent=2)}")
-    try:
-        if event["RequestType"] == "Delete":
-            logger.info(f'Request Type:{event["RequestType"]}')
-            bucket = os.environ["S3_SOURCE_BUCKET_NAME"]
-            delete_notification(bucket)
-            logger.info("Sending response to custom resource after Delete")
-
-        elif event["RequestType"] == "Create" or event["RequestType"] == "Update":
-            logger.info(f'Request Type: {event["RequestType"]}')
-            lambda_arn = os.environ["S3_TO_GLUE_FUNCTION_ARN"]
-            bucket = os.environ["S3_SOURCE_BUCKET_NAME"]
-            add_notification(lambda_arn, bucket)
-            logger.info("Sending response to custom resource")
-    except Exception as e:
-        logger.info(f"Failed to process:{e}")
+    if event["RequestType"] == "Delete":
+        logger.info(f'Request Type:{event["RequestType"]}')
+        bucket = os.environ["S3_SOURCE_BUCKET_NAME"]
+        delete_notification(s3, bucket)
+        logger.info("Sending response to custom resource after Delete")
+    elif event["RequestType"] in ["Update", "Create"]:
+        logger.info(f'Request Type: {event["RequestType"]}')
+        lambda_arn = os.environ["S3_TO_GLUE_FUNCTION_ARN"]
+        bucket = os.environ["S3_SOURCE_BUCKET_NAME"]
+        add_notification(s3, lambda_arn, bucket)
+        logger.info("Sending response to custom resource")
+    else:
+        err_msg =  f"The 'RequestType' key should have one of the following values: {REQUEST_TYPE_VALS}"
+        raise KeyError(err_msg)
 
 
-def add_notification(lambda_arn: str, bucket: str) -> None:
+def add_notification(
+    s3_resource: boto3.resources.base.ServiceResource, lambda_arn: str, bucket: str
+):
     """Adds the S3 notification configuration to an existing bucket
 
     Args:
+        s3_resource (boto3.resources.base.ServiceResource) : s3 resource to use for s3 event config
         lambda_arn (str): Arn of the lambda s3 event config function
         bucket (str): bucket name of the s3 bucket to add the config to
     """
-    bucket_notification = s3.BucketNotification(bucket)
+    bucket_notification = s3_resource.BucketNotification(bucket)
     response = bucket_notification.put(
         NotificationConfiguration={
             "LambdaFunctionConfigurations": [
@@ -56,12 +58,13 @@ def add_notification(lambda_arn: str, bucket: str) -> None:
     logger.info("Put request completed....")
 
 
-def delete_notification(bucket: str) -> None:
+def delete_notification(s3_resource: boto3, bucket: str):
     """Deletes the S3 notification configuration from an existing bucket
 
     Args:
+        s3_resource (boto3.resources.base.ServiceResource) : s3 resource to use for s3 event config
         bucket (str): bucket name of the s3 bucket to delete the config in
     """
-    bucket_notification = s3.BucketNotification(bucket)
+    bucket_notification = s3_resource.BucketNotification(bucket)
     response = bucket_notification.put(NotificationConfiguration={})
     logger.info("Delete request completed....")
diff --git a/src/lambda_function/s3_event_config/events/test-trigger-event.json b/src/lambda_function/s3_event_config/events/test-trigger-event.json
deleted file mode 100644
index a6f92aea..00000000
--- a/src/lambda_function/s3_event_config/events/test-trigger-event.json
+++ /dev/null
@@ -1,3 +0,0 @@
-{
-    "RequestType":"Create"
-}
diff --git a/src/lambda_function/s3_to_glue/README.md b/src/lambda_function/s3_to_glue/README.md
index 5d0a1145..878653bb 100644
--- a/src/lambda_function/s3_to_glue/README.md
+++ b/src/lambda_function/s3_to_glue/README.md
@@ -27,7 +27,7 @@ Use the SAM CLI to build and test your lambda locally.
 Build your application with the `sam build` command.
 
 ```bash
-cd src/lambda_function
+cd src/lambda_function/s3_to_glue/
 sam build
 ```
 
@@ -35,10 +35,10 @@ sam build
 
 ### Creating/modifying test events
 
-The file `single-record.json` in `src/lambda_function/events` contains a
+The file `single-record.json` in `src/lambda_function/s3_to_glue/events` contains a
 dummy event for an S3 event trigger. You can generate your own test events
 for single or multiple records with the script at
-`src/lambda_function/events/generate_test_event.py`.
+`src/lambda_function/s3_to_glue/events/generate_test_event.py`.
 
 ### Invoking test events
 
@@ -52,7 +52,7 @@ if you are testing a stack deployed as part of a feature branch.
 To invoke the lambda with the test event:
 
 ```bash
-cd src/lambda_function
+cd src/lambda_function/s3_to_glue
 sam local invoke -e events/single-record.json --env-vars test-env-vars.json
 ```
 
diff --git a/src/lambda_function/test-env-vars.json b/src/lambda_function/s3_to_glue/test-env-vars.json
similarity index 100%
rename from src/lambda_function/test-env-vars.json
rename to src/lambda_function/s3_to_glue/test-env-vars.json

From 50572e7c9778827e0e657058b1060c974d97af0e Mon Sep 17 00:00:00 2001
From: Rixing Xu <rxu@w197.local>
Date: Fri, 12 May 2023 14:46:36 -0700
Subject: [PATCH 5/7] add namespace for s3 notification filtering

---
 .../namespaced/s3-event-config-lambda.yaml    |  1 +
 .../namespaced/s3-event-config-lambda.yaml    |  1 +
 src/lambda_function/s3_event_config/app.py    | 32 ++++++++++++++-----
 .../s3_event_config/template.yaml             |  5 +++
 4 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/config/develop/namespaced/s3-event-config-lambda.yaml b/config/develop/namespaced/s3-event-config-lambda.yaml
index 8c800561..12a8958b 100644
--- a/config/develop/namespaced/s3-event-config-lambda.yaml
+++ b/config/develop/namespaced/s3-event-config-lambda.yaml
@@ -9,6 +9,7 @@ dependencies:
 stack_name: '{{ stack_group_config.namespace }}-lambda-S3EventConfig'
 stack_tags: {{ stack_group_config.default_stack_tags }}
 parameters:
+  Namespace: {{ stack_group_config.namespace }}
   S3ToGlueFunctionArn: !stack_output_external "{{ stack_group_config.namespace }}-lambda-S3ToGlue::S3ToGlueFunctionArn"
   S3EventConfigRoleArn: !stack_output_external "{{ stack_group_config.namespace }}-s3-event-config-lambda-role::RoleArn"
   S3SourceBucketName: {{ stack_group_config.input_bucket_name }}
diff --git a/config/prod/namespaced/s3-event-config-lambda.yaml b/config/prod/namespaced/s3-event-config-lambda.yaml
index 52fcd07f..9bb66735 100644
--- a/config/prod/namespaced/s3-event-config-lambda.yaml
+++ b/config/prod/namespaced/s3-event-config-lambda.yaml
@@ -9,6 +9,7 @@ dependencies:
 stack_name: '{{ stack_group_config.namespace }}-lambda-S3EventConfig'
 stack_tags: {{ stack_group_config.default_stack_tags }}
 parameters:
+  Namespace: {{ stack_group_config.namespace }}
   S3ToGlueFunctionArn: !stack_output_external "{{ stack_group_config.namespace }}-lambda-S3ToGlue::S3ToGlueFunctionArn"
   S3EventConfigRoleArn: !stack_output_external "{{ stack_group_config.namespace }}-s3-event-config-lambda-role::RoleArn"
   S3SourceBucketName: {{ stack_group_config.input_bucket_name }}
diff --git a/src/lambda_function/s3_event_config/app.py b/src/lambda_function/s3_event_config/app.py
index 15e45af9..5ae5f38f 100644
--- a/src/lambda_function/s3_event_config/app.py
+++ b/src/lambda_function/s3_event_config/app.py
@@ -23,22 +23,27 @@ def lambda_handler(event, context):
     logger.info(f"Received event: {json.dumps(event, indent=2)}")
     if event["RequestType"] == "Delete":
         logger.info(f'Request Type:{event["RequestType"]}')
-        bucket = os.environ["S3_SOURCE_BUCKET_NAME"]
-        delete_notification(s3, bucket)
+        delete_notification(s3, bucket=os.environ["S3_SOURCE_BUCKET_NAME"])
         logger.info("Sending response to custom resource after Delete")
     elif event["RequestType"] in ["Update", "Create"]:
         logger.info(f'Request Type: {event["RequestType"]}')
-        lambda_arn = os.environ["S3_TO_GLUE_FUNCTION_ARN"]
-        bucket = os.environ["S3_SOURCE_BUCKET_NAME"]
-        add_notification(s3, lambda_arn, bucket)
+        add_notification(
+            s3,
+            lambda_arn=os.environ["S3_TO_GLUE_FUNCTION_ARN"],
+            bucket=os.environ["S3_SOURCE_BUCKET_NAME"],
+            bucket_key_prefix=os.environ["BUCKET_KEY_PREFIX"],
+        )
         logger.info("Sending response to custom resource")
     else:
-        err_msg =  f"The 'RequestType' key should have one of the following values: {REQUEST_TYPE_VALS}"
+        err_msg = f"The 'RequestType' key should have one of the following values: {REQUEST_TYPE_VALS}"
         raise KeyError(err_msg)
 
 
 def add_notification(
-    s3_resource: boto3.resources.base.ServiceResource, lambda_arn: str, bucket: str
+    s3_resource: boto3.resources.base.ServiceResource,
+    lambda_arn: str,
+    bucket: str,
+    bucket_key_prefix: str,
 ):
     """Adds the S3 notification configuration to an existing bucket
 
@@ -46,12 +51,23 @@ def add_notification(
         s3_resource (boto3.resources.base.ServiceResource) : s3 resource to use for s3 event config
         lambda_arn (str): Arn of the lambda s3 event config function
         bucket (str): bucket name of the s3 bucket to add the config to
+        bucket_key_prefix (str): bucket key prefix for where to look for s3 object notifications
     """
     bucket_notification = s3_resource.BucketNotification(bucket)
     response = bucket_notification.put(
         NotificationConfiguration={
             "LambdaFunctionConfigurations": [
-                {"LambdaFunctionArn": lambda_arn, "Events": ["s3:ObjectCreated:*"]}
+                {
+                    "LambdaFunctionArn": lambda_arn,
+                    "Events": ["s3:ObjectCreated:*"],
+                    "Filter": {
+                        "Key": {
+                            "FilterRules": [
+                                {"Name": "prefix", "Value": bucket_key_prefix}
+                            ]
+                        }
+                    },
+                }
             ]
         }
     )
diff --git a/src/lambda_function/s3_event_config/template.yaml b/src/lambda_function/s3_event_config/template.yaml
index 0ecfd3fe..0f429141 100644
--- a/src/lambda_function/s3_event_config/template.yaml
+++ b/src/lambda_function/s3_event_config/template.yaml
@@ -6,6 +6,10 @@ Description: >
   SAM Template for s3-event-config lambda function
 
 Parameters:
+  Namespace:
+    Type: String
+    Description: >-
+      The namespace string used for the bucket key prefix
 
   S3ToGlueFunctionArn:
     Type: String
@@ -39,6 +43,7 @@ Resources:
         Variables:
           S3_SOURCE_BUCKET_NAME: !Ref S3SourceBucketName
           S3_TO_GLUE_FUNCTION_ARN: !Ref S3ToGlueFunctionArn
+          BUCKET_KEY_PREFIX: !Ref Namespace
 
 Outputs:
   S3EventConfigFunctionArn:

From 17e6e17b0af74e36dfbf3f2fb41c9b751302c80c Mon Sep 17 00:00:00 2001
From: Rixing Xu <rxu@w197.local>
Date: Mon, 15 May 2023 11:11:06 -0700
Subject: [PATCH 6/7] add tests for test_s3_event_config_lambda

---
 Pipfile                                    |  1 +
 src/lambda_function/s3_event_config/app.py | 21 +++----
 tests/README.md                            |  8 ++-
 tests/test_s3_event_config_lambda.py       | 67 ++++++++++++++++++++++
 4 files changed, 85 insertions(+), 12 deletions(-)
 create mode 100644 tests/test_s3_event_config_lambda.py

diff --git a/Pipfile b/Pipfile
index 93675b86..6e29e376 100644
--- a/Pipfile
+++ b/Pipfile
@@ -16,3 +16,4 @@ synapseclient = "~=2.7"
 pandas = "<1.5"
 moto = "~=4.1"
 datacompy = "~=0.8"
+docker = "~=6.1"
diff --git a/src/lambda_function/s3_event_config/app.py b/src/lambda_function/s3_event_config/app.py
index 5ae5f38f..5fb83ea4 100644
--- a/src/lambda_function/s3_event_config/app.py
+++ b/src/lambda_function/s3_event_config/app.py
@@ -19,7 +19,7 @@
 
 
 def lambda_handler(event, context):
-    s3 = boto3.resource("s3")
+    s3 = boto3.client("s3")
     logger.info(f"Received event: {json.dumps(event, indent=2)}")
     if event["RequestType"] == "Delete":
         logger.info(f'Request Type:{event["RequestType"]}')
@@ -40,7 +40,7 @@ def lambda_handler(event, context):
 
 
 def add_notification(
-    s3_resource: boto3.resources.base.ServiceResource,
+    s3_client: boto3.client,
     lambda_arn: str,
     bucket: str,
     bucket_key_prefix: str,
@@ -48,13 +48,13 @@ def add_notification(
     """Adds the S3 notification configuration to an existing bucket
 
     Args:
-        s3_resource (boto3.resources.base.ServiceResource) : s3 resource to use for s3 event config
+        s3_client (boto3.client) : s3 client to use for s3 event config
         lambda_arn (str): Arn of the lambda s3 event config function
         bucket (str): bucket name of the s3 bucket to add the config to
         bucket_key_prefix (str): bucket key prefix for where to look for s3 object notifications
     """
-    bucket_notification = s3_resource.BucketNotification(bucket)
-    response = bucket_notification.put(
+    s3_client.put_bucket_notification_configuration(
+        Bucket=bucket,
         NotificationConfiguration={
             "LambdaFunctionConfigurations": [
                 {
@@ -69,18 +69,19 @@ def add_notification(
                     },
                 }
             ]
-        }
+        },
     )
     logger.info("Put request completed....")
 
 
-def delete_notification(s3_resource: boto3, bucket: str):
+def delete_notification(s3_client: boto3.client, bucket: str):
     """Deletes the S3 notification configuration from an existing bucket
 
     Args:
-        s3_resource (boto3.resources.base.ServiceResource) : s3 resource to use for s3 event config
+        s3_client (boto3.client) : s3 client to use for s3 event config
         bucket (str): bucket name of the s3 bucket to delete the config in
     """
-    bucket_notification = s3_resource.BucketNotification(bucket)
-    response = bucket_notification.put(NotificationConfiguration={})
+    s3_client.put_bucket_notification_configuration(
+        Bucket=bucket, NotificationConfiguration={}
+    )
     logger.info("Delete request completed....")
diff --git a/tests/README.md b/tests/README.md
index 66bdf08b..6623efad 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -54,16 +54,20 @@ Here are the tests you can run locally using a pipenv. You'll run into an error
 pytest with other tests because they have to be run in a Dockerfile:
 
 - test_s3_to_glue_lambda.py
+- test_s3_event_config_lambda.py
 - test_setup_external_storage.py
 
-
 #### Running tests for lambda
-Run the following command from the repo root to run tests for the lambda function (in develop).
+Run the following command from the repo root to run tests for the lambda functions (in develop).
 
 ```shell script
 python3 -m pytest tests/test_s3_to_glue_lambda.py -v
 ```
 
+```shell script
+python3 -m pytest tests/test_s3_event_config_lambda.py -v
+```
+
 #### Running tests for setup external storage
 Run the following command from the repo root to run the integration test for the setup external storage script to check that the STS
 access has been set for a given synapse folder (in develop).
diff --git a/tests/test_s3_event_config_lambda.py b/tests/test_s3_event_config_lambda.py
new file mode 100644
index 00000000..5ccd5e7f
--- /dev/null
+++ b/tests/test_s3_event_config_lambda.py
@@ -0,0 +1,67 @@
+import zipfile
+import io
+import boto3
+from moto import mock_s3, mock_lambda, mock_iam, mock_logs
+import pytest
+
+from src.lambda_function.s3_event_config import app
+
+
+@pytest.fixture(scope="function")
+def mock_iam_role(mock_aws_credentials):
+    with mock_iam():
+        iam = boto3.client("iam")
+        yield iam.create_role(
+            RoleName="some-role",
+            AssumeRolePolicyDocument="some policy",
+            Path="/some-path/",
+        )["Role"]["Arn"]
+
+
+@pytest.fixture(scope="function")
+def mock_lambda_function(mock_aws_credentials, mock_iam_role):
+    with mock_lambda():
+        client = boto3.client("lambda")
+        client.create_function(
+            FunctionName="some_function",
+            Role=mock_iam_role,
+            Code={"ZipFile": "print('DONE')"},
+            Description="string",
+        )
+        yield client.get_function(FunctionName="some_function")
+
+
+@mock_s3
+def test_that_add_notification_adds_expected_settings(s3, mock_lambda_function):
+    s3.create_bucket(Bucket="some_bucket")
+    set_config = app.add_notification(
+        s3,
+        mock_lambda_function["Configuration"]["FunctionArn"],
+        "some_bucket",
+        "test_folder",
+    )
+    get_config = s3.get_bucket_notification_configuration(Bucket="some_bucket")
+    assert (
+        get_config["LambdaFunctionConfigurations"][0]["LambdaFunctionArn"]
+        == mock_lambda_function["Configuration"]["FunctionArn"]
+    )
+    assert get_config["LambdaFunctionConfigurations"][0]["Events"] == [
+        "s3:ObjectCreated:*"
+    ]
+    assert get_config["LambdaFunctionConfigurations"][0]["Filter"] == {
+        "Key": {"FilterRules": [{"Name": "prefix", "Value": "test_folder"}]}
+    }
+
+
+@mock_s3
+def test_that_delete_notification_is_successful(s3, mock_lambda_function):
+    s3.create_bucket(Bucket="some_bucket")
+    app.add_notification(
+        s3,
+        mock_lambda_function["Configuration"]["FunctionArn"],
+        "some_bucket",
+        "test_folder",
+    )
+    app.delete_notification(s3, "some_bucket")
+    get_config = s3.get_bucket_notification_configuration(Bucket="some_bucket")
+    assert "LambdaFunctionConfigurations" not in get_config

From 43dad9625e6533a8c264ac3d721a08e42eaed3a4 Mon Sep 17 00:00:00 2001
From: Rixing Xu <rxu@w197.local>
Date: Mon, 15 May 2023 11:51:12 -0700
Subject: [PATCH 7/7] update s3 event config lambda readme

---
 src/lambda_function/s3_event_config/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lambda_function/s3_event_config/README.md b/src/lambda_function/s3_event_config/README.md
index 9d15d82f..ccef795e 100644
--- a/src/lambda_function/s3_event_config/README.md
+++ b/src/lambda_function/s3_event_config/README.md
@@ -35,7 +35,7 @@ Note that they depend on the `s3 to json` lambda stacks.
 #### Launching in development
 
 Run the following command to create the lambda stack in your AWS account. Note this will
-also create the lambda s3 to glue IAM role stack as well:
+also create the lambda event config IAM role stack as well as well as any other dependencies of this stack:
 
 ```shell script
 sceptre --var namespace='test-namespace' launch develop/namespaced/s3-event-config-lambda.yaml