[ETL-397] Create s3-event-config lambda and dependencies to add s3 notification configuration

Pipfile

@@ -16,3 +16,4 @@ synapseclient = "~=2.7"
 pandas = "<1.5"
 moto = "~=4.1"
 datacompy = "~=0.8"
+docker = "~=6.1"

src/lambda_function/s3_event_config/app.py

@@ -19,7 +19,7 @@
 
 
 def lambda_handler(event, context):
-    s3 = boto3.resource("s3")
+    s3 = boto3.client("s3")
     logger.info(f"Received event: {json.dumps(event, indent=2)}")
     if event["RequestType"] == "Delete":
         logger.info(f'Request Type:{event["RequestType"]}')
@@ -40,21 +40,21 @@ def lambda_handler(event, context):
 
 
 def add_notification(
-    s3_resource: boto3.resources.base.ServiceResource,
+    s3_client: boto3.client,
     lambda_arn: str,
     bucket: str,
     bucket_key_prefix: str,
 ):
     """Adds the S3 notification configuration to an existing bucket
 
     Args:
-        s3_resource (boto3.resources.base.ServiceResource) : s3 resource to use for s3 event config
+        s3_client (boto3.client) : s3 client to use for s3 event config
         lambda_arn (str): Arn of the lambda s3 event config function
         bucket (str): bucket name of the s3 bucket to add the config to
         bucket_key_prefix (str): bucket key prefix for where to look for s3 object notifications
     """
-    bucket_notification = s3_resource.BucketNotification(bucket)
-    response = bucket_notification.put(
+    s3_client.put_bucket_notification_configuration(
+        Bucket=bucket,
         NotificationConfiguration={
             "LambdaFunctionConfigurations": [
                 {
@@ -69,18 +69,19 @@ def add_notification(
                     },
                 }
             ]
-        }
+        },
     )
     logger.info("Put request completed....")
 
 
-def delete_notification(s3_resource: boto3, bucket: str):
+def delete_notification(s3_client: boto3.client, bucket: str):
     """Deletes the S3 notification configuration from an existing bucket
 
     Args:
-        s3_resource (boto3.resources.base.ServiceResource) : s3 resource to use for s3 event config
+        s3_client (boto3.client) : s3 client to use for s3 event config
         bucket (str): bucket name of the s3 bucket to delete the config in
     """
-    bucket_notification = s3_resource.BucketNotification(bucket)
-    response = bucket_notification.put(NotificationConfiguration={})
+    s3_client.put_bucket_notification_configuration(
+        Bucket=bucket, NotificationConfiguration={}
+    )
     logger.info("Delete request completed....")

tests/README.md

@@ -54,16 +54,20 @@ Here are the tests you can run locally using a pipenv. You'll run into an error
 pytest with other tests because they have to be run in a Dockerfile:
 
 - test_s3_to_glue_lambda.py
+- test_s3_event_config_lambda.py
 - test_setup_external_storage.py
 
-
 #### Running tests for lambda
-Run the following command from the repo root to run tests for the lambda function (in develop).
+Run the following command from the repo root to run tests for the lambda functions (in develop).
 
 ```shell script
 python3 -m pytest tests/test_s3_to_glue_lambda.py -v
 ```
 
+```shell script
+python3 -m pytest tests/test_s3_event_config_lambda.py -v
+```
+
 #### Running tests for setup external storage
 Run the following command from the repo root to run the integration test for the setup external storage script to check that the STS
 access has been set for a given synapse folder (in develop).

tests/test_s3_event_config_lambda.py

@@ -0,0 +1,67 @@
+import zipfile
+import io
+import boto3
+from moto import mock_s3, mock_lambda, mock_iam, mock_logs
+import pytest
+
+from src.lambda_function.s3_event_config import app
+
+
+@pytest.fixture(scope="function")
+def mock_iam_role(mock_aws_credentials):
+    with mock_iam():
+        iam = boto3.client("iam")
+        yield iam.create_role(
+            RoleName="some-role",
+            AssumeRolePolicyDocument="some policy",
+            Path="/some-path/",
+        )["Role"]["Arn"]
+
+
+@pytest.fixture(scope="function")
+def mock_lambda_function(mock_aws_credentials, mock_iam_role):
+    with mock_lambda():
+        client = boto3.client("lambda")
+        client.create_function(
+            FunctionName="some_function",
+            Role=mock_iam_role,
+            Code={"ZipFile": "print('DONE')"},
+            Description="string",
+        )
+        yield client.get_function(FunctionName="some_function")
+
+
+@mock_s3
+def test_that_add_notification_adds_expected_settings(s3, mock_lambda_function):
+    s3.create_bucket(Bucket="some_bucket")
+    set_config = app.add_notification(
+        s3,
+        mock_lambda_function["Configuration"]["FunctionArn"],
+        "some_bucket",
+        "test_folder",
+    )
+    get_config = s3.get_bucket_notification_configuration(Bucket="some_bucket")
+    assert (
+        get_config["LambdaFunctionConfigurations"][0]["LambdaFunctionArn"]
+        == mock_lambda_function["Configuration"]["FunctionArn"]
+    )
+    assert get_config["LambdaFunctionConfigurations"][0]["Events"] == [
+        "s3:ObjectCreated:*"
+    ]
+    assert get_config["LambdaFunctionConfigurations"][0]["Filter"] == {
+        "Key": {"FilterRules": [{"Name": "prefix", "Value": "test_folder"}]}
+    }
+
+
+@mock_s3
+def test_that_delete_notification_is_successful(s3, mock_lambda_function):
+    s3.create_bucket(Bucket="some_bucket")
+    app.add_notification(
+        s3,
+        mock_lambda_function["Configuration"]["FunctionArn"],
+        "some_bucket",
+        "test_folder",
+    )
+    app.delete_notification(s3, "some_bucket")
+    get_config = s3.get_bucket_notification_configuration(Bucket="some_bucket")
+    assert "LambdaFunctionConfigurations" not in get_config