Merge pull request #37 from Sage-Bionetworks/etl-381

[ETL-381] Grant Synapse access to dev parquet bucket

config/develop/s3-processed-data-bucket-owner-txt.yaml

@@ -0,0 +1,13 @@
+template:
+  type: file
+  path: s3-owner-txt.yaml
+stack_name: "{{ stack_group_config.namespace }}-recover-dev-processed-data-bucket-owner-txt"
+dependencies:
+  - develop/cfn-s3objects-macro.yaml
+  - develop/s3-processed-data-bucket.yaml
+parameters:
+  BucketName: !stack_output_external recover-dev-processed-data-bucket::BucketName
+  SynapseIds: "3461799" # recoverETL
+  OwnerTxtKeyPrefix: "{{ stack_group_config.namespace }}/parquet"
+stack_tags:
+  {{ stack_group_config.default_stack_tags }}

config/develop/s3-processed-data-bucket.yaml

@@ -4,3 +4,6 @@ template:
 stack_name: recover-dev-processed-data-bucket
 parameters:
   BucketName: {{ stack_group_config.processed_data_bucket_name }}
+  ConnectToSynapse: "true"
+stack_tags:
+  {{ stack_group_config.default_stack_tags }}

config/prod/s3-processed-data-bucket-owner-txt.yaml

@@ -9,3 +9,5 @@ parameters:
   BucketName: !stack_output_external recover-processed-data-bucket::BucketName
   SynapseIds: "3461799" # recoverETL
   OwnerTxtKeyPrefix: "{{ stack_group_config.namespace }}/parquet"
+stack_tags:
+  {{ stack_group_config.default_stack_tags }}

src/scripts/setup_external_storage/setup_external_storage.py

@@ -5,6 +5,7 @@
 import os
 import json
 import argparse
+import boto3
 
 import synapseclient
 
@@ -35,13 +36,47 @@ def read_args():
         action="store_true",
         help="Whether this storage location should be STS enabled",
     )
+    parser.add_argument("--profile",
+                        help=("Optional. The AWS profile to use. Uses the default "
+                              "profile if not specified."))
+    parser.add_argument("--ssm-parameter",
+                        help=("Optional. The name of the SSM parameter containing "
+                              "the Synapse personal access token. "
+                              "If not provided, cached credentials are used"))
     args = parser.parse_args()
     return args
 
+def get_synapse_client(ssm_parameter=None, aws_session=None):
+    """
+    Return an authenticated Synapse client.
+
+    Args:
+        ssm_parameter (str): Name of the SSM parameter containing the
+            recoverETL Synapse authentication token.
+        aws_session (boto3.session.Session)
+
+    Returns:
+        synapseclient.Synapse
+    """
+    if ssm_parameter is not None:
+        ssm_client = aws_session.client("ssm")
+        token = ssm_client.get_parameter(
+            Name=ssm_parameter,
+            WithDecryption=True)
+        syn = synapseclient.Synapse()
+        syn.login(authToken=token["Parameter"]["Value"])
+    else: # try cached credentials
+        syn = synapseclient.login()
+    return syn
 
 def main():
     args = read_args()
-    syn = synapseclient.login()
+    aws_session = boto3.session.Session(
+            profile_name=args.profile,
+            region_name="us-east-1")
+    syn = get_synapse_client(
+            ssm_parameter=args.ssm_parameter,
+            aws_session=aws_session)
     synapse_folder, storage_location, synapse_project = syn.create_s3_storage_location(
         parent=args.synapse_parent,
         folder_name=args.synapse_folder_name,