diff --git a/config/develop/s3-processed-data-bucket-owner-txt.yaml b/config/develop/s3-processed-data-bucket-owner-txt.yaml new file mode 100644 index 00000000..187e1864 --- /dev/null +++ b/config/develop/s3-processed-data-bucket-owner-txt.yaml @@ -0,0 +1,13 @@ +template: + type: file + path: s3-owner-txt.yaml +stack_name: "{{ stack_group_config.namespace }}-recover-dev-processed-data-bucket-owner-txt" +dependencies: + - develop/cfn-s3objects-macro.yaml + - develop/s3-processed-data-bucket.yaml +parameters: + BucketName: !stack_output_external recover-dev-processed-data-bucket::BucketName + SynapseIds: "3461799" # recoverETL + OwnerTxtKeyPrefix: "{{ stack_group_config.namespace }}/parquet" +stack_tags: + {{ stack_group_config.default_stack_tags }} diff --git a/config/develop/s3-processed-data-bucket.yaml b/config/develop/s3-processed-data-bucket.yaml index 78594ba1..c3f6d23a 100644 --- a/config/develop/s3-processed-data-bucket.yaml +++ b/config/develop/s3-processed-data-bucket.yaml @@ -4,3 +4,6 @@ template: stack_name: recover-dev-processed-data-bucket parameters: BucketName: {{ stack_group_config.processed_data_bucket_name }} + ConnectToSynapse: "true" +stack_tags: + {{ stack_group_config.default_stack_tags }} diff --git a/config/prod/s3-processed-data-bucket-owner-txt.yaml b/config/prod/s3-processed-data-bucket-owner-txt.yaml index 329317c2..bb89f1fa 100644 --- a/config/prod/s3-processed-data-bucket-owner-txt.yaml +++ b/config/prod/s3-processed-data-bucket-owner-txt.yaml @@ -9,3 +9,5 @@ parameters: BucketName: !stack_output_external recover-processed-data-bucket::BucketName SynapseIds: "3461799" # recoverETL OwnerTxtKeyPrefix: "{{ stack_group_config.namespace }}/parquet" +stack_tags: + {{ stack_group_config.default_stack_tags }} diff --git a/src/scripts/setup_external_storage/setup_external_storage.py b/src/scripts/setup_external_storage/setup_external_storage.py index df24eae1..0f58e750 100644 --- a/src/scripts/setup_external_storage/setup_external_storage.py +++ b/src/scripts/setup_external_storage/setup_external_storage.py @@ -5,6 +5,7 @@ import os import json import argparse +import boto3 import synapseclient @@ -35,13 +36,47 @@ def read_args(): action="store_true", help="Whether this storage location should be STS enabled", ) + parser.add_argument("--profile", + help=("Optional. The AWS profile to use. Uses the default " + "profile if not specified.")) + parser.add_argument("--ssm-parameter", + help=("Optional. The name of the SSM parameter containing " + "the Synapse personal access token. " + "If not provided, cached credentials are used")) args = parser.parse_args() return args +def get_synapse_client(ssm_parameter=None, aws_session=None): + """ + Return an authenticated Synapse client. + + Args: + ssm_parameter (str): Name of the SSM parameter containing the + recoverETL Synapse authentication token. + aws_session (boto3.session.Session) + + Returns: + synapseclient.Synapse + """ + if ssm_parameter is not None: + ssm_client = aws_session.client("ssm") + token = ssm_client.get_parameter( + Name=ssm_parameter, + WithDecryption=True) + syn = synapseclient.Synapse() + syn.login(authToken=token["Parameter"]["Value"]) + else: # try cached credentials + syn = synapseclient.login() + return syn def main(): args = read_args() - syn = synapseclient.login() + aws_session = boto3.session.Session( + profile_name=args.profile, + region_name="us-east-1") + syn = get_synapse_client( + ssm_parameter=args.ssm_parameter, + aws_session=aws_session) synapse_folder, storage_location, synapse_project = syn.create_s3_storage_location( parent=args.synapse_parent, folder_name=args.synapse_folder_name,