Update Glue tables to use compressed JSON (#137)

Sage-Bionetworks · Aug 26, 2024 · fbd83a6 · fbd83a6
1 parent e807196
commit fbd83a6
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 4 deletions.
diff --git a/config/develop/namespaced/glue-tables.yaml b/config/develop/namespaced/glue-tables.yaml
@@ -4,6 +4,7 @@ stack_name: '{{ stack_group_config.namespace }}-glue-tables'
 parameters:
   Namespace: {{ stack_group_config.namespace }}
   S3IntermediateBucketName: {{ stack_group_config.intermediate_bucket_name }}
+  S3IntermediateBucketPrefix: 'compressed_json'
 stack_tags:
   {{ stack_group_config.default_stack_tags }}
 sceptre_user_data:

diff --git a/config/prod/namespaced/glue-tables.yaml b/config/prod/namespaced/glue-tables.yaml
@@ -4,6 +4,7 @@ stack_name: '{{ stack_group_config.namespace }}-glue-tables'
 parameters:
   Namespace: {{ stack_group_config.namespace }}
   S3IntermediateBucketName: {{ stack_group_config.intermediate_bucket_name }}
+  S3IntermediateBucketPrefix: 'compressed_json'
 stack_tags:
   {{ stack_group_config.default_stack_tags }}
 sceptre_user_data:

diff --git a/templates/glue-tables.j2 b/templates/glue-tables.j2
@@ -51,17 +51,16 @@ Resources:
         Name: {{ dataset.table_name }}
         Parameters:
           classification: json
-          compressionType: none
+          compressionType: gzip
           typeOfData: file
-        Retention: 0
         PartitionKeys: {{ dataset.partition_keys }}
         StorageDescriptor:
           Columns: {{ dataset.columns }}
-          Compressed: false
+          Compressed: true
           InputFormat: org.apache.hadoop.mapred.TextInputFormat
           Location: !Sub s3://${S3IntermediateBucketName}/${Namespace}/${S3IntermediateBucketPrefix}/dataset={{ dataset["type"] }}
           OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-          StoredAsSubDirectories: false
+          StoredAsSubDirectories: true
         TableType: EXTERNAL_TABLE
   {% endfor %}