add gx glue version as var in config

Sage-Bionetworks · Sep 12, 2024 · 0ec67b1 · 0ec67b1
1 parent 198ae92
commit 0ec67b1
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 2 deletions.
diff --git a/config/config.yaml b/config/config.yaml
@@ -7,6 +7,7 @@ template_key_prefix: "{{ var.namespace | default('main') }}/templates"
 glue_python_shell_python_version: "3.9"
 glue_python_shell_glue_version: "3.0"
 json_to_parquet_glue_version: "4.0"
+great_expectations_job_glue_version: "4.0"
 default_stack_tags:
   Department: DNT
   Project: recover

diff --git a/config/develop/namespaced/glue-job-run-great-expectations-on-parquet.yaml b/config/develop/namespaced/glue-job-run-great-expectations-on-parquet.yaml
@@ -10,7 +10,7 @@ parameters:
   TempS3Bucket: {{ stack_group_config.processed_data_bucket_name }}
   S3ScriptBucket: {{ stack_group_config.template_bucket_name }}
   S3ScriptKey: '{{ stack_group_config.namespace }}/src/glue/jobs/run_great_expectations_on_parquet.py'
-  GlueVersion: "{{ stack_group_config.json_to_parquet_glue_version }}"
+  GlueVersion: "{{ stack_group_config.great_expectations_job_glue_version }}"
   AdditionalPythonModules: "great_expectations~=0.18,urllib3<2"
 stack_tags:
   {{ stack_group_config.default_stack_tags }}

diff --git a/config/prod/namespaced/glue-job-run-great-expectations-on-parquet.yaml b/config/prod/namespaced/glue-job-run-great-expectations-on-parquet.yaml
@@ -10,7 +10,7 @@ parameters:
   TempS3Bucket: {{ stack_group_config.processed_data_bucket_name }}
   S3ScriptBucket: {{ stack_group_config.template_bucket_name }}
   S3ScriptKey: '{{ stack_group_config.namespace }}/src/glue/jobs/run_great_expectations_on_parquet.py'
-  GlueVersion: "{{ stack_group_config.json_to_parquet_glue_version }}"
+  GlueVersion: "{{ stack_group_config.great_expectations_job_glue_version }}"
   AdditionalPythonModules: "great_expectations~=0.18,urllib3<2"
 stack_tags:
   {{ stack_group_config.default_stack_tags }}