From 0ec67b14c3479f52bba2eec2cf54d4695f482e0e Mon Sep 17 00:00:00 2001
From: rxu17 <26471741+rxu17@users.noreply.github.com>
Date: Wed, 11 Sep 2024 23:52:27 -0700
Subject: [PATCH] add gx glue version as var in config

---
 config/config.yaml                                              | 1 +
 .../namespaced/glue-job-run-great-expectations-on-parquet.yaml  | 2 +-
 .../namespaced/glue-job-run-great-expectations-on-parquet.yaml  | 2 +-
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/config/config.yaml b/config/config.yaml
index 07930a8e..ff15f5f7 100644
--- a/config/config.yaml
+++ b/config/config.yaml
@@ -7,6 +7,7 @@ template_key_prefix: "{{ var.namespace | default('main') }}/templates"
 glue_python_shell_python_version: "3.9"
 glue_python_shell_glue_version: "3.0"
 json_to_parquet_glue_version: "4.0"
+great_expectations_job_glue_version: "4.0"
 default_stack_tags:
   Department: DNT
   Project: recover
diff --git a/config/develop/namespaced/glue-job-run-great-expectations-on-parquet.yaml b/config/develop/namespaced/glue-job-run-great-expectations-on-parquet.yaml
index 0f361e61..1e6d461e 100644
--- a/config/develop/namespaced/glue-job-run-great-expectations-on-parquet.yaml
+++ b/config/develop/namespaced/glue-job-run-great-expectations-on-parquet.yaml
@@ -10,7 +10,7 @@ parameters:
   TempS3Bucket: {{ stack_group_config.processed_data_bucket_name }}
   S3ScriptBucket: {{ stack_group_config.template_bucket_name }}
   S3ScriptKey: '{{ stack_group_config.namespace }}/src/glue/jobs/run_great_expectations_on_parquet.py'
-  GlueVersion: "{{ stack_group_config.json_to_parquet_glue_version }}"
+  GlueVersion: "{{ stack_group_config.great_expectations_job_glue_version }}"
   AdditionalPythonModules: "great_expectations~=0.18,urllib3<2"
 stack_tags:
   {{ stack_group_config.default_stack_tags }}
diff --git a/config/prod/namespaced/glue-job-run-great-expectations-on-parquet.yaml b/config/prod/namespaced/glue-job-run-great-expectations-on-parquet.yaml
index e01745ae..f0e8dd2a 100644
--- a/config/prod/namespaced/glue-job-run-great-expectations-on-parquet.yaml
+++ b/config/prod/namespaced/glue-job-run-great-expectations-on-parquet.yaml
@@ -10,7 +10,7 @@ parameters:
   TempS3Bucket: {{ stack_group_config.processed_data_bucket_name }}
   S3ScriptBucket: {{ stack_group_config.template_bucket_name }}
   S3ScriptKey: '{{ stack_group_config.namespace }}/src/glue/jobs/run_great_expectations_on_parquet.py'
-  GlueVersion: "{{ stack_group_config.json_to_parquet_glue_version }}"
+  GlueVersion: "{{ stack_group_config.great_expectations_job_glue_version }}"
   AdditionalPythonModules: "great_expectations~=0.18,urllib3<2"
 stack_tags:
   {{ stack_group_config.default_stack_tags }}