diff --git a/config/config.yaml b/config/config.yaml index 07930a8..ff15f5f 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -7,6 +7,7 @@ template_key_prefix: "{{ var.namespace | default('main') }}/templates" glue_python_shell_python_version: "3.9" glue_python_shell_glue_version: "3.0" json_to_parquet_glue_version: "4.0" +great_expectations_job_glue_version: "4.0" default_stack_tags: Department: DNT Project: recover diff --git a/config/develop/namespaced/glue-job-run-great-expectations-on-parquet.yaml b/config/develop/namespaced/glue-job-run-great-expectations-on-parquet.yaml index 0f361e6..1e6d461 100644 --- a/config/develop/namespaced/glue-job-run-great-expectations-on-parquet.yaml +++ b/config/develop/namespaced/glue-job-run-great-expectations-on-parquet.yaml @@ -10,7 +10,7 @@ parameters: TempS3Bucket: {{ stack_group_config.processed_data_bucket_name }} S3ScriptBucket: {{ stack_group_config.template_bucket_name }} S3ScriptKey: '{{ stack_group_config.namespace }}/src/glue/jobs/run_great_expectations_on_parquet.py' - GlueVersion: "{{ stack_group_config.json_to_parquet_glue_version }}" + GlueVersion: "{{ stack_group_config.great_expectations_job_glue_version }}" AdditionalPythonModules: "great_expectations~=0.18,urllib3<2" stack_tags: {{ stack_group_config.default_stack_tags }} diff --git a/config/prod/namespaced/glue-job-run-great-expectations-on-parquet.yaml b/config/prod/namespaced/glue-job-run-great-expectations-on-parquet.yaml index e01745a..f0e8dd2 100644 --- a/config/prod/namespaced/glue-job-run-great-expectations-on-parquet.yaml +++ b/config/prod/namespaced/glue-job-run-great-expectations-on-parquet.yaml @@ -10,7 +10,7 @@ parameters: TempS3Bucket: {{ stack_group_config.processed_data_bucket_name }} S3ScriptBucket: {{ stack_group_config.template_bucket_name }} S3ScriptKey: '{{ stack_group_config.namespace }}/src/glue/jobs/run_great_expectations_on_parquet.py' - GlueVersion: "{{ stack_group_config.json_to_parquet_glue_version }}" + GlueVersion: "{{ stack_group_config.great_expectations_job_glue_version }}" AdditionalPythonModules: "great_expectations~=0.18,urllib3<2" stack_tags: {{ stack_group_config.default_stack_tags }}