From cd8afee1424a56f3ab03c845f41ce2f52e9fa711 Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Tue, 16 Mar 2021 02:26:16 +0000
Subject: [PATCH 01/12] Add Python and Glue version parameters, add check mode

Available Python and Glue version can be found here:
https://docs.aws.amazon.com/glue/latest/dg/add-job.html

Example:

```
community.aws.aws_glue__jobs:
  - name: my-job
    description: My test job
    command_script_location: my-s3-bucket/script.py
    command_python_version: 3
    glue_version: "2.0"
    role: MyGlueJobRole
    state: present
```
---
 .../480-aws_glue_job-python-glue-version.yml  |   4 +
 plugins/modules/aws_glue_job.py               |  48 +++-
 .../integration/targets/aws_glue_job/aliases  |   2 +
 .../targets/aws_glue_job/defaults/main.yml    |   5 +
 .../targets/aws_glue_job/tasks/main.yml       | 271 ++++++++++++++++++
 5 files changed, 324 insertions(+), 6 deletions(-)
 create mode 100644 changelogs/fragments/480-aws_glue_job-python-glue-version.yml
 create mode 100644 tests/integration/targets/aws_glue_job/aliases
 create mode 100644 tests/integration/targets/aws_glue_job/defaults/main.yml
 create mode 100644 tests/integration/targets/aws_glue_job/tasks/main.yml

diff --git a/changelogs/fragments/480-aws_glue_job-python-glue-version.yml b/changelogs/fragments/480-aws_glue_job-python-glue-version.yml
new file mode 100644
index 00000000000..3f62ed5a3b6
--- /dev/null
+++ b/changelogs/fragments/480-aws_glue_job-python-glue-version.yml
@@ -0,0 +1,4 @@
+minor_changes:
+  - aws_glue_job - Added ``glue_version`` parameter (https://github.com/ansible-collections/community.aws/pull/480).
+  - aws_glue_job - Added ``command_python_version`` parameter (https://github.com/ansible-collections/community.aws/pull/480).
+  - aws_glue_job - Added support for check mode (https://github.com/ansible-collections/community.aws/pull/480).
diff --git a/plugins/modules/aws_glue_job.py b/plugins/modules/aws_glue_job.py
index 7f6af1f4d0c..05ace319797 100644
--- a/plugins/modules/aws_glue_job.py
+++ b/plugins/modules/aws_glue_job.py
@@ -27,6 +27,11 @@
       - The name of the job command. This must be 'glueetl'.
     default: glueetl
     type: str
+  command_python_version:
+    description:
+      - Python version being used to execute a Python shell job. Allowed values are '2' or '3'.
+    default: 2
+    type: str
   command_script_location:
     description:
       - The S3 path to a script that executes a job.
@@ -46,6 +51,12 @@
     description:
       - Description of the job being defined.
     type: str
+  glue_version:
+    description:
+      - AWS Glue version. This determines the available version of Apache Scala and Python as described here
+        U(https://docs.aws.amazon.com/glue/latest/dg/add-job.html).
+    default: 0.9
+    type: str
   max_concurrent_runs:
     description:
       - The maximum number of concurrent runs allowed for the job. The default is 1. An error is returned when
@@ -121,6 +132,11 @@
             returned: when state is present
             type: str
             sample: mybucket/myscript.py
+        python_version:
+            description: Specifies the Python version.
+            returned: when state is present
+            type: str
+            sample: 3
 connections:
     description: The connections used for this job.
     returned: when state is present
@@ -141,6 +157,11 @@
     returned: when state is present
     type: str
     sample: My first Glue job
+glue_version:
+    description: Glue version.
+    returned: when state is present
+    type: str
+    sample: 2.0
 job_name:
     description: The name of the AWS Glue job.
     returned: always
@@ -235,8 +256,11 @@ def _compare_glue_job_params(user_params, current_params):
 
     if 'AllocatedCapacity' in user_params and user_params['AllocatedCapacity'] != current_params['AllocatedCapacity']:
         return True
-    if 'Command' in user_params and user_params['Command']['ScriptLocation'] != current_params['Command']['ScriptLocation']:
-        return True
+    if 'Command' in user_params:
+        if user_params['Command']['ScriptLocation'] != current_params['Command']['ScriptLocation']:
+            return True
+        if user_params['Command']['PythonVersion'] != current_params['Command']['PythonVersion']:
+            return True
     if 'Connections' in user_params and set(user_params['Connections']) != set(current_params['Connections']):
         return True
     if 'DefaultArguments' in user_params and set(user_params['DefaultArguments']) != set(current_params['DefaultArguments']):
@@ -245,6 +269,8 @@ def _compare_glue_job_params(user_params, current_params):
         return True
     if 'ExecutionProperty' in user_params and user_params['ExecutionProperty']['MaxConcurrentRuns'] != current_params['ExecutionProperty']['MaxConcurrentRuns']:
         return True
+    if 'GlueVersion' in user_params and user_params['GlueVersion'] != current_params['GlueVersion']:
+        return True
     if 'MaxRetries' in user_params and user_params['MaxRetries'] != current_params['MaxRetries']:
         return True
     if 'Timeout' in user_params and user_params['Timeout'] != current_params['Timeout']:
@@ -271,12 +297,16 @@ def create_or_update_glue_job(connection, module, glue_job):
         params['AllocatedCapacity'] = module.params.get("allocated_capacity")
     if module.params.get("command_script_location") is not None:
         params['Command'] = {'Name': module.params.get("command_name"), 'ScriptLocation': module.params.get("command_script_location")}
+        if module.params.get("command_python_version") is not None:
+            params['Command']['PythonVersion'] = module.params.get("command_python_version")
     if module.params.get("connections") is not None:
         params['Connections'] = {'Connections': module.params.get("connections")}
     if module.params.get("default_arguments") is not None:
         params['DefaultArguments'] = module.params.get("default_arguments")
     if module.params.get("description") is not None:
         params['Description'] = module.params.get("description")
+    if module.params.get("glue_version") is not None:
+        params['GlueVersion'] = module.params.get("glue_version")
     if module.params.get("max_concurrent_runs") is not None:
         params['ExecutionProperty'] = {'MaxConcurrentRuns': module.params.get("max_concurrent_runs")}
     if module.params.get("max_retries") is not None:
@@ -291,13 +321,15 @@ def create_or_update_glue_job(connection, module, glue_job):
                 # Update job needs slightly modified params
                 update_params = {'JobName': params['Name'], 'JobUpdate': copy.deepcopy(params)}
                 del update_params['JobUpdate']['Name']
-                connection.update_job(**update_params)
+                if not module.check_mode:
+                    connection.update_job(**update_params)
                 changed = True
             except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
                 module.fail_json_aws(e)
     else:
         try:
-            connection.create_job(**params)
+            if not module.check_mode:
+                connection.create_job(**params)
             changed = True
         except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
             module.fail_json_aws(e)
@@ -323,7 +355,8 @@ def delete_glue_job(connection, module, glue_job):
 
     if glue_job:
         try:
-            connection.delete_job(JobName=glue_job['Name'])
+            if not module.check_mode:
+                connection.delete_job(JobName=glue_job['Name'])
             changed = True
         except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
             module.fail_json_aws(e)
@@ -337,10 +370,12 @@ def main():
         dict(
             allocated_capacity=dict(type='int'),
             command_name=dict(type='str', default='glueetl'),
+            command_python_version=dict(type='str', default='2'),
             command_script_location=dict(type='str'),
             connections=dict(type='list', elements='str'),
             default_arguments=dict(type='dict'),
             description=dict(type='str'),
+            glue_version=dict(type='str', default='0.9'),
             max_concurrent_runs=dict(type='int'),
             max_retries=dict(type='int'),
             name=dict(required=True, type='str'),
@@ -353,7 +388,8 @@ def main():
     module = AnsibleAWSModule(argument_spec=argument_spec,
                               required_if=[
                                   ('state', 'present', ['role', 'command_script_location'])
-                              ]
+                              ],
+                              supports_check_mode=True
                               )
 
     connection = module.client('glue')
diff --git a/tests/integration/targets/aws_glue_job/aliases b/tests/integration/targets/aws_glue_job/aliases
new file mode 100644
index 00000000000..6e3860bee23
--- /dev/null
+++ b/tests/integration/targets/aws_glue_job/aliases
@@ -0,0 +1,2 @@
+cloud/aws
+shippable/aws/group2
diff --git a/tests/integration/targets/aws_glue_job/defaults/main.yml b/tests/integration/targets/aws_glue_job/defaults/main.yml
new file mode 100644
index 00000000000..26557a1137a
--- /dev/null
+++ b/tests/integration/targets/aws_glue_job/defaults/main.yml
@@ -0,0 +1,5 @@
+---
+glue_job_name: '{{ resource_prefix }}'
+glue_job_role_name: 'ansible-test-{{ resource_prefix }}-glue-job'
+glue_job_description: Test job
+glue_job_command_script_location: test-s3-bucket-glue/job.py
diff --git a/tests/integration/targets/aws_glue_job/tasks/main.yml b/tests/integration/targets/aws_glue_job/tasks/main.yml
new file mode 100644
index 00000000000..d3122a2a4fd
--- /dev/null
+++ b/tests/integration/targets/aws_glue_job/tasks/main.yml
@@ -0,0 +1,271 @@
+---
+- name: aws_glue_job integration tests
+  collections:
+    - amazon.aws
+  module_defaults:
+    group/aws:
+      aws_access_key: "{{ aws_access_key }}"
+      aws_secret_key: "{{ aws_secret_key }}"
+      security_token: "{{ security_token | default(omit) }}"
+      region: "{{ aws_region }}"
+  block:
+    # AWS CLI is needed until there's a module to get info about Glue jobs
+    - name: Install AWS CLI
+      pip:
+        name: awscli
+        state: present
+
+    - name: Create minimal Glue job role
+      iam_role:
+        name: "{{ glue_job_role_name }}"
+        trust_policy:
+          Version: "2008-10-17"
+          Statement:
+            - Action: "sts:AssumeRole"
+              Effect: Allow
+              Principal:
+                Service: glue.amazonaws.com
+        create_instance_profile: false
+        managed_policies:
+          - "arn:aws:iam::aws:policy/AWSGlueServiceRole"
+
+    - name: Create Glue job (check mode)
+      aws_glue_job:
+        name: "{{ glue_job_name }}"
+        description: "{{ glue_job_description }}"
+        command_script_location: "{{ glue_job_command_script_location }}"
+        command_python_version: 3
+        glue_version: "2.0"
+        role: "{{ glue_job_role_name }}"
+        state: present
+      check_mode: true
+      register: glue_job_check
+
+    - name: Verity that Glue job was not created in check mode
+      assert:
+        that:
+          - glue_job_check.changed
+          - not glue_job_check.description
+
+    - name: Create Glue job
+      aws_glue_job:
+        name: "{{ glue_job_name }}"
+        description: "{{ glue_job_description }}"
+        command_script_location: "{{ glue_job_command_script_location }}"
+        command_python_version: 3
+        glue_version: "2.0"
+        role: "{{ glue_job_role_name }}"
+        state: present
+      register: glue_job
+
+    - name: Get info on Glue job
+      command: "aws glue get-job --job-name {{ glue_job_name }}"
+      environment:
+        AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
+        AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
+        AWS_SESSION_TOKEN: "{{ security_token | default('') }}"
+        AWS_DEFAULT_REGION: "{{ aws_region }}"
+      register: job_info_query
+
+    - name: Convert it to an object
+      set_fact:
+        job_info: "{{ job_info_query.stdout | from_json }}"
+
+    - name: Verity that Glue job was created
+      assert:
+        that:
+          - glue_job.changed"
+          - glue_job.description == job_info["Job"]["Description"]
+          - glue_job.role == job_info["Job"]["Role"]
+          - glue_job.command.script_location == job_info["Job"]["Command"]["ScriptLocation"]
+          - glue_job.command.python_version == job_info["Job"]["Command"]["PythonVersion"]
+          - glue_job.glue_version == job_info["Job"]["GlueVersion"]
+
+    - name: Create Glue job (idempotent) (check mode)
+      aws_glue_job:
+        name: "{{ glue_job_name }}"
+        description: "{{ glue_job_description }}"
+        command_script_location: "{{ glue_job_command_script_location }}"
+        command_python_version: 3
+        glue_version: "2.0"
+        role: "{{ glue_job_role_name }}"
+        state: present
+      check_mode: true
+      register: glue_job_idempotent_check
+
+    - name: Get info on Glue job
+      command: "aws glue get-job --job-name {{ glue_job_name }}"
+      environment:
+        AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
+        AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
+        AWS_SESSION_TOKEN: "{{ security_token | default('') }}"
+        AWS_DEFAULT_REGION: "{{ aws_region }}"
+      register: job_info_query_idempotent_check
+
+    - name: Convert it to an object
+      set_fact:
+        job_info_idempotent_check: "{{ job_info_query_idempotent_check.stdout | from_json }}"
+
+    - name: Verity that Glue job was not modified in check mode
+      assert:
+        that:
+          - not glue_job_idempotent_check.changed
+          - job_info["Job"]["Name"] == job_info_idempotent_check["Job"]["Name"]
+          - job_info["Job"]["Description"] == job_info_idempotent_check["Job"]["Description"]
+          - job_info["Job"]["Role"] == job_info_idempotent_check["Job"]["Role"]
+          - job_info["Job"]["GlueVersion"] == job_info_idempotent_check["Job"]["GlueVersion"]
+          - job_info["Job"]["Command"]['ScriptLocation'] == job_info_idempotent_check["Job"]["Command"]["ScriptLocation"]
+          - job_info["Job"]["Command"]['PythonVersion'] == job_info_idempotent_check["Job"]["Command"]["PythonVersion"]
+
+    - name: Create Glue job (idempotent)
+      aws_glue_job:
+        name: "{{ glue_job_name }}"
+        description: "{{ glue_job_description }}"
+        command_script_location: "{{ glue_job_command_script_location }}"
+        command_python_version: 3
+        glue_version: "2.0"
+        role: "{{ glue_job_role_name }}"
+        state: present
+      register: glue_job_idempotent
+
+    - name: Get info on Glue job
+      command: "aws glue get-job --job-name {{ glue_job_name }}"
+      environment:
+        AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
+        AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
+        AWS_SESSION_TOKEN: "{{ security_token | default('') }}"
+        AWS_DEFAULT_REGION: "{{ aws_region }}"
+      register: job_info_query_idempotent
+
+    - name: Convert it to an object
+      set_fact:
+        job_info_idempotent: "{{ job_info_query_idempotent.stdout | from_json }}"
+
+    - name: Verity that Glue job was not modified
+      assert:
+        that:
+          - not glue_job_idempotent.changed
+          - job_info["Job"]["Name"] == job_info_idempotent["Job"]["Name"]
+          - job_info["Job"]["Description"] == job_info_idempotent["Job"]["Description"]
+          - job_info["Job"]["Role"] == job_info_idempotent["Job"]["Role"]
+          - job_info["Job"]["GlueVersion"] == job_info_idempotent["Job"]["GlueVersion"]
+          - job_info["Job"]["Command"]['ScriptLocation'] == job_info_idempotent["Job"]["Command"]["ScriptLocation"]
+          - job_info["Job"]["Command"]['PythonVersion'] == job_info_idempotent["Job"]["Command"]["PythonVersion"]
+
+    - name: Update Glue job (check mode)
+      aws_glue_job:
+        name: "{{ glue_job_name }}"
+        description: "{{ glue_job_description }}"
+        command_script_location: "{{ glue_job_command_script_location }}"
+        command_python_version: 2
+        glue_version: "0.9"
+        role: "{{ glue_job_role_name }}"
+        state: present
+      check_mode: true
+      register: glue_job_update_check
+
+    - name: Get info on Glue job
+      command: "aws glue get-job --job-name {{ glue_job_name }}"
+      environment:
+        AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
+        AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
+        AWS_SESSION_TOKEN: "{{ security_token | default('') }}"
+        AWS_DEFAULT_REGION: "{{ aws_region }}"
+      register: job_info_query_update_check
+
+    - name: Convert it to an object
+      set_fact:
+        job_info_update_check: "{{ job_info_query_update_check.stdout | from_json }}"
+
+    - name: Verity that Glue job was not modified in check mode
+      assert:
+        that:
+          - glue_job_update_check.changed
+          - glue_job_update_check.description == job_info_update_check["Job"]["Description"]
+          - glue_job_update_check.role == job_info_update_check["Job"]["Role"]
+          - glue_job_update_check.command.script_location == job_info_update_check["Job"]["Command"]["ScriptLocation"]
+          - glue_job_update_check.command.python_version == "3"
+          - glue_job_update_check.glue_version == "2.0"
+
+    - name: Update Glue job
+      aws_glue_job:
+        name: "{{ glue_job_name }}"
+        description: "{{ glue_job_description }}"
+        command_script_location: "{{ glue_job_command_script_location }}"
+        command_python_version: 2
+        glue_version: "0.9"
+        role: "{{ glue_job_role_name }}"
+        state: present
+      register: glue_job_update
+
+    - name: Get info on Glue job
+      command: "aws glue get-job --job-name {{ glue_job_name }}"
+      environment:
+        AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
+        AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
+        AWS_SESSION_TOKEN: "{{ security_token | default('') }}"
+        AWS_DEFAULT_REGION: "{{ aws_region }}"
+      register: job_info_query_update
+
+    - name: Convert it to an object
+      set_fact:
+        job_info_update: "{{ job_info_query_update.stdout | from_json }}"
+
+    - name: Verity that Glue job was modified
+      assert:
+        that:
+          - glue_job_update.changed
+          - glue_job_update.description == job_info_update["Job"]["Description"]
+          - glue_job_update.role == job_info_update["Job"]["Role"]
+          - glue_job_update.command.script_location == job_info_update["Job"]["Command"]["ScriptLocation"]
+          - glue_job_update.command.python_version == job_info_update["Job"]["Command"]["PythonVersion"]
+          - glue_job_update.glue_version == job_info_update["Job"]["Command"]["GlueVersion"]
+
+    - name: Delete Glue job (check mode)
+      aws_glue_job:
+        name: "{{ glue_job_name }}"
+        state: absent
+      check_mode: true
+      register: glue_job_delete_check
+
+    - name: Get info on Glue job
+      command: "aws glue get-job --job-name {{ glue_job_name }}"
+      environment:
+        AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
+        AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
+        AWS_SESSION_TOKEN: "{{ security_token | default('') }}"
+        AWS_DEFAULT_REGION: "{{ aws_region }}"
+      register: job_info_query_delete_check
+
+    - name: Convert it to an object
+      set_fact:
+        job_info_delete_check: "{{ job_info_query_delete_check.stdout | from_json }}"
+
+    - name: Verity that Glue job was not deleted in check mode
+      assert:
+        that:
+          - glue_job_delete_check.changed
+          - job_info["Job"]["Name"] == job_info_delete_check["Job"]["Name"]
+
+    - name: Delete Glue job
+      aws_glue_job:
+        name: "{{ glue_job_name }}"
+        state: absent
+      register: glue_job_delete
+
+    - name: Verity that Glue job was deleted
+      assert:
+        that:
+          - glue_job_delete.changed
+
+  always:
+    - name: Delete Glue job
+      aws_glue_connection:
+        name: "{{ glue_job_name }}"
+        state: absent
+      ignore_errors: true
+    - name: Delete Glue job role
+      iam_role:
+        name: "{{ glue_job_role_name }}"
+        state: absent
+      ignore_errors: true

From a0e8af09d695592541641505a8c997e9f4fedfbc Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Tue, 16 Mar 2021 02:50:08 +0000
Subject: [PATCH 02/12] Fix parameter name - assume_role_policy_document

---
 tests/integration/targets/aws_glue_job/tasks/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/targets/aws_glue_job/tasks/main.yml b/tests/integration/targets/aws_glue_job/tasks/main.yml
index d3122a2a4fd..2c3c686b065 100644
--- a/tests/integration/targets/aws_glue_job/tasks/main.yml
+++ b/tests/integration/targets/aws_glue_job/tasks/main.yml
@@ -18,7 +18,7 @@
     - name: Create minimal Glue job role
       iam_role:
         name: "{{ glue_job_role_name }}"
-        trust_policy:
+        assume_role_policy_document:
           Version: "2008-10-17"
           Statement:
             - Action: "sts:AssumeRole"

From b3610ceaac7c61b9c22668e926f618c74cc87b4e Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Tue, 16 Mar 2021 03:31:26 +0000
Subject: [PATCH 03/12] Attempt with a different policy that appears to be
 allowed

---
 tests/integration/targets/aws_glue_job/tasks/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/targets/aws_glue_job/tasks/main.yml b/tests/integration/targets/aws_glue_job/tasks/main.yml
index 2c3c686b065..677c8ce3bb3 100644
--- a/tests/integration/targets/aws_glue_job/tasks/main.yml
+++ b/tests/integration/targets/aws_glue_job/tasks/main.yml
@@ -27,7 +27,7 @@
                 Service: glue.amazonaws.com
         create_instance_profile: false
         managed_policies:
-          - "arn:aws:iam::aws:policy/AWSGlueServiceRole"
+          - "arn:aws:iam::aws:policy/AWSXrayWriteOnlyAccess"
 
     - name: Create Glue job (check mode)
       aws_glue_job:

From 8f188264a22867a50cbe8ac9b8474ae9514ab2fc Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Tue, 16 Mar 2021 09:49:45 -0400
Subject: [PATCH 04/12] Remove default value for command_python_version

Co-authored-by: Mark Chappell <mchappel@redhat.com>
---
 plugins/modules/aws_glue_job.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/modules/aws_glue_job.py b/plugins/modules/aws_glue_job.py
index 05ace319797..d3388c21461 100644
--- a/plugins/modules/aws_glue_job.py
+++ b/plugins/modules/aws_glue_job.py
@@ -370,7 +370,7 @@ def main():
         dict(
             allocated_capacity=dict(type='int'),
             command_name=dict(type='str', default='glueetl'),
-            command_python_version=dict(type='str', default='2'),
+            command_python_version=dict(type='str'),
             command_script_location=dict(type='str'),
             connections=dict(type='list', elements='str'),
             default_arguments=dict(type='dict'),

From 1208d3e875d9b9d6d3b434f1e550812bc3f36653 Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Tue, 16 Mar 2021 13:51:24 +0000
Subject: [PATCH 05/12] Remove default values for command_python_version and
 glue_version

This is to avoid inadvertently breaking existing playbooks. Defaults
will be set by AWS.
---
 plugins/modules/aws_glue_job.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/plugins/modules/aws_glue_job.py b/plugins/modules/aws_glue_job.py
index d3388c21461..eacd85021c6 100644
--- a/plugins/modules/aws_glue_job.py
+++ b/plugins/modules/aws_glue_job.py
@@ -30,7 +30,6 @@
   command_python_version:
     description:
       - Python version being used to execute a Python shell job. Allowed values are '2' or '3'.
-    default: 2
     type: str
   command_script_location:
     description:
@@ -55,7 +54,6 @@
     description:
       - AWS Glue version. This determines the available version of Apache Scala and Python as described here
         U(https://docs.aws.amazon.com/glue/latest/dg/add-job.html).
-    default: 0.9
     type: str
   max_concurrent_runs:
     description:
@@ -375,7 +373,7 @@ def main():
             connections=dict(type='list', elements='str'),
             default_arguments=dict(type='dict'),
             description=dict(type='str'),
-            glue_version=dict(type='str', default='0.9'),
+            glue_version=dict(type='str'),
             max_concurrent_runs=dict(type='int'),
             max_retries=dict(type='int'),
             name=dict(required=True, type='str'),

From a70984ab2333beb5f6423bbdca0acf1451039cd1 Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Tue, 16 Mar 2021 14:48:49 +0000
Subject: [PATCH 06/12] Update IAM role name for Glue job to make sure it's
 shorter than 64 characters

---
 .../integration/targets/aws_glue_job/defaults/main.yml | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/tests/integration/targets/aws_glue_job/defaults/main.yml b/tests/integration/targets/aws_glue_job/defaults/main.yml
index 26557a1137a..51ed2fddc62 100644
--- a/tests/integration/targets/aws_glue_job/defaults/main.yml
+++ b/tests/integration/targets/aws_glue_job/defaults/main.yml
@@ -1,5 +1,11 @@
 ---
-glue_job_name: '{{ resource_prefix }}'
-glue_job_role_name: 'ansible-test-{{ resource_prefix }}-glue-job'
+glue_job_name: "{{ resource_prefix }}"
 glue_job_description: Test job
 glue_job_command_script_location: test-s3-bucket-glue/job.py
+# IAM role names have to be less than 64 characters
+# The 8 digit identifier at the end of resource_prefix helps determine during
+# which test something was created and allows tests to be run in parallel
+# Shippable resource_prefixes are in the format shippable-123456-123, so in those cases
+# we need both sets of digits to keep the resource name unique
+unique_id: "{{ resource_prefix | regex_search('(\\d+-?)(\\d+)$') }}"
+glue_job_role_name: "ansible-test-{{ unique_id }}-glue-job"

From 84f7090ad9602c93476c8b08e45765b3eb24a7b0 Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Sun, 21 Mar 2021 23:31:23 +0000
Subject: [PATCH 07/12] Add support for tags

---
 .../480-aws_glue_job-python-glue-version.yml  |  1 +
 plugins/modules/aws_glue_job.py               | 77 ++++++++++++++++++-
 .../targets/aws_glue_job/tasks/main.yml       | 70 ++++++++++-------
 3 files changed, 117 insertions(+), 31 deletions(-)

diff --git a/changelogs/fragments/480-aws_glue_job-python-glue-version.yml b/changelogs/fragments/480-aws_glue_job-python-glue-version.yml
index 3f62ed5a3b6..a277d0bcc3d 100644
--- a/changelogs/fragments/480-aws_glue_job-python-glue-version.yml
+++ b/changelogs/fragments/480-aws_glue_job-python-glue-version.yml
@@ -2,3 +2,4 @@ minor_changes:
   - aws_glue_job - Added ``glue_version`` parameter (https://github.com/ansible-collections/community.aws/pull/480).
   - aws_glue_job - Added ``command_python_version`` parameter (https://github.com/ansible-collections/community.aws/pull/480).
   - aws_glue_job - Added support for check mode (https://github.com/ansible-collections/community.aws/pull/480).
+  - aws_glue_job - Added support for tags (https://github.com/ansible-collections/community.aws/pull/480).
diff --git a/plugins/modules/aws_glue_job.py b/plugins/modules/aws_glue_job.py
index eacd85021c6..c5ceca4756a 100644
--- a/plugins/modules/aws_glue_job.py
+++ b/plugins/modules/aws_glue_job.py
@@ -80,6 +80,12 @@
     required: true
     choices: [ 'present', 'absent' ]
     type: str
+  tags:
+    description:
+      - A hash/dictionary of tags to be applied to the job.
+      - Remove completely or specify an empty dictionary to remove all tags.
+    default: {}
+    type: dict
   timeout:
     description:
       - The job timeout in minutes.
@@ -95,7 +101,10 @@
 
 # Create an AWS Glue job
 - community.aws.aws_glue_job:
-    command_script_location: s3bucket/script.py
+    command_script_location: "s3://s3bucket/script.py"
+    default_arguments:
+      "--extra-py-files": s3://s3bucket/script-package.zip
+      "--TempDir": "s3://s3bucket/temp/"
     name: my-glue-job
     role: my-iam-role
     state: present
@@ -215,8 +224,28 @@
 
 from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule
 from ansible_collections.amazon.aws.plugins.module_utils.core import is_boto3_error_code
+from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ansible_dict_to_boto3_tag_list
+from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry
+from ansible_collections.amazon.aws.plugins.module_utils.ec2 import compare_aws_tags
+from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto3_tag_list_to_ansible_dict
+from ansible_collections.amazon.aws.plugins.module_utils.ec2 import compare_aws_tags
+
+
+@AWSRetry.backoff(tries=5, delay=5, backoff=2.0)
+def _get_account_info(module):
+    """
+    Return the account information (account ID and partition) we are currently working on.
+    """
+    account_id = None
+    partition = None
+    sts_client = module.client('sts')
+    caller_identity = sts_client.get_caller_identity()
+    account_id = caller_identity.get('Account')
+    partition = caller_identity.get('Arn').split(':')[1]
+    return account_id, partition
 
 
+@AWSRetry.backoff(tries=5, delay=5, backoff=2.0)
 def _get_glue_job(connection, module, glue_job_name):
     """
     Get an AWS Glue job based on name. If not found, return None.
@@ -277,6 +306,44 @@ def _compare_glue_job_params(user_params, current_params):
     return False
 
 
+@AWSRetry.backoff(tries=5, delay=5, backoff=2.0)
+def ensure_tags(connection, module, glue_job):
+    changed = False
+
+    account_id, partition = _get_account_info(module)
+    arn = 'arn:{0}:glue:{1}:{2}:job/{3}'.format(partition, module.region, account_id, module.params.get('name'))
+
+    try:
+        response = connection.get_tags(ResourceArn=arn)
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+        module.fail_json_aws(e, msg='Unable to get tags for Glue job %s' % module.params.get('name'))
+
+    if module.params.get('tags') is None:
+        return False
+
+    existing_tags = response.get('Tags')
+    tags_to_add, tags_to_remove = compare_aws_tags(existing_tags, module.params.get('tags'), True)
+
+    if tags_to_remove:
+        changed = True
+        if not module.check_mode:
+            try:
+                connection.untag_resource(ResourceArn=arn, TagsToRemove=tags_to_remove)
+            except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+                module.fail_json_aws(e, msg='Unable to set tags for Glue job %s' % module.params.get('name'))
+
+    if tags_to_add:
+        changed = True
+        if not module.check_mode:
+            try:
+                connection.tag_resource(ResourceArn=arn, TagsToAdd=tags_to_add)
+            except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+                module.fail_json_aws(e, msg='Unable to set tags for Glue job %s' % module.params.get('name'))
+
+    return changed
+
+
+@AWSRetry.backoff(tries=5, delay=5, backoff=2.0)
 def create_or_update_glue_job(connection, module, glue_job):
     """
     Create or update an AWS Glue job
@@ -332,13 +399,14 @@ def create_or_update_glue_job(connection, module, glue_job):
         except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
             module.fail_json_aws(e)
 
-    # If changed, get the Glue job again
-    if changed:
-        glue_job = _get_glue_job(connection, module, params['Name'])
+    glue_job = _get_glue_job(connection, module, params['Name'])
+
+    changed |= ensure_tags(connection, module, glue_job)
 
     module.exit_json(changed=changed, **camel_dict_to_snake_dict(glue_job))
 
 
+@AWSRetry.backoff(tries=5, delay=5, backoff=2.0)
 def delete_glue_job(connection, module, glue_job):
     """
     Delete an AWS Glue job
@@ -379,6 +447,7 @@ def main():
             name=dict(required=True, type='str'),
             role=dict(type='str'),
             state=dict(required=True, choices=['present', 'absent'], type='str'),
+            tags=dict(type='dict', default={}),
             timeout=dict(type='int')
         )
     )
diff --git a/tests/integration/targets/aws_glue_job/tasks/main.yml b/tests/integration/targets/aws_glue_job/tasks/main.yml
index 677c8ce3bb3..b906539a0b4 100644
--- a/tests/integration/targets/aws_glue_job/tasks/main.yml
+++ b/tests/integration/targets/aws_glue_job/tasks/main.yml
@@ -32,11 +32,14 @@
     - name: Create Glue job (check mode)
       aws_glue_job:
         name: "{{ glue_job_name }}"
-        description: "{{ glue_job_description }}"
-        command_script_location: "{{ glue_job_command_script_location }}"
         command_python_version: 3
+        command_script_location: "{{ glue_job_command_script_location }}"
+        description: "{{ glue_job_description }}"
         glue_version: "2.0"
         role: "{{ glue_job_role_name }}"
+        tags:
+          Environment: Test
+          Product: Glue
         state: present
       check_mode: true
       register: glue_job_check
@@ -50,11 +53,14 @@
     - name: Create Glue job
       aws_glue_job:
         name: "{{ glue_job_name }}"
-        description: "{{ glue_job_description }}"
-        command_script_location: "{{ glue_job_command_script_location }}"
         command_python_version: 3
+        command_script_location: "{{ glue_job_command_script_location }}"
+        description: "{{ glue_job_description }}"
         glue_version: "2.0"
         role: "{{ glue_job_role_name }}"
+        tags:
+          Environment: Test
+          Product: Glue
         state: present
       register: glue_job
 
@@ -75,20 +81,23 @@
       assert:
         that:
           - glue_job.changed"
-          - glue_job.description == job_info["Job"]["Description"]
-          - glue_job.role == job_info["Job"]["Role"]
-          - glue_job.command.script_location == job_info["Job"]["Command"]["ScriptLocation"]
           - glue_job.command.python_version == job_info["Job"]["Command"]["PythonVersion"]
+          - glue_job.command.script_location == job_info["Job"]["Command"]["ScriptLocation"]
+          - glue_job.description == job_info["Job"]["Description"]
           - glue_job.glue_version == job_info["Job"]["GlueVersion"]
+          - glue_job.role == job_info["Job"]["Role"]
 
     - name: Create Glue job (idempotent) (check mode)
       aws_glue_job:
         name: "{{ glue_job_name }}"
-        description: "{{ glue_job_description }}"
-        command_script_location: "{{ glue_job_command_script_location }}"
         command_python_version: 3
+        command_script_location: "{{ glue_job_command_script_location }}"
+        description: "{{ glue_job_description }}"
         glue_version: "2.0"
         role: "{{ glue_job_role_name }}"
+        tags:
+          Environment: Test
+          Product: Glue
         state: present
       check_mode: true
       register: glue_job_idempotent_check
@@ -111,20 +120,23 @@
         that:
           - not glue_job_idempotent_check.changed
           - job_info["Job"]["Name"] == job_info_idempotent_check["Job"]["Name"]
+          - job_info["Job"]["Command"]['PythonVersion'] == job_info_idempotent_check["Job"]["Command"]["PythonVersion"]
+          - job_info["Job"]["Command"]['ScriptLocation'] == job_info_idempotent_check["Job"]["Command"]["ScriptLocation"]
           - job_info["Job"]["Description"] == job_info_idempotent_check["Job"]["Description"]
-          - job_info["Job"]["Role"] == job_info_idempotent_check["Job"]["Role"]
           - job_info["Job"]["GlueVersion"] == job_info_idempotent_check["Job"]["GlueVersion"]
-          - job_info["Job"]["Command"]['ScriptLocation'] == job_info_idempotent_check["Job"]["Command"]["ScriptLocation"]
-          - job_info["Job"]["Command"]['PythonVersion'] == job_info_idempotent_check["Job"]["Command"]["PythonVersion"]
+          - job_info["Job"]["Role"] == job_info_idempotent_check["Job"]["Role"]
 
     - name: Create Glue job (idempotent)
       aws_glue_job:
         name: "{{ glue_job_name }}"
-        description: "{{ glue_job_description }}"
-        command_script_location: "{{ glue_job_command_script_location }}"
         command_python_version: 3
+        command_script_location: "{{ glue_job_command_script_location }}"
+        description: "{{ glue_job_description }}"
         glue_version: "2.0"
         role: "{{ glue_job_role_name }}"
+        tags:
+          Environment: Test
+          Product: Glue
         state: present
       register: glue_job_idempotent
 
@@ -146,20 +158,22 @@
         that:
           - not glue_job_idempotent.changed
           - job_info["Job"]["Name"] == job_info_idempotent["Job"]["Name"]
+          - job_info["Job"]["Command"]['PythonVersion'] == job_info_idempotent["Job"]["Command"]["PythonVersion"]
+          - job_info["Job"]["Command"]['ScriptLocation'] == job_info_idempotent["Job"]["Command"]["ScriptLocation"]
           - job_info["Job"]["Description"] == job_info_idempotent["Job"]["Description"]
-          - job_info["Job"]["Role"] == job_info_idempotent["Job"]["Role"]
           - job_info["Job"]["GlueVersion"] == job_info_idempotent["Job"]["GlueVersion"]
-          - job_info["Job"]["Command"]['ScriptLocation'] == job_info_idempotent["Job"]["Command"]["ScriptLocation"]
-          - job_info["Job"]["Command"]['PythonVersion'] == job_info_idempotent["Job"]["Command"]["PythonVersion"]
+          - job_info["Job"]["Role"] == job_info_idempotent["Job"]["Role"]
 
     - name: Update Glue job (check mode)
       aws_glue_job:
         name: "{{ glue_job_name }}"
-        description: "{{ glue_job_description }}"
-        command_script_location: "{{ glue_job_command_script_location }}"
         command_python_version: 2
+        command_script_location: "{{ glue_job_command_script_location }}"
+        description: "{{ glue_job_description }}"
         glue_version: "0.9"
         role: "{{ glue_job_role_name }}"
+        tags:
+          Environment: Test
         state: present
       check_mode: true
       register: glue_job_update_check
@@ -181,20 +195,22 @@
       assert:
         that:
           - glue_job_update_check.changed
-          - glue_job_update_check.description == job_info_update_check["Job"]["Description"]
-          - glue_job_update_check.role == job_info_update_check["Job"]["Role"]
-          - glue_job_update_check.command.script_location == job_info_update_check["Job"]["Command"]["ScriptLocation"]
           - glue_job_update_check.command.python_version == "3"
+          - glue_job_update_check.command.script_location == job_info_update_check["Job"]["Command"]["ScriptLocation"]
+          - glue_job_update_check.description == job_info_update_check["Job"]["Description"]
           - glue_job_update_check.glue_version == "2.0"
+          - glue_job_update_check.role == job_info_update_check["Job"]["Role"]
 
     - name: Update Glue job
       aws_glue_job:
         name: "{{ glue_job_name }}"
-        description: "{{ glue_job_description }}"
-        command_script_location: "{{ glue_job_command_script_location }}"
         command_python_version: 2
+        command_script_location: "{{ glue_job_command_script_location }}"
+        description: "{{ glue_job_description }}"
         glue_version: "0.9"
         role: "{{ glue_job_role_name }}"
+        tags:
+          Environment: Test
         state: present
       register: glue_job_update
 
@@ -215,11 +231,11 @@
       assert:
         that:
           - glue_job_update.changed
-          - glue_job_update.description == job_info_update["Job"]["Description"]
-          - glue_job_update.role == job_info_update["Job"]["Role"]
-          - glue_job_update.command.script_location == job_info_update["Job"]["Command"]["ScriptLocation"]
           - glue_job_update.command.python_version == job_info_update["Job"]["Command"]["PythonVersion"]
+          - glue_job_update.command.script_location == job_info_update["Job"]["Command"]["ScriptLocation"]
+          - glue_job_update.description == job_info_update["Job"]["Description"]
           - glue_job_update.glue_version == job_info_update["Job"]["Command"]["GlueVersion"]
+          - glue_job_update.role == job_info_update["Job"]["Role"]
 
     - name: Delete Glue job (check mode)
       aws_glue_job:

From 704f6be0e2e8cf676df1f13ff7557264d0c554d8 Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Mon, 22 Mar 2021 16:42:44 +0000
Subject: [PATCH 08/12] Properly detect changes in DefaultArguments parameter

Add tests for DefaultArguments.
---
 plugins/modules/aws_glue_job.py               |  4 +--
 .../targets/aws_glue_job/defaults/main.yml    |  3 ++-
 .../targets/aws_glue_job/tasks/main.yml       | 25 ++++++++++++++++---
 3 files changed, 25 insertions(+), 7 deletions(-)

diff --git a/plugins/modules/aws_glue_job.py b/plugins/modules/aws_glue_job.py
index c5ceca4756a..d99c3e48b70 100644
--- a/plugins/modules/aws_glue_job.py
+++ b/plugins/modules/aws_glue_job.py
@@ -288,9 +288,9 @@ def _compare_glue_job_params(user_params, current_params):
             return True
         if user_params['Command']['PythonVersion'] != current_params['Command']['PythonVersion']:
             return True
-    if 'Connections' in user_params and set(user_params['Connections']) != set(current_params['Connections']):
+    if 'Connections' in user_params and user_params['Connections'] != current_params['Connections']:
         return True
-    if 'DefaultArguments' in user_params and set(user_params['DefaultArguments']) != set(current_params['DefaultArguments']):
+    if 'DefaultArguments' in user_params and user_params['DefaultArguments'] != current_params['DefaultArguments']:
         return True
     if 'Description' in user_params and user_params['Description'] != current_params['Description']:
         return True
diff --git a/tests/integration/targets/aws_glue_job/defaults/main.yml b/tests/integration/targets/aws_glue_job/defaults/main.yml
index 51ed2fddc62..19c44066b20 100644
--- a/tests/integration/targets/aws_glue_job/defaults/main.yml
+++ b/tests/integration/targets/aws_glue_job/defaults/main.yml
@@ -1,7 +1,8 @@
 ---
 glue_job_name: "{{ resource_prefix }}"
 glue_job_description: Test job
-glue_job_command_script_location: test-s3-bucket-glue/job.py
+glue_job_command_script_location: "s3://test-s3-bucket-glue/job.py"
+glue_job_temp_dir: "s3://test-s3-bucket-glue/temp/"
 # IAM role names have to be less than 64 characters
 # The 8 digit identifier at the end of resource_prefix helps determine during
 # which test something was created and allows tests to be run in parallel
diff --git a/tests/integration/targets/aws_glue_job/tasks/main.yml b/tests/integration/targets/aws_glue_job/tasks/main.yml
index b906539a0b4..3f6d8833c40 100644
--- a/tests/integration/targets/aws_glue_job/tasks/main.yml
+++ b/tests/integration/targets/aws_glue_job/tasks/main.yml
@@ -34,6 +34,8 @@
         name: "{{ glue_job_name }}"
         command_python_version: 3
         command_script_location: "{{ glue_job_command_script_location }}"
+        default_arguments:
+          "--TempDir": "{{ glue_job_temp_dir }}"
         description: "{{ glue_job_description }}"
         glue_version: "2.0"
         role: "{{ glue_job_role_name }}"
@@ -55,6 +57,8 @@
         name: "{{ glue_job_name }}"
         command_python_version: 3
         command_script_location: "{{ glue_job_command_script_location }}"
+        default_arguments:
+          "--TempDir": "{{ glue_job_temp_dir }}"
         description: "{{ glue_job_description }}"
         glue_version: "2.0"
         role: "{{ glue_job_role_name }}"
@@ -83,6 +87,7 @@
           - glue_job.changed"
           - glue_job.command.python_version == job_info["Job"]["Command"]["PythonVersion"]
           - glue_job.command.script_location == job_info["Job"]["Command"]["ScriptLocation"]
+          - glue_job.default_arguments == job_info["Job"]["DefaultArguments"]
           - glue_job.description == job_info["Job"]["Description"]
           - glue_job.glue_version == job_info["Job"]["GlueVersion"]
           - glue_job.role == job_info["Job"]["Role"]
@@ -92,6 +97,8 @@
         name: "{{ glue_job_name }}"
         command_python_version: 3
         command_script_location: "{{ glue_job_command_script_location }}"
+        default_arguments:
+          "--TempDir": "{{ glue_job_temp_dir }}"
         description: "{{ glue_job_description }}"
         glue_version: "2.0"
         role: "{{ glue_job_role_name }}"
@@ -122,6 +129,7 @@
           - job_info["Job"]["Name"] == job_info_idempotent_check["Job"]["Name"]
           - job_info["Job"]["Command"]['PythonVersion'] == job_info_idempotent_check["Job"]["Command"]["PythonVersion"]
           - job_info["Job"]["Command"]['ScriptLocation'] == job_info_idempotent_check["Job"]["Command"]["ScriptLocation"]
+          - job_info["Job"]["DefaultArguments"] == job_info_idempotent_check["Job"]["DefaultArguments"]
           - job_info["Job"]["Description"] == job_info_idempotent_check["Job"]["Description"]
           - job_info["Job"]["GlueVersion"] == job_info_idempotent_check["Job"]["GlueVersion"]
           - job_info["Job"]["Role"] == job_info_idempotent_check["Job"]["Role"]
@@ -131,6 +139,8 @@
         name: "{{ glue_job_name }}"
         command_python_version: 3
         command_script_location: "{{ glue_job_command_script_location }}"
+        default_arguments:
+          "--TempDir": "{{ glue_job_temp_dir }}"
         description: "{{ glue_job_description }}"
         glue_version: "2.0"
         role: "{{ glue_job_role_name }}"
@@ -158,8 +168,9 @@
         that:
           - not glue_job_idempotent.changed
           - job_info["Job"]["Name"] == job_info_idempotent["Job"]["Name"]
-          - job_info["Job"]["Command"]['PythonVersion'] == job_info_idempotent["Job"]["Command"]["PythonVersion"]
-          - job_info["Job"]["Command"]['ScriptLocation'] == job_info_idempotent["Job"]["Command"]["ScriptLocation"]
+          - job_info["Job"]["Command"]["PythonVersion"] == job_info_idempotent["Job"]["Command"]["PythonVersion"]
+          - job_info["Job"]["Command"]["ScriptLocation"] == job_info_idempotent["Job"]["Command"]["ScriptLocation"]
+          - job_info["Job"]["DefaultArguments"] == job_info_idempotent["Job"]["DefaultArguments"]
           - job_info["Job"]["Description"] == job_info_idempotent["Job"]["Description"]
           - job_info["Job"]["GlueVersion"] == job_info_idempotent["Job"]["GlueVersion"]
           - job_info["Job"]["Role"] == job_info_idempotent["Job"]["Role"]
@@ -169,6 +180,8 @@
         name: "{{ glue_job_name }}"
         command_python_version: 2
         command_script_location: "{{ glue_job_command_script_location }}"
+        default_arguments:
+          "--TempDir": "{{ glue_job_temp_dir }}subfolder/"
         description: "{{ glue_job_description }}"
         glue_version: "0.9"
         role: "{{ glue_job_role_name }}"
@@ -195,10 +208,11 @@
       assert:
         that:
           - glue_job_update_check.changed
-          - glue_job_update_check.command.python_version == "3"
+          - glue_job_update_check.command.python_version == job_info_update_check["Job"]["Command"]["PythonVersion"]
           - glue_job_update_check.command.script_location == job_info_update_check["Job"]["Command"]["ScriptLocation"]
+          - glue_job_update_check.default_arguments == job_info_update_check["Job"]["DefaultArguments"]
           - glue_job_update_check.description == job_info_update_check["Job"]["Description"]
-          - glue_job_update_check.glue_version == "2.0"
+          - glue_job_update_check.glue_version == job_info_update_check["Job"]["Command"]["GlueVersion"]
           - glue_job_update_check.role == job_info_update_check["Job"]["Role"]
 
     - name: Update Glue job
@@ -206,6 +220,8 @@
         name: "{{ glue_job_name }}"
         command_python_version: 2
         command_script_location: "{{ glue_job_command_script_location }}"
+        default_arguments:
+          "--TempDir": "{{ glue_job_temp_dir }}subfolder/"
         description: "{{ glue_job_description }}"
         glue_version: "0.9"
         role: "{{ glue_job_role_name }}"
@@ -233,6 +249,7 @@
           - glue_job_update.changed
           - glue_job_update.command.python_version == job_info_update["Job"]["Command"]["PythonVersion"]
           - glue_job_update.command.script_location == job_info_update["Job"]["Command"]["ScriptLocation"]
+          - glue_job_update.default_arguments == job_info_update["Job"]["DefaultArguments"]
           - glue_job_update.description == job_info_update["Job"]["Description"]
           - glue_job_update.glue_version == job_info_update["Job"]["Command"]["GlueVersion"]
           - glue_job_update.role == job_info_update["Job"]["Role"]

From 53b3fcd733cd1637f14ec7fd1e7f177b09ecfb8f Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Fri, 26 Mar 2021 19:50:20 +0000
Subject: [PATCH 09/12] Ensure that Role paramter is modified if necessary

Correctly handle response when no Glue job exists in check mode.
---
 plugins/modules/aws_glue_job.py                | 18 +++++++++++-------
 .../targets/aws_glue_job/tasks/main.yml        |  6 +++---
 2 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/plugins/modules/aws_glue_job.py b/plugins/modules/aws_glue_job.py
index d99c3e48b70..fefe2e9331e 100644
--- a/plugins/modules/aws_glue_job.py
+++ b/plugins/modules/aws_glue_job.py
@@ -300,6 +300,8 @@ def _compare_glue_job_params(user_params, current_params):
         return True
     if 'MaxRetries' in user_params and user_params['MaxRetries'] != current_params['MaxRetries']:
         return True
+    if 'Role' in user_params and user_params['Role'] != current_params['Role']:
+        return True
     if 'Timeout' in user_params and user_params['Timeout'] != current_params['Timeout']:
         return True
 
@@ -310,18 +312,20 @@ def _compare_glue_job_params(user_params, current_params):
 def ensure_tags(connection, module, glue_job):
     changed = False
 
+    if module.params.get('tags') is None:
+        return False
+
     account_id, partition = _get_account_info(module)
     arn = 'arn:{0}:glue:{1}:{2}:job/{3}'.format(partition, module.region, account_id, module.params.get('name'))
 
     try:
-        response = connection.get_tags(ResourceArn=arn)
+        existing_tags = connection.get_tags(ResourceArn=arn).get('Tags', {})
     except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
-        module.fail_json_aws(e, msg='Unable to get tags for Glue job %s' % module.params.get('name'))
-
-    if module.params.get('tags') is None:
-        return False
+        if module.check_mode:
+            existing_tags = {}
+        else:
+            module.fail_json_aws(e, msg='Unable to get tags for Glue job %s' % module.params.get('name'))
 
-    existing_tags = response.get('Tags')
     tags_to_add, tags_to_remove = compare_aws_tags(existing_tags, module.params.get('tags'), True)
 
     if tags_to_remove:
@@ -403,7 +407,7 @@ def create_or_update_glue_job(connection, module, glue_job):
 
     changed |= ensure_tags(connection, module, glue_job)
 
-    module.exit_json(changed=changed, **camel_dict_to_snake_dict(glue_job))
+    module.exit_json(changed=changed, **camel_dict_to_snake_dict(glue_job or {}))
 
 
 @AWSRetry.backoff(tries=5, delay=5, backoff=2.0)
diff --git a/tests/integration/targets/aws_glue_job/tasks/main.yml b/tests/integration/targets/aws_glue_job/tasks/main.yml
index 3f6d8833c40..8e8db677090 100644
--- a/tests/integration/targets/aws_glue_job/tasks/main.yml
+++ b/tests/integration/targets/aws_glue_job/tasks/main.yml
@@ -84,7 +84,7 @@
     - name: Verity that Glue job was created
       assert:
         that:
-          - glue_job.changed"
+          - glue_job.changed
           - glue_job.command.python_version == job_info["Job"]["Command"]["PythonVersion"]
           - glue_job.command.script_location == job_info["Job"]["Command"]["ScriptLocation"]
           - glue_job.default_arguments == job_info["Job"]["DefaultArguments"]
@@ -127,8 +127,8 @@
         that:
           - not glue_job_idempotent_check.changed
           - job_info["Job"]["Name"] == job_info_idempotent_check["Job"]["Name"]
-          - job_info["Job"]["Command"]['PythonVersion'] == job_info_idempotent_check["Job"]["Command"]["PythonVersion"]
-          - job_info["Job"]["Command"]['ScriptLocation'] == job_info_idempotent_check["Job"]["Command"]["ScriptLocation"]
+          - job_info["Job"]["Command"]["PythonVersion"] == job_info_idempotent_check["Job"]["Command"]["PythonVersion"]
+          - job_info["Job"]["Command"]["ScriptLocation"] == job_info_idempotent_check["Job"]["Command"]["ScriptLocation"]
           - job_info["Job"]["DefaultArguments"] == job_info_idempotent_check["Job"]["DefaultArguments"]
           - job_info["Job"]["Description"] == job_info_idempotent_check["Job"]["Description"]
           - job_info["Job"]["GlueVersion"] == job_info_idempotent_check["Job"]["GlueVersion"]

From d06e40a1f85711e6bd405b4f4d2fc8e2ea2d39e5 Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Sat, 27 Mar 2021 21:51:08 +0000
Subject: [PATCH 10/12] Use retries with jittered backoff

Reference:  https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/

Remove duplicate import.
---
 plugins/modules/aws_glue_job.py | 29 ++++++++++++-----------------
 1 file changed, 12 insertions(+), 17 deletions(-)

diff --git a/plugins/modules/aws_glue_job.py b/plugins/modules/aws_glue_job.py
index fefe2e9331e..5659272d97f 100644
--- a/plugins/modules/aws_glue_job.py
+++ b/plugins/modules/aws_glue_job.py
@@ -31,6 +31,7 @@
     description:
       - Python version being used to execute a Python shell job. Allowed values are '2' or '3'.
     type: str
+    version_added: 1.5.0
   command_script_location:
     description:
       - The S3 path to a script that executes a job.
@@ -55,6 +56,7 @@
       - AWS Glue version. This determines the available version of Apache Scala and Python as described here
         U(https://docs.aws.amazon.com/glue/latest/dg/add-job.html).
     type: str
+    version_added: 1.5.0
   max_concurrent_runs:
     description:
       - The maximum number of concurrent runs allowed for the job. The default is 1. An error is returned when
@@ -86,6 +88,7 @@
       - Remove completely or specify an empty dictionary to remove all tags.
     default: {}
     type: dict
+    version_added: 1.5.0
   timeout:
     description:
       - The job timeout in minutes.
@@ -226,12 +229,10 @@
 from ansible_collections.amazon.aws.plugins.module_utils.core import is_boto3_error_code
 from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ansible_dict_to_boto3_tag_list
 from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry
-from ansible_collections.amazon.aws.plugins.module_utils.ec2 import compare_aws_tags
 from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto3_tag_list_to_ansible_dict
 from ansible_collections.amazon.aws.plugins.module_utils.ec2 import compare_aws_tags
 
 
-@AWSRetry.backoff(tries=5, delay=5, backoff=2.0)
 def _get_account_info(module):
     """
     Return the account information (account ID and partition) we are currently working on.
@@ -245,7 +246,6 @@ def _get_account_info(module):
     return account_id, partition
 
 
-@AWSRetry.backoff(tries=5, delay=5, backoff=2.0)
 def _get_glue_job(connection, module, glue_job_name):
     """
     Get an AWS Glue job based on name. If not found, return None.
@@ -255,9 +255,8 @@ def _get_glue_job(connection, module, glue_job_name):
     :param glue_job_name: Name of Glue job to get
     :return: boto3 Glue job dict or None if not found
     """
-
     try:
-        return connection.get_job(JobName=glue_job_name)['Job']
+        return connection.get_job(aws_retry=True, JobName=glue_job_name)['Job']
     except is_boto3_error_code('EntityNotFoundException'):
         return None
     except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:  # pylint: disable=duplicate-except
@@ -272,7 +271,6 @@ def _compare_glue_job_params(user_params, current_params):
     :param current_params: the Glue job parameters currently configured
     :return: True if any parameter is mismatched else False
     """
-
     # Weirdly, boto3 doesn't return some keys if the value is empty e.g. Description
     # To counter this, add the key if it's missing with a blank value
 
@@ -308,7 +306,6 @@ def _compare_glue_job_params(user_params, current_params):
     return False
 
 
-@AWSRetry.backoff(tries=5, delay=5, backoff=2.0)
 def ensure_tags(connection, module, glue_job):
     changed = False
 
@@ -319,7 +316,7 @@ def ensure_tags(connection, module, glue_job):
     arn = 'arn:{0}:glue:{1}:{2}:job/{3}'.format(partition, module.region, account_id, module.params.get('name'))
 
     try:
-        existing_tags = connection.get_tags(ResourceArn=arn).get('Tags', {})
+        existing_tags = connection.get_tags(aws_retry=True, ResourceArn=arn).get('Tags', {})
     except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
         if module.check_mode:
             existing_tags = {}
@@ -332,7 +329,7 @@ def ensure_tags(connection, module, glue_job):
         changed = True
         if not module.check_mode:
             try:
-                connection.untag_resource(ResourceArn=arn, TagsToRemove=tags_to_remove)
+                connection.untag_resource(aws_retry=True, ResourceArn=arn, TagsToRemove=tags_to_remove)
             except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
                 module.fail_json_aws(e, msg='Unable to set tags for Glue job %s' % module.params.get('name'))
 
@@ -340,14 +337,13 @@ def ensure_tags(connection, module, glue_job):
         changed = True
         if not module.check_mode:
             try:
-                connection.tag_resource(ResourceArn=arn, TagsToAdd=tags_to_add)
+                connection.tag_resource(aws_retry=True, ResourceArn=arn, TagsToAdd=tags_to_add)
             except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
                 module.fail_json_aws(e, msg='Unable to set tags for Glue job %s' % module.params.get('name'))
 
     return changed
 
 
-@AWSRetry.backoff(tries=5, delay=5, backoff=2.0)
 def create_or_update_glue_job(connection, module, glue_job):
     """
     Create or update an AWS Glue job
@@ -391,14 +387,14 @@ def create_or_update_glue_job(connection, module, glue_job):
                 update_params = {'JobName': params['Name'], 'JobUpdate': copy.deepcopy(params)}
                 del update_params['JobUpdate']['Name']
                 if not module.check_mode:
-                    connection.update_job(**update_params)
+                    connection.update_job(aws_retry=True, **update_params)
                 changed = True
             except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
                 module.fail_json_aws(e)
     else:
         try:
             if not module.check_mode:
-                connection.create_job(**params)
+                connection.create_job(aws_retry=True, **params)
             changed = True
         except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
             module.fail_json_aws(e)
@@ -410,7 +406,6 @@ def create_or_update_glue_job(connection, module, glue_job):
     module.exit_json(changed=changed, **camel_dict_to_snake_dict(glue_job or {}))
 
 
-@AWSRetry.backoff(tries=5, delay=5, backoff=2.0)
 def delete_glue_job(connection, module, glue_job):
     """
     Delete an AWS Glue job
@@ -420,13 +415,12 @@ def delete_glue_job(connection, module, glue_job):
     :param glue_job: a dict of AWS Glue job parameters or None
     :return:
     """
-
     changed = False
 
     if glue_job:
         try:
             if not module.check_mode:
-                connection.delete_job(JobName=glue_job['Name'])
+                connection.delete_job(aws_retry=True, JobName=glue_job['Name'])
             changed = True
         except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
             module.fail_json_aws(e)
@@ -463,7 +457,8 @@ def main():
                               supports_check_mode=True
                               )
 
-    connection = module.client('glue')
+    retry_decorator = AWSRetry.jittered_backoff(retries=10)
+    connection = module.client('glue', retry_decorator=retry_decorator)
 
     state = module.params.get("state")
 

From 4c3f380deef94a3d43dee521231abca4bb723cc9 Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Fri, 2 Apr 2021 00:47:11 +0000
Subject: [PATCH 11/12] Do not convert keys in DefaultArguments dict to
 snake_case

---
 plugins/modules/aws_glue_job.py                       | 2 +-
 tests/integration/targets/aws_glue_job/tasks/main.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/plugins/modules/aws_glue_job.py b/plugins/modules/aws_glue_job.py
index 5659272d97f..02342d1e1ae 100644
--- a/plugins/modules/aws_glue_job.py
+++ b/plugins/modules/aws_glue_job.py
@@ -403,7 +403,7 @@ def create_or_update_glue_job(connection, module, glue_job):
 
     changed |= ensure_tags(connection, module, glue_job)
 
-    module.exit_json(changed=changed, **camel_dict_to_snake_dict(glue_job or {}))
+    module.exit_json(changed=changed, **camel_dict_to_snake_dict(glue_job or {}, ignore_list=['DefaultArguments']))
 
 
 def delete_glue_job(connection, module, glue_job):
diff --git a/tests/integration/targets/aws_glue_job/tasks/main.yml b/tests/integration/targets/aws_glue_job/tasks/main.yml
index 8e8db677090..4daf665730e 100644
--- a/tests/integration/targets/aws_glue_job/tasks/main.yml
+++ b/tests/integration/targets/aws_glue_job/tasks/main.yml
@@ -50,7 +50,7 @@
       assert:
         that:
           - glue_job_check.changed
-          - not glue_job_check.description
+          - "description" not in glue_job_check
 
     - name: Create Glue job
       aws_glue_job:

From 4b86cc6568018ae415e01d597392bdde19902d25 Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Fri, 2 Apr 2021 00:56:56 +0000
Subject: [PATCH 12/12] Remove duplicate glue_version argument in module
 definition

---
 plugins/modules/aws_glue_job.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/plugins/modules/aws_glue_job.py b/plugins/modules/aws_glue_job.py
index 56733e297c7..66f09882db1 100644
--- a/plugins/modules/aws_glue_job.py
+++ b/plugins/modules/aws_glue_job.py
@@ -469,7 +469,6 @@ def main():
             connections=dict(type='list', elements='str'),
             default_arguments=dict(type='dict'),
             description=dict(type='str'),
-            glue_version=dict(type='str'),
             max_concurrent_runs=dict(type='int'),
             max_retries=dict(type='int'),
             name=dict(required=True, type='str'),