Cleanup - use is_boto3_error_(message|code) (#268)

* Reorder imports
* Make use of is_boto3_error_message
* Mass-migration over to is_boto3_error_code
* Remove unused imports
* unused vars in exception
* Improve consistency around catching BotoCoreError and ClientError
* Remove unused imports
* Remove unused 'PolicyError' from iam_policy_info
* Avoid catching botocore.exceptions.ClientError when we only want some error codes
* Import camel_dict_to_snake_dict/snake_dict_to_camel_dict from ansible.module_utils.common.dict_transformations

changelogs/fragments/268-is_boto3_error.yml

@@ -0,0 +1,3 @@
+minor_changes:
+- various community.aws modules - cleanup error handling to use ``is_boto3_error_code`` and ``is_boto3_error_message`` helpers (https://github.com/ansible-collections/community.aws/pull/268).
+- various community.aws modules - improve consistency of handling Boto3 exceptions (https://github.com/ansible-collections/community.aws/pull/268).

plugins/modules/aws_elasticbeanstalk_app.py

@@ -85,11 +85,12 @@
 '''
 
 try:
-    from botocore.exceptions import BotoCoreError, ClientError
+    import botocore
 except ImportError:
     pass  # handled by AnsibleAWSModule
 
 from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule
+from ansible_collections.amazon.aws.plugins.module_utils.core import is_boto3_error_message
 
 
 def describe_app(ebs, app_name, module):
@@ -104,7 +105,7 @@ def list_apps(ebs, app_name, module):
             apps = ebs.describe_applications(ApplicationNames=[app_name])
         else:
             apps = ebs.describe_applications()
-    except (BotoCoreError, ClientError) as e:
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
         module.fail_json_aws(e, msg="Could not describe application")
 
     return apps.get("Applications", [])
@@ -175,7 +176,7 @@ def main():
             try:
                 create_app = ebs.create_application(**filter_empty(ApplicationName=app_name,
                                                     Description=description))
-            except (BotoCoreError, ClientError) as e:
+            except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
                 module.fail_json_aws(e, msg="Could not create application")
 
             app = describe_app(ebs, app_name, module)
@@ -188,7 +189,7 @@ def main():
                         ebs.update_application(ApplicationName=app_name)
                     else:
                         ebs.update_application(ApplicationName=app_name, Description=description)
-                except (BotoCoreError, ClientError) as e:
+                except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
                     module.fail_json_aws(e, msg="Could not update application")
 
                 app = describe_app(ebs, app_name, module)
@@ -208,13 +209,10 @@ def main():
                 else:
                     ebs.delete_application(ApplicationName=app_name)
                 changed = True
-            except BotoCoreError as e:
+            except is_boto3_error_message('It is currently pending deletion'):
+                changed = False
+            except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:  # pylint: disable=duplicate-except
                 module.fail_json_aws(e, msg="Cannot terminate app")
-            except ClientError as e:
-                if 'It is currently pending deletion.' not in e.response['Error']['Message']:
-                    module.fail_json_aws(e, msg="Cannot terminate app")
-                else:
-                    changed = False
 
             result = dict(changed=changed, app=app)
 

plugins/modules/aws_glue_connection.py

@@ -130,17 +130,20 @@
     sample: {'subnet-id':'subnet-aabbccddee'}
 '''
 
-from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule
-from ansible_collections.amazon.aws.plugins.module_utils.ec2 import camel_dict_to_snake_dict, get_ec2_security_group_ids_from_names
-
 # Non-ansible imports
 import copy
 import time
 try:
-    from botocore.exceptions import BotoCoreError, ClientError
+    import botocore
 except ImportError:
     pass
 
+from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict
+
+from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule
+from ansible_collections.amazon.aws.plugins.module_utils.core import is_boto3_error_code
+from ansible_collections.amazon.aws.plugins.module_utils.ec2 import get_ec2_security_group_ids_from_names
+
 
 def _get_glue_connection(connection, module):
     """
@@ -160,11 +163,8 @@ def _get_glue_connection(connection, module):
 
     try:
         return connection.get_connection(**params)['Connection']
-    except (BotoCoreError, ClientError) as e:
-        if e.response['Error']['Code'] == 'EntityNotFoundException':
-            return None
-        else:
-            raise e
+    except is_boto3_error_code('EntityNotFoundException'):
+        return None
 
 
 def _compare_glue_connection_params(user_params, current_params):
@@ -251,13 +251,13 @@ def create_or_update_glue_connection(connection, connection_ec2, module, glue_co
                 update_params['Name'] = update_params['ConnectionInput']['Name']
                 connection.update_connection(**update_params)
                 changed = True
-            except (BotoCoreError, ClientError) as e:
+            except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
                 module.fail_json_aws(e)
     else:
         try:
             connection.create_connection(**params)
             changed = True
-        except (BotoCoreError, ClientError) as e:
+        except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
             module.fail_json_aws(e)
 
     # If changed, get the Glue connection again
@@ -292,7 +292,7 @@ def delete_glue_connection(connection, module, glue_connection):
         try:
             connection.delete_connection(**params)
             changed = True
-        except (BotoCoreError, ClientError) as e:
+        except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
             module.fail_json_aws(e)
 
     module.exit_json(changed=changed)

plugins/modules/aws_glue_job.py

@@ -185,15 +185,17 @@
     sample: 300
 '''
 
-from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule
-from ansible_collections.amazon.aws.plugins.module_utils.ec2 import camel_dict_to_snake_dict
-
 # Non-ansible imports
 import copy
 try:
-    from botocore.exceptions import BotoCoreError, ClientError
+    import botocore
 except ImportError:
-    pass
+    pass  # Handled by AnsibleAWSModule
+
+from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict
+
+from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule
+from ansible_collections.amazon.aws.plugins.module_utils.core import is_boto3_error_code
 
 
 def _get_glue_job(connection, module, glue_job_name):
@@ -208,11 +210,10 @@ def _get_glue_job(connection, module, glue_job_name):
 
     try:
         return connection.get_job(JobName=glue_job_name)['Job']
-    except (BotoCoreError, ClientError) as e:
-        if e.response['Error']['Code'] == 'EntityNotFoundException':
-            return None
-        else:
-            module.fail_json_aws(e)
+    except is_boto3_error_code('EntityNotFoundException'):
+        return None
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:  # pylint: disable=duplicate-except
+        module.fail_json_aws(e)
 
 
 def _compare_glue_job_params(user_params, current_params):
@@ -292,13 +293,13 @@ def create_or_update_glue_job(connection, module, glue_job):
                 del update_params['JobUpdate']['Name']
                 connection.update_job(**update_params)
                 changed = True
-            except (BotoCoreError, ClientError) as e:
+            except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
                 module.fail_json_aws(e)
     else:
         try:
             connection.create_job(**params)
             changed = True
-        except (BotoCoreError, ClientError) as e:
+        except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
             module.fail_json_aws(e)
 
     # If changed, get the Glue job again
@@ -324,7 +325,7 @@ def delete_glue_job(connection, module, glue_job):
         try:
             connection.delete_job(JobName=glue_job['Name'])
             changed = True
-        except (BotoCoreError, ClientError) as e:
+        except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
             module.fail_json_aws(e)
 
     module.exit_json(changed=changed)

plugins/modules/aws_kms_info.py

@@ -290,7 +290,7 @@ def get_key_policy_with_backoff(connection, key_id, policy_name):
 def get_enable_key_rotation_with_backoff(connection, key_id):
     try:
         current_rotation_status = connection.get_key_rotation_status(KeyId=key_id)
-    except is_boto3_error_code('AccessDeniedException') as e:
+    except is_boto3_error_code('AccessDeniedException'):
         return None
 
     return current_rotation_status.get('KeyRotationEnabled')
@@ -306,11 +306,10 @@ def get_kms_tags(connection, module, key_id):
         try:
             tag_response = get_kms_tags_with_backoff(connection, key_id, **kwargs)
             tags.extend(tag_response['Tags'])
-        except botocore.exceptions.ClientError as e:
-            if e.response['Error']['Code'] != 'AccessDeniedException':
-                module.fail_json_aws(e, msg="Failed to obtain key tags")
-            else:
-                tag_response = {}
+        except is_boto3_error_code('AccessDeniedException'):
+            tag_response = {}
+        except botocore.exceptions.ClientError as e:  # pylint: disable=duplicate-except
+            module.fail_json_aws(e, msg="Failed to obtain key tags")
         if tag_response.get('NextMarker'):
             kwargs['Marker'] = tag_response['NextMarker']
         else:
@@ -323,11 +322,10 @@ def get_kms_policies(connection, module, key_id):
         policies = list_key_policies_with_backoff(connection, key_id)['PolicyNames']
         return [get_key_policy_with_backoff(connection, key_id, policy)['Policy'] for
                 policy in policies]
-    except botocore.exceptions.ClientError as e:
-        if e.response['Error']['Code'] != 'AccessDeniedException':
-            module.fail_json_aws(e, msg="Failed to obtain key policies")
-        else:
-            return []
+    except is_boto3_error_code('AccessDeniedException'):
+        return []
+    except botocore.exceptions.ClientError as e:  # pylint: disable=duplicate-except
+        module.fail_json_aws(e, msg="Failed to obtain key policies")
 
 
 def key_matches_filter(key, filtr):

plugins/modules/aws_ssm_parameter_store.py

@@ -127,13 +127,14 @@
     type: dict
 '''
 
-from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule
-
 try:
-    from botocore.exceptions import ClientError
+    import botocore
 except ImportError:
     pass  # Handled by AnsibleAWSModule
 
+from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule
+from ansible_collections.amazon.aws.plugins.module_utils.core import is_boto3_error_code
+
 
 def update_parameter(client, module, args):
     changed = False
@@ -142,7 +143,7 @@ def update_parameter(client, module, args):
     try:
         response = client.put_parameter(**args)
         changed = True
-    except ClientError as e:
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
         module.fail_json_aws(e, msg="setting parameter")
 
     return changed, response
@@ -195,7 +196,7 @@ def create_update_parameter(client, module):
                     describe_existing_parameter = describe_existing_parameter_paginator.paginate(
                         Filters=[{"Key": "Name", "Values": [args['Name']]}]).build_full_result()
 
-                except ClientError as e:
+                except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
                     module.fail_json_aws(e, msg="getting description value")
 
                 if describe_existing_parameter['Parameters'][0]['Description'] != args['Description']:
@@ -213,9 +214,9 @@ def delete_parameter(client, module):
         response = client.delete_parameter(
             Name=module.params.get('name')
         )
-    except ClientError as e:
-        if e.response['Error']['Code'] == 'ParameterNotFound':
-            return False, {}
+    except is_boto3_error_code('ParameterNotFound'):
+        return False, {}
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:  # pylint: disable=duplicate-except
         module.fail_json_aws(e, msg="deleting parameter")
 
     return True, response

plugins/modules/aws_step_functions_state_machine_execution.py

@@ -89,14 +89,16 @@
 '''
 
 
-from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule
-from ansible_collections.amazon.aws.plugins.module_utils.ec2 import camel_dict_to_snake_dict
-
 try:
-    from botocore.exceptions import ClientError, BotoCoreError
+    import botocore
 except ImportError:
     pass  # caught by AnsibleAWSModule
 
+from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict
+
+from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule
+from ansible_collections.amazon.aws.plugins.module_utils.core import is_boto3_error_code
+
 
 def start_execution(module, sfn_client):
     '''
@@ -123,10 +125,10 @@ def start_execution(module, sfn_client):
             name=name,
             input=execution_input
         )
-    except (ClientError, BotoCoreError) as e:
-        if e.response['Error']['Code'] == 'ExecutionAlreadyExists':
-            # this will never be executed anymore
-            module.exit_json(changed=False)
+    except is_boto3_error_code('ExecutionAlreadyExists'):
+        # this will never be executed anymore
+        module.exit_json(changed=False)
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:  # pylint: disable=duplicate-except
         module.fail_json_aws(e, msg="Failed to start execution.")
 
     module.exit_json(changed=True, **camel_dict_to_snake_dict(res_execution))
@@ -151,7 +153,7 @@ def stop_execution(module, sfn_client):
             cause=cause,
             error=error
         )
-    except (ClientError, BotoCoreError) as e:
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
         module.fail_json_aws(e, msg="Failed to stop execution.")
 
     module.exit_json(changed=True, **camel_dict_to_snake_dict(res))

plugins/modules/aws_waf_condition.py

@@ -402,10 +402,17 @@
 except ImportError:
     pass  # handled by AnsibleAWSModule
 
+from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict
+
 from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule
-from ansible_collections.amazon.aws.plugins.module_utils.ec2 import camel_dict_to_snake_dict, AWSRetry, compare_policies
-from ansible_collections.amazon.aws.plugins.module_utils.waf import run_func_with_change_token_backoff, MATCH_LOOKUP
-from ansible_collections.amazon.aws.plugins.module_utils.waf import get_rule_with_backoff, list_rules_with_backoff, list_regional_rules_with_backoff
+from ansible_collections.amazon.aws.plugins.module_utils.core import is_boto3_error_code
+from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry
+from ansible_collections.amazon.aws.plugins.module_utils.ec2 import compare_policies
+from ansible_collections.amazon.aws.plugins.module_utils.waf import MATCH_LOOKUP
+from ansible_collections.amazon.aws.plugins.module_utils.waf import run_func_with_change_token_backoff
+from ansible_collections.amazon.aws.plugins.module_utils.waf import get_rule_with_backoff
+from ansible_collections.amazon.aws.plugins.module_utils.waf import list_regional_rules_with_backoff
+from ansible_collections.amazon.aws.plugins.module_utils.waf import list_rules_with_backoff
 
 
 class Condition(object):
@@ -540,9 +547,9 @@ def delete_unused_regex_pattern(self, regex_pattern_set_id):
             run_func_with_change_token_backoff(self.client, self.module,
                                                {'RegexPatternSetId': regex_pattern_set_id},
                                                self.client.delete_regex_pattern_set, wait=True)
-        except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
-            if e.response['Error']['Code'] == 'WAFNonexistentItemException':
-                return
+        except is_boto3_error_code('WAFNonexistentItemException'):
+            return
+        except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:  # pylint: disable=duplicate-except
             self.module.fail_json_aws(e, msg='Could not delete regex pattern')
 
     def get_condition_by_name(self, name):