From fef93121dd385637abe3e3dba6a4ccd99048a757 Mon Sep 17 00:00:00 2001 From: Sean Marlow Date: Fri, 7 Dec 2018 16:15:10 -0700 Subject: [PATCH] Switch ec2 provider to boto3. Boto3 is better maintained api for ec2. --- ipa/ipa_ec2.py | 196 +++++++++++++++++++------------- package/python3-ipa.spec | 2 + requirements.txt | 1 + tests/test_ipa_ec2.py | 240 +++++++++++++++++++++++---------------- 4 files changed, 260 insertions(+), 179 deletions(-) diff --git a/ipa/ipa_ec2.py b/ipa/ipa_ec2.py index cf40a534..197b8491 100644 --- a/ipa/ipa_ec2.py +++ b/ipa/ipa_ec2.py @@ -2,7 +2,7 @@ """Provider module for testing AWS EC2 images.""" -# Copyright (c) 2017 SUSE LLC +# Copyright (c) 2018 SUSE LLC # # This file is part of ipa. Ipa provides an api and command line # utilities for testing images in the Public Cloud. @@ -20,6 +20,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import boto3 + from ipa import ipa_utils from ipa.ipa_constants import ( BASH_SSH_SCRIPT, @@ -28,14 +30,10 @@ EC2_DEFAULT_USER ) from ipa.ipa_exceptions import IpaException, EC2ProviderException -from ipa.ipa_libcloud import LibcloudProvider - -from libcloud.common.exceptions import BaseHTTPError -from libcloud.compute.types import Provider -from libcloud.compute.providers import get_driver +from ipa.ipa_provider import IpaProvider -class EC2Provider(LibcloudProvider): +class EC2Provider(IpaProvider): """Provider class for testing AWS EC2 images.""" def __init__(self, @@ -128,7 +126,9 @@ def __init__(self, ) self.security_group_id = ( security_group_id or - self._get_value('security_group_id') + self._get_value( + security_group_id, config_key='security_group_id' + ) ) self.ssh_key_name = ( ssh_key_name or @@ -153,26 +153,23 @@ def __init__(self, 'SSH private key file is required to connect to instance.' ) - self.compute_driver = self._get_driver() - - def _get_driver(self): - """Get authenticated EC2 driver.""" - if not self.access_key_id: - raise EC2ProviderException( - 'Access key id is required to authenticate EC2 driver.' + def _connect(self): + """Connect to ec2 resource.""" + resource = None + try: + resource = boto3.resource( + 'ec2', + aws_access_key_id=self.access_key_id, + aws_secret_access_key=self.secret_access_key, + region_name=self.region ) - - if not self.secret_access_key: + # boto3 resource is lazy so attempt method to test connection + resource.meta.client.describe_account_attributes() + except Exception: raise EC2ProviderException( - 'Secret access key is required to authenticate EC2 driver.' + 'Could not connect to region: %s' % self.region ) - - ComputeEngine = get_driver(Provider.EC2) - return ComputeEngine( - self.access_key_id, - self.secret_access_key, - region=self.region - ) + return resource def _get_from_ec2_config(self, entry): """Get config entry from ec2utils config file.""" @@ -186,28 +183,13 @@ def _get_from_ec2_config(self, entry): else: return None - def _get_image(self): - """Retrieve NodeImage given the image id.""" - try: - image = self.compute_driver.list_images( - ex_image_ids=[self.image_id] - )[0] - except (IndexError, BaseHTTPError): - raise EC2ProviderException( - 'Image with ID: {image_id} not found.'.format( - image_id=self.image_id - ) - ) - - return image - def _get_instance(self): """Retrieve instance matching instance_id.""" + resource = self._connect() + try: - instance = self.compute_driver.list_nodes( - ex_node_ids=[self.running_instance_id] - )[0] - except (IndexError, BaseHTTPError): + instance = resource.Instance(self.running_instance_id) + except Exception: raise EC2ProviderException( 'Instance with ID: {instance_id} not found.'.format( instance_id=self.running_instance_id @@ -215,36 +197,26 @@ def _get_instance(self): ) return instance - def _get_instance_size(self): - """Retrieve NodeSize given the instance type.""" - instance_type = self.instance_type or EC2_DEFAULT_TYPE - - try: - sizes = self.compute_driver.list_sizes() - size = [size for size in sizes if size.id == instance_type][0] - except IndexError: - raise EC2ProviderException( - 'Instance type: {instance_type} not found.'.format( - instance_type=instance_type - ) - ) - - return size + def _get_instance_state(self): + """ + Attempt to retrieve the state of the instance. + Raises: + EC2ProviderException: If the instance cannot be found. + """ + instance = self._get_instance() + state = None - def _get_subnet(self, subnet_id): - subnet = None try: - subnet = self.compute_driver.ex_list_subnets( - subnet_ids=[subnet_id] - )[0] + state = instance.state['Name'] except Exception: raise EC2ProviderException( - 'EC2 subnet: {subnet_id} not found.'.format( - subnet_id=subnet_id + 'Instance with id: {instance_id}, ' + 'cannot be found.'.format( + instance_id=self.running_instance_id ) ) - return subnet + return state def _get_user_data(self): """ @@ -259,34 +231,96 @@ def _get_user_data(self): script = BASH_SSH_SCRIPT.format(user=self.ssh_user, key=key) return script + def _is_instance_running(self): + """ + Return True if instance is in running state. + """ + return self._get_instance_state() == 'running' + def _launch_instance(self): """Launch an instance of the given image.""" + resource = self._connect() + + instance_name = ipa_utils.generate_instance_name('ec2-ipa-test') kwargs = { - 'name': ipa_utils.generate_instance_name('ec2-ipa-test'), - 'size': self._get_instance_size(), - 'image': self._get_image() + 'InstanceType': self.instance_type or EC2_DEFAULT_TYPE, + 'ImageId': self.image_id, + 'MaxCount': 1, + 'MinCount': 1, + 'TagSpecifications': [ + { + 'ResourceType': 'instance', + 'Tags': [ + { + 'Key': 'Name', + 'Value': instance_name + } + ] + } + ] } if self.ssh_key_name: - kwargs['ex_keyname'] = self.ssh_key_name + kwargs['KeyName'] = self.ssh_key_name else: - kwargs['ex_userdata'] = self._get_user_data() + kwargs['UserData'] = self._get_user_data() if self.subnet_id: - kwargs['ex_subnet'] = self._get_subnet(self.subnet_id) + kwargs['SubnetId'] = self.subnet_id if self.security_group_id: - kwargs['ex_security_group_ids'] = [self.security_group_id] + kwargs['SecurityGroupIds'] = [self.security_group_id] - instance = self.compute_driver.create_node(**kwargs) + try: + instances = resource.create_instances(**kwargs) + except Exception as error: + raise EC2ProviderException( + 'Unable to create instance: {0}.'.format(error) + ) - self.compute_driver.wait_until_running( - [instance], - timeout=self.timeout - ) - self.running_instance_id = instance.id + instances[0].wait_until_running() + self.running_instance_id = instances[0].instance_id def _set_image_id(self): """If existing image used get image id.""" instance = self._get_instance() - self.image_id = instance.extra['image_id'] + self.image_id = instance.image_id + + def _set_instance_ip(self): + """ + Retrieve instance ip and cache it. + + Current preference is for public ipv4, ipv6 and private. + """ + instance = self._get_instance() + + # ipv6 + try: + ipv6 = instance.network_interfaces[0].ipv6_addresses[0] + except IndexError: + ipv6 = None + + self.instance_ip = instance.public_ip_address or \ + ipv6 or instance.private_ip_address + + if not self.instance_ip: + raise EC2ProviderException( + 'IP address for instance cannot be found.' + ) + + def _start_instance(self): + """Start the instance.""" + instance = self._get_instance() + instance.start() + instance.wait_until_running() + + def _stop_instance(self): + """Stop the instance.""" + instance = self._get_instance() + instance.stop() + instance.wait_until_stopped() + + def _terminate_instance(self): + """Terminate the instance.""" + instance = self._get_instance() + instance.terminate() diff --git a/package/python3-ipa.spec b/package/python3-ipa.spec index 0dd4be6b..4d7d68bc 100644 --- a/package/python3-ipa.spec +++ b/package/python3-ipa.spec @@ -28,6 +28,7 @@ Source: https://files.pythonhosted.org/packages/source/p/python3-ipa/%{n BuildRequires: python3-devel BuildRequires: python3-setuptools Requires: python3-PyYAML +Requires: python3-boto3 Requires: python3-apache-libcloud Requires: python3-azure-common Requires: python3-azure-mgmt-compute @@ -43,6 +44,7 @@ Requires: python3-testinfra BuildArch: noarch %if %{with test} BuildRequires: python3-PyYAML +BuildRequires: python3-boto3 BuildRequires: python3-apache-libcloud BuildRequires: python3-azure-common BuildRequires: python3-azure-mgmt-compute diff --git a/requirements.txt b/requirements.txt index 3be6f1e0..90d1bdca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +boto3 apache-libcloud azure-common azure-mgmt-compute diff --git a/tests/test_ipa_ec2.py b/tests/test_ipa_ec2.py index 44ca51e6..f54d53f1 100644 --- a/tests/test_ipa_ec2.py +++ b/tests/test_ipa_ec2.py @@ -3,7 +3,7 @@ """Ipa ec2 provider unit tests.""" -# Copyright (c) 2017 SUSE LLC +# Copyright (c) 2018 SUSE LLC # # This file is part of ipa. Ipa provides an api and command line # utilities for testing images in the Public Cloud. @@ -21,6 +21,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import boto3 import pytest from ipa.ipa_ec2 import EC2Provider @@ -41,7 +42,8 @@ def setup_method(self, method): 'image_id': 'fakeimage', 'no_default_test_dirs': True, 'provider_config': 'tests/ec2/.ec2utils.conf', - 'test_files': ['test_image'] + 'test_files': ['test_image'], + 'ssh_key_name': 'test-key' } def test_ec2_exception_required_args(self): @@ -66,71 +68,89 @@ def test_ec2_exception_required_args(self): assert str(error.value) == msg self.kwargs['provider_config'] = 'tests/ec2/.ec2utils.conf' - @patch('libcloud.compute.drivers.ec2.EC2NodeDriver') - def test_gce_get_driver(self, mock_node_driver): - """Test ec2 get driver method.""" - driver = MagicMock() - mock_node_driver.return_value = driver + @patch.object(boto3, 'resource') + def test_ec2_bad_connection(self, mock_boto3): + """Test an exception is raised if boto3 unable to connect.""" + mock_boto3.side_effect = Exception('ERROR!') provider = EC2Provider(**self.kwargs) - assert driver == provider.compute_driver + msg = 'Could not connect to region: %s' % provider.region - @patch.object(EC2Provider, '_get_driver') - def test_ec2_get_instance(self, mock_get_driver): + # Test ssh private key file required + with pytest.raises(EC2ProviderException) as error: + provider._connect() + + assert str(error.value) == msg + assert mock_boto3.call_count > 0 + + @patch.object(EC2Provider, '_connect') + def test_ec2_get_instance(self, mock_connect): """Test get instance method.""" instance = MagicMock() - driver = MagicMock() - mock_get_driver.return_value = driver - driver.list_nodes.return_value = [instance] + resource = MagicMock() + resource.Instance.return_value = instance + mock_connect.return_value = resource provider = EC2Provider(**self.kwargs) val = provider._get_instance() assert val == instance - assert driver.list_nodes.call_count == 1 + assert mock_connect.call_count == 1 - @patch.object(EC2Provider, '_get_driver') - def test_ec2_get_instance_not_found(self, mock_get_driver): - """Test get instance method.""" - driver = MagicMock() - mock_get_driver.return_value = driver - driver.list_nodes.return_value = [] + @patch.object(EC2Provider, '_connect') + def test_ec2_get_instance_error(self, mock_connect): + """Test get instance method error.""" + resource = MagicMock() + resource.Instance.side_effect = Exception('Error!') + mock_connect.return_value = resource - self.kwargs['running_instance_id'] = 'i-123456789' provider = EC2Provider(**self.kwargs) + provider.running_instance_id = 'i-123456789' with pytest.raises(EC2ProviderException) as error: provider._get_instance() - assert str(error.value) == 'Instance with ID: i-123456789 not found.' - assert driver.list_nodes.call_count == 1 + msg = 'Instance with ID: i-123456789 not found.' + assert str(error.value) == msg - @patch.object(EC2Provider, '_get_driver') - def test_ec2_get_subnet(self, mock_get_driver): - """Test EC2 get subnetwork method.""" - subnetwork = MagicMock() - driver = MagicMock() - driver.ex_list_subnets.return_value = [subnetwork] - mock_get_driver.return_value = driver + @patch.object(EC2Provider, '_get_instance') + def test_ec2_get_instance_state(self, mock_get_instance): + """Test an exception is raised if boto3 unable to connect.""" + instance = MagicMock() + instance.state = {'Name': 'ReadyRole'} + mock_get_instance.return_value = instance provider = EC2Provider(**self.kwargs) - result = provider._get_subnet('test-subnet') + val = provider._get_instance_state() + assert val == 'ReadyRole' + assert mock_get_instance.call_count == 1 + + instance.state = {} + mock_get_instance.reset_mock() + msg = 'Instance with id: %s, cannot be found.' \ + % provider.running_instance_id + + # Test exception raised if instance state not found + with pytest.raises(EC2ProviderException) as error: + provider._get_instance_state() - assert result == subnetwork + assert str(error.value) == msg + assert mock_get_instance.call_count == 1 - @patch.object(EC2Provider, '_get_driver') - def test_ec2_get_subnet_exception(self, mock_get_driver): - """Test EC2 get subnetwork method.""" - driver = MagicMock() - driver.ex_list_subnets.side_effect = Exception('Cannot find subnet!') - mock_get_driver.return_value = driver + @patch.object(EC2Provider, '_get_instance_state') + def test_ec2_is_instance_running(self, mock_get_instance_state): + """Test ec2 provider is instance runnning method.""" + mock_get_instance_state.return_value = 'running' provider = EC2Provider(**self.kwargs) + assert provider._is_instance_running() + assert mock_get_instance_state.call_count == 1 - msg = 'EC2 subnet: test-subnet not found.' - with pytest.raises(EC2ProviderException) as error: - provider._get_subnet('test-subnet') + mock_get_instance_state.return_value = 'stopped' + mock_get_instance_state.reset_mock() - assert msg == str(error.value) + provider = EC2Provider(**self.kwargs) + assert not provider._is_instance_running() + assert mock_get_instance_state.call_count == 1 @patch('ipa.ipa_ec2.ipa_utils.generate_public_ssh_key') def test_ec2_get_user_data(self, mock_generate_ssh_key): @@ -143,87 +163,111 @@ def test_ec2_get_user_data(self, mock_generate_ssh_key): assert result == '#!/bin/bash\n' \ 'echo testkey12345 >> /home/ec2-user/.ssh/authorized_keys\n' - @patch.object(EC2Provider, '_get_user_data') - @patch.object(EC2Provider, '_get_subnet') - @patch.object(EC2Provider, '_get_driver') - def test_ec2_launch_instance( - self, mock_get_driver, mock_get_subnet, mock_get_user_data - ): + @patch.object(EC2Provider, '_connect') + def test_ec2_launch_instance(self, mock_connect): """Test ec2 provider launch instance method.""" - driver = MagicMock() + instance = MagicMock() + instance.instance_id = 'i-123456789' + instances = [instance] - size = MagicMock() - size.id = 't2.micro' - driver.list_sizes.return_value = [size] + resource = MagicMock() + resource.create_instances.return_value = instances - image = MagicMock() - image.id = 'fakeimage' - driver.list_images.return_value = [image] + mock_connect.return_value = resource - instance = MagicMock() - instance.id = 'i-123456789' - driver.create_node.return_value = instance + provider = EC2Provider(**self.kwargs) + provider.subnet_id = 'subnet-123456789' + provider.security_group_id = 'sg-123456789' + provider._launch_instance() - mock_get_driver.return_value = driver + assert instance.instance_id == provider.running_instance_id + assert resource.create_instances.call_count == 1 - subnet = MagicMock() - mock_get_subnet.return_value = subnet + @patch.object(EC2Provider, '_get_instance') + def test_ec2_set_image_id(self, mock_get_instance): + """Test ec2 provider set image id method.""" + instance = MagicMock() + instance.image_id = 'ami-123456' + mock_get_instance.return_value = instance provider = EC2Provider(**self.kwargs) - provider.subnet_id = 'test-subnet' - provider._launch_instance() + provider._set_image_id() - assert instance.id == provider.running_instance_id - assert driver.list_sizes.call_count == 1 - assert driver.list_images.call_count == 1 - assert driver.create_node.call_count == 1 + assert provider.image_id == instance.image_id + assert mock_get_instance.call_count == 1 - @patch.object(EC2Provider, '_get_driver') - def test_ec2_launch_instance_no_size(self, mock_get_driver): - """Test exception raised if instance type not found.""" - driver = MagicMock() - driver.list_sizes.return_value = [] + @patch.object(EC2Provider, '_get_instance') + def test_ec2_set_instance_ip(self, mock_get_instance): + """Test ec2 provider set image id method.""" + instance = MagicMock() + instance.public_ip_address = None + instance.private_ip_address = None + instance.network_interfaces = [] + mock_get_instance.return_value = instance - mock_get_driver.return_value = driver provider = EC2Provider(**self.kwargs) + msg = 'IP address for instance cannot be found.' with pytest.raises(EC2ProviderException) as error: - provider._launch_instance() + provider._set_instance_ip() + + assert str(error.value) == msg + assert mock_get_instance.call_count == 1 + mock_get_instance.reset_mock() - assert str(error.value) == 'Instance type: t2.micro not found.' - assert driver.list_sizes.call_count == 1 + instance.private_ip_address = '127.0.0.1' - @patch.object(EC2Provider, '_get_driver') - def test_ec2_launch_instance_no_image(self, mock_get_driver): - """Test exception raised if image not found.""" - driver = MagicMock() + provider._set_instance_ip() + assert provider.instance_ip == '127.0.0.1' + assert mock_get_instance.call_count == 1 + mock_get_instance.reset_mock() - size = MagicMock() - size.id = 't2.micro' - driver.list_sizes.return_value = [size] - driver.list_images.return_value = [] + network_interface = MagicMock() + network_interface.ipv6_addresses = ['127.0.0.2'] + instance.network_interfaces = [network_interface] - mock_get_driver.return_value = driver - provider = EC2Provider(**self.kwargs) + provider._set_instance_ip() + assert provider.instance_ip == '127.0.0.2' + assert mock_get_instance.call_count == 1 + mock_get_instance.reset_mock() - with pytest.raises(EC2ProviderException) as error: - provider._launch_instance() + instance.public_ip_address = '127.0.0.3' - assert str(error.value) == 'Image with ID: fakeimage not found.' - assert driver.list_sizes.call_count == 1 - assert driver.list_images.call_count == 1 + provider._set_instance_ip() + assert provider.instance_ip == '127.0.0.3' + assert mock_get_instance.call_count == 1 @patch.object(EC2Provider, '_get_instance') - @patch.object(EC2Provider, '_get_driver') - def test_ec2_set_image_id(self, mock_get_driver, mock_get_instance): - """Test ec2 provider set image id method.""" + def test_ec2_start_instance(self, mock_get_instance): + """Test ec2 start instance method.""" instance = MagicMock() - instance.extra = {'image_id': 'ami-123456'} + instance.start.return_value = None + instance.wait_until_running.return_value = None mock_get_instance.return_value = instance - mock_get_driver.return_value = None provider = EC2Provider(**self.kwargs) - provider._set_image_id() + provider._start_instance() + assert mock_get_instance.call_count == 1 + + @patch.object(EC2Provider, '_get_instance') + def test_ec2_stop_instance(self, mock_get_instance): + """Test ec2 stop instance method.""" + instance = MagicMock() + instance.stop.return_value = None + instance.wait_until_stopped.return_value = None + mock_get_instance.return_value = instance + + provider = EC2Provider(**self.kwargs) + provider._stop_instance() + assert mock_get_instance.call_count == 1 - assert provider.image_id == instance.extra['image_id'] + @patch.object(EC2Provider, '_get_instance') + def test_ec2_terminate_instance(self, mock_get_instance): + """Test ec2 terminate instance method.""" + instance = MagicMock() + instance.terminate.return_value = None + mock_get_instance.return_value = instance + + provider = EC2Provider(**self.kwargs) + provider._terminate_instance() assert mock_get_instance.call_count == 1