From 437791e42eb2fb5764ad5c28fe9460b5e6a682cd Mon Sep 17 00:00:00 2001 From: Sean Marlow Date: Tue, 20 Feb 2018 16:23:28 -0700 Subject: [PATCH 01/11] Migrate azure to python SDK for ARM. --- .gitignore | 1 + ipa/ipa_azure.py | 495 +++++++++++++----- ipa/ipa_constants.py | 2 +- ipa/ipa_controller.py | 5 +- ipa/ipa_ec2.py | 2 +- ipa/ipa_gce.py | 1 - ipa/scripts/cli.py | 15 +- .../lib/ipa/tests/SLES/test_sles_hostname.py | 4 +- 8 files changed, 383 insertions(+), 142 deletions(-) diff --git a/.gitignore b/.gitignore index 4dc16aac..b454e48b 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ var/ .tox/ .coverage .cache +.pytest_cache htmlcov/ nosetests.xml coverage.xml diff --git a/ipa/ipa_azure.py b/ipa/ipa_azure.py index ef4d0e79..35ae1fcd 100644 --- a/ipa/ipa_azure.py +++ b/ipa/ipa_azure.py @@ -20,13 +20,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import azurectl.logger +import json +import os -from azurectl.account.service import AzureAccount -from azurectl.config.parser import Config as AzurectlConfig -from azurectl.instance.cloud_service import CloudService -from azurectl.instance.virtual_machine import VirtualMachine -from azurectl.management.request_result import RequestResult +from azure.common.credentials import ServicePrincipalCredentials +from azure.mgmt.resource import ResourceManagementClient +from azure.mgmt.network import NetworkManagementClient +from azure.mgmt.compute import ComputeManagementClient from ipa import ipa_utils from ipa.ipa_constants import AZURE_DEFAULT_TYPE, AZURE_DEFAULT_USER @@ -59,7 +59,7 @@ def __init__(self, ssh_key_name=None, # Not used in Azure ssh_private_key=None, ssh_user=None, - storage_container=None, + subscription_id=None, subnet_id=None, # Not used in Azure test_dirs=None, test_files=None): @@ -82,176 +82,417 @@ def __init__(self, test_dirs, test_files) - azurectl.logger.init() - self.account_name = account_name + self.service_account_file = ( + service_account_file or + self._get_value( + service_account_file, + config_key='service_account_file' + ) + ) + if not self.service_account_file: + raise AzureProviderException( + 'Service account file is required to connect to Azure.' + ) + else: + self.service_account_file = os.path.expanduser( + self.service_account_file + ) - if not ssh_private_key: + self.ssh_private_key = ( + ssh_private_key or + self._get_value(ssh_private_key, config_key='ssh_private_key') + ) + if not self.ssh_private_key: raise AzureProviderException( 'SSH private key file is required to connect to instance.' ) else: - self.ssh_private_key = ssh_private_key + self.ssh_private_key = os.path.expanduser( + self.ssh_private_key + ) - self.ssh_user = ssh_user or AZURE_DEFAULT_USER - self.storage_container = storage_container - self.account = self._get_account() - self.vm = self._get_virtual_machine() - - def _create_cloud_service(self, instance_name): - """Create cloud service if it does not exist.""" - cloud_service = self._get_cloud_service() - request_id = cloud_service.create( - instance_name, - self.region + self.subscription_id = ( + subscription_id or + self._get_value(subscription_id, config_key='subscription_id') ) + if not self.subscription_id: + raise AzureProviderException( + 'Subscription ID is required to connect to instance.' + ) - if int(request_id, 16) > 0: - # Cloud service created - self._wait_on_request(request_id) + self.ssh_user = ssh_user or AZURE_DEFAULT_USER + self.ssh_public_key = self._get_ssh_public_key() - return cloud_service + self._get_service_account_info() + self.credentials = self._get_credentials() - def _get_account(self): - """Create an account object.""" - if self.provider_config: - self.logger.debug( - 'Using Azure config file: %s' % self.provider_config + self.compute = self._get_compute_management() + self.network = self._get_network_management() + self.resource = self._get_resource_management() + + if self.running_instance_id: + self._set_resource_names() + + def _create_network_interface(self): + """ + Create a new vnet, subnet, public ip and network interface. + + The instance requires an IP to be accessible via SSH for testing. + """ + try: + vnet_operation = self.network.virtual_networks.create_or_update( + self.running_instance_id, + self.vnet_name, + { + 'location': self.region, + 'address_space': { + 'address_prefixes': ['10.0.0.0/27'] + } + } + ) + except Exception as error: + raise AzureProviderException( + 'Unable to create vnet: {0}.'.format(error.message) ) - else: - self.logger.debug('Using default Azure config file') - config = AzurectlConfig( - account_name=self.account_name, - filename=self.provider_config, - region_name=self.region, - storage_container_name=self.storage_container - ) + vnet_operation.wait() - return AzureAccount(config) + try: + subnet_operation = self.network.subnets.create_or_update( + self.running_instance_id, + self.vnet_name, + self.subnet_name, + {'address_prefix': '10.0.0.0/29'} + ) + except Exception as error: + raise AzureProviderException( + 'Unable to create subnet: {0}.'.format(error.message) + ) - def _get_cloud_service(self): - """Return instance of CloudService class.""" - return CloudService(self.account) + subnet_info = subnet_operation.result() - def _get_instance_state(self): - """Retrieve state of instance.""" - return self.vm.instance_status(self.running_instance_id) + try: + public_ip_operation = \ + self.network.public_ip_addresses.create_or_update( + self.running_instance_id, + self.public_ip_name, + { + 'location': self.region, + 'public_ip_allocation_method': 'Dynamic' + } + ) + except Exception as error: + raise AzureProviderException( + 'Unable to create public IP: {0}.'.format(error.message) + ) - def _get_virtual_machine(self): - """Return instance of VirtualMachine class.""" - return VirtualMachine(self.account) + public_ip = public_ip_operation.result() - def _is_instance_running(self): + try: + nic_operation = self.network.network_interfaces.create_or_update( + self.running_instance_id, + self.nic_name, + { + 'location': self.region, + 'ip_configurations': [{ + 'name': self.ipa_config_name, + 'private_ip_allocation_method': 'Dynamic', + 'subnet': { + 'id': subnet_info.id + }, + 'public_ip_address': { + 'id': public_ip.id + }, + }] + } + ) + except Exception as error: + raise AzureProviderException( + 'Unable to create network interface: {0}.'.format( + error.message + ) + ) + + return nic_operation.result() + + def _create_resource_group(self): + """Create resource group if it does not exist.""" + try: + self.resource.resource_groups.create_or_update( + self.running_instance_id, {'location': self.region} + ) + except Exception as error: + raise AzureProviderException( + 'Unable to create resource group: {0}.'.format(error.message) + ) + + def _create_vm(self, vm_parameters): """ - Return True if instance is in running state. + Attempt to create or update VM instance based on vm_parameters config. + """ + try: + vm_operation = self.compute.virtual_machines.create_or_update( + self.running_instance_id, self.running_instance_id, + vm_parameters + ) + except Exception as error: + raise AzureProviderException( + 'An exception occurred creating virtual machine: {0}'.format( + error.message + ) + ) - Raises: - AzureProviderException: If state is Undefined. + vm_operation.wait() + + def _create_vm_parameters(self, interface): """ - state = self._get_instance_state() + Create the VM parameters dictionary. + + Requires an existing network interface object. + """ + # Split image ID into it's components. + self._process_image_id() + + return { + 'location': self.region, + 'os_profile': { + 'computer_name': self.running_instance_id, + 'admin_username': self.ssh_user, + 'linux_configuration': { + 'disable_password_authentication': True, + 'ssh': { + 'public_keys': [{ + 'path': '/home/{0}/.ssh/authorized_keys'.format( + self.ssh_user + ), + 'key_data': self.ssh_public_key + }] + } + } + }, + 'hardware_profile': { + 'vm_size': self.instance_type or AZURE_DEFAULT_TYPE + }, + 'storage_profile': { + 'image_reference': { + 'publisher': self.image_publisher, + 'offer': self.image_offer, + 'sku': self.image_sku, + 'version': self.image_version + }, + }, + 'network_profile': { + 'network_interfaces': [{ + 'id': interface.id, + 'primary': True + }] + } + } + + def _get_compute_management(self): + """Return instance of compute management class.""" + return ComputeManagementClient(self.credentials, self.subscription_id) + + def _get_credentials(self): + """Return instance of service principal credentials.""" + return ServicePrincipalCredentials( + client_id=self.client_id, + secret=self.client_secret, + tenant=self.tenant_id + ) + + def _get_network_management(self): + """Return instance of network management class.""" + return NetworkManagementClient(self.credentials, self.subscription_id) + + def _get_resource_management(self): + """Return instance of resource management class.""" + return ResourceManagementClient(self.credentials, self.subscription_id) + + def _get_service_account_info(self): + """Retrieve json dict from service account file.""" + try: + with open(self.service_account_file, 'r') as f: + info = json.load(f) + except Exception as error: + raise AzureProviderException( + 'Exception processing service account file: {0}.'.format(error) + ) - if state == 'Undefined': + try: + self.tenant_id = info['tenant'] + self.client_id = info['appId'] + self.client_secret = info['password'] + except KeyError as error: raise AzureProviderException( - 'Instance with name: %s, ' - 'cannot be found.' % self.running_instance_id + 'Invalid service account file, missing key: {0}.'.format(error) ) - return state == 'ReadyRole' + def _get_ssh_public_key(self): + """Generate SSH public key from private key.""" + key = ipa_utils.generate_public_ssh_key(self.ssh_private_key) + return key.decode() - def _launch_instance(self): - """Create new test instance in cloud service with same name.""" - instance_name = ipa_utils.generate_instance_name( - 'azure-ipa-test' - ) + def _get_instance(self): + """ + Return the instance matching the running_instance_id. + """ + try: + instance = self.compute.virtual_machines.get( + self.running_instance_id, self.running_instance_id, + expand='instanceView' + ) + except Exception as error: + raise AzureProviderException( + 'Unable to retrieve instance: {0}'.format(error.message) + ) - cloud_service = self._create_cloud_service(instance_name) - fingerprint = cloud_service.add_certificate( - instance_name, - self.ssh_private_key - ) + return instance - linux_configuration = self.vm.create_linux_configuration( - instance_name=instance_name, - fingerprint=fingerprint - ) + def _get_instance_state(self): + """Retrieve state of instance.""" + instance = self._get_instance() + statuses = instance.instance_view.statuses - ssh_endpoint = self.vm.create_network_endpoint( - name='SSH', - public_port=22, - local_port=22, - protocol='TCP' - ) + for status in statuses: + if status.code.startswith('PowerState'): + return status.display_status - network_configuration = self.vm.create_network_configuration( - [ssh_endpoint] - ) + def _is_instance_running(self): + """ + Return True if instance is in running state. + """ + return self._get_instance_state() == 'VM running' - self.vm.create_instance( - instance_name, - self.image_id, - linux_configuration, - network_config=network_configuration, - machine_size=self.instance_type or AZURE_DEFAULT_TYPE + def _launch_instance(self): + """Create new test instance in a resource group with the same name.""" + self.running_instance_id = ipa_utils.generate_instance_name( + 'azure-ipa-test' ) + self._set_resource_names(new_instance=True) + + try: + # Try block acts as a transaction. If an exception occurrs + # attempt to cleanup the resource group and all resources. + + # Create resourece group with same name as instance. + self._create_resource_group() + + # Setup network, subnet and interface in resource group. + interface = self._create_network_interface() + + # Get dictionary of VM parameters. + vm_parameters = self._create_vm_parameters(interface) + self._create_vm(vm_parameters) + except Exception: + try: + self._terminate_instance() + except Exception: + pass + raise + else: + # Ensure VM is in the running state. + self._wait_on_instance('VM running') + + def _process_image_id(self): + """ + Split image id into component values. - self.running_instance_id = instance_name - self._wait_on_instance('ReadyRole') + Example: SUSE:SLES:12-SP3:2018.01.04 + Publisher:Offer:Sku:Version + + Raises: + If image_id is not a valid format. + """ + try: + image_info = self.image_id.strip().split(':') + self.image_publisher = image_info[0] + self.image_offer = image_info[1] + self.image_sku = image_info[2] + self.image_version = image_info[3] + except Exception: + raise AzureProviderException( + 'Image ID is invalid. Format must match ' + '{Publisher}:{Offer}:{Sku}:{Version}.' + ) def _set_image_id(self): """If an existing instance is used get image id from deployment.""" + instance = self._get_instance() + image_info = instance.storage_profile.image_reference + + self.image_id = ':'.join([ + image_info.publisher, image_info.offer, + image_info.sku, image_info.version + ]) + + def _set_instance_ip(self): + """ + Get the public IP address based on instance ID. + """ try: - properties = self.vm.service.get_hosted_service_properties( - service_name=self.running_instance_id, - embed_detail=True + public_ip = self.network.public_ip_addresses.get( + self.running_instance_id, self.public_ip_name ) - self.image_id = properties.deployments[0].role_list[0]\ - .os_virtual_hard_disk.source_image_name - except IndexError: + except Exception as error: raise AzureProviderException( - 'Image name for instance cannot be found.' + 'Unable to retrieve instance public IP: {0}.'.format( + error.message + ) ) - def _set_instance_ip(self): + self.instance_ip = public_ip.ip_address + + def _set_resource_names(self, new_instance=False): """ - Get the first ip from first deployment. + Generate names for resources based on the running_instance_id. - There is only one vm in current cloud service. + If a new instance is created the new_instance flag will be true. + Otherwise for an existing instance only the public_ip_name is needed. """ - cloud_service = self._get_cloud_service() - service_info = cloud_service.get_properties(self.running_instance_id) + if new_instance: + self.ip_config_name = ''.join([ + self.running_instance_id, '-ip-config' + ]) + self.nic_name = ''.join([self.running_instance_id, '-nic']) + self.subnet_name = ''.join([self.running_instance_id, '-subnet']) + self.vnet_name = ''.join([self.running_instance_id, '-vnet']) + + self.public_ip_name = ''.join([self.running_instance_id, '-public-ip']) + def _start_instance(self): + """Start the instance.""" try: - self.instance_ip = \ - service_info['deployments'][0]['virtual_ips'][0]['address'] - except IndexError: + start_operation = self.compute.virtual_machines.start( + self.running_instance_id, self.running_instance_id + ) + except Exception as error: raise AzureProviderException( - 'IP address for instance cannot be found.' + 'Unable to start instance: {0}.'.format(error.message) ) - def _start_instance(self): - """Start the instance.""" - self.vm.start_instance( - cloud_service_name=self.running_instance_id, - instance_name=self.running_instance_id - ) - self._wait_on_instance('ReadyRole') + start_operation.wait() def _stop_instance(self): """Stop the instance.""" - self.vm.shutdown_instance( - cloud_service_name=self.running_instance_id, - instance_name=self.running_instance_id, - deallocate_resources=True - ) - self._wait_on_instance('StoppedDeallocated') + try: + stop_operation = self.compute.virtual_machines.power_off( + self.running_instance_id, self.running_instance_id + ) + except Exception as error: + raise AzureProviderException( + 'Unable to stop instance: {0}.'.format(error.message) + ) + + stop_operation.wait() def _terminate_instance(self): - """Terminate the cloud service and instance.""" - cloud_service = self._get_cloud_service() - cloud_service.delete(self.running_instance_id, complete=True) - - def _wait_on_request(self, request_id): - """Wait for request to complete.""" - service = self.account.get_management_service() - request_result = RequestResult(request_id) - request_result.wait_for_request_completion(service) + """Terminate the resource group and instance.""" + try: + self.resource.resource_groups.delete(self.running_instance_id) + except Exception as error: + raise AzureProviderException( + 'Unable to terminate resource group: {0}.'.format( + error.message + ) + ) diff --git a/ipa/ipa_constants.py b/ipa/ipa_constants.py index e6e73d19..401e2298 100644 --- a/ipa/ipa_constants.py +++ b/ipa/ipa_constants.py @@ -28,7 +28,7 @@ SUPPORTED_DISTROS = ('openSUSE_Leap', 'SLES') SUPPORTED_PROVIDERS = ('azure', 'ec2', 'gce') -AZURE_DEFAULT_TYPE = 'Small' +AZURE_DEFAULT_TYPE = 'Standard_B1ms' AZURE_DEFAULT_USER = 'azureuser' EC2_DEFAULT_TYPE = 't2.micro' EC2_DEFAULT_USER = 'ec2-user' diff --git a/ipa/ipa_controller.py b/ipa/ipa_controller.py index 542f4c9d..4005443d 100644 --- a/ipa/ipa_controller.py +++ b/ipa/ipa_controller.py @@ -55,8 +55,8 @@ def test_image(provider_name, ssh_key_name=None, ssh_private_key=None, ssh_user=None, - storage_container=None, subnet_id=None, + subscription_id=None, test_dirs=None, tests=None): """Creates a cloud provider instance and initiates testing.""" @@ -93,8 +93,7 @@ def test_image(provider_name, ssh_key_name=ssh_key_name, ssh_private_key=ssh_private_key, ssh_user=ssh_user, - storage_container=storage_container, - subnet_id=subnet_id, + subscription_id=subscription_id, test_dirs=test_dirs, test_files=tests ) diff --git a/ipa/ipa_ec2.py b/ipa/ipa_ec2.py index 054d47db..7bc7b261 100644 --- a/ipa/ipa_ec2.py +++ b/ipa/ipa_ec2.py @@ -59,8 +59,8 @@ def __init__(self, ssh_key_name=None, ssh_private_key=None, ssh_user=None, - storage_container=None, # Not used in EC2 subnet_id=None, + subscription_id=None, # Not used in EC2 test_dirs=None, test_files=None): """Initialize EC2 provider class.""" diff --git a/ipa/ipa_gce.py b/ipa/ipa_gce.py index b4c95aed..e41bcf82 100644 --- a/ipa/ipa_gce.py +++ b/ipa/ipa_gce.py @@ -61,7 +61,6 @@ def __init__(self, ssh_key_name=None, # Not used in GCE ssh_private_key=None, ssh_user=None, - storage_container=None, # Not used in GCE subnet_id=None, test_dirs=None, test_files=None): diff --git a/ipa/scripts/cli.py b/ipa/scripts/cli.py index 8a722bc2..8ff0d9de 100644 --- a/ipa/scripts/cli.py +++ b/ipa/scripts/cli.py @@ -192,15 +192,14 @@ def main(): '--ssh-user', help='SSH user for accessing instance.' ) -@click.option( - '-S', - '--storage-container', - help='Azure storage container to use.' -) @click.option( '--subnet-id', help='Subnet to launch the new instance into.' ) +@click.option( + '--subscription-id', + help='Subscription ID for Azure account.' +) @click.option( '--test-dirs', help='Directories to search for tests.' @@ -232,8 +231,8 @@ def test(access_key_id, ssh_key_name, ssh_private_key, ssh_user, - storage_container, subnet_id, + subscription_id, test_dirs, provider, tests): @@ -262,8 +261,8 @@ def test(access_key_id, ssh_key_name, ssh_private_key, ssh_user, - storage_container, subnet_id, + subscription_id, test_dirs, tests ) @@ -387,7 +386,7 @@ def results(clear, echo_log(log_file, no_color) else: echo_results_file( - log_file.split('.')[0] + '.results', + log_file.rsplit('.', 1)[0] + '.results', no_color, verbose ) diff --git a/usr/share/lib/ipa/tests/SLES/test_sles_hostname.py b/usr/share/lib/ipa/tests/SLES/test_sles_hostname.py index 9ebb74da..c6c42be9 100644 --- a/usr/share/lib/ipa/tests/SLES/test_sles_hostname.py +++ b/usr/share/lib/ipa/tests/SLES/test_sles_hostname.py @@ -1,2 +1,4 @@ def test_sles_hostname(host): - assert host.system_info.hostname != 'linux' + result = host.run('hostname') + print('*** %s ***' % result.stdout.strip()) + assert result.stdout.strip() != 'linux' From d7c0dfffaf8299afd62e68aae8b6829a91aa2613 Mon Sep 17 00:00:00 2001 From: Sean Marlow Date: Wed, 21 Feb 2018 17:22:45 -0700 Subject: [PATCH 02/11] Fix typo in ip config name. --- ipa/ipa_azure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipa/ipa_azure.py b/ipa/ipa_azure.py index 35ae1fcd..d9277d75 100644 --- a/ipa/ipa_azure.py +++ b/ipa/ipa_azure.py @@ -195,7 +195,7 @@ def _create_network_interface(self): { 'location': self.region, 'ip_configurations': [{ - 'name': self.ipa_config_name, + 'name': self.ip_config_name, 'private_ip_allocation_method': 'Dynamic', 'subnet': { 'id': subnet_info.id From fe897a735f2ada51f3d8f28b4678843535979c9e Mon Sep 17 00:00:00 2001 From: Sean Marlow Date: Thu, 22 Feb 2018 13:23:33 -0700 Subject: [PATCH 03/11] Use factory method to get clients. This accepts the json cred fiel directly which includes the proper resource management endpoint for each service principal. --- ipa/ipa_azure.py | 323 +++++++++--------- ipa/ipa_controller.py | 2 - ipa/ipa_ec2.py | 1 - ipa/scripts/cli.py | 6 - .../lib/ipa/tests/SLES/test_sles_hostname.py | 1 - 5 files changed, 169 insertions(+), 164 deletions(-) diff --git a/ipa/ipa_azure.py b/ipa/ipa_azure.py index d9277d75..6d4fb95f 100644 --- a/ipa/ipa_azure.py +++ b/ipa/ipa_azure.py @@ -23,7 +23,7 @@ import json import os -from azure.common.credentials import ServicePrincipalCredentials +from azure.common.client_factory import get_client_from_auth_file from azure.mgmt.resource import ResourceManagementClient from azure.mgmt.network import NetworkManagementClient from azure.mgmt.compute import ComputeManagementClient @@ -59,7 +59,6 @@ def __init__(self, ssh_key_name=None, # Not used in Azure ssh_private_key=None, ssh_user=None, - subscription_id=None, subnet_id=None, # Not used in Azure test_dirs=None, test_files=None): @@ -111,94 +110,36 @@ def __init__(self, self.ssh_private_key ) - self.subscription_id = ( - subscription_id or - self._get_value(subscription_id, config_key='subscription_id') - ) - if not self.subscription_id: - raise AzureProviderException( - 'Subscription ID is required to connect to instance.' - ) - self.ssh_user = ssh_user or AZURE_DEFAULT_USER self.ssh_public_key = self._get_ssh_public_key() - self._get_service_account_info() - self.credentials = self._get_credentials() - - self.compute = self._get_compute_management() - self.network = self._get_network_management() - self.resource = self._get_resource_management() + self.compute = self._get_management_client(ComputeManagementClient) + self.network = self._get_management_client(NetworkManagementClient) + self.resource = self._get_management_client(ResourceManagementClient) if self.running_instance_id: - self._set_resource_names() + self._set_default_resource_names() - def _create_network_interface(self): + def _create_network_interface( + self, ip_config_name, nic_name, public_ip, region, + resource_group_name, subnet + ): """ - Create a new vnet, subnet, public ip and network interface. + Create a network interface in the resource group. - The instance requires an IP to be accessible via SSH for testing. + Attach NIC to the subnet and public IP provided. """ - try: - vnet_operation = self.network.virtual_networks.create_or_update( - self.running_instance_id, - self.vnet_name, - { - 'location': self.region, - 'address_space': { - 'address_prefixes': ['10.0.0.0/27'] - } - } - ) - except Exception as error: - raise AzureProviderException( - 'Unable to create vnet: {0}.'.format(error.message) - ) - - vnet_operation.wait() - - try: - subnet_operation = self.network.subnets.create_or_update( - self.running_instance_id, - self.vnet_name, - self.subnet_name, - {'address_prefix': '10.0.0.0/29'} - ) - except Exception as error: - raise AzureProviderException( - 'Unable to create subnet: {0}.'.format(error.message) - ) - - subnet_info = subnet_operation.result() - - try: - public_ip_operation = \ - self.network.public_ip_addresses.create_or_update( - self.running_instance_id, - self.public_ip_name, - { - 'location': self.region, - 'public_ip_allocation_method': 'Dynamic' - } - ) - except Exception as error: - raise AzureProviderException( - 'Unable to create public IP: {0}.'.format(error.message) - ) - - public_ip = public_ip_operation.result() - try: nic_operation = self.network.network_interfaces.create_or_update( - self.running_instance_id, - self.nic_name, + resource_group_name, + nic_name, { - 'location': self.region, + 'location': region, 'ip_configurations': [{ - 'name': self.ip_config_name, + 'name': ip_config_name, 'private_ip_allocation_method': 'Dynamic', 'subnet': { - 'id': subnet_info.id + 'id': subnet.id }, 'public_ip_address': { 'id': public_ip.id @@ -209,21 +150,44 @@ def _create_network_interface(self): except Exception as error: raise AzureProviderException( 'Unable to create network interface: {0}.'.format( - error.message + error ) ) return nic_operation.result() - def _create_resource_group(self): - """Create resource group if it does not exist.""" + def _create_public_ip(self, public_ip_name, resource_group_name, region): + """ + Create dynamic public IP address in the resource group. + """ + try: + public_ip_operation = \ + self.network.public_ip_addresses.create_or_update( + resource_group_name, + public_ip_name, + { + 'location': region, + 'public_ip_allocation_method': 'Dynamic' + } + ) + except Exception as error: + raise AzureProviderException( + 'Unable to create public IP: {0}.'.format(error) + ) + + return public_ip_operation.result() + + def _create_resource_group(self, region, resource_group_name): + """ + Create resource group if it does not exist. + """ try: self.resource.resource_groups.create_or_update( - self.running_instance_id, {'location': self.region} + resource_group_name, {'location': region} ) except Exception as error: raise AzureProviderException( - 'Unable to create resource group: {0}.'.format(error.message) + 'Unable to create resource group: {0}.'.format(error) ) def _create_vm(self, vm_parameters): @@ -238,12 +202,52 @@ def _create_vm(self, vm_parameters): except Exception as error: raise AzureProviderException( 'An exception occurred creating virtual machine: {0}'.format( - error.message + error ) ) vm_operation.wait() + def _create_subnet(self, resource_group_name, subnet_name, vnet_name): + """ + Create a subnet in the provided vnet and resource group. + """ + try: + subnet_operation = self.network.subnets.create_or_update( + resource_group_name, + vnet_name, + subnet_name, + {'address_prefix': '10.0.0.0/29'} + ) + except Exception as error: + raise AzureProviderException( + 'Unable to create subnet: {0}.'.format(error) + ) + + return subnet_operation.result() + + def _create_virtual_network(self, region, resource_group_name, vnet_name): + """ + Create a vnet in the given resource group with default address space. + """ + try: + vnet_operation = self.network.virtual_networks.create_or_update( + resource_group_name, + vnet_name, + { + 'location': region, + 'address_space': { + 'address_prefixes': ['10.0.0.0/27'] + } + } + ) + except Exception as error: + raise AzureProviderException( + 'Unable to create vnet: {0}.'.format(error) + ) + + vnet_operation.wait() + def _create_vm_parameters(self, interface): """ Create the VM parameters dictionary. @@ -289,47 +293,34 @@ def _create_vm_parameters(self, interface): } } - def _get_compute_management(self): - """Return instance of compute management class.""" - return ComputeManagementClient(self.credentials, self.subscription_id) - - def _get_credentials(self): - """Return instance of service principal credentials.""" - return ServicePrincipalCredentials( - client_id=self.client_id, - secret=self.client_secret, - tenant=self.tenant_id - ) - - def _get_network_management(self): - """Return instance of network management class.""" - return NetworkManagementClient(self.credentials, self.subscription_id) - - def _get_resource_management(self): - """Return instance of resource management class.""" - return ResourceManagementClient(self.credentials, self.subscription_id) - - def _get_service_account_info(self): - """Retrieve json dict from service account file.""" + def _get_management_client(self, client_class): + """ + Return instance of resource management client. + """ try: - with open(self.service_account_file, 'r') as f: - info = json.load(f) - except Exception as error: + client = get_client_from_auth_file( + client_class, auth_path=self.service_account_file + ) + except json.JSONDecodeError as error: raise AzureProviderException( - 'Exception processing service account file: {0}.'.format(error) + 'Service account file format is invalid: {0}.'.format(error) ) - - try: - self.tenant_id = info['tenant'] - self.client_id = info['appId'] - self.client_secret = info['password'] except KeyError as error: raise AzureProviderException( - 'Invalid service account file, missing key: {0}.'.format(error) + 'Service account file missing key: {0}.'.format(error) ) + except Exception as error: + raise AzureProviderException( + 'Unable to create resource management client: ' + '{0}.'.format(error) + ) + + return client def _get_ssh_public_key(self): - """Generate SSH public key from private key.""" + """ + Generate SSH public key from private key. + """ key = ipa_utils.generate_public_ssh_key(self.ssh_private_key) return key.decode() @@ -344,13 +335,15 @@ def _get_instance(self): ) except Exception as error: raise AzureProviderException( - 'Unable to retrieve instance: {0}'.format(error.message) + 'Unable to retrieve instance: {0}'.format(error) ) return instance def _get_instance_state(self): - """Retrieve state of instance.""" + """ + Retrieve state of instance. + """ instance = self._get_instance() statuses = instance.instance_view.statuses @@ -365,23 +358,37 @@ def _is_instance_running(self): return self._get_instance_state() == 'VM running' def _launch_instance(self): - """Create new test instance in a resource group with the same name.""" + """ + Create new test instance in a resource group with the same name. + """ self.running_instance_id = ipa_utils.generate_instance_name( 'azure-ipa-test' ) - self._set_resource_names(new_instance=True) + self._set_default_resource_names(new_instance=True) try: - # Try block acts as a transaction. If an exception occurrs + # Try block acts as a transaction. If an exception is raised # attempt to cleanup the resource group and all resources. - # Create resourece group with same name as instance. - self._create_resource_group() + # Create resource group. + self._create_resource_group(self.region, self.running_instance_id) - # Setup network, subnet and interface in resource group. - interface = self._create_network_interface() + # Setup network, subnet, interface and public ip in resource group. + self._create_virtual_network( + self.region, self.running_instance_id, self.vnet_name + ) + subnet = self._create_subnet( + self.running_instance_id, self.subnet_name, self.vnet_name + ) + public_ip = self._create_public_ip( + self.public_ip_name, self.running_instance_id, self.region + ) + interface = self._create_network_interface( + self.ip_config_name, self.nic_name, public_ip, self.region, + self.running_instance_id, subnet + ) - # Get dictionary of VM parameters. + # Get dictionary of VM parameters and create instance. vm_parameters = self._create_vm_parameters(interface) self._create_vm(vm_parameters) except Exception: @@ -417,7 +424,9 @@ def _process_image_id(self): ) def _set_image_id(self): - """If an existing instance is used get image id from deployment.""" + """ + If an existing instance is used get image id from deployment. + """ instance = self._get_instance() image_info = instance.storage_profile.image_reference @@ -428,71 +437,77 @@ def _set_image_id(self): def _set_instance_ip(self): """ - Get the public IP address based on instance ID. + Get the IP address based on instance ID. + + If public IP address not found attempt to get private IP. """ try: - public_ip = self.network.public_ip_addresses.get( + ip_address = self.network.public_ip_addresses.get( self.running_instance_id, self.public_ip_name - ) - except Exception as error: - raise AzureProviderException( - 'Unable to retrieve instance public IP: {0}.'.format( - error.message + ).ip_address + except Exception: + try: + ip_address = self.network.network_interfaces.get( + self.running_instance_id, self.nic_name + ).ip_configurations[0].private_ip_address + except Exception as error: + raise AzureProviderException( + 'Unable to retrieve instance IP address: {0}.'.format( + error + ) ) - ) - self.instance_ip = public_ip.ip_address + self.instance_ip = ip_address - def _set_resource_names(self, new_instance=False): + def _set_default_resource_names(self): """ Generate names for resources based on the running_instance_id. - - If a new instance is created the new_instance flag will be true. - Otherwise for an existing instance only the public_ip_name is needed. """ - if new_instance: - self.ip_config_name = ''.join([ - self.running_instance_id, '-ip-config' - ]) - self.nic_name = ''.join([self.running_instance_id, '-nic']) - self.subnet_name = ''.join([self.running_instance_id, '-subnet']) - self.vnet_name = ''.join([self.running_instance_id, '-vnet']) - + self.ip_config_name = ''.join([ + self.running_instance_id, '-ip-config' + ]) + self.nic_name = ''.join([self.running_instance_id, '-nic']) + self.subnet_name = ''.join([self.running_instance_id, '-subnet']) + self.vnet_name = ''.join([self.running_instance_id, '-vnet']) self.public_ip_name = ''.join([self.running_instance_id, '-public-ip']) def _start_instance(self): - """Start the instance.""" + """ + Start the instance. + """ try: start_operation = self.compute.virtual_machines.start( self.running_instance_id, self.running_instance_id ) except Exception as error: raise AzureProviderException( - 'Unable to start instance: {0}.'.format(error.message) + 'Unable to start instance: {0}.'.format(error) ) start_operation.wait() def _stop_instance(self): - """Stop the instance.""" + """ + Stop the instance. + """ try: stop_operation = self.compute.virtual_machines.power_off( self.running_instance_id, self.running_instance_id ) except Exception as error: raise AzureProviderException( - 'Unable to stop instance: {0}.'.format(error.message) + 'Unable to stop instance: {0}.'.format(error) ) stop_operation.wait() def _terminate_instance(self): - """Terminate the resource group and instance.""" + """ + Terminate the resource group and instance. + """ try: self.resource.resource_groups.delete(self.running_instance_id) except Exception as error: raise AzureProviderException( - 'Unable to terminate resource group: {0}.'.format( - error.message - ) + 'Unable to terminate resource group: {0}.'.format(error) ) diff --git a/ipa/ipa_controller.py b/ipa/ipa_controller.py index 4005443d..942363f5 100644 --- a/ipa/ipa_controller.py +++ b/ipa/ipa_controller.py @@ -56,7 +56,6 @@ def test_image(provider_name, ssh_private_key=None, ssh_user=None, subnet_id=None, - subscription_id=None, test_dirs=None, tests=None): """Creates a cloud provider instance and initiates testing.""" @@ -93,7 +92,6 @@ def test_image(provider_name, ssh_key_name=ssh_key_name, ssh_private_key=ssh_private_key, ssh_user=ssh_user, - subscription_id=subscription_id, test_dirs=test_dirs, test_files=tests ) diff --git a/ipa/ipa_ec2.py b/ipa/ipa_ec2.py index 7bc7b261..03092ec7 100644 --- a/ipa/ipa_ec2.py +++ b/ipa/ipa_ec2.py @@ -60,7 +60,6 @@ def __init__(self, ssh_private_key=None, ssh_user=None, subnet_id=None, - subscription_id=None, # Not used in EC2 test_dirs=None, test_files=None): """Initialize EC2 provider class.""" diff --git a/ipa/scripts/cli.py b/ipa/scripts/cli.py index 8ff0d9de..6698ee9d 100644 --- a/ipa/scripts/cli.py +++ b/ipa/scripts/cli.py @@ -196,10 +196,6 @@ def main(): '--subnet-id', help='Subnet to launch the new instance into.' ) -@click.option( - '--subscription-id', - help='Subscription ID for Azure account.' -) @click.option( '--test-dirs', help='Directories to search for tests.' @@ -232,7 +228,6 @@ def test(access_key_id, ssh_private_key, ssh_user, subnet_id, - subscription_id, test_dirs, provider, tests): @@ -262,7 +257,6 @@ def test(access_key_id, ssh_private_key, ssh_user, subnet_id, - subscription_id, test_dirs, tests ) diff --git a/usr/share/lib/ipa/tests/SLES/test_sles_hostname.py b/usr/share/lib/ipa/tests/SLES/test_sles_hostname.py index c6c42be9..87bd3ed2 100644 --- a/usr/share/lib/ipa/tests/SLES/test_sles_hostname.py +++ b/usr/share/lib/ipa/tests/SLES/test_sles_hostname.py @@ -1,4 +1,3 @@ def test_sles_hostname(host): result = host.run('hostname') - print('*** %s ***' % result.stdout.strip()) assert result.stdout.strip() != 'linux' From 8b6beb22f8c12dd712340621200421f0764a90f3 Mon Sep 17 00:00:00 2001 From: Sean Marlow Date: Fri, 23 Feb 2018 10:39:53 -0700 Subject: [PATCH 04/11] Add args for optional subnet in azure. Supplying subnet-id vnet-name and vnet-resource-group launches instance into the given subnet. All created resources will reside in a new resource group for easy cleanup when testing is finished. --- ipa/ipa_azure.py | 41 ++++++++++++++++++++++++++++++----------- ipa/ipa_controller.py | 9 +++++++-- ipa/ipa_ec2.py | 5 ++++- ipa/ipa_gce.py | 5 ++++- ipa/scripts/cli.py | 14 +++++++++++++- setup.py | 7 +++++-- 6 files changed, 63 insertions(+), 18 deletions(-) diff --git a/ipa/ipa_azure.py b/ipa/ipa_azure.py index 6d4fb95f..6871d1fd 100644 --- a/ipa/ipa_azure.py +++ b/ipa/ipa_azure.py @@ -59,9 +59,11 @@ def __init__(self, ssh_key_name=None, # Not used in Azure ssh_private_key=None, ssh_user=None, - subnet_id=None, # Not used in Azure + subnet_id=None, test_dirs=None, - test_files=None): + test_files=None, + vnet_name=None, + vnet_resource_group=None): """Initialize Azure Provider class.""" super(AzureProvider, self).__init__('azure', cleanup, @@ -81,6 +83,12 @@ def __init__(self, test_dirs, test_files) + if subnet_id and not (vnet_name and vnet_resource_group): + raise AzureProviderException( + 'If subnet_id is provided vnet_resource_group and vnet_name' + ' are also required.' + ) + self.service_account_file = ( service_account_file or self._get_value( @@ -112,6 +120,9 @@ def __init__(self, self.ssh_user = ssh_user or AZURE_DEFAULT_USER self.ssh_public_key = self._get_ssh_public_key() + self.subnet_id = subnet_id + self.vnet_resource_group = vnet_resource_group + self.vnet_name = vnet_name self.compute = self._get_management_client(ComputeManagementClient) self.network = self._get_management_client(NetworkManagementClient) @@ -364,22 +375,30 @@ def _launch_instance(self): self.running_instance_id = ipa_utils.generate_instance_name( 'azure-ipa-test' ) - self._set_default_resource_names(new_instance=True) + self._set_default_resource_names() try: # Try block acts as a transaction. If an exception is raised - # attempt to cleanup the resource group and all resources. + # attempt to cleanup the resource group and all created resources. # Create resource group. self._create_resource_group(self.region, self.running_instance_id) - # Setup network, subnet, interface and public ip in resource group. - self._create_virtual_network( - self.region, self.running_instance_id, self.vnet_name - ) - subnet = self._create_subnet( - self.running_instance_id, self.subnet_name, self.vnet_name - ) + if self.subnet_id: + # Use existing vnet/subnet. + subnet = self.network.subnets.get( + self.vnet_resource_group, self.vnet_name, self.subnet_id + ) + else: + # Create new vnet/subnet. + self._create_virtual_network( + self.region, self.running_instance_id, self.vnet_name + ) + subnet = self._create_subnet( + self.running_instance_id, self.subnet_name, self.vnet_name + ) + + # Setup interface and public ip in resource group. public_ip = self._create_public_ip( self.public_ip_name, self.running_instance_id, self.region ) diff --git a/ipa/ipa_controller.py b/ipa/ipa_controller.py index 942363f5..ce127bd4 100644 --- a/ipa/ipa_controller.py +++ b/ipa/ipa_controller.py @@ -57,7 +57,9 @@ def test_image(provider_name, ssh_user=None, subnet_id=None, test_dirs=None, - tests=None): + tests=None, + vnet_name=None, + vnet_resource_group=None): """Creates a cloud provider instance and initiates testing.""" if provider_name == 'azure': provider_class = AzureProvider @@ -92,8 +94,11 @@ def test_image(provider_name, ssh_key_name=ssh_key_name, ssh_private_key=ssh_private_key, ssh_user=ssh_user, + subnet_id=subnet_id, test_dirs=test_dirs, - test_files=tests + test_files=tests, + vnet_name=vnet_name, + vnet_resource_group=vnet_resource_group ) return provider.test_image() diff --git a/ipa/ipa_ec2.py b/ipa/ipa_ec2.py index 03092ec7..2955a21e 100644 --- a/ipa/ipa_ec2.py +++ b/ipa/ipa_ec2.py @@ -61,7 +61,10 @@ def __init__(self, ssh_user=None, subnet_id=None, test_dirs=None, - test_files=None): + test_files=None, + vnet_name=None, # Not used in EC2 + vnet_resource_group=None # Not used in EC2 + ): """Initialize EC2 provider class.""" super(EC2Provider, self).__init__('ec2', cleanup, diff --git a/ipa/ipa_gce.py b/ipa/ipa_gce.py index e41bcf82..b109cd02 100644 --- a/ipa/ipa_gce.py +++ b/ipa/ipa_gce.py @@ -63,7 +63,10 @@ def __init__(self, ssh_user=None, subnet_id=None, test_dirs=None, - test_files=None): + test_files=None, + vnet_name=None, # Not used in GCE + vnet_resource_group=None # Not used in GCE + ): super(GCEProvider, self).__init__('gce', cleanup, config, diff --git a/ipa/scripts/cli.py b/ipa/scripts/cli.py index 6698ee9d..e3e1d1b1 100644 --- a/ipa/scripts/cli.py +++ b/ipa/scripts/cli.py @@ -200,6 +200,14 @@ def main(): '--test-dirs', help='Directories to search for tests.' ) +@click.option( + '--vnet-name', + help='Azure virtual network name to attach network interface.' +) +@click.option( + '--vnet-resource-group', + help='Azure resource group name where virtual network is found.' +) @click.argument( 'provider', type=click.Choice(SUPPORTED_PROVIDERS) @@ -229,6 +237,8 @@ def test(access_key_id, ssh_user, subnet_id, test_dirs, + vnet_name, + vnet_resource_group, provider, tests): """Test image in the given framework using the supplied test files.""" @@ -258,7 +268,9 @@ def test(access_key_id, ssh_user, subnet_id, test_dirs, - tests + tests, + vnet_name, + vnet_resource_group ) echo_results(results, no_color) sys.exit(status) diff --git a/setup.py b/setup.py index 08db7f82..d399f9db 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,9 @@ requirements = [ 'apache-libcloud', - 'azurectl>=3.0.1', + 'azure-mgmt-compute', + 'azure-mgmt-network', + 'azure-mgmt-resource', 'Click', 'cryptography', 'paramiko', @@ -87,7 +89,8 @@ 'Environment :: Console', 'Intended Audience :: Developers', 'Topic :: Software Development :: Testing', - 'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)', + 'License :: OSI Approved :: ' + 'GNU General Public License v3 or later (GPLv3+)', 'Natural Language :: English', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.4', From f1a9a4f9bede958623566d93c52d474022ea803a Mon Sep 17 00:00:00 2001 From: Sean Marlow Date: Fri, 23 Feb 2018 10:50:05 -0700 Subject: [PATCH 05/11] Add azure common dependency. Required by all azure packages but explicitly imported for client factory function. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index d399f9db..27a6ab5a 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,7 @@ requirements = [ 'apache-libcloud', + 'azure-common', 'azure-mgmt-compute', 'azure-mgmt-network', 'azure-mgmt-resource', From 22512c3e4f911a2975c8c3e8512c0a0c1aea3e4d Mon Sep 17 00:00:00 2001 From: Sean Marlow Date: Wed, 28 Feb 2018 10:39:36 -0700 Subject: [PATCH 06/11] Update unit tests for Azure ARM backend. --- tests/azure/azure.config | 13 - tests/azure/publishsettings | 6 - tests/azure/test-sa.json | 12 + tests/test_ipa_azure.py | 542 ++++++++++++++---------------------- 4 files changed, 222 insertions(+), 351 deletions(-) delete mode 100644 tests/azure/azure.config delete mode 100644 tests/azure/publishsettings create mode 100644 tests/azure/test-sa.json diff --git a/tests/azure/azure.config b/tests/azure/azure.config deleted file mode 100644 index cb79f155..00000000 --- a/tests/azure/azure.config +++ /dev/null @@ -1,13 +0,0 @@ -[DEFAULT] -default_account = account:bob -default_region = region:West US - -[region:West US] -default_storage_account = bob -default_storage_container = bar - -[account:bob] -publishsettings = tests/azure/publishsettings - -[account:foo] -publishsettings = tests/azure/publishsettings diff --git a/tests/azure/publishsettings b/tests/azure/publishsettings deleted file mode 100644 index aeff8ec2..00000000 --- a/tests/azure/publishsettings +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - diff --git a/tests/azure/test-sa.json b/tests/azure/test-sa.json new file mode 100644 index 00000000..28b3686d --- /dev/null +++ b/tests/azure/test-sa.json @@ -0,0 +1,12 @@ +{ + "clientId": "12345", + "clientSecret": "12345", + "subscriptionId": "12345", + "tenantId": "12345", + "activeDirectoryEndpointUrl": "https://login.microsoftonline.com", + "resourceManagerEndpointUrl": "https://management.azure.com/", + "activeDirectoryGraphResourceId": "https://graph.windows.net/", + "sqlManagementEndpointUrl": "https://management.core.windows.net:8443/", + "galleryEndpointUrl": "https://gallery.azure.com/", + "managementEndpointUrl": "https://management.core.windows.net/" +} diff --git a/tests/test_ipa_azure.py b/tests/test_ipa_azure.py index 3be20089..3609d04c 100644 --- a/tests/test_ipa_azure.py +++ b/tests/test_ipa_azure.py @@ -21,31 +21,41 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import json +import pytest + from ipa.ipa_azure import AzureProvider from ipa.ipa_exceptions import AzureProviderException -import pytest - from unittest.mock import MagicMock, patch class TestAzureProvider(object): """Test Azure provider class.""" + def setup(self): + self.client = MagicMock() + def setup_method(self, method): """Set up kwargs dict.""" self.kwargs = { 'config': 'tests/data/config', - 'distro_name': 'SLES', - 'image_id': 'fakeimage', + 'distro_name': 'sles', + 'image_id': 'another:fake:image:id', 'no_default_test_dirs': True, - 'provider_config': 'tests/azure/azure.config', 'running_instance_id': 'fakeinstance', + 'service_account_file': 'tests/azure/test-sa.json', 'ssh_private_key': 'tests/data/ida_test', 'test_dirs': 'tests/data/tests', 'test_files': ['test_image'] } + @patch.object(AzureProvider, '_get_management_client') + @patch.object(AzureProvider, '_get_ssh_public_key') + def helper_get_provider(self, mock_get_ssh_pub_key, mock_get_client): + mock_get_client.return_value = self.client + return AzureProvider(**self.kwargs) + def test_azure_exception_required_args(self): """Test an exception is raised if required args missing.""" self.kwargs['ssh_private_key'] = None @@ -57,374 +67,242 @@ def test_azure_exception_required_args(self): assert str(error.value) == msg - @patch.object(AzureProvider, '_wait_on_request') - @patch.object(AzureProvider, '_get_virtual_machine') - @patch.object(AzureProvider, '_get_cloud_service') - @patch.object(AzureProvider, '_get_account') - def test_azure_create_cloud_service(self, - mock_get_account, - mock_get_cloud_service, - mock_get_vm, - mock_wait_on_request): - """Test create cloud service method.""" - account = MagicMock() - cloud_service = MagicMock() - cloud_service.create.return_value = '283e65be259bac8d9beb' - vm = MagicMock() - - mock_get_account.return_value = account - mock_get_cloud_service.return_value = cloud_service - mock_get_vm.return_value = vm - mock_wait_on_request.return_value = None - - provider = AzureProvider(**self.kwargs) - service = provider._create_cloud_service('azure-test-instance') - assert cloud_service == service - assert mock_get_cloud_service.call_count == 1 - - @patch.object(AzureProvider, '_get_virtual_machine') - @patch('ipa.ipa_azure.AzurectlConfig') - @patch('ipa.ipa_azure.AzureAccount') - def test_azure_get_account(self, - mock_azure_account, - mock_azure_config, - mock_get_vm): - """Test get account method.""" - account = MagicMock() - config = MagicMock() - vm = MagicMock() - - mock_azure_account.return_value = account - mock_azure_config.return_value = config - mock_get_vm.return_value = vm - - provider = AzureProvider(**self.kwargs) - assert account == provider.account - assert mock_azure_account.call_count == 1 - - @patch.object(AzureProvider, '_get_virtual_machine') - @patch.object(AzureProvider, '_get_account') - @patch('ipa.ipa_azure.CloudService') - def test_azure_get_cloud_service(self, - mock_cloud_service, - mock_get_account, - mock_get_vm): - """Test get cloud service method.""" - account = MagicMock() - cloud_service = MagicMock() - vm = MagicMock() - - mock_get_account.return_value = account - mock_cloud_service.return_value = cloud_service - mock_get_vm.return_value = vm - - provider = AzureProvider(**self.kwargs) - service = provider._get_cloud_service() - assert service == cloud_service - assert mock_cloud_service.call_count == 1 - - @patch.object(AzureProvider, '_get_virtual_machine') - @patch.object(AzureProvider, '_get_account') - def test_azure_get_instance_state(self, - mock_get_account, - mock_get_vm): - """Test get instance state method.""" - account = MagicMock() - vm = MagicMock() - vm.instance_status.return_value = 'ReadyRole' - - mock_get_account.return_value = account - mock_get_vm.return_value = vm - - provider = AzureProvider(**self.kwargs) - assert provider._get_instance_state() == 'ReadyRole' - assert vm.instance_status.call_count == 1 - - @patch.object(AzureProvider, '_get_account') - @patch('ipa.ipa_azure.VirtualMachine') - def test_azure_get_virtual_machine(self, - mock_virtual_machine, - mock_get_account): - """Test get virtual machine method.""" - account = MagicMock() - vm = MagicMock() - - mock_get_account.return_value = account - mock_virtual_machine.return_value = vm - - provider = AzureProvider(**self.kwargs) - assert vm == provider.vm - assert mock_virtual_machine.call_count == 1 + @patch('ipa.ipa_azure.get_client_from_auth_file') + def test_get_management_client(self, mock_get_client): + client = MagicMock() + mock_get_client.return_value = client + + client_class = MagicMock() + + provider = self.helper_get_provider() + result = provider._get_management_client(client_class) + + assert result == client + + @patch('ipa.ipa_azure.get_client_from_auth_file') + def test_get_management_client_json_error(self, mock_get_client): + client_class = MagicMock() + mock_get_client.side_effect = json.JSONDecodeError( + 'Not valid', 'tests/azure/test-sa.json', 34 + ) + + provider = self.helper_get_provider() + + with pytest.raises(AzureProviderException) as error: + provider._get_management_client(client_class) + + assert str(error.value) == 'Service account file format is invalid: ' \ + 'Not valid: line 1 column 35 (char 34).' + + @patch('ipa.ipa_azure.get_client_from_auth_file') + def test_get_management_client_key_error(self, mock_get_client): + client_class = MagicMock() + mock_get_client.side_effect = KeyError('subscriptionId') + + provider = self.helper_get_provider() + + with pytest.raises(AzureProviderException) as error: + provider._get_management_client(client_class) + + assert str(error.value) == "Service account file missing key: " \ + "'subscriptionId'." + + @patch('ipa.ipa_azure.get_client_from_auth_file') + def test_get_management_client_exception(self, mock_get_client): + client_class = MagicMock() + mock_get_client.side_effect = Exception('Not valid') + + provider = self.helper_get_provider() + + with pytest.raises(AzureProviderException) as error: + provider._get_management_client(client_class) + + assert str(error.value) == 'Unable to create resource management ' \ + 'client: Not valid.' + + @patch('ipa.ipa_azure.ipa_utils.generate_public_ssh_key') + def test_get_ssh_public_key(self, mock_generate_pub_key): + mock_generate_pub_key.return_value = b'pub-key' + provider = self.helper_get_provider() + key = provider._get_ssh_public_key() + + assert key == 'pub-key' + + def test_azure_is_instance_running(self): + """Test is instance running method.""" + instance = MagicMock() + status = MagicMock() + status.code = 'PowerState' + status.display_status = 'VM running' + instance.instance_view.statuses = [status] + + self.client.virtual_machines.get.return_value = instance + + provider = self.helper_get_provider() + assert provider._is_instance_running() @patch.object(AzureProvider, '_wait_on_instance') - @patch.object(AzureProvider, '_get_virtual_machine') - @patch.object(AzureProvider, '_get_account') - @patch.object(AzureProvider, '_create_cloud_service') @patch('ipa.ipa_utils.generate_instance_name') - def test_azure_launch_instance(self, - mock_generate_instance_name, - mock_get_account, - mock_create_cloud_service, - mock_get_vm, - mock_wait_on_instance): + def test_azure_launch_instance( + self, mock_generate_instance_name, mock_wait_on_instance + ): """Test launch instance method.""" - account = MagicMock() - cloud_service = MagicMock() - config = MagicMock() - fingerprint = MagicMock() - net_config = MagicMock() - ssh_endpoint = MagicMock() - vm = MagicMock() - - cloud_service.add_certificate.return_value = fingerprint - - vm.create_linux_configuration.return_value = config - vm.create_instance.return_value = None - vm.create_network_configuration.return_value = net_config - vm.create_network_endpoint.return_value = ssh_endpoint - - mock_create_cloud_service.return_value = cloud_service - mock_get_account.return_value = account - mock_get_vm.return_value = vm - mock_wait_on_instance.return_value = None mock_generate_instance_name.return_value = 'azure-test-instance' - provider = AzureProvider(**self.kwargs) + provider = self.helper_get_provider() provider._launch_instance() + + assert self.client.network_interfaces.create_or_update.call_count == 1 + assert self.client.public_ip_addresses.create_or_update.call_count == 1 + assert self.client.resource_groups.create_or_update.call_count == 1 + assert self.client.virtual_machines.create_or_update.call_count == 1 + assert self.client.subnets.create_or_update.call_count == 1 + assert self.client.virtual_networks.create_or_update.call_count == 1 assert provider.running_instance_id == 'azure-test-instance' - assert vm.create_instance.call_count == 1 - - @patch.object(AzureProvider, '_get_virtual_machine') - @patch.object(AzureProvider, '_get_instance_state') - @patch.object(AzureProvider, '_get_account') - def test_azure_instance_runnning_undefined(self, - mock_get_account, - mock_get_instance_state, - mock_get_vm): - """Test instance running undefined state.""" - account = MagicMock() - vm = MagicMock() - mock_get_account.return_value = account - mock_get_instance_state.return_value = 'Undefined' - mock_get_vm.return_value = vm - - provider = AzureProvider(**self.kwargs) - with pytest.raises(AzureProviderException) as error: - provider._is_instance_running() + def test_process_image_id(self): + provider = self.helper_get_provider() + provider._process_image_id() - assert str(error.value) == \ - 'Instance with name: fakeinstance, cannot be found.' - assert mock_get_instance_state.call_count == 1 - - @patch.object(AzureProvider, '_get_virtual_machine') - @patch.object(AzureProvider, '_get_instance_state') - @patch.object(AzureProvider, '_get_account') - def test_azure_instance_runnning(self, - mock_get_account, - mock_get_instance_state, - mock_get_vm): - """Test instance running undefined state.""" - account = MagicMock() - vm = MagicMock() - mock_get_account.return_value = account - mock_get_instance_state.return_value = 'ReadyRole' - mock_get_vm.return_value = vm - - provider = AzureProvider(**self.kwargs) + assert provider.image_publisher == 'another' + assert provider.image_offer == 'fake' + assert provider.image_sku == 'image' + assert provider.image_version == 'id' - assert provider._is_instance_running() - assert mock_get_instance_state.call_count == 1 - mock_get_instance_state.reset_mock() - - mock_get_instance_state.return_value = 'StoppedDeallocated' - assert not provider._is_instance_running() - assert mock_get_instance_state.call_count == 1 - - @patch.object(AzureProvider, '_get_virtual_machine') - @patch.object(AzureProvider, '_get_account') - def test_azure_set_image_id_exception(self, - mock_get_account, - mock_get_vm): - """Test set image id exception.""" - account = MagicMock() - vm = MagicMock() - properties = MagicMock() - properties.deployments = [] - vm.service.get_hosted_service_properties.return_value = properties - - mock_get_account.return_value = account - mock_get_vm.return_value = vm - - provider = AzureProvider(**self.kwargs) + def test_process_image_id_invalid(self): + provider = self.helper_get_provider() + provider.image_id = 'invalid:id' with pytest.raises(AzureProviderException) as error: - provider._set_image_id() + provider._process_image_id() - assert str(error.value) == \ - 'Image name for instance cannot be found.' + assert str(error.value) == 'Image ID is invalid. Format must match ' \ + '{Publisher}:{Offer}:{Sku}:{Version}.' + + def test_set_default_resource_names(self): + provider = self.helper_get_provider() + provider._set_default_resource_names() - @patch.object(AzureProvider, '_get_virtual_machine') - @patch.object(AzureProvider, '_get_account') - def test_azure_set_image_id(self, - mock_get_account, - mock_get_vm): + assert provider.ip_config_name == 'fakeinstance-ip-config' + assert provider.nic_name == 'fakeinstance-nic' + assert provider.subnet_name == 'fakeinstance-subnet' + assert provider.vnet_name == 'fakeinstance-vnet' + assert provider.public_ip_name == 'fakeinstance-public-ip' + + @patch.object(AzureProvider, '_get_instance') + def test_azure_set_image_id(self, mock_get_instance): """Test set image id method.""" - account = MagicMock() - vm = MagicMock() - properties = MagicMock() - deployment = MagicMock() - role = MagicMock() - role.os_virtual_hard_disk.source_image_name = 'fakeimage' - deployment.role_list = [role] - properties.deployments = [deployment] - vm.service.get_hosted_service_properties.return_value = properties - - mock_get_account.return_value = account - mock_get_vm.return_value = vm - - provider = AzureProvider(**self.kwargs) - provider._set_image_id() + self.kwargs['image_id'] = None - assert provider.image_id == 'fakeimage' + image_reference = MagicMock() + image_reference.publisher = 'another' + image_reference.offer = 'fake' + image_reference.sku = 'image' + image_reference.version = 'id' - @patch.object(AzureProvider, '_get_virtual_machine') - @patch.object(AzureProvider, '_get_cloud_service') - @patch.object(AzureProvider, '_get_account') - def test_azure_set_instance_ip_exception(self, - mock_get_account, - mock_get_cloud_service, - mock_get_vm): - """Test set instance ip exception.""" - cloud_service = MagicMock() - cloud_service.get_properties.return_value = {'deployments': []} - mock_get_cloud_service.return_value = cloud_service + instance = MagicMock() + instance.storage_profile.image_reference = image_reference - account = MagicMock() - mock_get_account.return_value = account + mock_get_instance.return_value = instance - vm = MagicMock() - mock_get_vm.return_value = vm + provider = self.helper_get_provider() + provider._set_image_id() - provider = AzureProvider(**self.kwargs) + assert provider.image_id == 'another:fake:image:id' + + @patch('ipa.ipa_utils.generate_instance_name') + def test_azure_set_instance_ip_exception( + self, mock_generate_instance_name + ): + """Test set instance ip exception.""" + mock_generate_instance_name.return_value = 'azure-test-instance' + provider = self.helper_get_provider() + + self.client.public_ip_addresses.get.side_effect = Exception( + 'IP not found' + ) + self.client.network_interfaces.get.side_effect = Exception( + 'IP not found' + ) with pytest.raises(AzureProviderException) as error: provider._set_instance_ip() assert str(error.value) == \ - 'IP address for instance cannot be found.' - assert mock_get_cloud_service.call_count == 1 - - @patch.object(AzureProvider, '_get_virtual_machine') - @patch.object(AzureProvider, '_get_cloud_service') - @patch.object(AzureProvider, '_get_account') - def test_azure_set_instance_ip(self, - mock_get_account, - mock_get_cloud_service, - mock_get_vm): - """Test set instance ip method.""" - cloud_service = MagicMock() - cloud_service.get_properties.return_value = { - 'deployments': [{'virtual_ips': [{'address': '127.0.0.1'}]}] - } - mock_get_cloud_service.return_value = cloud_service + 'Unable to retrieve instance IP address: IP not found.' - account = MagicMock() - mock_get_account.return_value = account + @patch('ipa.ipa_utils.generate_instance_name') + def test_azure_set_instance_ip(self, mock_generate_instance_name): + """Test set instance ip method.""" + mock_generate_instance_name.return_value = 'azure-test-instance' + provider = self.helper_get_provider() - vm = MagicMock() - mock_get_vm.return_value = vm + ip_address = MagicMock() + ip_address.ip_address = '10.0.0.1' + self.client.public_ip_addresses.get.return_value = ip_address - provider = AzureProvider(**self.kwargs) provider._set_instance_ip() + assert provider.instance_ip == '10.0.0.1' - assert provider.instance_ip == '127.0.0.1' - assert mock_get_cloud_service.call_count == 1 - - @patch.object(AzureProvider, '_wait_on_instance') - @patch.object(AzureProvider, '_get_virtual_machine') - @patch.object(AzureProvider, '_get_account') - def test_azure_start_instance(self, - mock_get_account, - mock_get_vm, - mock_wait_on_instance): + def test_azure_start_instance(self): """Test start instance method.""" - account = MagicMock() - vm = MagicMock() + provider = self.helper_get_provider() + provider.running_instance_id = 'ipa-test-instance' - vm.start_instance.return_value = None + provider._start_instance() + self.client.virtual_machines.start.assert_called_once_with( + 'ipa-test-instance', 'ipa-test-instance' + ) - mock_get_account.return_value = account - mock_get_vm.return_value = vm - mock_wait_on_instance.return_value = None + # Test exception + self.client.virtual_machines.start.side_effect = Exception( + 'Instance not found' + ) - provider = AzureProvider(**self.kwargs) - provider._start_instance() - assert vm.start_instance.call_count == 1 + with pytest.raises(AzureProviderException) as error: + provider._start_instance() - @patch.object(AzureProvider, '_wait_on_instance') - @patch.object(AzureProvider, '_get_virtual_machine') - @patch.object(AzureProvider, '_get_account') - def test_azure_stop_instance(self, - mock_get_account, - mock_get_vm, - mock_wait_on_instance): + assert str(error.value) == 'Unable to start instance: ' \ + 'Instance not found.' + + def test_azure_stop_instance(self): """Test stop instance method.""" - account = MagicMock() - vm = MagicMock() + provider = self.helper_get_provider() + provider.running_instance_id = 'ipa-test-instance' + + provider._stop_instance() + self.client.virtual_machines.power_off.assert_called_once_with( + 'ipa-test-instance', 'ipa-test-instance' + ) + + # Test exception + self.client.virtual_machines.power_off.side_effect = Exception( + 'Instance not found' + ) - vm.shutdown_instance.return_value = None + with pytest.raises(AzureProviderException) as error: + provider._stop_instance() - mock_get_account.return_value = account - mock_get_vm.return_value = vm - mock_wait_on_instance.return_value = None + assert str(error.value) == 'Unable to stop instance: ' \ + 'Instance not found.' - provider = AzureProvider(**self.kwargs) - provider._stop_instance() - assert vm.shutdown_instance.call_count == 1 - - @patch.object(AzureProvider, '_get_cloud_service') - @patch.object(AzureProvider, '_get_virtual_machine') - @patch.object(AzureProvider, '_get_account') - def test_azure_terminate_instance(self, - mock_get_account, - mock_get_vm, - mock_get_cloud_service): + def test_azure_terminate_instance(self): """Test terminate instance method.""" - account = MagicMock() - cloud_service = MagicMock() - vm = MagicMock() + provider = self.helper_get_provider() + provider.running_instance_id = 'ipa-test-instance' - cloud_service.delete.return_value = None + provider._terminate_instance() + self.client.resource_groups.delete.assert_called_once_with( + 'ipa-test-instance' + ) - mock_get_account.return_value = account - mock_get_cloud_service.return_value = cloud_service - mock_get_vm.return_value = vm + # Test exception + self.client.resource_groups.delete.side_effect = Exception( + 'Instance not found' + ) - provider = AzureProvider(**self.kwargs) - provider._terminate_instance() - assert cloud_service.delete.call_count == 1 - - @patch('ipa.ipa_azure.RequestResult') - @patch.object(AzureProvider, '_get_virtual_machine') - @patch.object(AzureProvider, '_get_account') - def test_azure_wait_on_request(self, - mock_get_account, - mock_get_vm, - mock_request_result): - """Test wait on request method.""" - account = MagicMock() - request_result = MagicMock() - service = MagicMock() - vm = MagicMock() - - account.get_management_service.return_value = service - request_result.wait_for_request_completion.return_value = None - - mock_get_account.return_value = account - mock_get_vm.return_value = vm - mock_request_result.return_value = request_result - - provider = AzureProvider(**self.kwargs) - provider._wait_on_request('12345') - assert request_result.wait_for_request_completion.call_count == 1 + with pytest.raises(AzureProviderException) as error: + provider._terminate_instance() + + assert str(error.value) == 'Unable to terminate resource group: ' \ + 'Instance not found.' From 7491206e8c9e55046ccd64562b0dc545a87cb4fa Mon Sep 17 00:00:00 2001 From: Sean Marlow Date: Wed, 28 Feb 2018 10:52:26 -0700 Subject: [PATCH 07/11] Jsondecodeerror not in python 3.4. Extends from ValueError. Thus catch ValueError instead. --- ipa/ipa_azure.py | 3 +-- tests/test_ipa_azure.py | 7 +++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/ipa/ipa_azure.py b/ipa/ipa_azure.py index 6871d1fd..74ca3381 100644 --- a/ipa/ipa_azure.py +++ b/ipa/ipa_azure.py @@ -20,7 +20,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import json import os from azure.common.client_factory import get_client_from_auth_file @@ -312,7 +311,7 @@ def _get_management_client(self, client_class): client = get_client_from_auth_file( client_class, auth_path=self.service_account_file ) - except json.JSONDecodeError as error: + except ValueError as error: raise AzureProviderException( 'Service account file format is invalid: {0}.'.format(error) ) diff --git a/tests/test_ipa_azure.py b/tests/test_ipa_azure.py index 3609d04c..001cffc4 100644 --- a/tests/test_ipa_azure.py +++ b/tests/test_ipa_azure.py @@ -21,7 +21,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import json import pytest from ipa.ipa_azure import AzureProvider @@ -82,8 +81,8 @@ def test_get_management_client(self, mock_get_client): @patch('ipa.ipa_azure.get_client_from_auth_file') def test_get_management_client_json_error(self, mock_get_client): client_class = MagicMock() - mock_get_client.side_effect = json.JSONDecodeError( - 'Not valid', 'tests/azure/test-sa.json', 34 + mock_get_client.side_effect = ValueError( + 'Not valid' ) provider = self.helper_get_provider() @@ -92,7 +91,7 @@ def test_get_management_client_json_error(self, mock_get_client): provider._get_management_client(client_class) assert str(error.value) == 'Service account file format is invalid: ' \ - 'Not valid: line 1 column 35 (char 34).' + 'Not valid.' @patch('ipa.ipa_azure.get_client_from_auth_file') def test_get_management_client_key_error(self, mock_get_client): From ee63bd722e7fb533403d2df5f021bd87bf89676d Mon Sep 17 00:00:00 2001 From: Sean Marlow Date: Wed, 28 Feb 2018 13:07:26 -0700 Subject: [PATCH 08/11] Move config dicts outside of function calls. --- ipa/ipa_azure.py | 85 +++++++++++++++++++++++++----------------------- 1 file changed, 44 insertions(+), 41 deletions(-) diff --git a/ipa/ipa_azure.py b/ipa/ipa_azure.py index 74ca3381..a51037c5 100644 --- a/ipa/ipa_azure.py +++ b/ipa/ipa_azure.py @@ -139,23 +139,23 @@ def _create_network_interface( Attach NIC to the subnet and public IP provided. """ + nic_config = { + 'location': region, + 'ip_configurations': [{ + 'name': ip_config_name, + 'private_ip_allocation_method': 'Dynamic', + 'subnet': { + 'id': subnet.id + }, + 'public_ip_address': { + 'id': public_ip.id + }, + }] + } + try: nic_operation = self.network.network_interfaces.create_or_update( - resource_group_name, - nic_name, - { - 'location': region, - 'ip_configurations': [{ - 'name': ip_config_name, - 'private_ip_allocation_method': 'Dynamic', - 'subnet': { - 'id': subnet.id - }, - 'public_ip_address': { - 'id': public_ip.id - }, - }] - } + resource_group_name, nic_name, nic_config ) except Exception as error: raise AzureProviderException( @@ -170,15 +170,15 @@ def _create_public_ip(self, public_ip_name, resource_group_name, region): """ Create dynamic public IP address in the resource group. """ + public_ip_config = { + 'location': region, + 'public_ip_allocation_method': 'Dynamic' + } + try: public_ip_operation = \ self.network.public_ip_addresses.create_or_update( - resource_group_name, - public_ip_name, - { - 'location': region, - 'public_ip_allocation_method': 'Dynamic' - } + resource_group_name, public_ip_name, public_ip_config ) except Exception as error: raise AzureProviderException( @@ -191,23 +191,25 @@ def _create_resource_group(self, region, resource_group_name): """ Create resource group if it does not exist. """ + resource_group_config = {'location': region} + try: self.resource.resource_groups.create_or_update( - resource_group_name, {'location': region} + resource_group_name, resource_group_config ) except Exception as error: raise AzureProviderException( 'Unable to create resource group: {0}.'.format(error) ) - def _create_vm(self, vm_parameters): + def _create_vm(self, vm_config): """ Attempt to create or update VM instance based on vm_parameters config. """ try: vm_operation = self.compute.virtual_machines.create_or_update( self.running_instance_id, self.running_instance_id, - vm_parameters + vm_config ) except Exception as error: raise AzureProviderException( @@ -222,12 +224,11 @@ def _create_subnet(self, resource_group_name, subnet_name, vnet_name): """ Create a subnet in the provided vnet and resource group. """ + subnet_config = {'address_prefix': '10.0.0.0/29'} + try: subnet_operation = self.network.subnets.create_or_update( - resource_group_name, - vnet_name, - subnet_name, - {'address_prefix': '10.0.0.0/29'} + resource_group_name, vnet_name, subnet_name, subnet_config ) except Exception as error: raise AzureProviderException( @@ -240,16 +241,16 @@ def _create_virtual_network(self, region, resource_group_name, vnet_name): """ Create a vnet in the given resource group with default address space. """ + vnet_config = { + 'location': region, + 'address_space': { + 'address_prefixes': ['10.0.0.0/27'] + } + } + try: vnet_operation = self.network.virtual_networks.create_or_update( - resource_group_name, - vnet_name, - { - 'location': region, - 'address_space': { - 'address_prefixes': ['10.0.0.0/27'] - } - } + resource_group_name, vnet_name, vnet_config ) except Exception as error: raise AzureProviderException( @@ -258,16 +259,16 @@ def _create_virtual_network(self, region, resource_group_name, vnet_name): vnet_operation.wait() - def _create_vm_parameters(self, interface): + def _create_vm_config(self, interface): """ - Create the VM parameters dictionary. + Create the VM config dictionary. Requires an existing network interface object. """ # Split image ID into it's components. self._process_image_id() - return { + vm_config = { 'location': self.region, 'os_profile': { 'computer_name': self.running_instance_id, @@ -303,6 +304,8 @@ def _create_vm_parameters(self, interface): } } + return vm_config + def _get_management_client(self, client_class): """ Return instance of resource management client. @@ -407,8 +410,8 @@ def _launch_instance(self): ) # Get dictionary of VM parameters and create instance. - vm_parameters = self._create_vm_parameters(interface) - self._create_vm(vm_parameters) + vm_config = self._create_vm_config(interface) + self._create_vm(vm_config) except Exception: try: self._terminate_instance() From 48f1818b7cf2a7536f91a070700ae401d1db874d Mon Sep 17 00:00:00 2001 From: Sean Marlow Date: Wed, 28 Feb 2018 13:09:07 -0700 Subject: [PATCH 09/11] Fix alpha order of methods. --- ipa/ipa_azure.py | 114 +++++++++++++++++++++++------------------------ 1 file changed, 57 insertions(+), 57 deletions(-) diff --git a/ipa/ipa_azure.py b/ipa/ipa_azure.py index a51037c5..3aaef202 100644 --- a/ipa/ipa_azure.py +++ b/ipa/ipa_azure.py @@ -202,24 +202,6 @@ def _create_resource_group(self, region, resource_group_name): 'Unable to create resource group: {0}.'.format(error) ) - def _create_vm(self, vm_config): - """ - Attempt to create or update VM instance based on vm_parameters config. - """ - try: - vm_operation = self.compute.virtual_machines.create_or_update( - self.running_instance_id, self.running_instance_id, - vm_config - ) - except Exception as error: - raise AzureProviderException( - 'An exception occurred creating virtual machine: {0}'.format( - error - ) - ) - - vm_operation.wait() - def _create_subnet(self, resource_group_name, subnet_name, vnet_name): """ Create a subnet in the provided vnet and resource group. @@ -259,6 +241,24 @@ def _create_virtual_network(self, region, resource_group_name, vnet_name): vnet_operation.wait() + def _create_vm(self, vm_config): + """ + Attempt to create or update VM instance based on vm_parameters config. + """ + try: + vm_operation = self.compute.virtual_machines.create_or_update( + self.running_instance_id, self.running_instance_id, + vm_config + ) + except Exception as error: + raise AzureProviderException( + 'An exception occurred creating virtual machine: {0}'.format( + error + ) + ) + + vm_operation.wait() + def _create_vm_config(self, interface): """ Create the VM config dictionary. @@ -306,6 +306,33 @@ def _create_vm_config(self, interface): return vm_config + def _get_instance(self): + """ + Return the instance matching the running_instance_id. + """ + try: + instance = self.compute.virtual_machines.get( + self.running_instance_id, self.running_instance_id, + expand='instanceView' + ) + except Exception as error: + raise AzureProviderException( + 'Unable to retrieve instance: {0}'.format(error) + ) + + return instance + + def _get_instance_state(self): + """ + Retrieve state of instance. + """ + instance = self._get_instance() + statuses = instance.instance_view.statuses + + for status in statuses: + if status.code.startswith('PowerState'): + return status.display_status + def _get_management_client(self, client_class): """ Return instance of resource management client. @@ -337,33 +364,6 @@ def _get_ssh_public_key(self): key = ipa_utils.generate_public_ssh_key(self.ssh_private_key) return key.decode() - def _get_instance(self): - """ - Return the instance matching the running_instance_id. - """ - try: - instance = self.compute.virtual_machines.get( - self.running_instance_id, self.running_instance_id, - expand='instanceView' - ) - except Exception as error: - raise AzureProviderException( - 'Unable to retrieve instance: {0}'.format(error) - ) - - return instance - - def _get_instance_state(self): - """ - Retrieve state of instance. - """ - instance = self._get_instance() - statuses = instance.instance_view.statuses - - for status in statuses: - if status.code.startswith('PowerState'): - return status.display_status - def _is_instance_running(self): """ Return True if instance is in running state. @@ -444,6 +444,18 @@ def _process_image_id(self): '{Publisher}:{Offer}:{Sku}:{Version}.' ) + def _set_default_resource_names(self): + """ + Generate names for resources based on the running_instance_id. + """ + self.ip_config_name = ''.join([ + self.running_instance_id, '-ip-config' + ]) + self.nic_name = ''.join([self.running_instance_id, '-nic']) + self.subnet_name = ''.join([self.running_instance_id, '-subnet']) + self.vnet_name = ''.join([self.running_instance_id, '-vnet']) + self.public_ip_name = ''.join([self.running_instance_id, '-public-ip']) + def _set_image_id(self): """ If an existing instance is used get image id from deployment. @@ -480,18 +492,6 @@ def _set_instance_ip(self): self.instance_ip = ip_address - def _set_default_resource_names(self): - """ - Generate names for resources based on the running_instance_id. - """ - self.ip_config_name = ''.join([ - self.running_instance_id, '-ip-config' - ]) - self.nic_name = ''.join([self.running_instance_id, '-nic']) - self.subnet_name = ''.join([self.running_instance_id, '-subnet']) - self.vnet_name = ''.join([self.running_instance_id, '-vnet']) - self.public_ip_name = ''.join([self.running_instance_id, '-public-ip']) - def _start_instance(self): """ Start the instance. From 14bc8a9678ce497e2e31b991385fa0ea483a1d61 Mon Sep 17 00:00:00 2001 From: Sean Marlow Date: Wed, 28 Feb 2018 13:33:53 -0700 Subject: [PATCH 10/11] Rename operations to setup. Split VM config into separate dictionary components. --- ipa/ipa_azure.py | 100 +++++++++++++++++++++++++---------------------- 1 file changed, 54 insertions(+), 46 deletions(-) diff --git a/ipa/ipa_azure.py b/ipa/ipa_azure.py index 3aaef202..afb749af 100644 --- a/ipa/ipa_azure.py +++ b/ipa/ipa_azure.py @@ -154,7 +154,7 @@ def _create_network_interface( } try: - nic_operation = self.network.network_interfaces.create_or_update( + nic_setup = self.network.network_interfaces.create_or_update( resource_group_name, nic_name, nic_config ) except Exception as error: @@ -164,7 +164,7 @@ def _create_network_interface( ) ) - return nic_operation.result() + return nic_setup.result() def _create_public_ip(self, public_ip_name, resource_group_name, region): """ @@ -176,7 +176,7 @@ def _create_public_ip(self, public_ip_name, resource_group_name, region): } try: - public_ip_operation = \ + public_ip_setup = \ self.network.public_ip_addresses.create_or_update( resource_group_name, public_ip_name, public_ip_config ) @@ -185,7 +185,7 @@ def _create_public_ip(self, public_ip_name, resource_group_name, region): 'Unable to create public IP: {0}.'.format(error) ) - return public_ip_operation.result() + return public_ip_setup.result() def _create_resource_group(self, region, resource_group_name): """ @@ -209,7 +209,7 @@ def _create_subnet(self, resource_group_name, subnet_name, vnet_name): subnet_config = {'address_prefix': '10.0.0.0/29'} try: - subnet_operation = self.network.subnets.create_or_update( + subnet_setup = self.network.subnets.create_or_update( resource_group_name, vnet_name, subnet_name, subnet_config ) except Exception as error: @@ -217,7 +217,7 @@ def _create_subnet(self, resource_group_name, subnet_name, vnet_name): 'Unable to create subnet: {0}.'.format(error) ) - return subnet_operation.result() + return subnet_setup.result() def _create_virtual_network(self, region, resource_group_name, vnet_name): """ @@ -231,7 +231,7 @@ def _create_virtual_network(self, region, resource_group_name, vnet_name): } try: - vnet_operation = self.network.virtual_networks.create_or_update( + vnet_setup = self.network.virtual_networks.create_or_update( resource_group_name, vnet_name, vnet_config ) except Exception as error: @@ -239,14 +239,14 @@ def _create_virtual_network(self, region, resource_group_name, vnet_name): 'Unable to create vnet: {0}.'.format(error) ) - vnet_operation.wait() + vnet_setup.wait() def _create_vm(self, vm_config): """ Attempt to create or update VM instance based on vm_parameters config. """ try: - vm_operation = self.compute.virtual_machines.create_or_update( + vm_setup = self.compute.virtual_machines.create_or_update( self.running_instance_id, self.running_instance_id, vm_config ) @@ -257,7 +257,7 @@ def _create_vm(self, vm_config): ) ) - vm_operation.wait() + vm_setup.wait() def _create_vm_config(self, interface): """ @@ -268,42 +268,50 @@ def _create_vm_config(self, interface): # Split image ID into it's components. self._process_image_id() - vm_config = { - 'location': self.region, - 'os_profile': { - 'computer_name': self.running_instance_id, - 'admin_username': self.ssh_user, - 'linux_configuration': { - 'disable_password_authentication': True, - 'ssh': { - 'public_keys': [{ - 'path': '/home/{0}/.ssh/authorized_keys'.format( - self.ssh_user - ), - 'key_data': self.ssh_public_key - }] - } - } - }, - 'hardware_profile': { - 'vm_size': self.instance_type or AZURE_DEFAULT_TYPE - }, - 'storage_profile': { - 'image_reference': { - 'publisher': self.image_publisher, - 'offer': self.image_offer, - 'sku': self.image_sku, - 'version': self.image_version - }, + hardware_profile = { + 'vm_size': self.instance_type or AZURE_DEFAULT_TYPE + } + + network_profile = { + 'network_interfaces': [{ + 'id': interface.id, + 'primary': True + }] + } + + storage_profile = { + 'image_reference': { + 'publisher': self.image_publisher, + 'offer': self.image_offer, + 'sku': self.image_sku, + 'version': self.image_version }, - 'network_profile': { - 'network_interfaces': [{ - 'id': interface.id, - 'primary': True - }] + } + + os_profile = { + 'computer_name': self.running_instance_id, + 'admin_username': self.ssh_user, + 'linux_configuration': { + 'disable_password_authentication': True, + 'ssh': { + 'public_keys': [{ + 'path': '/home/{0}/.ssh/authorized_keys'.format( + self.ssh_user + ), + 'key_data': self.ssh_public_key + }] + } } } + vm_config = { + 'location': self.region, + 'os_profile': os_profile, + 'hardware_profile': hardware_profile, + 'storage_profile': storage_profile, + 'network_profile': network_profile + } + return vm_config def _get_instance(self): @@ -497,7 +505,7 @@ def _start_instance(self): Start the instance. """ try: - start_operation = self.compute.virtual_machines.start( + vm_start = self.compute.virtual_machines.start( self.running_instance_id, self.running_instance_id ) except Exception as error: @@ -505,14 +513,14 @@ def _start_instance(self): 'Unable to start instance: {0}.'.format(error) ) - start_operation.wait() + vm_start.wait() def _stop_instance(self): """ Stop the instance. """ try: - stop_operation = self.compute.virtual_machines.power_off( + vm_stop = self.compute.virtual_machines.power_off( self.running_instance_id, self.running_instance_id ) except Exception as error: @@ -520,7 +528,7 @@ def _stop_instance(self): 'Unable to stop instance: {0}.'.format(error) ) - stop_operation.wait() + vm_stop.wait() def _terminate_instance(self): """ From 91aab6fde3e0161579b2da8a1c1b781ccdd40484 Mon Sep 17 00:00:00 2001 From: Sean Marlow Date: Thu, 15 Mar 2018 09:13:07 -0600 Subject: [PATCH 11/11] Update spec with azure mgmt requirements. Remove azurectl. --- package/python3-ipa.spec | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/package/python3-ipa.spec b/package/python3-ipa.spec index bf2500af..4f0bb23f 100644 --- a/package/python3-ipa.spec +++ b/package/python3-ipa.spec @@ -28,7 +28,10 @@ BuildRequires: python3-devel BuildRequires: python3-setuptools %if %{with test} BuildRequires: python3-apache-libcloud -BuildRequires: python3-azurectl >= 3.0.1 +BuildRequires: python3-azure-common +BuildRequires: python3-azure-mgmt-compute +BuildRequires: python3-azure-mgmt-network +BuildRequires: python3-azure-mgmt-resource BuildRequires: python3-click BuildRequires: python3-coverage BuildRequires: python3-cryptography @@ -40,7 +43,10 @@ BuildRequires: python3-PyYAML BuildRequires: python3-testinfra %endif Requires: python3-apache-libcloud -Requires: python3-azurectl >= 3.0.1 +Requires: python3-azure-common +Requires: python3-azure-mgmt-compute +Requires: python3-azure-mgmt-network +Requires: python3-azure-mgmt-resource Requires: python3-click Requires: python3-cryptography Requires: python3-paramiko